x86_64/entry: Mark C noinstr __noendbr Mark all the C noinstr entry functions __noendbr where possible. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 14cd43d..b498fdd 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c
@@ -15,7 +15,7 @@ * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered * the VM-Exit is held pending until it's unblocked in the host. */ -noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) +noinstr __noendbr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector) { if (event_type == EVENT_TYPE_EXTINT) { #ifdef CONFIG_X86_64
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c index fbe2d10..b5a2834 100644 --- a/arch/x86/entry/entry_fred.c +++ b/arch/x86/entry/entry_fred.c
@@ -235,7 +235,7 @@ static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code) } } -__visible noinstr void fred_entry_from_user(struct pt_regs *regs) +__visible noinstr __noendbr void fred_entry_from_user(struct pt_regs *regs) { unsigned long error_code = regs->orig_ax; @@ -267,7 +267,7 @@ __visible noinstr void fred_entry_from_user(struct pt_regs *regs) return fred_bad_type(regs, error_code); } -__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) +__visible noinstr __noendbr void fred_entry_from_kernel(struct pt_regs *regs) { unsigned long error_code = regs->orig_ax; @@ -296,7 +296,7 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs) } #if IS_ENABLED(CONFIG_KVM_INTEL) -__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs) +__visible noinstr __noendbr void __fred_entry_from_kvm(struct pt_regs *regs) { switch (regs->fred_ss.type) { case EVENT_TYPE_EXTINT:
diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c index 632a0c7..303b671 100644 --- a/arch/x86/entry/syscall_32.c +++ b/arch/x86/entry/syscall_32.c
@@ -131,7 +131,7 @@ static __always_inline bool int80_is_external(void) * eax: system call number * ebx, ecx, edx, esi, edi, ebp: arg1 - arg 6 */ -__visible noinstr void do_int80_emulation(struct pt_regs *regs) +__visible noinstr __noendbr void do_int80_emulation(struct pt_regs *regs) { int nr; @@ -248,7 +248,7 @@ DEFINE_FREDENTRY_RAW(int80_emulation) #else /* CONFIG_IA32_EMULATION */ /* Handles int $0x80 on a 32bit kernel */ -__visible noinstr void do_int80_syscall_32(struct pt_regs *regs) +__visible noinstr __noendbr void do_int80_syscall_32(struct pt_regs *regs) { int nr = syscall_32_enter(regs); @@ -268,7 +268,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) } #endif /* !CONFIG_IA32_EMULATION */ -static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) +static noinstr __noendbr bool __do_fast_syscall_32(struct pt_regs *regs) { int nr = syscall_32_enter(regs); int res; @@ -317,7 +317,7 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) } /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */ -__visible noinstr bool do_fast_syscall_32(struct pt_regs *regs) +__visible noinstr __noendbr bool do_fast_syscall_32(struct pt_regs *regs) { /* * Called using the internal vDSO SYSENTER/SYSCALL32 calling @@ -364,7 +364,7 @@ __visible noinstr bool do_fast_syscall_32(struct pt_regs *regs) } /* Returns true to return using SYSEXIT/SYSRETL, or false to use IRET */ -__visible noinstr bool do_SYSENTER_32(struct pt_regs *regs) +__visible noinstr __noendbr bool do_SYSENTER_32(struct pt_regs *regs) { /* SYSENTER loses RSP, but the vDSO saved it in RBP. */ regs->sp = regs->bp;
diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c index 378ce80..2ab6ae8 100644 --- a/arch/x86/entry/syscall_64.c +++ b/arch/x86/entry/syscall_64.c
@@ -33,7 +33,7 @@ const sys_call_ptr_t sys_call_table[] = { #undef __SYSCALL #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs); -static noinstr long x64_sys_call(const struct pt_regs *regs, unsigned int nr) +static noinstr __noendbr long x64_sys_call(const struct pt_regs *regs, unsigned int nr) { /* * Because -fno-jump-tables, this compiles into a binary branch tree @@ -51,7 +51,7 @@ static noinstr long x64_sys_call(const struct pt_regs *regs, unsigned int nr) } #ifdef CONFIG_X86_X32_ABI -static noinstr long x32_sys_call(const struct pt_regs *regs, unsigned int nr) +static noinstr __noendbr long x32_sys_call(const struct pt_regs *regs, unsigned int nr) { instrumentation_begin(); switch (nr) { @@ -100,7 +100,7 @@ static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr) } /* Returns true to return using SYSRET, or false to use IRET */ -__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr) +__visible noinstr __noendbr bool do_syscall_64(struct pt_regs *regs, int nr) { nr = syscall_enter_from_user_mode(regs, nr);
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h index eca24b5..0586f01 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h
@@ -97,6 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void) } #define arch_exit_to_user_mode arch_exit_to_user_mode -extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector); +extern __noendbr void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector); #endif
diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h index 18a2f81..17600c4 100644 --- a/arch/x86/include/asm/fred.h +++ b/arch/x86/include/asm/fred.h
@@ -67,9 +67,9 @@ void asm_fred_entrypoint_user(void); void asm_fred_entrypoint_kernel(void); void asm_fred_entry_from_kvm(struct fred_ss); -__visible void fred_entry_from_user(struct pt_regs *regs); -__visible void fred_entry_from_kernel(struct pt_regs *regs); -__visible void __fred_entry_from_kvm(struct pt_regs *regs); +__visible __noendbr void fred_entry_from_user(struct pt_regs *regs); +__visible __noendbr void fred_entry_from_kernel(struct pt_regs *regs); +__visible __noendbr void __fred_entry_from_kvm(struct pt_regs *regs); /* Can be called from noinstr code, thus __always_inline */ static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector)
diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h index 59a4060..f3ecf4a 100644 --- a/arch/x86/include/asm/syscall.h +++ b/arch/x86/include/asm/syscall.h
@@ -164,13 +164,13 @@ static inline int syscall_get_arch(struct task_struct *task) ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64; } -bool do_syscall_64(struct pt_regs *regs, int nr); -void do_int80_emulation(struct pt_regs *regs); +__noendbr bool do_syscall_64(struct pt_regs *regs, int nr); +__noendbr void do_int80_emulation(struct pt_regs *regs); #endif /* CONFIG_X86_32 */ -void do_int80_syscall_32(struct pt_regs *regs); -bool do_fast_syscall_32(struct pt_regs *regs); -bool do_SYSENTER_32(struct pt_regs *regs); +__noendbr void do_int80_syscall_32(struct pt_regs *regs); +__noendbr bool do_fast_syscall_32(struct pt_regs *regs); +__noendbr bool do_SYSENTER_32(struct pt_regs *regs); #endif /* _ASM_X86_SYSCALL_H */