x86/entry/64: Teach idtentry to use the IRQ stack
We don't specifically need IST for things like kprobes, but we do
want to avoid rare, surprising extra stack usage if a kprobe hits
with a deep stack.
Teach idtentry to use the IRQ stack for selected entries.
This implementation uses the IRQ stack even if we entered from user
mode. This disallows tricks like ist_begin_non_atomic. If we ever
need such a trick in one of these entries, we can rework this. For
now, let's keep it simple.
Signed-off-by: Andy Lutomirski <luto@kernel.org>
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d366fb8..5a129f5 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -756,13 +756,17 @@
*/
#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8)
-.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
+.macro idtentry sym do_sym has_error_code:req irqstack=0 paranoid=0 shift_ist=-1
ENTRY(\sym)
/* Sanity check */
.if \shift_ist != -1 && \paranoid == 0
.error "using shift_ist requires paranoid=1"
.endif
+ .if \irqstack && \paranoid
+ .error "using irqstack requires !paranoid"
+ .endif
+
ASM_CLAC
PARAVIRT_ADJUST_EXCEPTION_FRAME
@@ -804,8 +808,16 @@
subq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
.endif
+ .if \irqstack
+ ENTER_IRQ_STACK old_rsp=%rcx
+ .endif
+
call \do_sym
+ .if \irqstack
+ LEAVE_IRQ_STACK
+ .endif
+
.if \shift_ist != -1
addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
.endif
