x86: continued work on thread_info separation from stack
More work on this.
Signed-off-by: Peter P Waskiewicz Jr <peter.p.waskiewicz.jr@intel.com>
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 4299eb0..85c98a8 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -135,7 +135,12 @@
CFI_REL_OFFSET rsp,0
pushfq_cfi
/*CFI_REL_OFFSET rflags,0*/
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ movl TI_sysenter_return(%rcx),%r10d
+#else
movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
+#endif
CFI_REGISTER rip,r10
pushq_cfi $__USER32_CS
/*CFI_REL_OFFSET cs,0*/
@@ -151,8 +156,14 @@
1: movl (%rbp),%ebp
_ASM_EXTABLE(1b,ia32_badarg)
ASM_CLAC
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ orl $TS_COMPAT,TI_status(%rcx)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
+#else
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
CFI_REMEMBER_STATE
jnz sysenter_tracesys
cmpq $(IA32_NR_syscalls-1),%rax
@@ -164,10 +175,20 @@
movq %rax,RAX-ARGOFFSET(%rsp)
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $_TIF_ALLWORK_MASK,TI_flags(%rcx)
+#else
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jnz sysexit_audit
sysexit_from_sys_call:
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ andl $~TS_COMPAT,TI_status(%rcx)
+#else
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
/* clear IF, that popfq doesn't enable interrupts early */
andl $~0x200,EFLAGS-R11(%rsp)
movl RIP-R11(%rsp),%edx /* User %eip */
@@ -204,7 +225,12 @@
.endm
.macro auditsys_exit exit
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
+#else
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jnz ia32_ret_from_sys_call
TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_NONE)
@@ -219,7 +245,12 @@
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl %edi,TI_flags(%rcx)
+#else
testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jz \exit
CLEAR_RREGS -ARGOFFSET
jmp int_with_check
@@ -237,7 +268,12 @@
sysenter_tracesys:
#ifdef CONFIG_AUDITSYSCALL
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
+#else
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jz sysenter_auditsys
#endif
SAVE_REST
@@ -308,8 +344,14 @@
1: movl (%r8),%r9d
_ASM_EXTABLE(1b,ia32_badarg)
ASM_CLAC
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ orl $TS_COMPAT,TI_status(%rcx)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
+#else
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
CFI_REMEMBER_STATE
jnz cstar_tracesys
cmpq $IA32_NR_syscalls-1,%rax
@@ -321,10 +363,20 @@
movq %rax,RAX-ARGOFFSET(%rsp)
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $_TIF_ALLWORK_MASK,TI_flags(%rcx)
+#else
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jnz sysretl_audit
sysretl_from_sys_call:
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ andl $~TS_COMPAT,TI_status(%rcx)
+#else
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
RESTORE_ARGS 0,-ARG_SKIP,0,0,0
movl RIP-ARGOFFSET(%rsp),%ecx
CFI_REGISTER rip,rcx
@@ -352,7 +404,12 @@
cstar_tracesys:
#ifdef CONFIG_AUDITSYSCALL
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
+#else
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jz cstar_auditsys
#endif
xchgl %r9d,%ebp
@@ -418,8 +475,14 @@
/* note the registers are not zero extended to the sf.
this could be a problem. */
SAVE_ARGS 0,1,0
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ orl $TS_COMPAT,TI_status(%rcx)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
+#else
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jnz ia32_tracesys
cmpq $(IA32_NR_syscalls-1),%rax
ja ia32_badsys
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ff3adc0..dda23b7 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -204,21 +204,33 @@
*/
#ifndef __ASSEMBLY__
DECLARE_PER_CPU(unsigned long, kernel_stack);
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+DECLARE_PER_CPU(struct thread_info *, current_ti);
+#endif
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ ti = (void *)(this_cpu_read_stable(current_ti));
+#else
ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
+#endif
return ti;
}
#else /* !__ASSEMBLY__ */
/* how to get the thread information struct from ASM */
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+#define GET_THREAD_INFO(reg) \
+ movq PER_CPU_VAR(current_ti),reg
+#else
#define GET_THREAD_INFO(reg) \
movq PER_CPU_VAR(kernel_stack),reg ; \
subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
+#endif /* CONFIG_ARCH_TASK_THREAD_MERGED */
/*
* Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 439bbd4..fe66f13 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1095,6 +1095,10 @@
&GET_INIT_TASK;
EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(struct thread_info *, current_ti) ____cacheline_aligned =
+ &init_task_thread.thread_info;
+EXPORT_PER_CPU_SYMBOL(current_ti);
+
DEFINE_PER_CPU(unsigned long, kernel_stack) =
(unsigned long)&init_stack;
EXPORT_PER_CPU_SYMBOL(kernel_stack);
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1e96c36..171edb1 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -613,9 +613,15 @@
ENABLE_INTERRUPTS(CLBR_NONE)
SAVE_ARGS 8,0
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
+ /* pjw XXX - probably drop this? */
movq %rcx,RIP-ARGOFFSET(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
+#else
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jnz tracesys
system_call_fastpath:
#if __SYSCALL_MASK == ~0
@@ -639,7 +645,12 @@
LOCKDEP_SYS_EXIT
DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ movl TI_flags(%rcx),%edx
+#else
movl TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET),%edx
+#endif
andl %edi,%edx
jnz sysret_careful
CFI_REMEMBER_STATE
@@ -722,7 +733,12 @@
/* Do syscall tracing */
tracesys:
#ifdef CONFIG_AUDITSYSCALL
+#ifdef CONFIG_ARCH_TASK_THREAD_MERGED
+ GET_THREAD_INFO(%rcx)
+ testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
+#else
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
+#endif
jz auditsys
#endif
SAVE_REST
