| Subject: x86-preempt-lazy.patch |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Thu, 01 Nov 2012 11:03:47 +0100 |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| arch/x86/Kconfig | 1 + |
| arch/x86/include/asm/thread_info.h | 6 ++++++ |
| arch/x86/kernel/asm-offsets.c | 1 + |
| arch/x86/kernel/entry_32.S | 18 +++++++++++++----- |
| arch/x86/kernel/entry_64.S | 24 +++++++++++++++--------- |
| 5 files changed, 36 insertions(+), 14 deletions(-) |
| |
| --- a/arch/x86/Kconfig |
| +++ b/arch/x86/Kconfig |
| @@ -121,6 +121,7 @@ config X86 |
| select OLD_SIGACTION if X86_32 |
| select COMPAT_OLD_SIGACTION if IA32_EMULATION |
| select RTC_LIB |
| + select HAVE_PREEMPT_LAZY |
| |
| config INSTRUCTION_DECODER |
| def_bool y |
| --- a/arch/x86/include/asm/thread_info.h |
| +++ b/arch/x86/include/asm/thread_info.h |
| @@ -30,6 +30,8 @@ struct thread_info { |
| __u32 cpu; /* current CPU */ |
| int preempt_count; /* 0 => preemptable, |
| <0 => BUG */ |
| + int preempt_lazy_count; /* 0 => lazy preemptable, |
| + <0 => BUG */ |
| mm_segment_t addr_limit; |
| struct restart_block restart_block; |
| void __user *sysenter_return; |
| @@ -81,6 +83,7 @@ struct thread_info { |
| #define TIF_SYSCALL_EMU 6 /* syscall emulation active */ |
| #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ |
| #define TIF_SECCOMP 8 /* secure computing */ |
| +#define TIF_NEED_RESCHED_LAZY 9 /* lazy rescheduling necessary */ |
| #define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ |
| #define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */ |
| #define TIF_UPROBE 12 /* breakpointed or singlestepping */ |
| @@ -106,6 +109,7 @@ struct thread_info { |
| #define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) |
| #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) |
| #define _TIF_SECCOMP (1 << TIF_SECCOMP) |
| +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) |
| #define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) |
| #define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY) |
| #define _TIF_UPROBE (1 << TIF_UPROBE) |
| @@ -156,6 +160,8 @@ struct thread_info { |
| #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY) |
| #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) |
| |
| +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) |
| + |
| #define PREEMPT_ACTIVE 0x10000000 |
| |
| #ifdef CONFIG_X86_32 |
| --- a/arch/x86/kernel/asm-offsets.c |
| +++ b/arch/x86/kernel/asm-offsets.c |
| @@ -33,6 +33,7 @@ void common(void) { |
| OFFSET(TI_status, thread_info, status); |
| OFFSET(TI_addr_limit, thread_info, addr_limit); |
| OFFSET(TI_preempt_count, thread_info, preempt_count); |
| + OFFSET(TI_preempt_lazy_count, thread_info, preempt_lazy_count); |
| |
| BLANK(); |
| OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); |
| --- a/arch/x86/kernel/entry_32.S |
| +++ b/arch/x86/kernel/entry_32.S |
| @@ -364,14 +364,22 @@ ENTRY(resume_kernel) |
| DISABLE_INTERRUPTS(CLBR_ANY) |
| cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? |
| jnz restore_all |
| -need_resched: |
| movl TI_flags(%ebp), %ecx # need_resched set ? |
| testb $_TIF_NEED_RESCHED, %cl |
| + jnz 1f |
| + |
| + cmpl $0,TI_preempt_lazy_count(%ebp) # non-zero preempt_lazy_count ? |
| + jnz restore_all |
| + testl $_TIF_NEED_RESCHED_LAZY, %ecx |
| jz restore_all |
| - testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? |
| + |
| +1: testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? |
| jz restore_all |
| call preempt_schedule_irq |
| - jmp need_resched |
| + movl TI_flags(%ebp), %ecx # need_resched set ? |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| + jnz 1b |
| + jmp restore_all |
| END(resume_kernel) |
| #endif |
| CFI_ENDPROC |
| @@ -607,7 +615,7 @@ ENDPROC(system_call) |
| ALIGN |
| RING0_PTREGS_FRAME # can't unwind into user space anyway |
| work_pending: |
| - testb $_TIF_NEED_RESCHED, %cl |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| jz work_notifysig |
| work_resched: |
| call schedule |
| @@ -620,7 +628,7 @@ work_resched: |
| andl $_TIF_WORK_MASK, %ecx # is there any work to be done other |
| # than syscall tracing? |
| jz restore_all |
| - testb $_TIF_NEED_RESCHED, %cl |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| jnz work_resched |
| |
| work_notifysig: # deal with pending signals and |
| --- a/arch/x86/kernel/entry_64.S |
| +++ b/arch/x86/kernel/entry_64.S |
| @@ -673,8 +673,8 @@ sysret_check: |
| /* Handle reschedules */ |
| /* edx: work, edi: workmask */ |
| sysret_careful: |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc sysret_signal |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz sysret_signal |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -786,8 +786,8 @@ GLOBAL(int_with_check) |
| /* First do a reschedule test. */ |
| /* edx: work, edi: workmask */ |
| int_careful: |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc int_very_careful |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz int_very_careful |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -1086,8 +1086,8 @@ bad_iret: |
| /* edi: workmask, edx: work */ |
| retint_careful: |
| CFI_RESTORE_STATE |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc retint_signal |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz retint_signal |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -1120,9 +1120,15 @@ retint_signal: |
| ENTRY(retint_kernel) |
| cmpl $0,TI_preempt_count(%rcx) |
| jnz retint_restore_args |
| - bt $TIF_NEED_RESCHED,TI_flags(%rcx) |
| + bt $TIF_NEED_RESCHED,TI_flags(%rcx) |
| + jc 1f |
| + |
| + cmpl $0,TI_preempt_lazy_count(%rcx) |
| + jnz retint_restore_args |
| + bt $TIF_NEED_RESCHED_LAZY,TI_flags(%rcx) |
| jnc retint_restore_args |
| - bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ |
| + |
| +1: bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ |
| jnc retint_restore_args |
| call preempt_schedule_irq |
| jmp exit_intr |
| @@ -1524,7 +1530,7 @@ paranoid_userspace: |
| movq %rsp,%rdi /* &pt_regs */ |
| call sync_regs |
| movq %rax,%rsp /* switch stack for scheduling */ |
| - testl $_TIF_NEED_RESCHED,%ebx |
| + testl $_TIF_NEED_RESCHED_MASK,%ebx |
| jnz paranoid_schedule |
| movl %ebx,%edx /* arg3: thread flags */ |
| TRACE_IRQS_ON |