| Subject: x86-preempt-lazy.patch |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Thu, 01 Nov 2012 11:03:47 +0100 |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| arch/x86/Kconfig | 1 + |
| arch/x86/include/asm/thread_info.h | 6 ++++++ |
| arch/x86/kernel/asm-offsets.c | 1 + |
| arch/x86/kernel/entry_32.S | 18 +++++++++++++----- |
| arch/x86/kernel/entry_64.S | 24 +++++++++++++++--------- |
| 5 files changed, 36 insertions(+), 14 deletions(-) |
| |
| Index: linux-stable/arch/x86/Kconfig |
| =================================================================== |
| --- linux-stable.orig/arch/x86/Kconfig |
| +++ linux-stable/arch/x86/Kconfig |
| @@ -97,6 +97,7 @@ config X86 |
| select KTIME_SCALAR if X86_32 |
| select GENERIC_STRNCPY_FROM_USER |
| select GENERIC_STRNLEN_USER |
| + select HAVE_PREEMPT_LAZY |
| |
| config INSTRUCTION_DECODER |
| def_bool (KPROBES || PERF_EVENTS || UPROBES) |
| Index: linux-stable/arch/x86/include/asm/thread_info.h |
| =================================================================== |
| --- linux-stable.orig/arch/x86/include/asm/thread_info.h |
| +++ linux-stable/arch/x86/include/asm/thread_info.h |
| @@ -31,6 +31,8 @@ struct thread_info { |
| __u32 cpu; /* current CPU */ |
| int preempt_count; /* 0 => preemptable, |
| <0 => BUG */ |
| + int preempt_lazy_count; /* 0 => lazy preemptable, |
| + <0 => BUG */ |
| mm_segment_t addr_limit; |
| struct restart_block restart_block; |
| void __user *sysenter_return; |
| @@ -83,6 +85,7 @@ struct thread_info { |
| #define TIF_SYSCALL_EMU 6 /* syscall emulation active */ |
| #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ |
| #define TIF_SECCOMP 8 /* secure computing */ |
| +#define TIF_NEED_RESCHED_LAZY 9 /* lazy rescheduling necessary */ |
| #define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */ |
| #define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */ |
| #define TIF_UPROBE 12 /* breakpointed or singlestepping */ |
| @@ -108,6 +111,7 @@ struct thread_info { |
| #define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) |
| #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) |
| #define _TIF_SECCOMP (1 << TIF_SECCOMP) |
| +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY) |
| #define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY) |
| #define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY) |
| #define _TIF_UPROBE (1 << TIF_UPROBE) |
| @@ -155,6 +159,8 @@ struct thread_info { |
| #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY) |
| #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) |
| |
| +#define _TIF_NEED_RESCHED_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY) |
| + |
| #define PREEMPT_ACTIVE 0x10000000 |
| |
| #ifdef CONFIG_X86_32 |
| Index: linux-stable/arch/x86/kernel/asm-offsets.c |
| =================================================================== |
| --- linux-stable.orig/arch/x86/kernel/asm-offsets.c |
| +++ linux-stable/arch/x86/kernel/asm-offsets.c |
| @@ -33,6 +33,7 @@ void common(void) { |
| OFFSET(TI_status, thread_info, status); |
| OFFSET(TI_addr_limit, thread_info, addr_limit); |
| OFFSET(TI_preempt_count, thread_info, preempt_count); |
| + OFFSET(TI_preempt_lazy_count, thread_info, preempt_lazy_count); |
| |
| BLANK(); |
| OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); |
| Index: linux-stable/arch/x86/kernel/entry_32.S |
| =================================================================== |
| --- linux-stable.orig/arch/x86/kernel/entry_32.S |
| +++ linux-stable/arch/x86/kernel/entry_32.S |
| @@ -349,14 +349,22 @@ ENTRY(resume_kernel) |
| DISABLE_INTERRUPTS(CLBR_ANY) |
| cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? |
| jnz restore_all |
| -need_resched: |
| movl TI_flags(%ebp), %ecx # need_resched set ? |
| testb $_TIF_NEED_RESCHED, %cl |
| + jnz 1f |
| + |
| + cmpl $0,TI_preempt_lazy_count(%ebp) # non-zero preempt_lazy_count ? |
| + jnz restore_all |
| + testl $_TIF_NEED_RESCHED_LAZY, %ecx |
| jz restore_all |
| - testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? |
| + |
| +1: testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? |
| jz restore_all |
| call preempt_schedule_irq |
| - jmp need_resched |
| + movl TI_flags(%ebp), %ecx # need_resched set ? |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| + jnz 1b |
| + jmp restore_all |
| END(resume_kernel) |
| #endif |
| CFI_ENDPROC |
| @@ -589,7 +597,7 @@ ENDPROC(system_call) |
| ALIGN |
| RING0_PTREGS_FRAME # can't unwind into user space anyway |
| work_pending: |
| - testb $_TIF_NEED_RESCHED, %cl |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| jz work_notifysig |
| work_resched: |
| call schedule |
| @@ -602,7 +610,7 @@ work_resched: |
| andl $_TIF_WORK_MASK, %ecx # is there any work to be done other |
| # than syscall tracing? |
| jz restore_all |
| - testb $_TIF_NEED_RESCHED, %cl |
| + testl $_TIF_NEED_RESCHED_MASK, %ecx |
| jnz work_resched |
| |
| work_notifysig: # deal with pending signals and |
| Index: linux-stable/arch/x86/kernel/entry_64.S |
| =================================================================== |
| --- linux-stable.orig/arch/x86/kernel/entry_64.S |
| +++ linux-stable/arch/x86/kernel/entry_64.S |
| @@ -560,8 +560,8 @@ sysret_check: |
| /* Handle reschedules */ |
| /* edx: work, edi: workmask */ |
| sysret_careful: |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc sysret_signal |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz sysret_signal |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -673,8 +673,8 @@ GLOBAL(int_with_check) |
| /* First do a reschedule test. */ |
| /* edx: work, edi: workmask */ |
| int_careful: |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc int_very_careful |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz int_very_careful |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -969,8 +969,8 @@ bad_iret: |
| /* edi: workmask, edx: work */ |
| retint_careful: |
| CFI_RESTORE_STATE |
| - bt $TIF_NEED_RESCHED,%edx |
| - jnc retint_signal |
| + testl $_TIF_NEED_RESCHED_MASK,%edx |
| + jz retint_signal |
| TRACE_IRQS_ON |
| ENABLE_INTERRUPTS(CLBR_NONE) |
| pushq_cfi %rdi |
| @@ -1003,9 +1003,15 @@ retint_signal: |
| ENTRY(retint_kernel) |
| cmpl $0,TI_preempt_count(%rcx) |
| jnz retint_restore_args |
| - bt $TIF_NEED_RESCHED,TI_flags(%rcx) |
| + bt $TIF_NEED_RESCHED,TI_flags(%rcx) |
| + jc 1f |
| + |
| + cmpl $0,TI_preempt_lazy_count(%rcx) |
| + jnz retint_restore_args |
| + bt $TIF_NEED_RESCHED_LAZY,TI_flags(%rcx) |
| jnc retint_restore_args |
| - bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ |
| + |
| +1: bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ |
| jnc retint_restore_args |
| call preempt_schedule_irq |
| jmp exit_intr |
| @@ -1437,7 +1443,7 @@ paranoid_userspace: |
| movq %rsp,%rdi /* &pt_regs */ |
| call sync_regs |
| movq %rax,%rsp /* switch stack for scheduling */ |
| - testl $_TIF_NEED_RESCHED,%ebx |
| + testl $_TIF_NEED_RESCHED_MASK,%ebx |
| jnz paranoid_schedule |
| movl %ebx,%edx /* arg3: thread flags */ |
| TRACE_IRQS_ON |