| From a56aad45284b7019ba3e8362b284b430f9f218b2 Mon Sep 17 00:00:00 2001 |
| From: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Date: Thu, 11 Aug 2011 15:31:31 +0200 |
| Subject: [PATCH 034/262] mm: pagefault_disabled() |
| |
| Wrap the test for pagefault_disabled() into a helper, this allows us |
| to remove the need for current->pagefault_disabled on !-rt kernels. |
| |
| Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org |
| --- |
| arch/alpha/mm/fault.c | 2 +- |
| arch/arm/mm/fault.c | 2 +- |
| arch/avr32/mm/fault.c | 3 +-- |
| arch/cris/mm/fault.c | 2 +- |
| arch/frv/mm/fault.c | 2 +- |
| arch/ia64/mm/fault.c | 2 +- |
| arch/m32r/mm/fault.c | 2 +- |
| arch/m68k/mm/fault.c | 2 +- |
| arch/microblaze/mm/fault.c | 2 +- |
| arch/mips/mm/fault.c | 2 +- |
| arch/mn10300/mm/fault.c | 2 +- |
| arch/parisc/mm/fault.c | 2 +- |
| arch/powerpc/mm/fault.c | 2 +- |
| arch/s390/mm/fault.c | 6 +++--- |
| arch/score/mm/fault.c | 2 +- |
| arch/sh/mm/fault_32.c | 2 +- |
| arch/sparc/mm/fault_32.c | 2 +- |
| arch/sparc/mm/fault_64.c | 2 +- |
| arch/tile/mm/fault.c | 2 +- |
| arch/um/kernel/trap.c | 2 +- |
| arch/x86/mm/fault.c | 2 +- |
| arch/xtensa/mm/fault.c | 2 +- |
| include/linux/sched.h | 14 ++++++++++++++ |
| kernel/fork.c | 2 ++ |
| 24 files changed, 40 insertions(+), 25 deletions(-) |
| |
| diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c |
| index 0ddf6c0..ab6b9d13 100644 |
| --- a/arch/alpha/mm/fault.c |
| +++ b/arch/alpha/mm/fault.c |
| @@ -106,7 +106,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, |
| |
| /* If we're in an interrupt context, or have no user context, |
| we must not take the fault. */ |
| - if (!mm || in_atomic() || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| #ifdef CONFIG_ALPHA_LARGE_VMALLOC |
| diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c |
| index 40bd40f..17a9f4a 100644 |
| --- a/arch/arm/mm/fault.c |
| +++ b/arch/arm/mm/fault.c |
| @@ -279,7 +279,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| /* |
| diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c |
| index 623a027..155ad8d 100644 |
| --- a/arch/avr32/mm/fault.c |
| +++ b/arch/avr32/mm/fault.c |
| @@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs) |
| * If we're in an interrupt or have no user context, we must |
| * not take the fault... |
| */ |
| - if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) || |
| - current->pagefault_disabled) |
| + if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled()) |
| goto no_context; |
| |
| local_irq_enable(); |
| diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c |
| index 1c1630e..886be8e 100644 |
| --- a/arch/cris/mm/fault.c |
| +++ b/arch/cris/mm/fault.c |
| @@ -112,7 +112,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs, |
| * user context, we must not take the fault. |
| */ |
| |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c |
| index 6372088..e87972c 100644 |
| --- a/arch/frv/mm/fault.c |
| +++ b/arch/frv/mm/fault.c |
| @@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c |
| index 721dbb9..8ca850e 100644 |
| --- a/arch/ia64/mm/fault.c |
| +++ b/arch/ia64/mm/fault.c |
| @@ -88,7 +88,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re |
| /* |
| * If we're in an interrupt or have no user context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| #ifdef CONFIG_VIRTUAL_MEM_MAP |
| diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c |
| index 1eec8af..6945056 100644 |
| --- a/arch/m32r/mm/fault.c |
| +++ b/arch/m32r/mm/fault.c |
| @@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, |
| * If we're in an interrupt or have no user context or are running in an |
| * atomic region then we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto bad_area_nosemaphore; |
| |
| /* When running in the kernel we expect faults to occur only to |
| diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c |
| index 7a4cc68..46b8cce 100644 |
| --- a/arch/m68k/mm/fault.c |
| +++ b/arch/m68k/mm/fault.c |
| @@ -84,7 +84,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c |
| index 621ad49..a438434 100644 |
| --- a/arch/microblaze/mm/fault.c |
| +++ b/arch/microblaze/mm/fault.c |
| @@ -106,7 +106,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, |
| if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) |
| is_write = 0; |
| |
| - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { |
| + if (unlikely(!mm || pagefault_disabled())) { |
| if (kernel_mode(regs)) |
| goto bad_area_nosemaphore; |
| |
| diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c |
| index 3468e69..39a3180 100644 |
| --- a/arch/mips/mm/fault.c |
| +++ b/arch/mips/mm/fault.c |
| @@ -89,7 +89,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, unsigned long writ |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto bad_area_nosemaphore; |
| |
| retry: |
| diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c |
| index ff5acf5..5d9e10f 100644 |
| --- a/arch/mn10300/mm/fault.c |
| +++ b/arch/mn10300/mm/fault.c |
| @@ -167,7 +167,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code, |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c |
| index 09ecc8a..df22f39 100644 |
| --- a/arch/parisc/mm/fault.c |
| +++ b/arch/parisc/mm/fault.c |
| @@ -176,7 +176,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, |
| unsigned long acc_type; |
| int fault; |
| |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c |
| index 6642dff..7bd8f27 100644 |
| --- a/arch/powerpc/mm/fault.c |
| +++ b/arch/powerpc/mm/fault.c |
| @@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, |
| if (!arch_irq_disabled_regs(regs)) |
| local_irq_enable(); |
| |
| - if (in_atomic() || mm == NULL || current->pagefault_disabled) { |
| + if (!mm || pagefault_disabled()) { |
| if (!user_mode(regs)) |
| return SIGSEGV; |
| /* in_atomic() in user mode is really bad, |
| diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c |
| index a089ec9..eef4110 100644 |
| --- a/arch/s390/mm/fault.c |
| +++ b/arch/s390/mm/fault.c |
| @@ -283,8 +283,8 @@ static inline int do_exception(struct pt_regs *regs, int access) |
| * user context. |
| */ |
| fault = VM_FAULT_BADCONTEXT; |
| - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || |
| - tsk->pagefault_disabled)) |
| + if (unlikely(!user_space_fault(trans_exc_code) || |
| + !mm || pagefault_disabled())) |
| goto out; |
| |
| address = trans_exc_code & __FAIL_ADDR_MASK; |
| @@ -417,7 +417,7 @@ void __kprobes do_asce_exception(struct pt_regs *regs) |
| |
| trans_exc_code = regs->int_parm_long; |
| if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || |
| - current->pagefault_disabled)) |
| + pagefault_disabled())) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c |
| index 4c12824..59fccbe 100644 |
| --- a/arch/score/mm/fault.c |
| +++ b/arch/score/mm/fault.c |
| @@ -72,7 +72,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto bad_area_nosemaphore; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c |
| index 8c3ae42..1aca948 100644 |
| --- a/arch/sh/mm/fault_32.c |
| +++ b/arch/sh/mm/fault_32.c |
| @@ -166,7 +166,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, |
| * If we're in an interrupt, have no user context or are running |
| * in an atomic region then we must not take the fault: |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c |
| index ffaa99e..77b37e0 100644 |
| --- a/arch/sparc/mm/fault_32.c |
| +++ b/arch/sparc/mm/fault_32.c |
| @@ -248,7 +248,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto no_context; |
| |
| perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
| diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c |
| index fbd5e11..ea4e14b 100644 |
| --- a/arch/sparc/mm/fault_64.c |
| +++ b/arch/sparc/mm/fault_64.c |
| @@ -323,7 +323,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) |
| * If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_enabled) |
| + if (!mm || pagefault_disabled()) |
| goto intr_or_no_mm; |
| |
| perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
| diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c |
| index f683aad..3e85178 100644 |
| --- a/arch/tile/mm/fault.c |
| +++ b/arch/tile/mm/fault.c |
| @@ -355,7 +355,7 @@ static int handle_page_fault(struct pt_regs *regs, |
| * If we're in an interrupt, have no user context or are running in an |
| * atomic region then we must not take the fault. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) { |
| + if (!mm || pagefault_disabled()) { |
| vma = NULL; /* happy compiler */ |
| goto bad_area_nosemaphore; |
| } |
| diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c |
| index a283400..7878069 100644 |
| --- a/arch/um/kernel/trap.c |
| +++ b/arch/um/kernel/trap.c |
| @@ -37,7 +37,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, |
| * If the fault was during atomic operation, don't take the fault, just |
| * fail. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) |
| + if (!mm || pagefault_disabled()) |
| goto out_nosemaphore; |
| |
| down_read(&mm->mmap_sem); |
| diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c |
| index bb891f2..9d57357 100644 |
| --- a/arch/x86/mm/fault.c |
| +++ b/arch/x86/mm/fault.c |
| @@ -1094,7 +1094,7 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code) |
| * If we're in an interrupt, have no user context or are running |
| * in an atomic region then we must not take the fault: |
| */ |
| - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { |
| + if (unlikely(!mm || pagefault_disabled())) { |
| bad_area_nosemaphore(regs, error_code, address); |
| return; |
| } |
| diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c |
| index cc58b7c..93d33ee 100644 |
| --- a/arch/xtensa/mm/fault.c |
| +++ b/arch/xtensa/mm/fault.c |
| @@ -56,7 +56,7 @@ void do_page_fault(struct pt_regs *regs) |
| /* If we're in an interrupt or have no user |
| * context, we must not take the fault.. |
| */ |
| - if (in_atomic() || !mm || current->pagefault_disabled) { |
| + if (!mm || pagefault_disabled()) { |
| bad_page_fault(regs, address, SIGSEGV); |
| return; |
| } |
| diff --git a/include/linux/sched.h b/include/linux/sched.h |
| index a1b9db2..cc4c59f 100644 |
| --- a/include/linux/sched.h |
| +++ b/include/linux/sched.h |
| @@ -90,6 +90,7 @@ struct sched_param { |
| #include <linux/latencytop.h> |
| #include <linux/cred.h> |
| #include <linux/llist.h> |
| +#include <linux/hardirq.h> |
| |
| #include <asm/processor.h> |
| |
| @@ -1478,7 +1479,9 @@ struct task_struct { |
| /* mutex deadlock detection */ |
| struct mutex_waiter *blocked_on; |
| #endif |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| int pagefault_disabled; |
| +#endif |
| #ifdef CONFIG_TRACE_IRQFLAGS |
| unsigned int irq_events; |
| unsigned long hardirq_enable_ip; |
| @@ -1628,6 +1631,17 @@ struct task_struct { |
| /* Future-safe accessor for struct task_struct's cpus_allowed. */ |
| #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) |
| |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| +static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; } |
| +#else |
| +static inline bool cur_pf_disabled(void) { return false; } |
| +#endif |
| + |
| +static inline bool pagefault_disabled(void) |
| +{ |
| + return in_atomic() || cur_pf_disabled(); |
| +} |
| + |
| /* |
| * Priority of a process goes from 0..MAX_PRIO-1, valid RT |
| * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH |
| diff --git a/kernel/fork.c b/kernel/fork.c |
| index fdf0795..4c9def3 100644 |
| --- a/kernel/fork.c |
| +++ b/kernel/fork.c |
| @@ -1265,7 +1265,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, |
| p->hardirq_context = 0; |
| p->softirq_context = 0; |
| #endif |
| +#ifdef CONFIG_PREEMPT_RT_FULL |
| p->pagefault_disabled = 0; |
| +#endif |
| #ifdef CONFIG_LOCKDEP |
| p->lockdep_depth = 0; /* no locks held yet */ |
| p->curr_chain_key = 0; |
| -- |
| 1.7.10.4 |
| |