| Subject: sched: Add support for lazy preemption |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Fri, 26 Oct 2012 18:50:54 +0100 |
| |
| It has become an obsession to mitigate the determinism vs. throughput |
| loss of RT. Looking at the mainline semantics of preemption points |
| gives a hint why RT sucks throughput wise for ordinary SCHED_OTHER |
| tasks. One major issue is the wakeup of tasks which are right away |
| preempting the waking task while the waking task holds a lock on which |
| the woken task will block right after having preempted the wakee. In |
| mainline this is prevented due to the implicit preemption disable of |
| spin/rw_lock held regions. On RT this is not possible due to the fully |
| preemptible nature of sleeping spinlocks. |
| |
| Though for a SCHED_OTHER task preempting another SCHED_OTHER task this |
| is really not a correctness issue. RT folks are concerned about |
| SCHED_FIFO/RR tasks preemption and not about the purely fairness |
| driven SCHED_OTHER preemption latencies. |
| |
| So I introduced a lazy preemption mechanism which only applies to |
| SCHED_OTHER tasks preempting another SCHED_OTHER task. Aside of the |
| existing preempt_count each tasks sports now a preempt_lazy_count |
| which is manipulated on lock acquiry and release. This is slightly |
| incorrect as for lazyness reasons I coupled this on |
| migrate_disable/enable so some other mechanisms get the same treatment |
| (e.g. get_cpu_light). |
| |
| Now on the scheduler side instead of setting NEED_RESCHED this sets |
| NEED_RESCHED_LAZY in case of a SCHED_OTHER/SCHED_OTHER preemption and |
| therefor allows to exit the waking task the lock held region before |
| the woken task preempts. That also works better for cross CPU wakeups |
| as the other side can stay in the adaptive spinning loop. |
| |
| For RT class preemption there is no change. This simply sets |
| NEED_RESCHED and forgoes the lazy preemption counter. |
| |
| Initial test do not expose any observable latency increasement, but |
| history shows that I've been proven wrong before :) |
| |
| The lazy preemption mode is per default on, but with |
| CONFIG_SCHED_DEBUG enabled it can be disabled via: |
| |
| # echo NO_PREEMPT_LAZY >/sys/kernel/debug/sched_features |
| |
| and reenabled via |
| |
| # echo PREEMPT_LAZY >/sys/kernel/debug/sched_features |
| |
| The test results so far are very machine and workload dependent, but |
| there is a clear trend that it enhances the non RT workload |
| performance. |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| include/linux/ftrace_event.h | 1 |
| include/linux/preempt.h | 38 ++++++++++++++++++++++++++- |
| include/linux/sched.h | 51 ++++++++++++++++++++++++++++++++---- |
| kernel/Kconfig.preempt | 6 ++++ |
| kernel/sched/core.c | 60 ++++++++++++++++++++++++++++++++++++++++++- |
| kernel/sched/fair.c | 16 +++++------ |
| kernel/sched/features.h | 4 ++ |
| kernel/sched/sched.h | 9 ++++++ |
| kernel/trace/trace.c | 41 +++++++++++++++++------------ |
| kernel/trace/trace.h | 2 + |
| kernel/trace/trace_output.c | 13 +++++++-- |
| 11 files changed, 207 insertions(+), 34 deletions(-) |
| |
| Index: linux-stable/include/linux/ftrace_event.h |
| =================================================================== |
| --- linux-stable.orig/include/linux/ftrace_event.h |
| +++ linux-stable/include/linux/ftrace_event.h |
| @@ -51,6 +51,7 @@ struct trace_entry { |
| int pid; |
| unsigned short migrate_disable; |
| unsigned short padding; |
| + unsigned char preempt_lazy_count; |
| }; |
| |
| #define FTRACE_MAX_EVENT \ |
| Index: linux-stable/include/linux/preempt.h |
| =================================================================== |
| --- linux-stable.orig/include/linux/preempt.h |
| +++ linux-stable/include/linux/preempt.h |
| @@ -23,15 +23,38 @@ |
| |
| #define preempt_count() (current_thread_info()->preempt_count) |
| |
| +#ifdef CONFIG_PREEMPT_LAZY |
| +#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0) |
| +#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0) |
| +#define inc_preempt_lazy_count() add_preempt_lazy_count(1) |
| +#define dec_preempt_lazy_count() sub_preempt_lazy_count(1) |
| +#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count) |
| +#else |
| +#define add_preempt_lazy_count(val) do { } while (0) |
| +#define sub_preempt_lazy_count(val) do { } while (0) |
| +#define inc_preempt_lazy_count() do { } while (0) |
| +#define dec_preempt_lazy_count() do { } while (0) |
| +#define preempt_lazy_count() (0) |
| +#endif |
| + |
| #ifdef CONFIG_PREEMPT |
| |
| asmlinkage void preempt_schedule(void); |
| |
| +# ifdef CONFIG_PREEMPT_LAZY |
| +#define preempt_check_resched() \ |
| +do { \ |
| + if (unlikely(test_thread_flag(TIF_NEED_RESCHED) || \ |
| + test_thread_flag(TIF_NEED_RESCHED_LAZY))) \ |
| + preempt_schedule(); \ |
| +} while (0) |
| +# else |
| #define preempt_check_resched() \ |
| do { \ |
| - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ |
| + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ |
| preempt_schedule(); \ |
| } while (0) |
| +# endif |
| |
| #else /* !CONFIG_PREEMPT */ |
| |
| @@ -48,6 +71,12 @@ do { \ |
| barrier(); \ |
| } while (0) |
| |
| +#define preempt_lazy_disable() \ |
| +do { \ |
| + inc_preempt_lazy_count(); \ |
| + barrier(); \ |
| +} while (0) |
| + |
| #define sched_preempt_enable_no_resched() \ |
| do { \ |
| barrier(); \ |
| @@ -69,6 +98,13 @@ do { \ |
| preempt_check_resched(); \ |
| } while (0) |
| |
| +#define preempt_lazy_enable() \ |
| +do { \ |
| + dec_preempt_lazy_count(); \ |
| + barrier(); \ |
| + preempt_check_resched(); \ |
| +} while (0) |
| + |
| /* For debugging and tracer internals only! */ |
| #define add_preempt_count_notrace(val) \ |
| do { preempt_count() += (val); } while (0) |
| Index: linux-stable/include/linux/sched.h |
| =================================================================== |
| --- linux-stable.orig/include/linux/sched.h |
| +++ linux-stable/include/linux/sched.h |
| @@ -2660,6 +2660,52 @@ static inline int test_tsk_need_resched( |
| return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); |
| } |
| |
| +#ifdef CONFIG_PREEMPT_LAZY |
| +static inline void set_tsk_need_resched_lazy(struct task_struct *tsk) |
| +{ |
| + set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); |
| +} |
| + |
| +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) |
| +{ |
| + clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); |
| +} |
| + |
| +static inline int test_tsk_need_resched_lazy(struct task_struct *tsk) |
| +{ |
| + return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY)); |
| +} |
| + |
| +static inline int need_resched_lazy(void) |
| +{ |
| + return test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +} |
| + |
| +static inline int need_resched_now(void) |
| +{ |
| + return test_thread_flag(TIF_NEED_RESCHED); |
| +} |
| + |
| +static inline int need_resched(void) |
| +{ |
| + return test_thread_flag(TIF_NEED_RESCHED) || |
| + test_thread_flag(TIF_NEED_RESCHED_LAZY); |
| +} |
| +#else |
| +static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { } |
| +static inline int need_resched_lazy(void) { return 0; } |
| + |
| +static inline int need_resched_now(void) |
| +{ |
| + return test_thread_flag(TIF_NEED_RESCHED); |
| +} |
| + |
| +static inline int need_resched(void) |
| +{ |
| + return test_thread_flag(TIF_NEED_RESCHED); |
| +} |
| +#endif |
| + |
| static inline int restart_syscall(void) |
| { |
| set_tsk_thread_flag(current, TIF_SIGPENDING); |
| @@ -2691,11 +2737,6 @@ static inline int signal_pending_state(l |
| return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); |
| } |
| |
| -static inline int need_resched(void) |
| -{ |
| - return unlikely(test_thread_flag(TIF_NEED_RESCHED)); |
| -} |
| - |
| /* |
| * cond_resched() and cond_resched_lock(): latency reduction via |
| * explicit rescheduling in places that are safe. The return |
| Index: linux-stable/kernel/Kconfig.preempt |
| =================================================================== |
| --- linux-stable.orig/kernel/Kconfig.preempt |
| +++ linux-stable/kernel/Kconfig.preempt |
| @@ -6,6 +6,12 @@ config PREEMPT_RT_BASE |
| bool |
| select PREEMPT |
| |
| +config HAVE_PREEMPT_LAZY |
| + bool |
| + |
| +config PREEMPT_LAZY |
| + def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL |
| + |
| choice |
| prompt "Preemption Model" |
| default PREEMPT_NONE |
| Index: linux-stable/kernel/sched/core.c |
| =================================================================== |
| --- linux-stable.orig/kernel/sched/core.c |
| +++ linux-stable/kernel/sched/core.c |
| @@ -534,6 +534,37 @@ void resched_task(struct task_struct *p) |
| smp_send_reschedule(cpu); |
| } |
| |
| +#ifdef CONFIG_PREEMPT_LAZY |
| +void resched_task_lazy(struct task_struct *p) |
| +{ |
| + int cpu; |
| + |
| + if (!sched_feat(PREEMPT_LAZY)) { |
| + resched_task(p); |
| + return; |
| + } |
| + |
| + assert_raw_spin_locked(&task_rq(p)->lock); |
| + |
| + if (test_tsk_need_resched(p)) |
| + return; |
| + |
| + if (test_tsk_need_resched_lazy(p)) |
| + return; |
| + |
| + set_tsk_need_resched_lazy(p); |
| + |
| + cpu = task_cpu(p); |
| + if (cpu == smp_processor_id()) |
| + return; |
| + |
| + /* NEED_RESCHED_LAZY must be visible before we test polling */ |
| + smp_mb(); |
| + if (!tsk_is_polling(p)) |
| + smp_send_reschedule(cpu); |
| +} |
| +#endif |
| + |
| void resched_cpu(int cpu) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| @@ -650,6 +681,17 @@ void resched_task(struct task_struct *p) |
| assert_raw_spin_locked(&task_rq(p)->lock); |
| set_tsk_need_resched(p); |
| } |
| +#ifdef CONFIG_PREEMPT_LAZY |
| +void resched_task_lazy(struct task_struct *p) |
| +{ |
| + if (!sched_feat(PREEMPT_LAZY)) { |
| + resched_task(p); |
| + return; |
| + } |
| + assert_raw_spin_locked(&task_rq(p)->lock); |
| + set_tsk_need_resched_lazy(p); |
| +} |
| +#endif |
| #endif /* CONFIG_SMP */ |
| |
| #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ |
| @@ -1838,6 +1880,9 @@ void sched_fork(struct task_struct *p) |
| /* Want to start with kernel preemption disabled. */ |
| task_thread_info(p)->preempt_count = 1; |
| #endif |
| +#ifdef CONFIG_HAVE_PREEMPT_LAZY |
| + task_thread_info(p)->preempt_lazy_count = 0; |
| +#endif |
| #ifdef CONFIG_SMP |
| plist_node_init(&p->pushable_tasks, MAX_PRIO); |
| #endif |
| @@ -3448,6 +3493,7 @@ void migrate_disable(void) |
| return; |
| } |
| |
| + preempt_lazy_disable(); |
| pin_current_cpu(); |
| p->migrate_disable = 1; |
| preempt_enable(); |
| @@ -3503,6 +3549,7 @@ void migrate_enable(void) |
| |
| unpin_current_cpu(); |
| preempt_enable(); |
| + preempt_lazy_enable(); |
| } |
| EXPORT_SYMBOL(migrate_enable); |
| #else |
| @@ -3603,6 +3650,7 @@ need_resched: |
| put_prev_task(rq, prev); |
| next = pick_next_task(rq); |
| clear_tsk_need_resched(prev); |
| + clear_tsk_need_resched_lazy(prev); |
| rq->skip_clock_update = 0; |
| |
| if (likely(prev != next)) { |
| @@ -3724,6 +3772,14 @@ asmlinkage void __sched notrace preempt_ |
| if (likely(ti->preempt_count || irqs_disabled())) |
| return; |
| |
| +#ifdef CONFIG_PREEMPT_LAZY |
| + /* |
| + * Check for lazy preemption |
| + */ |
| + if (ti->preempt_lazy_count && !test_thread_flag(TIF_NEED_RESCHED)) |
| + return; |
| +#endif |
| + |
| do { |
| add_preempt_count_notrace(PREEMPT_ACTIVE); |
| /* |
| @@ -5331,7 +5387,9 @@ void __cpuinit init_idle(struct task_str |
| |
| /* Set the preempt count _outside_ the spinlocks! */ |
| task_thread_info(idle)->preempt_count = 0; |
| - |
| +#ifdef CONFIG_HAVE_PREEMPT_LAZY |
| + task_thread_info(idle)->preempt_lazy_count = 0; |
| +#endif |
| /* |
| * The idle tasks have their own, simple scheduling class: |
| */ |
| Index: linux-stable/kernel/sched/fair.c |
| =================================================================== |
| --- linux-stable.orig/kernel/sched/fair.c |
| +++ linux-stable/kernel/sched/fair.c |
| @@ -1222,7 +1222,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq |
| ideal_runtime = sched_slice(cfs_rq, curr); |
| delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; |
| if (delta_exec > ideal_runtime) { |
| - resched_task(rq_of(cfs_rq)->curr); |
| + resched_task_lazy(rq_of(cfs_rq)->curr); |
| /* |
| * The current task ran long enough, ensure it doesn't get |
| * re-elected due to buddy favours. |
| @@ -1246,7 +1246,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq |
| return; |
| |
| if (delta > ideal_runtime) |
| - resched_task(rq_of(cfs_rq)->curr); |
| + resched_task_lazy(rq_of(cfs_rq)->curr); |
| } |
| |
| static void |
| @@ -1363,7 +1363,7 @@ entity_tick(struct cfs_rq *cfs_rq, struc |
| * validating it and just reschedule. |
| */ |
| if (queued) { |
| - resched_task(rq_of(cfs_rq)->curr); |
| + resched_task_lazy(rq_of(cfs_rq)->curr); |
| return; |
| } |
| /* |
| @@ -1543,7 +1543,7 @@ static void __account_cfs_rq_runtime(str |
| * hierarchy can be throttled |
| */ |
| if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) |
| - resched_task(rq_of(cfs_rq)->curr); |
| + resched_task_lazy(rq_of(cfs_rq)->curr); |
| } |
| |
| static __always_inline |
| @@ -2129,7 +2129,7 @@ static void hrtick_start_fair(struct rq |
| |
| if (delta < 0) { |
| if (rq->curr == p) |
| - resched_task(p); |
| + resched_task_lazy(p); |
| return; |
| } |
| |
| @@ -2954,7 +2954,7 @@ static void check_preempt_wakeup(struct |
| return; |
| |
| preempt: |
| - resched_task(curr); |
| + resched_task_lazy(curr); |
| /* |
| * Only set the backward buddy when the current task is still |
| * on the rq. This can happen when a wakeup gets interleaved |
| @@ -5027,7 +5027,7 @@ static void task_fork_fair(struct task_s |
| * 'current' within the tree based on its new key value. |
| */ |
| swap(curr->vruntime, se->vruntime); |
| - resched_task(rq->curr); |
| + resched_task_lazy(rq->curr); |
| } |
| |
| se->vruntime -= cfs_rq->min_vruntime; |
| @@ -5052,7 +5052,7 @@ prio_changed_fair(struct rq *rq, struct |
| */ |
| if (rq->curr == p) { |
| if (p->prio > oldprio) |
| - resched_task(rq->curr); |
| + resched_task_lazy(rq->curr); |
| } else |
| check_preempt_curr(rq, p, 0); |
| } |
| Index: linux-stable/kernel/sched/features.h |
| =================================================================== |
| --- linux-stable.orig/kernel/sched/features.h |
| +++ linux-stable/kernel/sched/features.h |
| @@ -68,6 +68,9 @@ SCHED_FEAT(NONTASK_POWER, true) |
| SCHED_FEAT(TTWU_QUEUE, true) |
| #else |
| SCHED_FEAT(TTWU_QUEUE, false) |
| +# ifdef CONFIG_PREEMPT_LAZY |
| +SCHED_FEAT(PREEMPT_LAZY, true) |
| +# endif |
| #endif |
| |
| SCHED_FEAT(FORCE_SD_OVERLAP, false) |
| Index: linux-stable/kernel/sched/sched.h |
| =================================================================== |
| --- linux-stable.orig/kernel/sched/sched.h |
| +++ linux-stable/kernel/sched/sched.h |
| @@ -876,6 +876,15 @@ extern void init_sched_fair_class(void); |
| extern void resched_task(struct task_struct *p); |
| extern void resched_cpu(int cpu); |
| |
| +#ifdef CONFIG_PREEMPT_LAZY |
| +extern void resched_task_lazy(struct task_struct *tsk); |
| +#else |
| +static inline void resched_task_lazy(struct task_struct *tsk) |
| +{ |
| + resched_task(tsk); |
| +} |
| +#endif |
| + |
| extern struct rt_bandwidth def_rt_bandwidth; |
| extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); |
| |
| Index: linux-stable/kernel/trace/trace.c |
| =================================================================== |
| --- linux-stable.orig/kernel/trace/trace.c |
| +++ linux-stable/kernel/trace/trace.c |
| @@ -1152,6 +1152,7 @@ tracing_generic_entry_update(struct trac |
| struct task_struct *tsk = current; |
| |
| entry->preempt_count = pc & 0xff; |
| + entry->preempt_lazy_count = preempt_lazy_count(); |
| entry->pid = (tsk) ? tsk->pid : 0; |
| entry->padding = 0; |
| entry->flags = |
| @@ -1162,7 +1163,8 @@ tracing_generic_entry_update(struct trac |
| #endif |
| ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | |
| ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | |
| - (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); |
| + (need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) | |
| + (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0); |
| |
| entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; |
| } |
| @@ -1985,15 +1987,17 @@ get_total_entries(struct trace_array *tr |
| |
| static void print_lat_help_header(struct seq_file *m) |
| { |
| - seq_puts(m, "# _------=> CPU# \n"); |
| - seq_puts(m, "# / _-----=> irqs-off \n"); |
| - seq_puts(m, "# | / _----=> need-resched \n"); |
| - seq_puts(m, "# || / _---=> hardirq/softirq \n"); |
| - seq_puts(m, "# ||| / _--=> preempt-depth \n"); |
| - seq_puts(m, "# |||| / _--=> migrate-disable\n"); |
| - seq_puts(m, "# ||||| / delay \n"); |
| - seq_puts(m, "# cmd pid |||||| time | caller \n"); |
| - seq_puts(m, "# \\ / ||||| \\ | / \n"); |
| + seq_puts(m, "# _--------=> CPU# \n"); |
| + seq_puts(m, "# / _-------=> irqs-off \n"); |
| + seq_puts(m, "# | / _------=> need-resched \n"); |
| + seq_puts(m, "# || / _-----=> need-resched_lazy \n"); |
| + seq_puts(m, "# ||| / _----=> hardirq/softirq \n"); |
| + seq_puts(m, "# |||| / _---=> preempt-depth \n"); |
| + seq_puts(m, "# ||||| / _--=> preempt-lazy-depth\n"); |
| + seq_puts(m, "# |||||| / _-=> migrate-disable \n"); |
| + seq_puts(m, "# ||||||| / delay \n"); |
| + seq_puts(m, "# cmd pid |||||||| time | caller \n"); |
| + seq_puts(m, "# \\ / |||||||| \\ | / \n"); |
| } |
| |
| static void print_event_info(struct trace_array *tr, struct seq_file *m) |
| @@ -2017,13 +2021,16 @@ static void print_func_help_header(struc |
| static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m) |
| { |
| print_event_info(tr, m); |
| - seq_puts(m, "# _-----=> irqs-off\n"); |
| - seq_puts(m, "# / _----=> need-resched\n"); |
| - seq_puts(m, "# | / _---=> hardirq/softirq\n"); |
| - seq_puts(m, "# || / _--=> preempt-depth\n"); |
| - seq_puts(m, "# ||| / delay\n"); |
| - seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"); |
| - seq_puts(m, "# | | | |||| | |\n"); |
| + seq_puts(m, "# _-------=> irqs-off \n"); |
| + seq_puts(m, "# / _------=> need-resched \n"); |
| + seq_puts(m, "# |/ _-----=> need-resched_lazy \n"); |
| + seq_puts(m, "# ||/ _----=> hardirq/softirq \n"); |
| + seq_puts(m, "# |||/ _---=> preempt-depth \n"); |
| + seq_puts(m, "# ||||/ _--=> preempt-lazy-depth\n"); |
| + seq_puts(m, "# ||||| / _-=> migrate-disable \n"); |
| + seq_puts(m, "# |||||| / delay\n"); |
| + seq_puts(m, "# TASK-PID CPU# ||||||| TIMESTAMP FUNCTION\n"); |
| + seq_puts(m, "# | | | ||||||| | |\n"); |
| } |
| |
| void |
| Index: linux-stable/kernel/trace/trace.h |
| =================================================================== |
| --- linux-stable.orig/kernel/trace/trace.h |
| +++ linux-stable/kernel/trace/trace.h |
| @@ -116,6 +116,7 @@ struct uprobe_trace_entry_head { |
| * NEED_RESCHED - reschedule is requested |
| * HARDIRQ - inside an interrupt handler |
| * SOFTIRQ - inside a softirq handler |
| + * NEED_RESCHED_LAZY - lazy reschedule is requested |
| */ |
| enum trace_flag_type { |
| TRACE_FLAG_IRQS_OFF = 0x01, |
| @@ -123,6 +124,7 @@ enum trace_flag_type { |
| TRACE_FLAG_NEED_RESCHED = 0x04, |
| TRACE_FLAG_HARDIRQ = 0x08, |
| TRACE_FLAG_SOFTIRQ = 0x10, |
| + TRACE_FLAG_NEED_RESCHED_LAZY = 0x20, |
| }; |
| |
| #define TRACE_BUF_SIZE 1024 |
| Index: linux-stable/kernel/trace/trace_output.c |
| =================================================================== |
| --- linux-stable.orig/kernel/trace/trace_output.c |
| +++ linux-stable/kernel/trace/trace_output.c |
| @@ -564,6 +564,7 @@ int trace_print_lat_fmt(struct trace_seq |
| { |
| char hardsoft_irq; |
| char need_resched; |
| + char need_resched_lazy; |
| char irqs_off; |
| int hardirq; |
| int softirq; |
| @@ -578,14 +579,17 @@ int trace_print_lat_fmt(struct trace_seq |
| '.'; |
| need_resched = |
| (entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'; |
| + need_resched_lazy = |
| + (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.'; |
| hardsoft_irq = |
| (hardirq && softirq) ? 'H' : |
| hardirq ? 'h' : |
| softirq ? 's' : |
| '.'; |
| |
| - if (!trace_seq_printf(s, "%c%c%c", |
| - irqs_off, need_resched, hardsoft_irq)) |
| + if (!trace_seq_printf(s, "%c%c%c%c", |
| + irqs_off, need_resched, need_resched_lazy, |
| + hardsoft_irq)) |
| return 0; |
| |
| if (entry->preempt_count) |
| @@ -593,6 +597,11 @@ int trace_print_lat_fmt(struct trace_seq |
| else |
| ret = trace_seq_putc(s, '.'); |
| |
| + if (entry->preempt_lazy_count) |
| + ret = trace_seq_printf(s, "%x", entry->preempt_lazy_count); |
| + else |
| + ret = trace_seq_putc(s, '.'); |
| + |
| if (entry->migrate_disable) |
| ret = trace_seq_printf(s, "%x", entry->migrate_disable); |
| else |