| Subject: timekeeping: Split timekeeper_lock into lock and seqcount |
| From: Thomas Gleixner <tglx@linutronix.de> |
| Date: Fri, 15 Feb 2013 15:03:17 +0100 |
| |
| We want to shorten the seqcount write hold time. So split the seqlock |
| into a lock and a seqcount. |
| |
| Open code the seqwrite_lock in the places which matter and drop the |
| sequence counter update where it's pointless. |
| |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| --- |
| kernel/time/timekeeping.c | 118 +++++++++++++++++++++++++--------------------- |
| 1 file changed, 65 insertions(+), 53 deletions(-) |
| |
| --- a/kernel/time/timekeeping.c |
| +++ b/kernel/time/timekeeping.c |
| @@ -26,7 +26,8 @@ |
| #include "tick-internal.h" |
| |
| static struct timekeeper timekeeper; |
| -static DEFINE_SEQLOCK(timekeeper_lock); |
| +static DEFINE_RAW_SPINLOCK(timekeeper_lock); |
| +static seqcount_t timekeeper_seq; |
| |
| /* flag for if timekeeping is suspended */ |
| int __read_mostly timekeeping_suspended; |
| @@ -186,8 +187,6 @@ static void update_pvclock_gtod(struct t |
| |
| /** |
| * pvclock_gtod_register_notifier - register a pvclock timedata update listener |
| - * |
| - * Must hold write on timekeeper.lock |
| */ |
| int pvclock_gtod_register_notifier(struct notifier_block *nb) |
| { |
| @@ -195,11 +194,10 @@ int pvclock_gtod_register_notifier(struc |
| unsigned long flags; |
| int ret; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); |
| - /* update timekeeping data */ |
| update_pvclock_gtod(tk); |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| return ret; |
| } |
| @@ -208,23 +206,21 @@ EXPORT_SYMBOL_GPL(pvclock_gtod_register_ |
| /** |
| * pvclock_gtod_unregister_notifier - unregister a pvclock |
| * timedata update listener |
| - * |
| - * Must hold write on timekeeper.lock |
| */ |
| int pvclock_gtod_unregister_notifier(struct notifier_block *nb) |
| { |
| unsigned long flags; |
| int ret; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); |
| |
| -/* must hold write on timekeeper.lock */ |
| +/* must hold timekeeper_lock */ |
| static void timekeeping_update(struct timekeeper *tk, bool clearntp) |
| { |
| if (clearntp) { |
| @@ -279,12 +275,12 @@ void getnstimeofday(struct timespec *ts) |
| WARN_ON(timekeeping_suspended); |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| ts->tv_sec = tk->xtime_sec; |
| nsecs = timekeeping_get_ns(tk); |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| ts->tv_nsec = 0; |
| timespec_add_ns(ts, nsecs); |
| @@ -300,11 +296,11 @@ ktime_t ktime_get(void) |
| WARN_ON(timekeeping_suspended); |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; |
| nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| /* |
| * Use ktime_set/ktime_add_ns to create a proper ktime on |
| * 32-bit architectures without CONFIG_KTIME_SCALAR. |
| @@ -331,12 +327,12 @@ void ktime_get_ts(struct timespec *ts) |
| WARN_ON(timekeeping_suspended); |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| ts->tv_sec = tk->xtime_sec; |
| nsec = timekeeping_get_ns(tk); |
| tomono = tk->wall_to_monotonic; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| ts->tv_sec += tomono.tv_sec; |
| ts->tv_nsec = 0; |
| @@ -364,7 +360,7 @@ void getnstime_raw_and_real(struct times |
| WARN_ON_ONCE(timekeeping_suspended); |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| *ts_raw = tk->raw_time; |
| ts_real->tv_sec = tk->xtime_sec; |
| @@ -373,7 +369,7 @@ void getnstime_raw_and_real(struct times |
| nsecs_raw = timekeeping_get_ns_raw(tk); |
| nsecs_real = timekeeping_get_ns(tk); |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| timespec_add_ns(ts_raw, nsecs_raw); |
| timespec_add_ns(ts_real, nsecs_real); |
| @@ -413,7 +409,8 @@ int do_settimeofday(const struct timespe |
| if (!timespec_valid_strict(tv)) |
| return -EINVAL; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| timekeeping_forward_now(tk); |
| |
| @@ -427,7 +424,8 @@ int do_settimeofday(const struct timespe |
| |
| timekeeping_update(tk, true); |
| |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| /* signal hrtimers about time change */ |
| clock_was_set(); |
| @@ -452,7 +450,8 @@ int timekeeping_inject_offset(struct tim |
| if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) |
| return -EINVAL; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| timekeeping_forward_now(tk); |
| |
| @@ -469,7 +468,8 @@ int timekeeping_inject_offset(struct tim |
| error: /* even if we error out, we forwarded the time, so call update */ |
| timekeeping_update(tk, true); |
| |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| /* signal hrtimers about time change */ |
| clock_was_set(); |
| @@ -491,7 +491,8 @@ static int change_clocksource(void *data |
| |
| new = (struct clocksource *) data; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| timekeeping_forward_now(tk); |
| if (!new->enable || new->enable(new) == 0) { |
| @@ -502,7 +503,8 @@ static int change_clocksource(void *data |
| } |
| timekeeping_update(tk, true); |
| |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| return 0; |
| } |
| @@ -552,11 +554,11 @@ void getrawmonotonic(struct timespec *ts |
| s64 nsecs; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| nsecs = timekeeping_get_ns_raw(tk); |
| *ts = tk->raw_time; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| timespec_add_ns(ts, nsecs); |
| } |
| @@ -572,11 +574,11 @@ int timekeeping_valid_for_hres(void) |
| int ret; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| return ret; |
| } |
| @@ -591,11 +593,11 @@ u64 timekeeping_max_deferment(void) |
| u64 ret; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| ret = tk->clock->max_idle_ns; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| return ret; |
| } |
| @@ -658,7 +660,8 @@ void __init timekeeping_init(void) |
| |
| ntp_init(); |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| clock = clocksource_default_clock(); |
| if (clock->enable) |
| clock->enable(clock); |
| @@ -677,7 +680,8 @@ void __init timekeeping_init(void) |
| tmp.tv_nsec = 0; |
| tk_set_sleep_time(tk, tmp); |
| |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| } |
| |
| /* time in seconds when suspend began */ |
| @@ -724,7 +728,8 @@ void timekeeping_inject_sleeptime(struct |
| if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) |
| return; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| timekeeping_forward_now(tk); |
| |
| @@ -732,7 +737,8 @@ void timekeeping_inject_sleeptime(struct |
| |
| timekeeping_update(tk, true); |
| |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| /* signal hrtimers about time change */ |
| clock_was_set(); |
| @@ -756,7 +762,8 @@ static void timekeeping_resume(void) |
| clockevents_resume(); |
| clocksource_resume(); |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { |
| ts = timespec_sub(ts, timekeeping_suspend_time); |
| @@ -767,7 +774,8 @@ static void timekeeping_resume(void) |
| tk->ntp_error = 0; |
| timekeeping_suspended = 0; |
| timekeeping_update(tk, false); |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| touch_softlockup_watchdog(); |
| |
| @@ -786,7 +794,8 @@ static int timekeeping_suspend(void) |
| |
| read_persistent_clock(&timekeeping_suspend_time); |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| timekeeping_forward_now(tk); |
| timekeeping_suspended = 1; |
| |
| @@ -809,7 +818,8 @@ static int timekeeping_suspend(void) |
| timekeeping_suspend_time = |
| timespec_add(timekeeping_suspend_time, delta_delta); |
| } |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); |
| clocksource_suspend(); |
| @@ -1147,7 +1157,8 @@ static void update_wall_time(void) |
| int shift = 0, maxshift; |
| unsigned long flags; |
| |
| - write_seqlock_irqsave(&timekeeper_lock, flags); |
| + raw_spin_lock_irqsave(&timekeeper_lock, flags); |
| + write_seqcount_begin(&timekeeper_seq); |
| |
| /* Make sure we're fully resumed: */ |
| if (unlikely(timekeeping_suspended)) |
| @@ -1202,7 +1213,8 @@ static void update_wall_time(void) |
| timekeeping_update(tk, false); |
| |
| out: |
| - write_sequnlock_irqrestore(&timekeeper_lock, flags); |
| + write_seqcount_end(&timekeeper_seq); |
| + raw_spin_unlock_irqrestore(&timekeeper_lock, flags); |
| |
| } |
| |
| @@ -1250,13 +1262,13 @@ void get_monotonic_boottime(struct times |
| WARN_ON(timekeeping_suspended); |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| ts->tv_sec = tk->xtime_sec; |
| nsec = timekeeping_get_ns(tk); |
| tomono = tk->wall_to_monotonic; |
| sleep = tk->total_sleep_time; |
| |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| ts->tv_sec += tomono.tv_sec + sleep.tv_sec; |
| ts->tv_nsec = 0; |
| @@ -1315,10 +1327,10 @@ struct timespec current_kernel_time(void |
| unsigned long seq; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| now = tk_xtime(tk); |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| return now; |
| } |
| @@ -1331,11 +1343,11 @@ struct timespec get_monotonic_coarse(voi |
| unsigned long seq; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| now = tk_xtime(tk); |
| mono = tk->wall_to_monotonic; |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, |
| now.tv_nsec + mono.tv_nsec); |
| @@ -1366,11 +1378,11 @@ void get_xtime_and_monotonic_and_sleep_o |
| unsigned long seq; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| *xtim = tk_xtime(tk); |
| *wtom = tk->wall_to_monotonic; |
| *sleep = tk->total_sleep_time; |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| } |
| |
| #ifdef CONFIG_HIGH_RES_TIMERS |
| @@ -1390,14 +1402,14 @@ ktime_t ktime_get_update_offsets(ktime_t |
| u64 secs, nsecs; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| |
| secs = tk->xtime_sec; |
| nsecs = timekeeping_get_ns(tk); |
| |
| *offs_real = tk->offs_real; |
| *offs_boot = tk->offs_boot; |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| now = ktime_add_ns(ktime_set(secs, 0), nsecs); |
| now = ktime_sub(now, *offs_real); |
| @@ -1415,9 +1427,9 @@ ktime_t ktime_get_monotonic_offset(void) |
| struct timespec wtom; |
| |
| do { |
| - seq = read_seqbegin(&timekeeper_lock); |
| + seq = read_seqcount_begin(&timekeeper_seq); |
| wtom = tk->wall_to_monotonic; |
| - } while (read_seqretry(&timekeeper_lock, seq)); |
| + } while (read_seqcount_retry(&timekeeper_seq, seq)); |
| |
| return timespec_to_ktime(wtom); |
| } |