| From: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| Date: Fri, 5 Feb 2016 18:26:11 +0100 |
| Subject: rtmutex: push down migrate_disable() into rt_spin_lock() |
| |
| No point in having the migrate disable/enable invocations in all the |
| macro/inlines. That's just more code for no win as we do a function |
| call anyway. Move it to the core code and save quite some text size. |
| |
| text data bss dec filename |
| 11034127 3676912 14901248 29612287 vmlinux.before |
| 10990437 3676848 14901248 29568533 vmlinux.after |
| |
| ~-40KiB |
| |
| Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> |
| --- |
| include/linux/locallock.h | 6 +++--- |
| include/linux/spinlock_rt.h | 25 +++++++------------------ |
| kernel/cpu.c | 4 ++-- |
| kernel/locking/lglock.c | 2 +- |
| kernel/locking/rt.c | 2 -- |
| kernel/locking/rtmutex.c | 44 +++++++++++++++++++++++++++++++++++++++++--- |
| 6 files changed, 54 insertions(+), 29 deletions(-) |
| |
| --- a/include/linux/locallock.h |
| +++ b/include/linux/locallock.h |
| @@ -43,9 +43,9 @@ struct local_irq_lock { |
| * for CONFIG_PREEMPT_BASE map to the normal spin_* calls. |
| */ |
| #ifdef CONFIG_PREEMPT_RT_FULL |
| -# define spin_lock_local(lock) rt_spin_lock(lock) |
| -# define spin_trylock_local(lock) rt_spin_trylock(lock) |
| -# define spin_unlock_local(lock) rt_spin_unlock(lock) |
| +# define spin_lock_local(lock) rt_spin_lock__no_mg(lock) |
| +# define spin_trylock_local(lock) rt_spin_trylock__no_mg(lock) |
| +# define spin_unlock_local(lock) rt_spin_unlock__no_mg(lock) |
| #else |
| # define spin_lock_local(lock) spin_lock(lock) |
| # define spin_trylock_local(lock) spin_trylock(lock) |
| --- a/include/linux/spinlock_rt.h |
| +++ b/include/linux/spinlock_rt.h |
| @@ -18,6 +18,10 @@ do { \ |
| __rt_spin_lock_init(slock, #slock, &__key); \ |
| } while (0) |
| |
| +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock); |
| +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock); |
| +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock); |
| + |
| extern void __lockfunc rt_spin_lock(spinlock_t *lock); |
| extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); |
| extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); |
| @@ -32,20 +36,16 @@ extern int atomic_dec_and_spin_lock(atom |
| * lockdep-less calls, for derived types like rwlock: |
| * (for trylock they can use rt_mutex_trylock() directly. |
| */ |
| +extern void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock); |
| extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); |
| extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); |
| extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock); |
| |
| -#define spin_lock(lock) \ |
| - do { \ |
| - migrate_disable(); \ |
| - rt_spin_lock(lock); \ |
| - } while (0) |
| +#define spin_lock(lock) rt_spin_lock(lock) |
| |
| #define spin_lock_bh(lock) \ |
| do { \ |
| local_bh_disable(); \ |
| - migrate_disable(); \ |
| rt_spin_lock(lock); \ |
| } while (0) |
| |
| @@ -56,24 +56,19 @@ extern int __lockfunc __rt_spin_trylock( |
| #define spin_trylock(lock) \ |
| ({ \ |
| int __locked; \ |
| - migrate_disable(); \ |
| __locked = spin_do_trylock(lock); \ |
| - if (!__locked) \ |
| - migrate_enable(); \ |
| __locked; \ |
| }) |
| |
| #ifdef CONFIG_LOCKDEP |
| # define spin_lock_nested(lock, subclass) \ |
| do { \ |
| - migrate_disable(); \ |
| rt_spin_lock_nested(lock, subclass); \ |
| } while (0) |
| |
| #define spin_lock_bh_nested(lock, subclass) \ |
| do { \ |
| local_bh_disable(); \ |
| - migrate_disable(); \ |
| rt_spin_lock_nested(lock, subclass); \ |
| } while (0) |
| |
| @@ -81,7 +76,6 @@ extern int __lockfunc __rt_spin_trylock( |
| do { \ |
| typecheck(unsigned long, flags); \ |
| flags = 0; \ |
| - migrate_disable(); \ |
| rt_spin_lock_nested(lock, subclass); \ |
| } while (0) |
| #else |
| @@ -117,16 +111,11 @@ static inline unsigned long spin_lock_tr |
| /* FIXME: we need rt_spin_lock_nest_lock */ |
| #define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0) |
| |
| -#define spin_unlock(lock) \ |
| - do { \ |
| - rt_spin_unlock(lock); \ |
| - migrate_enable(); \ |
| - } while (0) |
| +#define spin_unlock(lock) rt_spin_unlock(lock) |
| |
| #define spin_unlock_bh(lock) \ |
| do { \ |
| rt_spin_unlock(lock); \ |
| - migrate_enable(); \ |
| local_bh_enable(); \ |
| } while (0) |
| |
| --- a/kernel/cpu.c |
| +++ b/kernel/cpu.c |
| @@ -127,8 +127,8 @@ struct hotplug_pcp { |
| }; |
| |
| #ifdef CONFIG_PREEMPT_RT_FULL |
| -# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock) |
| -# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock) |
| +# define hotplug_lock(hp) rt_spin_lock__no_mg(&(hp)->lock) |
| +# define hotplug_unlock(hp) rt_spin_unlock__no_mg(&(hp)->lock) |
| #else |
| # define hotplug_lock(hp) mutex_lock(&(hp)->mutex) |
| # define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex) |
| --- a/kernel/locking/lglock.c |
| +++ b/kernel/locking/lglock.c |
| @@ -10,7 +10,7 @@ |
| # define lg_do_unlock(l) arch_spin_unlock(l) |
| #else |
| # define lg_lock_ptr struct rt_mutex |
| -# define lg_do_lock(l) __rt_spin_lock(l) |
| +# define lg_do_lock(l) __rt_spin_lock__no_mg(l) |
| # define lg_do_unlock(l) __rt_spin_unlock(l) |
| #endif |
| /* |
| --- a/kernel/locking/rt.c |
| +++ b/kernel/locking/rt.c |
| @@ -235,7 +235,6 @@ EXPORT_SYMBOL(rt_read_trylock); |
| void __lockfunc rt_write_lock(rwlock_t *rwlock) |
| { |
| rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); |
| - migrate_disable(); |
| __rt_spin_lock(&rwlock->lock); |
| } |
| EXPORT_SYMBOL(rt_write_lock); |
| @@ -249,7 +248,6 @@ void __lockfunc rt_read_lock(rwlock_t *r |
| * recursive read locks succeed when current owns the lock |
| */ |
| if (rt_mutex_owner(lock) != current) { |
| - migrate_disable(); |
| rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); |
| __rt_spin_lock(lock); |
| } |
| --- a/kernel/locking/rtmutex.c |
| +++ b/kernel/locking/rtmutex.c |
| @@ -1103,8 +1103,16 @@ static void noinline __sched rt_spin_lo |
| rt_mutex_adjust_prio(current); |
| } |
| |
| +void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock) |
| +{ |
| + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); |
| + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
| +} |
| +EXPORT_SYMBOL(rt_spin_lock__no_mg); |
| + |
| void __lockfunc rt_spin_lock(spinlock_t *lock) |
| { |
| + migrate_disable(); |
| rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); |
| spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); |
| } |
| @@ -1112,24 +1120,41 @@ EXPORT_SYMBOL(rt_spin_lock); |
| |
| void __lockfunc __rt_spin_lock(struct rt_mutex *lock) |
| { |
| + migrate_disable(); |
| rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); |
| } |
| EXPORT_SYMBOL(__rt_spin_lock); |
| |
| +void __lockfunc __rt_spin_lock__no_mg(struct rt_mutex *lock) |
| +{ |
| + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); |
| +} |
| +EXPORT_SYMBOL(__rt_spin_lock__no_mg); |
| + |
| #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) |
| { |
| + migrate_disable(); |
| rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); |
| spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); |
| } |
| EXPORT_SYMBOL(rt_spin_lock_nested); |
| #endif |
| |
| +void __lockfunc rt_spin_unlock__no_mg(spinlock_t *lock) |
| +{ |
| + /* NOTE: we always pass in '1' for nested, for simplicity */ |
| + spin_release(&lock->dep_map, 1, _RET_IP_); |
| + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); |
| +} |
| +EXPORT_SYMBOL(rt_spin_unlock__no_mg); |
| + |
| void __lockfunc rt_spin_unlock(spinlock_t *lock) |
| { |
| /* NOTE: we always pass in '1' for nested, for simplicity */ |
| spin_release(&lock->dep_map, 1, _RET_IP_); |
| rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); |
| + migrate_enable(); |
| } |
| EXPORT_SYMBOL(rt_spin_unlock); |
| |
| @@ -1156,12 +1181,27 @@ int __lockfunc __rt_spin_trylock(struct |
| return rt_mutex_trylock(lock); |
| } |
| |
| +int __lockfunc rt_spin_trylock__no_mg(spinlock_t *lock) |
| +{ |
| + int ret; |
| + |
| + ret = rt_mutex_trylock(&lock->lock); |
| + if (ret) |
| + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); |
| + return ret; |
| +} |
| +EXPORT_SYMBOL(rt_spin_trylock__no_mg); |
| + |
| int __lockfunc rt_spin_trylock(spinlock_t *lock) |
| { |
| - int ret = rt_mutex_trylock(&lock->lock); |
| + int ret; |
| |
| + migrate_disable(); |
| + ret = rt_mutex_trylock(&lock->lock); |
| if (ret) |
| spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); |
| + else |
| + migrate_enable(); |
| return ret; |
| } |
| EXPORT_SYMBOL(rt_spin_trylock); |
| @@ -1200,12 +1240,10 @@ int atomic_dec_and_spin_lock(atomic_t *a |
| /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ |
| if (atomic_add_unless(atomic, -1, 1)) |
| return 0; |
| - migrate_disable(); |
| rt_spin_lock(lock); |
| if (atomic_dec_and_test(atomic)) |
| return 1; |
| rt_spin_unlock(lock); |
| - migrate_enable(); |
| return 0; |
| } |
| EXPORT_SYMBOL(atomic_dec_and_spin_lock); |