From ba404fe58481603a411847fc718eb3366ccd4fc6 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@elte.hu>
Date: Fri, 3 Jul 2009 08:29:51 -0500
Subject: [PATCH] percpu: add percpu locked infrastructure

commit 9566018270e0c6cee0388e1a577ae8b6f15f4e8d in tip.

RT needs per cpu data structures protected by per cpu locks instead of
disabling preemption. Add the infrastructure for per cpu locked data.

[PG: fold in a couple small deltas from the big 33rt merge commit,
 also note the removal of per_cpu_## in dd17c8f72993f9461e9c19250]

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>

diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 04f91c2..d1d3488 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -5,6 +5,9 @@
 #include <linux/threads.h>
 #include <linux/percpu-defs.h>
 
+#define __per_cpu_var_lock(var)	per_cpu__lock_##var##_locked
+#define __per_cpu_var_lock_var(var) per_cpu__##var##_locked
+
 #ifdef CONFIG_SMP
 
 /*
@@ -60,10 +63,18 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 #define __raw_get_cpu_var(var) \
 	(*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
 
+#define per_cpu_lock(var, cpu) \
+	(*SHIFT_PERCPU_PTR(&__per_cpu_var_lock(var), per_cpu_offset(cpu)))
+#define per_cpu_var_locked(var, cpu) \
+	(*SHIFT_PERCPU_PTR(&__per_cpu_var_lock_var(var), per_cpu_offset(cpu)))
+#define __get_cpu_lock(var, cpu) \
+		per_cpu_lock(var, cpu)
+#define __get_cpu_var_locked(var, cpu) \
+		per_cpu_var_locked(var, cpu)
+
 #define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
 #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
 
-
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
 #endif
@@ -71,6 +82,10 @@ extern void setup_per_cpu_areas(void);
 #else /* ! SMP */
 
 #define per_cpu(var, cpu)			(*((void)(cpu), &(var)))
+#define per_cpu_var_locked(var, cpu) \
+	(*((void)(cpu), &__per_cpu_var_lock_var(var)))
+#define __get_cpu_lock(var, cpu)		__per_cpu_var_lock(var)
+#define __get_cpu_var_locked(var, cpu)		__per_cpu_var_lock_var(var)
 #define __get_cpu_var(var)			(var)
 #define __raw_get_cpu_var(var)			(var)
 #define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 68567c0..2187c24 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -91,6 +91,22 @@
 	DEFINE_PER_CPU_SECTION(type, name, "")
 
 /*
+ * next three added for RT patch
+ * (wonder if we need corresponding DECLARE_*'s?) (clrkwllms)
+ */
+#define DEFINE_PER_CPU_SPINLOCK(name, sec)				\
+	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES			\
+	__DEFINE_SPINLOCK(per_cpu__lock_##name##_locked)
+
+#define DECLARE_PER_CPU_LOCKED(type, name)				\
+	extern PER_CPU_ATTRIBUTES spinlock_t __per_cpu_var_lock(name);	\
+	extern PER_CPU_ATTRIBUTES __typeof__(type) __per_cpu_var_lock_var(name)
+
+#define DEFINE_PER_CPU_LOCKED(type, name)				\
+	DEFINE_PER_CPU_SPINLOCK(name, "");				\
+	DEFINE_PER_CPU_SECTION(type, name##_locked, "")
+
+/*
  * Declaration/definition used for per-CPU variables that must come first in
  * the set of variables.
  */
@@ -145,10 +161,14 @@
  */
 #ifndef __CHECKER__
 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL(var) EXPORT_SYMBOL(var##_locked)
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var##_locked)
 #else
 #define EXPORT_PER_CPU_SYMBOL(var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL(var##_locked)
 #define EXPORT_PER_CPU_SYMBOL_GPL(var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL_GPL(var##_locked)
 #endif
 
 #endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index a93e5bf..35f9a30 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -39,6 +39,29 @@
 	preempt_enable();				\
 } while (0)
 
+/*
+ * Per-CPU data structures with an additional lock - useful for
+ * PREEMPT_RT code that wants to reschedule but also wants per-CPU
+ * data structures.
+ *
+ * 'cpu' gets updated with the CPU the task is currently executing on.
+ *
+ * NOTE: on normal !PREEMPT_RT kernels these per-CPU variables are the
+ * same as the normal per-CPU variables, so there is no runtime
+ * overhead.
+ */
+#define get_cpu_var_locked(var, cpuptr)			\
+(*({							\
+	int __cpu = raw_smp_processor_id();		\
+							\
+	*(cpuptr) = __cpu;				\
+	spin_lock(&__get_cpu_lock(var, __cpu));		\
+	&__get_cpu_var_locked(var, __cpu);		\
+}))
+
+#define put_cpu_var_locked(var, cpu) \
+	 do { (void)cpu; spin_unlock(&__get_cpu_lock(var, cpu)); } while (0)
+
 #ifdef CONFIG_SMP
 
 /* minimum unit size, also is the maximum supported allocation size */
-- 
1.7.1.1