include/linux/hardirq.h - pub/scm/linux/kernel/git/arjan/linux-hardening - Git at Google

 #ifndef LINUX_HARDIRQ_H
 #define LINUX_HARDIRQ_H

 #include <linux/preempt.h>
 #include <linux/lockdep.h>
 #include <linux/ftrace_irq.h>
 #include <asm/hardirq.h>

 /*
  * We put the hardirq and softirq counter into the preemption
  * counter. The bitmask has the following meaning:
  *
  * - bits 0-7 are the preemption count (max preemption depth: 256)
  * - bits 8-15 are the softirq count (max # of softirqs: 256)
  *
  * The hardirq count can in theory reach the same as NR_IRQS.
  * In reality, the number of nested IRQS is limited to the stack
  * size as well. For archs with over 1000 IRQS it is not practical
  * to expect that they will all nest. We give a max of 10 bits for
  * hardirq nesting. An arch may choose to give less than 10 bits.
  * m68k expects it to be 8.
  *
  * - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
  * - bit 26 is the NMI_MASK
  * - bit 28 is the PREEMPT_ACTIVE flag
  *
  * PREEMPT_MASK: 0x000000ff
  * SOFTIRQ_MASK: 0x0000ff00
  * HARDIRQ_MASK: 0x03ff0000
  *     NMI_MASK: 0x04000000
  */
 #define PREEMPT_BITS	8
 #define SOFTIRQ_BITS	8
 #define NMI_BITS	1

 #define MAX_HARDIRQ_BITS 10

 #ifndef HARDIRQ_BITS
 # define HARDIRQ_BITS	MAX_HARDIRQ_BITS
 #endif

 #if HARDIRQ_BITS > MAX_HARDIRQ_BITS
 #error HARDIRQ_BITS too high!
 #endif

 #define PREEMPT_SHIFT	0
 #define SOFTIRQ_SHIFT	(PREEMPT_SHIFT + PREEMPT_BITS)
 #define HARDIRQ_SHIFT	(SOFTIRQ_SHIFT + SOFTIRQ_BITS)
 #define NMI_SHIFT	(HARDIRQ_SHIFT + HARDIRQ_BITS)

 #define __IRQ_MASK(x)	((1UL << (x))-1)

 #define PREEMPT_MASK	(__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
 #define SOFTIRQ_MASK	(__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
 #define HARDIRQ_MASK	(__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
 #define NMI_MASK	(__IRQ_MASK(NMI_BITS)     << NMI_SHIFT)

 #define PREEMPT_OFFSET	(1UL << PREEMPT_SHIFT)
 #define SOFTIRQ_OFFSET	(1UL << SOFTIRQ_SHIFT)
 #define HARDIRQ_OFFSET	(1UL << HARDIRQ_SHIFT)
 #define NMI_OFFSET	(1UL << NMI_SHIFT)

 #define SOFTIRQ_DISABLE_OFFSET	(2 * SOFTIRQ_OFFSET)

 #ifndef PREEMPT_ACTIVE
 #define PREEMPT_ACTIVE_BITS	1
 #define PREEMPT_ACTIVE_SHIFT	(NMI_SHIFT + NMI_BITS)
 #define PREEMPT_ACTIVE	(__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
 #endif

 #if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
 #error PREEMPT_ACTIVE is too low!
 #endif

 #define hardirq_count()	(preempt_count() & HARDIRQ_MASK)
 #define softirq_count()	(preempt_count() & SOFTIRQ_MASK)
 #define irq_count()	(preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \
 				 | NMI_MASK))

 /*
  * Are we doing bottom half or hardware interrupt processing?
  * Are we in a softirq context? Interrupt context?
  * in_softirq - Are we currently processing softirq or have bh disabled?
  * in_serving_softirq - Are we currently processing softirq?
  */
 #define in_irq()		(hardirq_count())
 #define in_softirq()		(softirq_count())
 #define in_interrupt()		(irq_count())
 #define in_serving_softirq()	(softirq_count() & SOFTIRQ_OFFSET)

 /*
  * Are we in NMI context?
  */
 #define in_nmi()	(preempt_count() & NMI_MASK)

 #if defined(CONFIG_PREEMPT_COUNT)
 # define PREEMPT_CHECK_OFFSET 1
 #else
 # define PREEMPT_CHECK_OFFSET 0
 #endif

 /*
  * Are we running in atomic context?  WARNING: this macro cannot
  * always detect atomic context; in particular, it cannot know about
  * held spinlocks in non-preemptible kernels.  Thus it should not be
  * used in the general case to determine whether sleeping is possible.
  * Do not use in_atomic() in driver code.
  */
 #define in_atomic()	((preempt_count() & ~PREEMPT_ACTIVE) != 0)

 /*
  * Check whether we were atomic before we did preempt_disable():
  * (used by the scheduler, *after* releasing the kernel lock)
  */
 #define in_atomic_preempt_off() \
 		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)

 #ifdef CONFIG_PREEMPT_COUNT
 # define preemptible()	(preempt_count() == 0 && !irqs_disabled())
 # define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
 #else
 # define preemptible()	0
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif

 #if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS)
 extern void synchronize_irq(unsigned int irq);
 #else
 # define synchronize_irq(irq)	barrier()
 #endif

 struct task_struct;

 #if !defined(CONFIG_VIRT_CPU_ACCOUNTING) && !defined(CONFIG_IRQ_TIME_ACCOUNTING)
 static inline void account_system_vtime(struct task_struct *tsk)
 {
 }
 #else
 extern void account_system_vtime(struct task_struct *tsk);
 #endif

 #if defined(CONFIG_NO_HZ)
 #if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
 extern void rcu_enter_nohz(void);
 extern void rcu_exit_nohz(void);

 static inline void rcu_irq_enter(void)
 {
 	rcu_exit_nohz();
 }

 static inline void rcu_irq_exit(void)
 {
 	rcu_enter_nohz();
 }

 static inline void rcu_nmi_enter(void)
 {
 }

 static inline void rcu_nmi_exit(void)
 {
 }

 #else
 extern void rcu_irq_enter(void);
 extern void rcu_irq_exit(void);
 extern void rcu_nmi_enter(void);
 extern void rcu_nmi_exit(void);
 #endif
 #else
 # define rcu_irq_enter() do { } while (0)
 # define rcu_irq_exit() do { } while (0)
 # define rcu_nmi_enter() do { } while (0)
 # define rcu_nmi_exit() do { } while (0)
 #endif /* #if defined(CONFIG_NO_HZ) */

 /*
  * It is safe to do non-atomic ops on ->hardirq_context,
  * because NMI handlers may not preempt and the ops are
  * always balanced, so the interrupted value of ->hardirq_context
  * will always be restored.
  */
 #define __irq_enter()					\
 	do {						\
 		account_system_vtime(current);		\
 		add_preempt_count(HARDIRQ_OFFSET);	\
 		trace_hardirq_enter();			\
 	} while (0)

 /*
  * Enter irq context (on NO_HZ, update jiffies):
  */
 extern void irq_enter(void);

 /*
  * Exit irq context without processing softirqs:
  */
 #define __irq_exit()					\
 	do {						\
 		trace_hardirq_exit();			\
 		account_system_vtime(current);		\
 		sub_preempt_count(HARDIRQ_OFFSET);	\
 	} while (0)

 /*
  * Exit irq context and process softirqs if needed:
  */
 extern void irq_exit(void);

 #define nmi_enter()						\
 	do {							\
 		ftrace_nmi_enter();				\
 		BUG_ON(in_nmi());				\
 		add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
 		lockdep_off();					\
 		rcu_nmi_enter();				\
 		trace_hardirq_enter();				\
 	} while (0)

 #define nmi_exit()						\
 	do {							\
 		trace_hardirq_exit();				\
 		rcu_nmi_exit();					\
 		lockdep_on();					\
 		BUG_ON(!in_nmi());				\
 		sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
 		ftrace_nmi_exit();				\
 	} while (0)

 #endif /* LINUX_HARDIRQ_H */
	#ifndef LINUX_HARDIRQ_H
	#define LINUX_HARDIRQ_H

	#include <linux/preempt.h>
	#include <linux/lockdep.h>
	#include <linux/ftrace_irq.h>
	#include <asm/hardirq.h>

	/*
	* We put the hardirq and softirq counter into the preemption
	* counter. The bitmask has the following meaning:
	*
	* - bits 0-7 are the preemption count (max preemption depth: 256)
	* - bits 8-15 are the softirq count (max # of softirqs: 256)
	*
	* The hardirq count can in theory reach the same as NR_IRQS.
	* In reality, the number of nested IRQS is limited to the stack
	* size as well. For archs with over 1000 IRQS it is not practical
	* to expect that they will all nest. We give a max of 10 bits for
	* hardirq nesting. An arch may choose to give less than 10 bits.
	* m68k expects it to be 8.
	*
	* - bits 16-25 are the hardirq count (max # of nested hardirqs: 1024)
	* - bit 26 is the NMI_MASK
	* - bit 28 is the PREEMPT_ACTIVE flag
	*
	* PREEMPT_MASK: 0x000000ff
	* SOFTIRQ_MASK: 0x0000ff00
	* HARDIRQ_MASK: 0x03ff0000
	* NMI_MASK: 0x04000000
	*/
	#define PREEMPT_BITS 8
	#define SOFTIRQ_BITS 8
	#define NMI_BITS 1

	#define MAX_HARDIRQ_BITS 10

	#ifndef HARDIRQ_BITS
	# define HARDIRQ_BITS MAX_HARDIRQ_BITS
	#endif

	#if HARDIRQ_BITS > MAX_HARDIRQ_BITS
	#error HARDIRQ_BITS too high!
	#endif

	#define PREEMPT_SHIFT 0
	#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
	#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
	#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)

	#define __IRQ_MASK(x) ((1UL << (x))-1)

	#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
	#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
	#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
	#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)

	#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
	#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
	#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
	#define NMI_OFFSET (1UL << NMI_SHIFT)

	#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET)

	#ifndef PREEMPT_ACTIVE
	#define PREEMPT_ACTIVE_BITS 1
	#define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS)
	#define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT)
	#endif

	#if PREEMPT_ACTIVE < (1 << (NMI_SHIFT + NMI_BITS))
	#error PREEMPT_ACTIVE is too low!
	#endif

	#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
	#define softirq_count() (preempt_count() & SOFTIRQ_MASK)
	#define irq_count() (preempt_count() & (HARDIRQ_MASK \| SOFTIRQ_MASK \
	\| NMI_MASK))

	/*
	* Are we doing bottom half or hardware interrupt processing?
	* Are we in a softirq context? Interrupt context?
	* in_softirq - Are we currently processing softirq or have bh disabled?
	* in_serving_softirq - Are we currently processing softirq?
	*/
	#define in_irq() (hardirq_count())
	#define in_softirq() (softirq_count())
	#define in_interrupt() (irq_count())
	#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET)

	/*
	* Are we in NMI context?
	*/
	#define in_nmi() (preempt_count() & NMI_MASK)

	#if defined(CONFIG_PREEMPT_COUNT)
	# define PREEMPT_CHECK_OFFSET 1
	#else
	# define PREEMPT_CHECK_OFFSET 0
	#endif

	/*
	* Are we running in atomic context? WARNING: this macro cannot
	* always detect atomic context; in particular, it cannot know about
	* held spinlocks in non-preemptible kernels. Thus it should not be
	* used in the general case to determine whether sleeping is possible.
	* Do not use in_atomic() in driver code.
	*/
	#define in_atomic() ((preempt_count() & ~PREEMPT_ACTIVE) != 0)

	/*
	* Check whether we were atomic before we did preempt_disable():
	* (used by the scheduler, after releasing the kernel lock)
	*/
	#define in_atomic_preempt_off() \
	((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)

	#ifdef CONFIG_PREEMPT_COUNT
	# define preemptible() (preempt_count() == 0 && !irqs_disabled())
	# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
	#else
	# define preemptible() 0
	# define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
	#endif

	#if defined(CONFIG_SMP) \|\| defined(CONFIG_GENERIC_HARDIRQS)
	extern void synchronize_irq(unsigned int irq);
	#else
	# define synchronize_irq(irq) barrier()
	#endif

	struct task_struct;

	#if !defined(CONFIG_VIRT_CPU_ACCOUNTING) && !defined(CONFIG_IRQ_TIME_ACCOUNTING)
	static inline void account_system_vtime(struct task_struct *tsk)
	{
	}
	#else
	extern void account_system_vtime(struct task_struct *tsk);
	#endif

	#if defined(CONFIG_NO_HZ)
	#if defined(CONFIG_TINY_RCU) \|\| defined(CONFIG_TINY_PREEMPT_RCU)
	extern void rcu_enter_nohz(void);
	extern void rcu_exit_nohz(void);

	static inline void rcu_irq_enter(void)
	{
	rcu_exit_nohz();
	}

	static inline void rcu_irq_exit(void)
	{
	rcu_enter_nohz();
	}

	static inline void rcu_nmi_enter(void)
	{
	}

	static inline void rcu_nmi_exit(void)
	{
	}

	#else
	extern void rcu_irq_enter(void);
	extern void rcu_irq_exit(void);
	extern void rcu_nmi_enter(void);
	extern void rcu_nmi_exit(void);
	#endif
	#else
	# define rcu_irq_enter() do { } while (0)
	# define rcu_irq_exit() do { } while (0)
	# define rcu_nmi_enter() do { } while (0)
	# define rcu_nmi_exit() do { } while (0)
	#endif /* #if defined(CONFIG_NO_HZ) */

	/*
	* It is safe to do non-atomic ops on ->hardirq_context,
	* because NMI handlers may not preempt and the ops are
	* always balanced, so the interrupted value of ->hardirq_context
	* will always be restored.
	*/
	#define __irq_enter() \
	do { \
	account_system_vtime(current); \
	add_preempt_count(HARDIRQ_OFFSET); \
	trace_hardirq_enter(); \
	} while (0)

	/*
	* Enter irq context (on NO_HZ, update jiffies):
	*/
	extern void irq_enter(void);

	/*
	* Exit irq context without processing softirqs:
	*/
	#define __irq_exit() \
	do { \
	trace_hardirq_exit(); \
	account_system_vtime(current); \
	sub_preempt_count(HARDIRQ_OFFSET); \
	} while (0)

	/*
	* Exit irq context and process softirqs if needed:
	*/
	extern void irq_exit(void);

	#define nmi_enter() \
	do { \
	ftrace_nmi_enter(); \
	BUG_ON(in_nmi()); \
	add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \
	lockdep_off(); \
	rcu_nmi_enter(); \
	trace_hardirq_enter(); \
	} while (0)

	#define nmi_exit() \
	do { \
	trace_hardirq_exit(); \
	rcu_nmi_exit(); \
	lockdep_on(); \
	BUG_ON(!in_nmi()); \
	sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \
	ftrace_nmi_exit(); \
	} while (0)

	#endif /* LINUX_HARDIRQ_H */