trace kprot patch
Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index 2d14ae4..c9b97ac 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -96,6 +96,7 @@
{
long ret;
+ trace_printk("syscall: Enter from user\n");
enter_from_user_mode(regs, HT_PROTECT_SYSCALL);
instrumentation_begin();
@@ -299,11 +300,14 @@
local_irq_disable_exit_to_user();
exit_to_user_mode_prepare(regs, HT_PROTECT_SYSCALL);
instrumentation_end();
+ trace_printk("syscall: Entering exit_to_user_mode\n");
exit_to_user_mode();
+ trace_printk("syscall: Leaving exit_to_user_mode\n");
}
noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
{
+ trace_printk("irq: Enter from user\n");
enter_from_user_mode(regs, HT_PROTECT_IRQ);
}
@@ -312,7 +316,9 @@
instrumentation_begin();
exit_to_user_mode_prepare(regs, HT_PROTECT_IRQ);
instrumentation_end();
+ trace_printk("irq: Entering exit_to_user_mode\n");
exit_to_user_mode();
+ trace_printk("irq: Entering exit_to_user_mode\n");
}
noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index fbff25a..cd18de2 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -102,8 +102,10 @@
if (cpu != smp_processor_id()) {
/* Arch remote IPI send/receive backend aren't NMI safe */
WARN_ON_ONCE(in_nmi());
+ trace_printk("Queuing on cpu %d\n", cpu);
__smp_call_single_queue(cpu, &work->node.llist);
} else {
+ trace_printk("Queuing local\n");
__irq_work_queue_local(work);
}
preempt_enable();
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 508dbce..8ebb894 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5194,6 +5194,17 @@
*/
static void sched_core_irq_work(struct irq_work *work)
{
+ struct rq *rq;
+ int cpu;
+ cpu = smp_processor_id();
+ rq = cpu_rq(cpu);
+
+ WARN_ON_ONCE(!rq->pause_pending);
+
+ trace_printk("Entering irq work hnd\n");
+ trace_printk("Exit irq work hnd\n");
+
+ rq->pause_pending = false;
}
static inline void init_sched_core_irq_work(struct rq *rq)
@@ -5217,6 +5228,7 @@
bool restart = false;
struct rq *rq;
int cpu;
+ int loops = 0;
/* If arch doesn't define the TIF flag, don't do anything. */
if (!TIF_UNSAFE_RET)
@@ -5235,12 +5247,13 @@
preempt_disable();
local_irq_enable();
+ trace_printk("Enter wait\n");
/*
* Wait till the core of this HT is not in an unsafe state.
*
* Pair with raw_spin_lock/unlock() in sched_core_unsafe_enter/exit().
*/
- while (smp_load_acquire(&rq->core->core_unsafe_nest) > 0) {
+ while (smp_load_acquire(&rq->core->core_unsafe_nest) > 0 && loops++ < 100000000) {
cpu_relax();
if (READ_ONCE(current_thread_info()->flags) & ti_check) {
restart = true;
@@ -5252,6 +5265,11 @@
local_irq_disable();
preempt_enable();
+ trace_printk("Exit wait\n");
+
+ if (WARN_ON_ONCE(loops >= 100000000))
+ panic("excessive spinning\n");
+
ret:
if (!restart)
clear_tsk_thread_flag(current, TIF_UNSAFE_RET);
@@ -5303,6 +5321,11 @@
/* Count unsafe_enter() calls received without unsafe_exit() on this CPU. */
rq->core_this_unsafe_nest++;
+ trace_printk("enter: unsafe this nest now: %d\n", rq->core_this_unsafe_nest);
+ if (rq->core_this_unsafe_nest < 0) {
+ trace_printk("issue stop\n");
+ tracing_stop();
+ }
/*
* Should not nest: enter() should only pair with exit(). Both are done
@@ -5322,16 +5345,23 @@
* counter's smp_load_acquire() in sched_core_wait_till_safe().
*/
WRITE_ONCE(rq->core->core_unsafe_nest, rq->core->core_unsafe_nest + 1);
+ trace_printk("enter: unsafe nest now: %d\n", rq->core->core_unsafe_nest);
+ if (rq->core->core_unsafe_nest < 0) {
+ trace_printk("issue stop core-wide\n");
+ tracing_stop();
+ }
if (WARN_ON_ONCE(rq->core->core_unsafe_nest == UINT_MAX))
goto unlock;
if (irq_work_is_busy(&rq->core_irq_work)) {
+ WARN_ON_ONCE(!rq->pause_pending);
/*
* Do nothing more since we are in an IPI sent from another
* sibling to enforce safety. That sibling would have sent IPIs
* to all of the HTs.
*/
+ trace_printk("We are in IPI, do nothing more.\n");
goto unlock;
}
@@ -5339,8 +5369,10 @@
* If we are not the first ones on the core to enter core-wide unsafe
* state, do nothing.
*/
- if (rq->core->core_unsafe_nest > 1)
+ if (rq->core->core_unsafe_nest > 1) {
+ trace_printk("Inner core-wide nest.\n");
goto unlock;
+ }
/* Do nothing more if the core is not tagged. */
if (!rq->core->core_cookie)
@@ -5364,6 +5396,8 @@
* pending, no new IPIs are sent. This is Ok since the receiver
* would already be in the kernel, or on its way to it.
*/
+ trace_printk("Queuing irq_work on %d\n", i);
+ srq->pause_pending = true;
irq_work_queue_on(&srq->core_irq_work, i);
}
unlock:
@@ -5404,6 +5438,11 @@
goto ret;
rq->core_this_unsafe_nest--;
+ trace_printk("exit: unsafe this nest now: %d\n", rq->core_this_unsafe_nest);
+ if (rq->core_this_unsafe_nest < 0) {
+ trace_printk("issue stop\n");
+ tracing_stop();
+ }
/* enter() should be paired with exit() only. */
if (WARN_ON_ONCE(rq->core_this_unsafe_nest != 0))
@@ -5418,6 +5457,7 @@
WARN_ON_ONCE(!nest);
WRITE_ONCE(rq->core->core_unsafe_nest, nest - 1);
+ trace_printk("exit: unsafe nest now: %d\n", rq->core->core_unsafe_nest);
/*
* The raw_spin_unlock release semantics pairs with the nest counter's
* smp_load_acquire() in sched_core_wait_till_safe().
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 225654a..f97b91f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1085,6 +1085,7 @@
struct rb_root core_tree;
struct irq_work core_irq_work; /* To force HT into kernel */
unsigned int core_this_unsafe_nest;
+ bool pause_pending;
/* shared state */
unsigned int core_task_seq;
