sched/eevdf: Re-arrange current protection in pick_eevdf()
The way pick_eevdf() limits preemption is by explicitly picking
current if it is still eligible. It does this without consideration of
the best in-tree task.
Move current protection to after the tree selection such that a follow
up patch can change the conditions.
Should be an semantics no-op at this point.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6b835d7..72f0ba5 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -903,6 +903,26 @@ static inline void cancel_protect_slice(struct sched_entity *se)
se->vlag = se->deadline + 1;
}
+static inline bool __pick_eevdf_curr(struct cfs_rq *cfs_rq, struct sched_entity *best)
+{
+ struct sched_entity *curr = cfs_rq->curr;
+
+ /* only called when this is already checked */
+ WARN_ON_ONCE(!curr || !curr->on_rq);
+
+ /*
+ * Strictly speaking we should allow current to run until its
+ * deadline. However allow (wakeup) preemption once it is no longer
+ * eligible.
+ */
+ if (sched_feat(RUN_TO_PARITY) &&
+ protect_slice(curr) &&
+ entity_eligible(cfs_rq, curr))
+ return true;
+
+ return entity_before(curr, best);
+}
+
/*
* Earliest Eligible Virtual Deadline First
*
@@ -929,18 +949,15 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
struct sched_entity *curr = cfs_rq->curr;
struct sched_entity *best = NULL;
+ if (curr && !curr->on_rq)
+ curr = NULL;
+
/*
* We can safely skip eligibility check if there is only one entity
* in this cfs_rq, saving some cycles.
*/
if (cfs_rq->nr_queued == 1)
- return curr && curr->on_rq ? curr : se;
-
- if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr)))
- curr = NULL;
-
- if (sched_feat(RUN_TO_PARITY) && curr && protect_slice(curr))
- return curr;
+ return curr ?: se;
/* Pick the leftmost entity if it's eligible */
if (se && entity_eligible(cfs_rq, se)) {
@@ -977,7 +994,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
node = node->rb_right;
}
found:
- if (!best || (curr && entity_before(curr, best)))
+ if (!best || (curr && __pick_eevdf_curr(cfs_rq, best)))
best = curr;
return best;
