Google Git
Sign in
kernel/pub/scm/linux/kernel/git/peterz/queue/refs/heads/sched/prep^!/.
commit
2f088fa8f3d9bb2beb0a3f8c5f4f0a6e2bd1ea72
[log]
author
Peter Zijlstra <peterz@infradead.org>
Sat Aug 10 00:11:56 2024 +0200
committer
Peter Zijlstra <peterz@infradead.org>
Wed Aug 14 00:24:59 2024 +0200
tree
8bbd4debd3f310ff0342e3235e9597c56e7ac6a4
parent
d85299ba8ed3a9bf38b1157760854071522e637d [diff]
sched: Add pick_task(.core)

In order to distinguish between a regular vs a core pick_task()
invocation, add a boolean argument.

Notably SCX seems to need this, since its core pick

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 57cf27a..86b0826 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h

@@ -601,7 +601,7 @@ struct sched_rt_entity {
 } __randomize_layout;
 
 typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *);
-typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *);
+typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *, bool);
 
 struct sched_dl_entity {
 	struct rb_node			rb_node;

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ffcd637..5365217 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c

@@ -5885,7 +5885,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 
 		/* Assume the next prioritized class is idle_sched_class */
 		if (!p) {
-			p = pick_task_idle(rq);
+			p = pick_task_idle(rq, false);
 			put_prev_set_next_task(rq, prev, p);
 		}
 
@@ -5901,7 +5901,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			if (p)
 				return p;
 		} else {
-			p = class->pick_task(rq);
+			p = class->pick_task(rq, false);
 			if (p) {
 				put_prev_set_next_task(rq, prev, p);
 				return p;
@@ -5939,7 +5939,7 @@ static inline struct task_struct *pick_task(struct rq *rq)
 	rq->dl_server = NULL;
 
 	for_each_class(class) {
-		p = class->pick_task(rq);
+		p = class->pick_task(rq, true);
 		if (p)
 			return p;
 	}
@@ -6092,7 +6092,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			if (cookie)
 				p = sched_core_find(rq_i, cookie);
 			if (!p)
-				p = idle_sched_class.pick_task(rq_i);
+				p = idle_sched_class.pick_task(rq_i, true);
 		}
 
 		rq_i->core_pick = p;

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 8c48bb9..00e341e 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c

@@ -2409,7 +2409,7 @@ static struct sched_dl_entity *pick_next_dl_entity(struct dl_rq *dl_rq)
  * __pick_next_task_dl - Helper to pick the next -deadline task to run.
  * @rq: The runqueue to pick the next task from.
  */
-static struct task_struct *__pick_task_dl(struct rq *rq)
+static struct task_struct *pick_task_dl(struct rq *rq, bool core)
 {
 	struct sched_dl_entity *dl_se;
 	struct dl_rq *dl_rq = &rq->dl;
@@ -2423,7 +2423,7 @@ static struct task_struct *__pick_task_dl(struct rq *rq)
 	WARN_ON_ONCE(!dl_se);
 
 	if (dl_server(dl_se)) {
-		p = dl_se->server_pick_task(dl_se);
+		p = dl_se->server_pick_task(dl_se, core);
 		if (!p) {
 			dl_se->dl_yielded = 1;
 			update_curr_dl_se(rq, dl_se, 0);
@@ -2437,11 +2437,6 @@ static struct task_struct *__pick_task_dl(struct rq *rq)
 	return p;
 }
 
-static struct task_struct *pick_task_dl(struct rq *rq)
-{
-	return __pick_task_dl(rq);
-}
-
 static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_struct *next)
 {
 	struct sched_dl_entity *dl_se = &p->dl;

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7b03764..cf63875 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c

@@ -8723,7 +8723,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
 	resched_curr(rq);
 }
 
-static struct task_struct *pick_task_fair(struct rq *rq)
+static struct task_struct *pick_task_fair(struct rq *rq, bool core)
 {
 	struct sched_entity *se;
 	struct cfs_rq *cfs_rq;
@@ -8761,7 +8761,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
 	int new_tasks;
 
 again:
-	p = pick_task_fair(rq);
+	p = pick_task_fair(rq, false);
 	if (!p)
 		goto idle;
 	se = &p->se;
@@ -8850,9 +8850,10 @@ static bool fair_server_has_tasks(struct sched_dl_entity *dl_se)
 	return !!dl_se->rq->cfs.nr_running;
 }
 
-static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se)
+static struct task_struct *
+fair_server_pick_task(struct sched_dl_entity *dl_se, bool core)
 {
-	return pick_task_fair(dl_se->rq);
+	return pick_task_fair(dl_se->rq, core);
 }
 
 void fair_server_init(struct rq *rq)

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 7a105a0..d8c5667 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c

@@ -462,7 +462,7 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir
 	next->se.exec_start = rq_clock_task(rq);
 }
 
-struct task_struct *pick_task_idle(struct rq *rq)
+struct task_struct *pick_task_idle(struct rq *rq, bool core)
 {
 	return rq->idle;
 }

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 172c588..83a5d72 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c

@@ -1736,7 +1736,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
 	return rt_task_of(rt_se);
 }
 
-static struct task_struct *pick_task_rt(struct rq *rq)
+static struct task_struct *pick_task_rt(struct rq *rq, bool core)
 {
 	struct task_struct *p;
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 3744f16..8a560f0 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h

@@ -2302,7 +2302,7 @@ struct sched_class {
 
 	void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags);
 
-	struct task_struct *(*pick_task)(struct rq *rq);
+	struct task_struct *(*pick_task)(struct rq *rq, bool core);
 	/*
 	 * Optional! When implemented pick_next_task() should be equivalent to:
 	 *
@@ -2451,7 +2451,7 @@ static inline bool sched_fair_runnable(struct rq *rq)
 }
 
 extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
-extern struct task_struct *pick_task_idle(struct rq *rq);
+extern struct task_struct *pick_task_idle(struct rq *rq, bool core);
 
 #define SCA_CHECK		0x01
 #define SCA_MIGRATE_DISABLE	0x02

diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 058dd42..a147dcd 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c

@@ -33,7 +33,7 @@ static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool fir
 	stop->se.exec_start = rq_clock_task(rq);
 }
 
-static struct task_struct *pick_task_stop(struct rq *rq)
+static struct task_struct *pick_task_stop(struct rq *rq, bool core)
 {
 	if (!sched_stop_runnable(rq))
 		return NULL;