autonuma: Support to scan page table asynchronously In AutoNUMA, the page tables of processes are scanned periodically to trigger the NUMA hint page faults. The scanning is done synchronously. This has many advantages compared with asynchronous scanning, including - The processes which benefit from the AutoNUMA will pay the overhead - The hot pages will be scanned more often, while the cold pages will not - Reduces the cache ping-pong for page table itself, and TLB flushing One drawback is that this may introduce the latency outliers. With default configuration, scanning code could take several milliseconds to complete in tests. This is acceptable for most workloads, but may be not desirable for some other workloads. One possible solution is to make it possible to trigger page table scanning synchronously, and offload the actual page table scanning to some kernel threads (in fact work queue). And users can switch between synchronous and asynchronous scanning at run time via a sysfs knob. The patch has been tested with pmbench (which can measure memory access latency) on a 2-socket server machine with 256 GB memory. Latency (ms) Base (count) Async (count) 0.5-1 2399 1 1-2 13436 0 2-4 7435 1 In test, the pmbench score has no measurable changes between base and patched kernel with asynchronous scanning. But as in the above table the number of the latency outliers reduces from tens thousands to nearly 0. The test time is 3600s. TODO: ABI document Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
diff --git a/include/linux/sched.h b/include/linux/sched.h index afe01e2..9262c82 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h
@@ -1103,6 +1103,7 @@ struct task_struct { u64 last_task_numa_placement; u64 last_sum_exec_runtime; struct callback_head numa_work; + struct work_struct numa_async_work; /* * This pointer is only modified for current in syscall and
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 1b51723..5b6cff4 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h
@@ -64,6 +64,8 @@ extern unsigned int sysctl_numa_balancing_force_enable; #define sysctl_numa_balancing_force_enable 0 #endif +extern unsigned int sysctl_numa_balancing_scan_async; + #ifdef CONFIG_SCHED_DEBUG extern __read_mostly unsigned int sysctl_sched_migration_cost; extern __read_mostly unsigned int sysctl_sched_nr_migrate;
diff --git a/kernel/exit.c b/kernel/exit.c index 733e80f..b327f8f 100644 --- a/kernel/exit.c +++ b/kernel/exit.c
@@ -803,6 +803,9 @@ void __noreturn do_exit(long code) if (group_dead) disassociate_ctty(1); exit_task_namespaces(tsk); +#ifdef CONFIG_NUMA_BALANCING + cancel_work_sync(&tsk->numa_async_work); +#endif exit_task_work(tsk); exit_thread(tsk);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 92279a1..2f8ec84 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c
@@ -1093,6 +1093,9 @@ unsigned int sysctl_numa_balancing_write_bias = 2; */ unsigned int sysctl_numa_balancing_force_enable = 0; +/* Scan asynchronously via work queue */ +unsigned int sysctl_numa_balancing_scan_async; + struct numa_group { refcount_t refcount; @@ -2886,24 +2889,17 @@ static void reset_ptenuma_scan(struct task_struct *p) p->mm->numa_scan_offset = 0; } -/* - * The expensive part of numa migration is done from task_work context. - * Triggered from task_tick_numa(). - */ -static noinline void task_numa_work(struct callback_head *work) +static void numa_scan(struct task_struct *p) { unsigned long migrate, next_scan, now = jiffies; - struct task_struct *p = current; + struct task_struct *pc = current; struct mm_struct *mm = p->mm; - u64 runtime = p->se.sum_exec_runtime; + u64 runtime = pc->se.sum_exec_runtime; struct vm_area_struct *vma; unsigned long start, end; unsigned long nr_pte_updates = 0; long pages, virtpages; - SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); - - work->next = work; /* * Who cares about NUMA placement when they're dying. * @@ -3026,12 +3022,36 @@ static noinline void task_numa_work(struct callback_head *work) * Usually update_task_scan_period slows down scanning enough; on an * overloaded system we need to limit overhead on a per task basis. */ - if (unlikely(p->se.sum_exec_runtime != runtime)) { - u64 diff = p->se.sum_exec_runtime - runtime; + if (unlikely(pc->se.sum_exec_runtime != runtime)) { + u64 diff = pc->se.sum_exec_runtime - runtime; p->node_stamp += 32 * diff; } } +/* + * The expensive part of numa migration is done from task_work context. + * Triggered from task_tick_numa(). + */ +static noinline void task_numa_work(struct callback_head *work) +{ + struct task_struct *p = current; + + SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work)); + + work->next = work; + + numa_scan(p); +} + +static void numa_async_work(struct work_struct *work) +{ + struct task_struct *p; + + p = container_of(work, struct task_struct, numa_async_work); + + numa_scan(p); +} + void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) { int mm_users = 0; @@ -3055,6 +3075,7 @@ void init_numa_balancing(unsigned long clone_flags, struct task_struct *p) p->last_sum_exec_runtime = 0; init_task_work(&p->numa_work, task_numa_work); + INIT_WORK(&p->numa_async_work, numa_async_work); /* New address space, reset the preferred nid */ if (!(clone_flags & CLONE_VM)) { @@ -3104,8 +3125,14 @@ static void task_tick_numa(struct rq *rq, struct task_struct *curr) curr->numa_scan_period = task_scan_start(curr); curr->node_stamp += period; - if (!time_before(jiffies, curr->mm->numa_next_scan)) - task_work_add(curr, work, true); + if (!time_before(jiffies, curr->mm->numa_next_scan)) { + if (sysctl_numa_balancing_scan_async) + queue_work_node(numa_node_id(), + system_unbound_wq, + &curr->numa_async_work); + else + task_work_add(curr, work, true); + } } }
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 21fedb8..305e01a 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c
@@ -1755,6 +1755,15 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ONE, }, { + .procname = "numa_balancing_scan_async", + .data = &sysctl_numa_balancing_scan_async, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { .procname = "numa_balancing_hot_threshold_ms", .data = &sysctl_numa_balancing_hot_threshold, .maxlen = sizeof(unsigned int),