kernel/sched/isolation.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 *  Housekeeping management. Manage the targets for routine code that can run on
 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
 *
 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
 * Copyright (C) 2017-2022 SUSE, Frederic Weisbecker
 *
 */

enum hk_flags {
	HK_FLAG_NOHZ_FULL	= BIT(HK_TYPE_NOHZ_FULL),
	HK_FLAG_SCHED		= BIT(HK_TYPE_SCHED),
	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
};

DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
EXPORT_SYMBOL_GPL(housekeeping_overridden);

struct housekeeping {
	struct cpumask __rcu *non_nohz_full;
	cpumask_var_t cpumasks[HK_TYPE_MAX];
	unsigned long flags;
};

static struct housekeeping housekeeping;

DEFINE_PERCPU_RWSEM(housekeeping_rwsem);

bool housekeeping_enabled(enum hk_type type)
{
	if (type == HK_TYPE_NOHZ_FULL)
		return !!rcu_dereference(housekeeping.non_nohz_full);
	return !!(housekeeping.flags & BIT(type));
}
EXPORT_SYMBOL_GPL(housekeeping_enabled);

int housekeeping_any_cpu(enum hk_type type)
{
	if (static_branch_unlikely(&housekeeping_overridden)) {
		struct cpumask *cpumask;
		int cpu;

		if (type == HK_TYPE_NOHZ_FULL) {
			cpumask = rcu_dereference(housekeeping.non_nohz_full);
			if (!cpumask)
				return smp_processor_id();
		} else if (housekeeping.flags & BIT(type)) {
			cpumask = housekeeping.cpumasks[type];
		} else {
			return smp_processor_id();
		}

		cpu = sched_numa_find_closest(cpumask, smp_processor_id());
		if (cpu < nr_cpu_ids)
			return cpu;

		return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
	}
	return smp_processor_id();
}
EXPORT_SYMBOL_GPL(housekeeping_any_cpu);

const struct cpumask *housekeeping_cpumask(enum hk_type type)
{
	if (static_branch_unlikely(&housekeeping_overridden)) {
		if (type == HK_TYPE_NOHZ_FULL) {
			struct cpumask *cpumask;
			cpumask = rcu_dereference(housekeeping.non_nohz_full);
			if (!cpumask)
				return cpu_possible_mask;
		} else if (housekeeping.flags & BIT(type)) {
			return housekeeping.cpumasks[type];
		}
	}
	return cpu_possible_mask;
}
EXPORT_SYMBOL_GPL(housekeeping_cpumask);

void housekeeping_affine(struct task_struct *t, enum hk_type type)
{
	if (static_branch_unlikely(&housekeeping_overridden)) {
		hk_down_read();
		if (type == HK_TYPE_NOHZ_FULL) {
			struct cpumask *cpumask;
			cpumask = rcu_dereference(housekeeping.non_nohz_full);
			if (cpumask)
				set_cpus_allowed_ptr(t, cpumask);
		} else if (housekeeping.flags & BIT(type)) {
			set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
		}
		hk_up_read();
	}
}
EXPORT_SYMBOL_GPL(housekeeping_affine);

bool housekeeping_test_cpu(int cpu, enum hk_type type)
{
	if (static_branch_unlikely(&housekeeping_overridden)) {
		if (type == HK_TYPE_NOHZ_FULL) {
			struct cpumask *cpumask;
			cpumask = rcu_dereference(housekeeping.non_nohz_full);
			if (cpumask)
				cpumask_test_cpu(cpu, cpumask);
		} else if (housekeeping.flags & BIT(type)) {
			return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
		}
	}
	return true;
}
EXPORT_SYMBOL_GPL(housekeeping_test_cpu);

static int housekeeping_cpumask_update(struct cpumask *cpumask,
				       enum hk_type type, bool on)
{
	int err;

	switch (type) {
	case HK_TYPE_NOHZ_FULL:
		err = rcu_nocb_cpumask_update(cpumask, on);
		break;
	default:
		err = -EINVAL;
	}

	if (err >= 0) {
		if (on) {
			cpumask_or(housekeeping.cpumasks[type],
				   housekeeping.cpumasks[type],
				   cpumask);
		} else {
			cpumask_andnot(housekeeping.cpumasks[type],
				       housekeeping.cpumasks[type],
				       cpumask);
		}
	}

	return err;
}

int housekeeping_cpumask_set(struct cpumask *cpumask, enum hk_type type)
{
	return housekeeping_cpumask_update(cpumask, type, true);
}

int housekeeping_cpumask_clear(struct cpumask *cpumask, enum hk_type type)
{
	return housekeeping_cpumask_update(cpumask, type, false);
}

void __init housekeeping_init(void)
{
	enum hk_type type;

	if (!housekeeping.flags)
		return;

	static_branch_enable(&housekeeping_overridden);

	if (housekeeping.flags & HK_FLAG_NOHZ_FULL)
		sched_tick_offload_init();

	for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
		if (type == HK_TYPE_NOHZ_FULL) {
			struct cpumask *cpumask;
			cpumask = rcu_dereference(housekeeping.non_nohz_full);
			if (!WARN_ON_ONCE(!cpumask))
				WARN_ON_ONCE(cpumask_empty(cpumask));
		}
		/* We need at least one CPU to handle housekeeping work */
		WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
	}
}

static void __init housekeeping_setup_type(enum hk_type type,
					   cpumask_var_t housekeeping_staging)
{
	if (type == HK_TYPE_NOHZ_FULL) {
		struct cpumask *cpumask = kzalloc(sizeof(*cpumask), GFP_KERNEL);
		if (WARN_ON_ONCE(!cpumask))
			return;

		cpumask_copy(cpumask, housekeeping_staging);
		rcu_assign_pointer(housekeeping.non_nohz_full, cpumask);
	} else {
		alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
		cpumask_copy(housekeeping.cpumasks[type],
			     housekeeping_staging);
	}
}

static int __init housekeeping_setup(char *str, unsigned long flags)
{
	cpumask_var_t non_housekeeping_mask, housekeeping_staging;
	int err = 0;

	if ((flags & HK_FLAG_NOHZ_FULL) && !housekeeping_enabled(HK_TYPE_NOHZ_FULL)) {
		if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
			pr_warn("Housekeeping: nohz unsupported."
				" Build with CONFIG_NO_HZ_FULL\n");
			return 0;
		}
	}

	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
		goto free_non_housekeeping_mask;
	}

	alloc_bootmem_cpumask_var(&housekeeping_staging);
	cpumask_andnot(housekeeping_staging,
		       cpu_possible_mask, non_housekeeping_mask);

	if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
		__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
		__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
		pr_warn("Housekeeping: must include one present CPU, "
			"using boot CPU:%d\n", smp_processor_id());
	}

	if (!housekeeping.flags && !housekeeping_enabled(HK_TYPE_NOHZ_FULL)) {
		/* First setup call ("nohz_full=" or "isolcpus=") */
		enum hk_type type;

		for_each_set_bit(type, &flags, HK_TYPE_MAX)
			housekeeping_setup_type(type, housekeeping_staging);
	} else {
		/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
		enum hk_type type;
		unsigned long iter_flags;
		unsigned long oflags = housekeeping.flags;

		if (housekeeping_enabled(HK_TYPE_NOHZ_FULL))
			oflags |= HK_FLAG_NOHZ_FULL;

		/* First check that nohz_full= matches isolcpus=nohz */
		iter_flags = flags & oflags;
		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
			struct cpumask *ocpumask;

			if (type == HK_TYPE_NOHZ_FULL) {
				ocpumask = rcu_dereference(housekeeping.non_nohz_full);
				if (WARN_ON_ONCE(!ocpumask))
					goto free_housekeeping_staging;
			} else {
				ocpumask = housekeeping.cpumasks[type];
			}
			if (!cpumask_equal(housekeeping_staging, ocpumask)) {
				pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
				goto free_housekeeping_staging;
			}
		}

		/* Second, handle the newcomers */
		iter_flags = flags & ~housekeeping.flags;
		if (housekeeping_enabled(HK_TYPE_NOHZ_FULL))
			iter_flags &= ~HK_FLAG_NOHZ_FULL;

		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
			housekeeping_setup_type(type, housekeeping_staging);
	}

	if ((flags & HK_FLAG_NOHZ_FULL) && !housekeeping_enabled(HK_TYPE_NOHZ_FULL))
		tick_nohz_full_setup(non_housekeeping_mask);

	housekeeping.flags |= flags & ~HK_FLAG_NOHZ_FULL;
	err = 1;

free_housekeeping_staging:
	free_bootmem_cpumask_var(housekeeping_staging);
free_non_housekeeping_mask:
	free_bootmem_cpumask_var(non_housekeeping_mask);

	return err;
}

static int __init housekeeping_nohz_full_setup(char *str)
{
	return housekeeping_setup(str, HK_FLAG_NOHZ_FULL);
}
__setup("nohz_full=", housekeeping_nohz_full_setup);

static int __init housekeeping_isolcpus_setup(char *str)
{
	unsigned long flags = 0;
	bool illegal = false;
	char *par;
	int len;

	while (isalpha(*str)) {
		if (!strncmp(str, "nohz,", 5)) {
			str += 5;
			flags |= HK_FLAG_NOHZ_FULL;
			continue;
		}

		if (!strncmp(str, "domain,", 7)) {
			str += 7;
			flags |= HK_FLAG_DOMAIN;
			continue;
		}

		if (!strncmp(str, "managed_irq,", 12)) {
			str += 12;
			flags |= HK_FLAG_MANAGED_IRQ;
			continue;
		}

		/*
		 * Skip unknown sub-parameter and validate that it is not
		 * containing an invalid character.
		 */
		for (par = str, len = 0; *str && *str != ','; str++, len++) {
			if (!isalpha(*str) && *str != '_')
				illegal = true;
		}

		if (illegal) {
			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
			return 0;
		}

		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
		str++;
	}

	/* Default behaviour for isolcpus without flags */
	if (!flags)
		flags |= HK_FLAG_DOMAIN;

	return housekeeping_setup(str, flags);
}
__setup("isolcpus=", housekeeping_isolcpus_setup);