Merge branch 'pm-cpufreq-next' into linux-next * pm-cpufreq-next: cpufreq: intel_pstate: Avoid using DESIRED_PERF when DEC is enabled cpufreq: intel_pstate: Consolidate HWP P-states initialization cpufreq: schedutil: Fix rate limit overflow
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 7ce5981..ceb340f 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c
@@ -303,6 +303,7 @@ static bool per_cpu_limits __ro_after_init; static bool hwp_forced __ro_after_init; static bool hwp_boost __read_mostly; static bool hwp_is_hybrid; +static u32 hwp_desired_mask __read_mostly = ~0U; static struct cpufreq_driver *intel_pstate_driver __read_mostly; @@ -555,47 +556,6 @@ static int intel_pstate_freq_to_hwp(struct cpudata *cpu, int freq) return intel_pstate_freq_to_hwp_rel(cpu, freq, CPUFREQ_RELATION_L); } -/** - * intel_pstate_hybrid_hwp_adjust - Calibrate HWP performance levels. - * @cpu: Target CPU. - * - * On hybrid processors, HWP may expose more performance levels than there are - * P-states accessible through the PERF_CTL interface. If that happens, the - * scaling factor between HWP performance levels and CPU frequency will be less - * than the scaling factor between P-state values and CPU frequency. - * - * In that case, adjust the CPU parameters used in computations accordingly. - */ -static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) -{ - int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; - int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; - int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu); - int scaling = cpu->pstate.scaling; - int freq; - - pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); - pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpu->cpu, perf_ctl_turbo); - pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpu->cpu, perf_ctl_scaling); - pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); - pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate); - pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling); - - if (scaling == perf_ctl_scaling) - return; - - hwp_is_hybrid = true; - - freq = perf_ctl_max_phys * perf_ctl_scaling; - cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq); - - /* - * Cast the min P-state value retrieved via pstate_funcs.get_min() to - * the effective range of HWP performance levels. - */ - cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, cpu->pstate.min_freq); -} - static bool turbo_is_disabled(void) { u64 misc_en; @@ -2293,34 +2253,70 @@ static int hwp_get_cpu_scaling(int cpu) return intel_pstate_cppc_get_scaling(cpu); } +static void intel_pstate_get_hwp_pstates(struct cpudata *cpu) +{ + int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; + int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; + int perf_ctl_turbo = cpu->pstate.turbo_pstate; + int cpuid = cpu->cpu; + + __intel_pstate_get_hwp_cap(cpu); + + if (!pstate_funcs.get_cpu_scaling) + return; + + pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpuid, perf_ctl_max_phys); + pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpuid, perf_ctl_turbo); + pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpuid, perf_ctl_scaling); + pr_debug("CPU%d: PERF_CTL min = %d\n", cpuid, cpu->pstate.min_pstate); + pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpuid, cpu->pstate.max_pstate); + pr_debug("CPU%d: HWP_CAP highest = %d\n", cpuid, cpu->pstate.turbo_pstate); + + cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpuid); + + pr_debug("CPU%d: HWP-to-frequency scaling = %d\n", cpuid, cpu->pstate.scaling); + + /* + * On hybrid processors, HWP may expose more performance levels than + * there are P-states accessible through the PERF_CTL interface. If + * that happens, the scaling between HWP performance levels and CPU + * frequency will be less than the scaling between P-state values and + * CPU frequency. In that case, update the maximum physical non-turbo + * performance level accordingly. + */ + if (cpu->pstate.scaling != perf_ctl_scaling) { + int freq; + + freq = perf_ctl_max_phys * perf_ctl_scaling; + cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq); + + freq = cpu->pstate.min_freq; + cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq); + + hwp_is_hybrid = true; + } + /* + * If the CPU is going online for the first time and it was offline + * initially, asym capacity scaling may need to be updated. + */ + hybrid_update_capacity(cpu); +} + static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { int perf_ctl_scaling = pstate_funcs.get_scaling(); cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu); + cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling; cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; + cpu->pstate.scaling = perf_ctl_scaling; - if (hwp_active) { - __intel_pstate_get_hwp_cap(cpu); - - if (pstate_funcs.get_cpu_scaling) { - cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu); - intel_pstate_hybrid_hwp_adjust(cpu); - } else { - cpu->pstate.scaling = perf_ctl_scaling; - } - /* - * If the CPU is going online for the first time and it was - * offline initially, asym capacity scaling needs to be updated. - */ - hybrid_update_capacity(cpu); - } else { - cpu->pstate.scaling = perf_ctl_scaling; + if (hwp_active) + intel_pstate_get_hwp_pstates(cpu); + else cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu); - cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); - } intel_pstate_update_freq_limits(cpu); @@ -3155,7 +3151,7 @@ static void intel_cpufreq_hwp_update(struct cpudata *cpu, u32 min, u32 max, value |= HWP_MAX_PERF(max); value &= ~HWP_DESIRED_PERF(~0L); - value |= HWP_DESIRED_PERF(desired); + value |= HWP_DESIRED_PERF(desired & hwp_desired_mask); if (value == prev) return; @@ -3765,9 +3761,9 @@ static bool hwp_check_epp(void) static bool hwp_check_dec(void) { - u64 power_ctl; + u64 power_ctl = 0; - rdmsrq(MSR_IA32_POWER_CTL, power_ctl); + rdmsrq_safe(MSR_IA32_POWER_CTL, &power_ctl); return !!(power_ctl & BIT(POWER_CTL_DEC_ENABLE)); } @@ -3792,6 +3788,7 @@ static int __init intel_pstate_init(void) id = x86_match_cpu(hwp_support_ids); if (id) { bool epp_present = hwp_check_epp(); + bool dec_present = hwp_check_dec(); /* * If HWP is enabled already, there is no choice but to deal @@ -3803,7 +3800,7 @@ static int __init intel_pstate_init(void) no_hwp = 0; } else if (no_load) { return -ENODEV; - } else if (!epp_present && !hwp_check_dec()) { + } else if (!epp_present && !dec_present) { /* * Avoid enabling HWP for processors without EPP support * unless the Dynamic Efficiency Control (DEC) enable @@ -3825,6 +3822,9 @@ static int __init intel_pstate_init(void) if (!default_driver) default_driver = &intel_pstate; + if (dec_present) + hwp_desired_mask = 0; + if (!id->driver_data) pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index a178275..49ccd6f 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c
@@ -62,6 +62,17 @@ static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); /************************ Governor internals ***********************/ +static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) +{ + /* + * Cast rate_limit_us before multiplication to force 64-bit arithmetic. + * Otherwise, on 32-bit platforms, both operands are converted to + * 32-bit unsigned long and the multiplication may overflow. + */ + sg_policy->freq_update_delay_ns = + (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; +} + static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; @@ -608,7 +619,7 @@ rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count tunables->rate_limit_us = rate_limit_us; list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) - sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); return count; } @@ -850,7 +861,7 @@ static int sugov_start(struct cpufreq_policy *policy) void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); unsigned int cpu; - sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); sg_policy->last_freq_update_time = 0; sg_policy->next_freq = 0; sg_policy->work_in_progress = false;