path: root/drivers/cpufreq/cppc_cpufreq.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * CPPC (Collaborative Processor Performance Control) driver for
 * interfacing with the CPUfreq layer and governors. See
 * cppc_acpi.c for CPPC specific methods.
 *
 * (C) Copyright 2014, 2015 Linaro Ltd.
 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>
 */

#define pr_fmt(fmt)	"CPPC Cpufreq:"	fmt

#include <linux/arch_topology.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/cpu.h>
#include <linux/cpufreq.h>
#include <linux/irq_work.h>
#include <linux/kthread.h>
#include <linux/time.h>
#include <linux/vmalloc.h>
#include <uapi/linux/sched/types.h>

#include <linux/unaligned.h>

#include <acpi/cppc_acpi.h>

static struct cpufreq_driver cppc_cpufreq_driver;

#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
static enum {
	FIE_UNSET = -1,
	FIE_ENABLED,
	FIE_DISABLED
} fie_disabled = FIE_UNSET;

module_param(fie_disabled, int, 0444);
MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");

/* Frequency invariance support */
struct cppc_freq_invariance {
	int cpu;
	struct irq_work irq_work;
	struct kthread_work work;
	struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
	struct cppc_cpudata *cpu_data;
};

static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
static struct kthread_worker *kworker_fie;

static int cppc_perf_from_fbctrs(u64 reference_perf,
				 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
				 struct cppc_perf_fb_ctrs *fb_ctrs_t1);

/**
 * __cppc_scale_freq_tick - CPPC arch_freq_scale updater for frequency invariance
 * @cppc_fi: per-cpu CPPC FIE data.
 *
 * The CPPC driver registers itself with the topology core to provide its own
 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
 * gets called by the scheduler on every tick.
 *
 * Note that the arch specific counters have higher priority than CPPC counters,
 * if available, though the CPPC driver doesn't need to have any special
 * handling for that.
 */
static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi)
{
	struct cppc_perf_fb_ctrs fb_ctrs = {0};
	struct cppc_cpudata *cpu_data;
	unsigned long local_freq_scale;
	u64 perf, ref_perf;

	cpu_data = cppc_fi->cpu_data;

	if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
		pr_warn("%s: failed to read perf counters\n", __func__);
		return;
	}

	ref_perf = cpu_data->perf_caps.reference_perf;
	perf = cppc_perf_from_fbctrs(ref_perf,
				     &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs);
	if (!perf)
		return;

	cppc_fi->prev_perf_fb_ctrs = fb_ctrs;

	perf <<= SCHED_CAPACITY_SHIFT;
	local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);

	/* This can happen due to counter's overflow */
	if (unlikely(local_freq_scale > 1024))
		local_freq_scale = 1024;

	per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
}

static void cppc_scale_freq_tick(void)
{
	__cppc_scale_freq_tick(&per_cpu(cppc_freq_inv, smp_processor_id()));
}

static struct scale_freq_data cppc_sftd = {
	.source = SCALE_FREQ_SOURCE_CPPC,
	.set_freq_scale = cppc_scale_freq_tick,
};

static void cppc_scale_freq_workfn(struct kthread_work *work)
{
	struct cppc_freq_invariance *cppc_fi;

	cppc_fi = container_of(work, struct cppc_freq_invariance, work);
	__cppc_scale_freq_tick(cppc_fi);
}

static void cppc_irq_work(struct irq_work *irq_work)
{
	struct cppc_freq_invariance *cppc_fi;

	cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
	kthread_queue_work(kworker_fie, &cppc_fi->work);
}

/*
 * Reading perf counters may sleep if the CPC regs are in PCC.  Thus, we
 * schedule an irq work in scale_freq_tick (since we reach here from hard-irq
 * context), which then schedules a normal work item cppc_scale_freq_workfn()
 * that updates the per_cpu arch_freq_scale variable based on the counter
 * updates since the last tick.
 */
static void cppc_scale_freq_tick_pcc(void)
{
	struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());

	/*
	 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
	 * context.
	 */
	irq_work_queue(&cppc_fi->irq_work);
}

static struct scale_freq_data cppc_sftd_pcc = {
	.source = SCALE_FREQ_SOURCE_CPPC,
	.set_freq_scale = cppc_scale_freq_tick_pcc,
};

static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
{
	struct scale_freq_data *sftd = &cppc_sftd;
	struct cppc_freq_invariance *cppc_fi;
	int cpu, ret;

	if (