917 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * CPPC (Collaborative Processor Performance Control) driver for4 * interfacing with the CPUfreq layer and governors. See5 * cppc_acpi.c for CPPC specific methods.6 *7 * (C) Copyright 2014, 2015 Linaro Ltd.8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>9 */10 11#define pr_fmt(fmt) "CPPC Cpufreq:" fmt12 13#include <linux/arch_topology.h>14#include <linux/kernel.h>15#include <linux/module.h>16#include <linux/delay.h>17#include <linux/cpu.h>18#include <linux/cpufreq.h>19#include <linux/irq_work.h>20#include <linux/kthread.h>21#include <linux/time.h>22#include <linux/vmalloc.h>23#include <uapi/linux/sched/types.h>24 25#include <linux/unaligned.h>26 27#include <acpi/cppc_acpi.h>28 29/*30 * This list contains information parsed from per CPU ACPI _CPC and _PSD31 * structures: e.g. the highest and lowest supported performance, capabilities,32 * desired performance, level requested etc. Depending on the share_type, not33 * all CPUs will have an entry in the list.34 */35static LIST_HEAD(cpu_data_list);36 37static bool boost_supported;38 39struct cppc_workaround_oem_info {40 char oem_id[ACPI_OEM_ID_SIZE + 1];41 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];42 u32 oem_revision;43};44 45static struct cppc_workaround_oem_info wa_info[] = {46 {47 .oem_id = "HISI ",48 .oem_table_id = "HIP07 ",49 .oem_revision = 0,50 }, {51 .oem_id = "HISI ",52 .oem_table_id = "HIP08 ",53 .oem_revision = 0,54 }55};56 57static struct cpufreq_driver cppc_cpufreq_driver;58 59static enum {60 FIE_UNSET = -1,61 FIE_ENABLED,62 FIE_DISABLED63} fie_disabled = FIE_UNSET;64 65#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE66module_param(fie_disabled, int, 0444);67MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");68 69/* Frequency invariance support */70struct cppc_freq_invariance {71 int cpu;72 struct irq_work irq_work;73 struct kthread_work work;74 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;75 struct cppc_cpudata *cpu_data;76};77 78static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);79static struct kthread_worker *kworker_fie;80 81static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu);82static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,83 struct cppc_perf_fb_ctrs *fb_ctrs_t0,84 struct cppc_perf_fb_ctrs *fb_ctrs_t1);85 86/**87 * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance88 * @work: The work item.89 *90 * The CPPC driver register itself with the topology core to provide its own91 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which92 * gets called by the scheduler on every tick.93 *94 * Note that the arch specific counters have higher priority than CPPC counters,95 * if available, though the CPPC driver doesn't need to have any special96 * handling for that.97 *98 * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we99 * reach here from hard-irq context), which then schedules a normal work item100 * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable101 * based on the counter updates since the last tick.102 */103static void cppc_scale_freq_workfn(struct kthread_work *work)104{105 struct cppc_freq_invariance *cppc_fi;106 struct cppc_perf_fb_ctrs fb_ctrs = {0};107 struct cppc_cpudata *cpu_data;108 unsigned long local_freq_scale;109 u64 perf;110 111 cppc_fi = container_of(work, struct cppc_freq_invariance, work);112 cpu_data = cppc_fi->cpu_data;113 114 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {115 pr_warn("%s: failed to read perf counters\n", __func__);116 return;117 }118 119 perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs,120 &fb_ctrs);121 cppc_fi->prev_perf_fb_ctrs = fb_ctrs;122 123 perf <<= SCHED_CAPACITY_SHIFT;124 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);125 126 /* This can happen due to counter's overflow */127 if (unlikely(local_freq_scale > 1024))128 local_freq_scale = 1024;129 130 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;131}132 133static void cppc_irq_work(struct irq_work *irq_work)134{135 struct cppc_freq_invariance *cppc_fi;136 137 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);138 kthread_queue_work(kworker_fie, &cppc_fi->work);139}140 141static void cppc_scale_freq_tick(void)142{143 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());144 145 /*146 * cppc_get_perf_ctrs() can potentially sleep, call that from the right147 * context.148 */149 irq_work_queue(&cppc_fi->irq_work);150}151 152static struct scale_freq_data cppc_sftd = {153 .source = SCALE_FREQ_SOURCE_CPPC,154 .set_freq_scale = cppc_scale_freq_tick,155};156 157static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)158{159 struct cppc_freq_invariance *cppc_fi;160 int cpu, ret;161 162 if (fie_disabled)163 return;164 165 for_each_cpu(cpu, policy->cpus) {166 cppc_fi = &per_cpu(cppc_freq_inv, cpu);167 cppc_fi->cpu = cpu;168 cppc_fi->cpu_data = policy->driver_data;169 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);170 init_irq_work(&cppc_fi->irq_work, cppc_irq_work);171 172 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);173 if (ret) {174 pr_warn("%s: failed to read perf counters for cpu:%d: %d\n",175 __func__, cpu, ret);176 177 /*178 * Don't abort if the CPU was offline while the driver179 * was getting registered.180 */181 if (cpu_online(cpu))182 return;183 }184 }185 186 /* Register for freq-invariance */187 topology_set_scale_freq_source(&cppc_sftd, policy->cpus);188}189 190/*191 * We free all the resources on policy's removal and not on CPU removal as the192 * irq-work are per-cpu and the hotplug core takes care of flushing the pending193 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work194 * fires on another CPU after the concerned CPU is removed, it won't harm.195 *196 * We just need to make sure to remove them all on policy->exit().197 */198static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)199{200 struct cppc_freq_invariance *cppc_fi;201 int cpu;202 203 if (fie_disabled)204 return;205 206 /* policy->cpus will be empty here, use related_cpus instead */207 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);208 209 for_each_cpu(cpu, policy->related_cpus) {210 cppc_fi = &per_cpu(cppc_freq_inv, cpu);211 irq_work_sync(&cppc_fi->irq_work);212 kthread_cancel_work_sync(&cppc_fi->work);213 }214}215 216static void __init cppc_freq_invariance_init(void)217{218 struct sched_attr attr = {219 .size = sizeof(struct sched_attr),220 .sched_policy = SCHED_DEADLINE,221 .sched_nice = 0,222 .sched_priority = 0,223 /*224 * Fake (unused) bandwidth; workaround to "fix"225 * priority inheritance.226 */227 .sched_runtime = NSEC_PER_MSEC,228 .sched_deadline = 10 * NSEC_PER_MSEC,229 .sched_period = 10 * NSEC_PER_MSEC,230 };231 int ret;232 233 if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {234 fie_disabled = FIE_ENABLED;235 if (cppc_perf_ctrs_in_pcc()) {236 pr_info("FIE not enabled on systems with registers in PCC\n");237 fie_disabled = FIE_DISABLED;238 }239 }240 241 if (fie_disabled)242 return;243 244 kworker_fie = kthread_create_worker(0, "cppc_fie");245 if (IS_ERR(kworker_fie)) {246 pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,247 PTR_ERR(kworker_fie));248 fie_disabled = FIE_DISABLED;249 return;250 }251 252 ret = sched_setattr_nocheck(kworker_fie->task, &attr);253 if (ret) {254 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,255 ret);256 kthread_destroy_worker(kworker_fie);257 fie_disabled = FIE_DISABLED;258 }259}260 261static void cppc_freq_invariance_exit(void)262{263 if (fie_disabled)264 return;265 266 kthread_destroy_worker(kworker_fie);267}268 269#else270static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)271{272}273 274static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)275{276}277 278static inline void cppc_freq_invariance_init(void)279{280}281 282static inline void cppc_freq_invariance_exit(void)283{284}285#endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */286 287static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,288 unsigned int target_freq,289 unsigned int relation)290{291 struct cppc_cpudata *cpu_data = policy->driver_data;292 unsigned int cpu = policy->cpu;293 struct cpufreq_freqs freqs;294 int ret = 0;295 296 cpu_data->perf_ctrls.desired_perf =297 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);298 freqs.old = policy->cur;299 freqs.new = target_freq;300 301 cpufreq_freq_transition_begin(policy, &freqs);302 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);303 cpufreq_freq_transition_end(policy, &freqs, ret != 0);304 305 if (ret)306 pr_debug("Failed to set target on CPU:%d. ret:%d\n",307 cpu, ret);308 309 return ret;310}311 312static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,313 unsigned int target_freq)314{315 struct cppc_cpudata *cpu_data = policy->driver_data;316 unsigned int cpu = policy->cpu;317 u32 desired_perf;318 int ret;319 320 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);321 cpu_data->perf_ctrls.desired_perf = desired_perf;322 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);323 324 if (ret) {325 pr_debug("Failed to set target on CPU:%d. ret:%d\n",326 cpu, ret);327 return 0;328 }329 330 return target_freq;331}332 333static int cppc_verify_policy(struct cpufreq_policy_data *policy)334{335 cpufreq_verify_within_cpu_limits(policy);336 return 0;337}338 339/*340 * The PCC subspace describes the rate at which platform can accept commands341 * on the shared PCC channel (including READs which do not count towards freq342 * transition requests), so ideally we need to use the PCC values as a fallback343 * if we don't have a platform specific transition_delay_us344 */345#ifdef CONFIG_ARM64346#include <asm/cputype.h>347 348static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)349{350 unsigned long implementor = read_cpuid_implementor();351 unsigned long part_num = read_cpuid_part_number();352 353 switch (implementor) {354 case ARM_CPU_IMP_QCOM:355 switch (part_num) {356 case QCOM_CPU_PART_FALKOR_V1:357 case QCOM_CPU_PART_FALKOR:358 return 10000;359 }360 }361 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;362}363#else364static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)365{366 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;367}368#endif369 370#if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)371 372static DEFINE_PER_CPU(unsigned int, efficiency_class);373static void cppc_cpufreq_register_em(struct cpufreq_policy *policy);374 375/* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */376#define CPPC_EM_CAP_STEP (20)377/* Increase the cost value by CPPC_EM_COST_STEP every performance state. */378#define CPPC_EM_COST_STEP (1)379/* Add a cost gap correspnding to the energy of 4 CPUs. */380#define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \381 / CPPC_EM_CAP_STEP)382 383static unsigned int get_perf_level_count(struct cpufreq_policy *policy)384{385 struct cppc_perf_caps *perf_caps;386 unsigned int min_cap, max_cap;387 struct cppc_cpudata *cpu_data;388 int cpu = policy->cpu;389 390 cpu_data = policy->driver_data;391 perf_caps = &cpu_data->perf_caps;392 max_cap = arch_scale_cpu_capacity(cpu);393 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,394 perf_caps->highest_perf);395 if ((min_cap == 0) || (max_cap < min_cap))396 return 0;397 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;398}399 400/*401 * The cost is defined as:402 * cost = power * max_frequency / frequency403 */404static inline unsigned long compute_cost(int cpu, int step)405{406 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +407 step * CPPC_EM_COST_STEP;408}409 410static int cppc_get_cpu_power(struct device *cpu_dev,411 unsigned long *power, unsigned long *KHz)412{413 unsigned long perf_step, perf_prev, perf, perf_check;414 unsigned int min_step, max_step, step, step_check;415 unsigned long prev_freq = *KHz;416 unsigned int min_cap, max_cap;417 struct cpufreq_policy *policy;418 419 struct cppc_perf_caps *perf_caps;420 struct cppc_cpudata *cpu_data;421 422 policy = cpufreq_cpu_get_raw(cpu_dev->id);423 cpu_data = policy->driver_data;424 perf_caps = &cpu_data->perf_caps;425 max_cap = arch_scale_cpu_capacity(cpu_dev->id);426 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,427 perf_caps->highest_perf);428 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,429 max_cap);430 min_step = min_cap / CPPC_EM_CAP_STEP;431 max_step = max_cap / CPPC_EM_CAP_STEP;432 433 perf_prev = cppc_khz_to_perf(perf_caps, *KHz);434 step = perf_prev / perf_step;435 436 if (step > max_step)437 return -EINVAL;438 439 if (min_step == max_step) {440 step = max_step;441 perf = perf_caps->highest_perf;442 } else if (step < min_step) {443 step = min_step;444 perf = perf_caps->lowest_perf;445 } else {446 step++;447 if (step == max_step)448 perf = perf_caps->highest_perf;449 else450 perf = step * perf_step;451 }452 453 *KHz = cppc_perf_to_khz(perf_caps, perf);454 perf_check = cppc_khz_to_perf(perf_caps, *KHz);455 step_check = perf_check / perf_step;456 457 /*458 * To avoid bad integer approximation, check that new frequency value459 * increased and that the new frequency will be converted to the460 * desired step value.461 */462 while ((*KHz == prev_freq) || (step_check != step)) {463 perf++;464 *KHz = cppc_perf_to_khz(perf_caps, perf);465 perf_check = cppc_khz_to_perf(perf_caps, *KHz);466 step_check = perf_check / perf_step;467 }468 469 /*470 * With an artificial EM, only the cost value is used. Still the power471 * is populated such as 0 < power < EM_MAX_POWER. This allows to add472 * more sense to the artificial performance states.473 */474 *power = compute_cost(cpu_dev->id, step);475 476 return 0;477}478 479static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,480 unsigned long *cost)481{482 unsigned long perf_step, perf_prev;483 struct cppc_perf_caps *perf_caps;484 struct cpufreq_policy *policy;485 struct cppc_cpudata *cpu_data;486 unsigned int max_cap;487 int step;488 489 policy = cpufreq_cpu_get_raw(cpu_dev->id);490 cpu_data = policy->driver_data;491 perf_caps = &cpu_data->perf_caps;492 max_cap = arch_scale_cpu_capacity(cpu_dev->id);493 494 perf_prev = cppc_khz_to_perf(perf_caps, KHz);495 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;496 step = perf_prev / perf_step;497 498 *cost = compute_cost(cpu_dev->id, step);499 500 return 0;501}502 503static int populate_efficiency_class(void)504{505 struct acpi_madt_generic_interrupt *gicc;506 DECLARE_BITMAP(used_classes, 256) = {};507 int class, cpu, index;508 509 for_each_possible_cpu(cpu) {510 gicc = acpi_cpu_get_madt_gicc(cpu);511 class = gicc->efficiency_class;512 bitmap_set(used_classes, class, 1);513 }514 515 if (bitmap_weight(used_classes, 256) <= 1) {516 pr_debug("Efficiency classes are all equal (=%d). "517 "No EM registered", class);518 return -EINVAL;519 }520 521 /*522 * Squeeze efficiency class values on [0:#efficiency_class-1].523 * Values are per spec in [0:255].524 */525 index = 0;526 for_each_set_bit(class, used_classes, 256) {527 for_each_possible_cpu(cpu) {528 gicc = acpi_cpu_get_madt_gicc(cpu);529 if (gicc->efficiency_class == class)530 per_cpu(efficiency_class, cpu) = index;531 }532 index++;533 }534 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;535 536 return 0;537}538 539static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)540{541 struct cppc_cpudata *cpu_data;542 struct em_data_callback em_cb =543 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);544 545 cpu_data = policy->driver_data;546 em_dev_register_perf_domain(get_cpu_device(policy->cpu),547 get_perf_level_count(policy), &em_cb,548 cpu_data->shared_cpu_map, 0);549}550 551#else552static int populate_efficiency_class(void)553{554 return 0;555}556#endif557 558static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)559{560 struct cppc_cpudata *cpu_data;561 int ret;562 563 cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);564 if (!cpu_data)565 goto out;566 567 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))568 goto free_cpu;569 570 ret = acpi_get_psd_map(cpu, cpu_data);571 if (ret) {572 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);573 goto free_mask;574 }575 576 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);577 if (ret) {578 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);579 goto free_mask;580 }581 582 list_add(&cpu_data->node, &cpu_data_list);583 584 return cpu_data;585 586free_mask:587 free_cpumask_var(cpu_data->shared_cpu_map);588free_cpu:589 kfree(cpu_data);590out:591 return NULL;592}593 594static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)595{596 struct cppc_cpudata *cpu_data = policy->driver_data;597 598 list_del(&cpu_data->node);599 free_cpumask_var(cpu_data->shared_cpu_map);600 kfree(cpu_data);601 policy->driver_data = NULL;602}603 604static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)605{606 unsigned int cpu = policy->cpu;607 struct cppc_cpudata *cpu_data;608 struct cppc_perf_caps *caps;609 int ret;610 611 cpu_data = cppc_cpufreq_get_cpu_data(cpu);612 if (!cpu_data) {613 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);614 return -ENODEV;615 }616 caps = &cpu_data->perf_caps;617 policy->driver_data = cpu_data;618 619 /*620 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see621 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)622 */623 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);624 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);625 626 /*627 * Set cpuinfo.min_freq to Lowest to make the full range of performance628 * available if userspace wants to use any perf between lowest & lowest629 * nonlinear perf630 */631 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);632 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf);633 634 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);635 policy->shared_type = cpu_data->shared_type;636 637 switch (policy->shared_type) {638 case CPUFREQ_SHARED_TYPE_HW:639 case CPUFREQ_SHARED_TYPE_NONE:640 /* Nothing to be done - we'll have a policy for each CPU */641 break;642 case CPUFREQ_SHARED_TYPE_ANY:643 /*644 * All CPUs in the domain will share a policy and all cpufreq645 * operations will use a single cppc_cpudata structure stored646 * in policy->driver_data.647 */648 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);649 break;650 default:651 pr_debug("Unsupported CPU co-ord type: %d\n",652 policy->shared_type);653 ret = -EFAULT;654 goto out;655 }656 657 policy->fast_switch_possible = cppc_allow_fast_switch();658 policy->dvfs_possible_from_any_cpu = true;659 660 /*661 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost662 * is supported.663 */664 if (caps->highest_perf > caps->nominal_perf)665 boost_supported = true;666 667 /* Set policy->cur to max now. The governors will adjust later. */668 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);669 cpu_data->perf_ctrls.desired_perf = caps->highest_perf;670 671 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);672 if (ret) {673 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",674 caps->highest_perf, cpu, ret);675 goto out;676 }677 678 cppc_cpufreq_cpu_fie_init(policy);679 return 0;680 681out:682 cppc_cpufreq_put_cpu_data(policy);683 return ret;684}685 686static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)687{688 struct cppc_cpudata *cpu_data = policy->driver_data;689 struct cppc_perf_caps *caps = &cpu_data->perf_caps;690 unsigned int cpu = policy->cpu;691 int ret;692 693 cppc_cpufreq_cpu_fie_exit(policy);694 695 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;696 697 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);698 if (ret)699 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",700 caps->lowest_perf, cpu, ret);701 702 cppc_cpufreq_put_cpu_data(policy);703}704 705static inline u64 get_delta(u64 t1, u64 t0)706{707 if (t1 > t0 || t0 > ~(u32)0)708 return t1 - t0;709 710 return (u32)t1 - (u32)t0;711}712 713static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,714 struct cppc_perf_fb_ctrs *fb_ctrs_t0,715 struct cppc_perf_fb_ctrs *fb_ctrs_t1)716{717 u64 delta_reference, delta_delivered;718 u64 reference_perf;719 720 reference_perf = fb_ctrs_t0->reference_perf;721 722 delta_reference = get_delta(fb_ctrs_t1->reference,723 fb_ctrs_t0->reference);724 delta_delivered = get_delta(fb_ctrs_t1->delivered,725 fb_ctrs_t0->delivered);726 727 /* Check to avoid divide-by zero and invalid delivered_perf */728 if (!delta_reference || !delta_delivered)729 return cpu_data->perf_ctrls.desired_perf;730 731 return (reference_perf * delta_delivered) / delta_reference;732}733 734static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)735{736 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};737 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);738 struct cppc_cpudata *cpu_data;739 u64 delivered_perf;740 int ret;741 742 if (!policy)743 return -ENODEV;744 745 cpu_data = policy->driver_data;746 747 cpufreq_cpu_put(policy);748 749 ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0);750 if (ret)751 return 0;752 753 udelay(2); /* 2usec delay between sampling */754 755 ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1);756 if (ret)757 return 0;758 759 delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0,760 &fb_ctrs_t1);761 762 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);763}764 765static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)766{767 struct cppc_cpudata *cpu_data = policy->driver_data;768 struct cppc_perf_caps *caps = &cpu_data->perf_caps;769 int ret;770 771 if (!boost_supported) {772 pr_err("BOOST not supported by CPU or firmware\n");773 return -EINVAL;774 }775 776 if (state)777 policy->max = cppc_perf_to_khz(caps, caps->highest_perf);778 else779 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);780 policy->cpuinfo.max_freq = policy->max;781 782 ret = freq_qos_update_request(policy->max_freq_req, policy->max);783 if (ret < 0)784 return ret;785 786 return 0;787}788 789static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)790{791 struct cppc_cpudata *cpu_data = policy->driver_data;792 793 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);794}795cpufreq_freq_attr_ro(freqdomain_cpus);796 797static struct freq_attr *cppc_cpufreq_attr[] = {798 &freqdomain_cpus,799 NULL,800};801 802static struct cpufreq_driver cppc_cpufreq_driver = {803 .flags = CPUFREQ_CONST_LOOPS,804 .verify = cppc_verify_policy,805 .target = cppc_cpufreq_set_target,806 .get = cppc_cpufreq_get_rate,807 .fast_switch = cppc_cpufreq_fast_switch,808 .init = cppc_cpufreq_cpu_init,809 .exit = cppc_cpufreq_cpu_exit,810 .set_boost = cppc_cpufreq_set_boost,811 .attr = cppc_cpufreq_attr,812 .name = "cppc_cpufreq",813};814 815/*816 * HISI platform does not support delivered performance counter and817 * reference performance counter. It can calculate the performance using the818 * platform specific mechanism. We reuse the desired performance register to819 * store the real performance calculated by the platform.820 */821static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu)822{823 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);824 struct cppc_cpudata *cpu_data;825 u64 desired_perf;826 int ret;827 828 if (!policy)829 return -ENODEV;830 831 cpu_data = policy->driver_data;832 833 cpufreq_cpu_put(policy);834 835 ret = cppc_get_desired_perf(cpu, &desired_perf);836 if (ret < 0)837 return -EIO;838 839 return cppc_perf_to_khz(&cpu_data->perf_caps, desired_perf);840}841 842static void cppc_check_hisi_workaround(void)843{844 struct acpi_table_header *tbl;845 acpi_status status = AE_OK;846 int i;847 848 status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl);849 if (ACPI_FAILURE(status) || !tbl)850 return;851 852 for (i = 0; i < ARRAY_SIZE(wa_info); i++) {853 if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) &&854 !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&855 wa_info[i].oem_revision == tbl->oem_revision) {856 /* Overwrite the get() callback */857 cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate;858 fie_disabled = FIE_DISABLED;859 break;860 }861 }862 863 acpi_put_table(tbl);864}865 866static int __init cppc_cpufreq_init(void)867{868 int ret;869 870 if (!acpi_cpc_valid())871 return -ENODEV;872 873 cppc_check_hisi_workaround();874 cppc_freq_invariance_init();875 populate_efficiency_class();876 877 ret = cpufreq_register_driver(&cppc_cpufreq_driver);878 if (ret)879 cppc_freq_invariance_exit();880 881 return ret;882}883 884static inline void free_cpu_data(void)885{886 struct cppc_cpudata *iter, *tmp;887 888 list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) {889 free_cpumask_var(iter->shared_cpu_map);890 list_del(&iter->node);891 kfree(iter);892 }893 894}895 896static void __exit cppc_cpufreq_exit(void)897{898 cpufreq_unregister_driver(&cppc_cpufreq_driver);899 cppc_freq_invariance_exit();900 901 free_cpu_data();902}903 904module_exit(cppc_cpufreq_exit);905MODULE_AUTHOR("Ashwin Chaugule");906MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");907MODULE_LICENSE("GPL");908 909late_initcall(cppc_cpufreq_init);910 911static const struct acpi_device_id cppc_acpi_ids[] __used = {912 {ACPI_PROCESSOR_DEVICE_HID, },913 {}914};915 916MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);917