1166 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * POWERNV cpufreq driver for the IBM POWER processors4 *5 * (C) Copyright IBM 20146 *7 * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>8 */9 10#define pr_fmt(fmt) "powernv-cpufreq: " fmt11 12#include <linux/kernel.h>13#include <linux/sysfs.h>14#include <linux/cpumask.h>15#include <linux/module.h>16#include <linux/cpufreq.h>17#include <linux/smp.h>18#include <linux/of.h>19#include <linux/reboot.h>20#include <linux/slab.h>21#include <linux/cpu.h>22#include <linux/hashtable.h>23#include <trace/events/power.h>24 25#include <asm/cputhreads.h>26#include <asm/firmware.h>27#include <asm/reg.h>28#include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */29#include <asm/opal.h>30#include <linux/timer.h>31 32#define POWERNV_MAX_PSTATES_ORDER 833#define POWERNV_MAX_PSTATES (1UL << (POWERNV_MAX_PSTATES_ORDER))34#define PMSR_PSAFE_ENABLE (1UL << 30)35#define PMSR_SPR_EM_DISABLE (1UL << 31)36#define MAX_PSTATE_SHIFT 3237#define LPSTATE_SHIFT 4838#define GPSTATE_SHIFT 5639#define MAX_NR_CHIPS 3240 41#define MAX_RAMP_DOWN_TIME 512042/*43 * On an idle system we want the global pstate to ramp-down from max value to44 * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and45 * then ramp-down rapidly later on.46 *47 * This gives a percentage rampdown for time elapsed in milliseconds.48 * ramp_down_percentage = ((ms * ms) >> 18)49 * ~= 3.8 * (sec * sec)50 *51 * At 0 ms ramp_down_percent = 052 * At 5120 ms ramp_down_percent = 10053 */54#define ramp_down_percent(time) ((time * time) >> 18)55 56/* Interval after which the timer is queued to bring down global pstate */57#define GPSTATE_TIMER_INTERVAL 200058 59/**60 * struct global_pstate_info - Per policy data structure to maintain history of61 * global pstates62 * @highest_lpstate_idx: The local pstate index from which we are63 * ramping down64 * @elapsed_time: Time in ms spent in ramping down from65 * highest_lpstate_idx66 * @last_sampled_time: Time from boot in ms when global pstates were67 * last set68 * @last_lpstate_idx: Last set value of local pstate and global69 * @last_gpstate_idx: pstate in terms of cpufreq table index70 * @timer: Is used for ramping down if cpu goes idle for71 * a long time with global pstate held high72 * @gpstate_lock: A spinlock to maintain synchronization between73 * routines called by the timer handler and74 * governer's target_index calls75 * @policy: Associated CPUFreq policy76 */77struct global_pstate_info {78 int highest_lpstate_idx;79 unsigned int elapsed_time;80 unsigned int last_sampled_time;81 int last_lpstate_idx;82 int last_gpstate_idx;83 spinlock_t gpstate_lock;84 struct timer_list timer;85 struct cpufreq_policy *policy;86};87 88static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];89 90static DEFINE_HASHTABLE(pstate_revmap, POWERNV_MAX_PSTATES_ORDER);91/**92 * struct pstate_idx_revmap_data: Entry in the hashmap pstate_revmap93 * indexed by a function of pstate id.94 *95 * @pstate_id: pstate id for this entry.96 *97 * @cpufreq_table_idx: Index into the powernv_freqs98 * cpufreq_frequency_table for frequency99 * corresponding to pstate_id.100 *101 * @hentry: hlist_node that hooks this entry into the pstate_revmap102 * hashtable103 */104struct pstate_idx_revmap_data {105 u8 pstate_id;106 unsigned int cpufreq_table_idx;107 struct hlist_node hentry;108};109 110static bool rebooting, throttled, occ_reset;111 112static const char * const throttle_reason[] = {113 "No throttling",114 "Power Cap",115 "Processor Over Temperature",116 "Power Supply Failure",117 "Over Current",118 "OCC Reset"119};120 121enum throttle_reason_type {122 NO_THROTTLE = 0,123 POWERCAP,124 CPU_OVERTEMP,125 POWER_SUPPLY_FAILURE,126 OVERCURRENT,127 OCC_RESET_THROTTLE,128 OCC_MAX_REASON129};130 131static struct chip {132 unsigned int id;133 bool throttled;134 bool restore;135 u8 throttle_reason;136 cpumask_t mask;137 struct work_struct throttle;138 int throttle_turbo;139 int throttle_sub_turbo;140 int reason[OCC_MAX_REASON];141} *chips;142 143static int nr_chips;144static DEFINE_PER_CPU(struct chip *, chip_info);145 146/*147 * Note:148 * The set of pstates consists of contiguous integers.149 * powernv_pstate_info stores the index of the frequency table for150 * max, min and nominal frequencies. It also stores number of151 * available frequencies.152 *153 * powernv_pstate_info.nominal indicates the index to the highest154 * non-turbo frequency.155 */156static struct powernv_pstate_info {157 unsigned int min;158 unsigned int max;159 unsigned int nominal;160 unsigned int nr_pstates;161 bool wof_enabled;162} powernv_pstate_info;163 164static inline u8 extract_pstate(u64 pmsr_val, unsigned int shift)165{166 return ((pmsr_val >> shift) & 0xFF);167}168 169#define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT)170#define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT)171#define extract_max_pstate(x) extract_pstate(x, MAX_PSTATE_SHIFT)172 173/* Use following functions for conversions between pstate_id and index */174 175/*176 * idx_to_pstate : Returns the pstate id corresponding to the177 * frequency in the cpufreq frequency table178 * powernv_freqs indexed by @i.179 *180 * If @i is out of bound, this will return the pstate181 * corresponding to the nominal frequency.182 */183static inline u8 idx_to_pstate(unsigned int i)184{185 if (unlikely(i >= powernv_pstate_info.nr_pstates)) {186 pr_warn_once("idx_to_pstate: index %u is out of bound\n", i);187 return powernv_freqs[powernv_pstate_info.nominal].driver_data;188 }189 190 return powernv_freqs[i].driver_data;191}192 193/*194 * pstate_to_idx : Returns the index in the cpufreq frequencytable195 * powernv_freqs for the frequency whose corresponding196 * pstate id is @pstate.197 *198 * If no frequency corresponding to @pstate is found,199 * this will return the index of the nominal200 * frequency.201 */202static unsigned int pstate_to_idx(u8 pstate)203{204 unsigned int key = pstate % POWERNV_MAX_PSTATES;205 struct pstate_idx_revmap_data *revmap_data;206 207 hash_for_each_possible(pstate_revmap, revmap_data, hentry, key) {208 if (revmap_data->pstate_id == pstate)209 return revmap_data->cpufreq_table_idx;210 }211 212 pr_warn_once("pstate_to_idx: pstate 0x%x not found\n", pstate);213 return powernv_pstate_info.nominal;214}215 216static inline void reset_gpstates(struct cpufreq_policy *policy)217{218 struct global_pstate_info *gpstates = policy->driver_data;219 220 gpstates->highest_lpstate_idx = 0;221 gpstates->elapsed_time = 0;222 gpstates->last_sampled_time = 0;223 gpstates->last_lpstate_idx = 0;224 gpstates->last_gpstate_idx = 0;225}226 227/*228 * Initialize the freq table based on data obtained229 * from the firmware passed via device-tree230 */231static int init_powernv_pstates(void)232{233 struct device_node *power_mgt;234 int i, nr_pstates = 0;235 const __be32 *pstate_ids, *pstate_freqs;236 u32 len_ids, len_freqs;237 u32 pstate_min, pstate_max, pstate_nominal;238 u32 pstate_turbo, pstate_ultra_turbo;239 int rc = -ENODEV;240 241 power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");242 if (!power_mgt) {243 pr_warn("power-mgt node not found\n");244 return -ENODEV;245 }246 247 if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {248 pr_warn("ibm,pstate-min node not found\n");249 goto out;250 }251 252 if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {253 pr_warn("ibm,pstate-max node not found\n");254 goto out;255 }256 257 if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",258 &pstate_nominal)) {259 pr_warn("ibm,pstate-nominal not found\n");260 goto out;261 }262 263 if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo",264 &pstate_ultra_turbo)) {265 powernv_pstate_info.wof_enabled = false;266 goto next;267 }268 269 if (of_property_read_u32(power_mgt, "ibm,pstate-turbo",270 &pstate_turbo)) {271 powernv_pstate_info.wof_enabled = false;272 goto next;273 }274 275 if (pstate_turbo == pstate_ultra_turbo)276 powernv_pstate_info.wof_enabled = false;277 else278 powernv_pstate_info.wof_enabled = true;279 280next:281 pr_info("cpufreq pstate min 0x%x nominal 0x%x max 0x%x\n", pstate_min,282 pstate_nominal, pstate_max);283 pr_info("Workload Optimized Frequency is %s in the platform\n",284 (powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");285 286 pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);287 if (!pstate_ids) {288 pr_warn("ibm,pstate-ids not found\n");289 goto out;290 }291 292 pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",293 &len_freqs);294 if (!pstate_freqs) {295 pr_warn("ibm,pstate-frequencies-mhz not found\n");296 goto out;297 }298 299 if (len_ids != len_freqs) {300 pr_warn("Entries in ibm,pstate-ids and "301 "ibm,pstate-frequencies-mhz does not match\n");302 }303 304 nr_pstates = min(len_ids, len_freqs) / sizeof(u32);305 if (!nr_pstates) {306 pr_warn("No PStates found\n");307 goto out;308 }309 310 powernv_pstate_info.nr_pstates = nr_pstates;311 pr_debug("NR PStates %d\n", nr_pstates);312 313 for (i = 0; i < nr_pstates; i++) {314 u32 id = be32_to_cpu(pstate_ids[i]);315 u32 freq = be32_to_cpu(pstate_freqs[i]);316 struct pstate_idx_revmap_data *revmap_data;317 unsigned int key;318 319 pr_debug("PState id %d freq %d MHz\n", id, freq);320 powernv_freqs[i].frequency = freq * 1000; /* kHz */321 powernv_freqs[i].driver_data = id & 0xFF;322 323 revmap_data = kmalloc(sizeof(*revmap_data), GFP_KERNEL);324 if (!revmap_data) {325 rc = -ENOMEM;326 goto out;327 }328 329 revmap_data->pstate_id = id & 0xFF;330 revmap_data->cpufreq_table_idx = i;331 key = (revmap_data->pstate_id) % POWERNV_MAX_PSTATES;332 hash_add(pstate_revmap, &revmap_data->hentry, key);333 334 if (id == pstate_max)335 powernv_pstate_info.max = i;336 if (id == pstate_nominal)337 powernv_pstate_info.nominal = i;338 if (id == pstate_min)339 powernv_pstate_info.min = i;340 341 if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {342 int j;343 344 for (j = i - 1; j >= (int)powernv_pstate_info.max; j--)345 powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ;346 }347 }348 349 /* End of list marker entry */350 powernv_freqs[i].frequency = CPUFREQ_TABLE_END;351 352 of_node_put(power_mgt);353 return 0;354out:355 of_node_put(power_mgt);356 return rc;357}358 359/* Returns the CPU frequency corresponding to the pstate_id. */360static unsigned int pstate_id_to_freq(u8 pstate_id)361{362 int i;363 364 i = pstate_to_idx(pstate_id);365 if (i >= powernv_pstate_info.nr_pstates || i < 0) {366 pr_warn("PState id 0x%x outside of PState table, reporting nominal id 0x%x instead\n",367 pstate_id, idx_to_pstate(powernv_pstate_info.nominal));368 i = powernv_pstate_info.nominal;369 }370 371 return powernv_freqs[i].frequency;372}373 374/*375 * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by376 * the firmware377 */378static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,379 char *buf)380{381 return sprintf(buf, "%u\n",382 powernv_freqs[powernv_pstate_info.nominal].frequency);383}384 385static struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =386 __ATTR_RO(cpuinfo_nominal_freq);387 388#define SCALING_BOOST_FREQS_ATTR_INDEX 2389 390static struct freq_attr *powernv_cpu_freq_attr[] = {391 &cpufreq_freq_attr_scaling_available_freqs,392 &cpufreq_freq_attr_cpuinfo_nominal_freq,393 &cpufreq_freq_attr_scaling_boost_freqs,394 NULL,395};396 397#define throttle_attr(name, member) \398static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \399{ \400 struct chip *chip = per_cpu(chip_info, policy->cpu); \401 \402 return sprintf(buf, "%u\n", chip->member); \403} \404 \405static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \406 407throttle_attr(unthrottle, reason[NO_THROTTLE]);408throttle_attr(powercap, reason[POWERCAP]);409throttle_attr(overtemp, reason[CPU_OVERTEMP]);410throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);411throttle_attr(overcurrent, reason[OVERCURRENT]);412throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);413throttle_attr(turbo_stat, throttle_turbo);414throttle_attr(sub_turbo_stat, throttle_sub_turbo);415 416static struct attribute *throttle_attrs[] = {417 &throttle_attr_unthrottle.attr,418 &throttle_attr_powercap.attr,419 &throttle_attr_overtemp.attr,420 &throttle_attr_supply_fault.attr,421 &throttle_attr_overcurrent.attr,422 &throttle_attr_occ_reset.attr,423 &throttle_attr_turbo_stat.attr,424 &throttle_attr_sub_turbo_stat.attr,425 NULL,426};427 428static const struct attribute_group throttle_attr_grp = {429 .name = "throttle_stats",430 .attrs = throttle_attrs,431};432 433/* Helper routines */434 435/* Access helpers to power mgt SPR */436 437static inline unsigned long get_pmspr(unsigned long sprn)438{439 switch (sprn) {440 case SPRN_PMCR:441 return mfspr(SPRN_PMCR);442 443 case SPRN_PMICR:444 return mfspr(SPRN_PMICR);445 446 case SPRN_PMSR:447 return mfspr(SPRN_PMSR);448 }449 BUG();450}451 452static inline void set_pmspr(unsigned long sprn, unsigned long val)453{454 switch (sprn) {455 case SPRN_PMCR:456 mtspr(SPRN_PMCR, val);457 return;458 459 case SPRN_PMICR:460 mtspr(SPRN_PMICR, val);461 return;462 }463 BUG();464}465 466/*467 * Use objects of this type to query/update468 * pstates on a remote CPU via smp_call_function.469 */470struct powernv_smp_call_data {471 unsigned int freq;472 u8 pstate_id;473 u8 gpstate_id;474};475 476/*477 * powernv_read_cpu_freq: Reads the current frequency on this CPU.478 *479 * Called via smp_call_function.480 *481 * Note: The caller of the smp_call_function should pass an argument of482 * the type 'struct powernv_smp_call_data *' along with this function.483 *484 * The current frequency on this CPU will be returned via485 * ((struct powernv_smp_call_data *)arg)->freq;486 */487static void powernv_read_cpu_freq(void *arg)488{489 unsigned long pmspr_val;490 struct powernv_smp_call_data *freq_data = arg;491 492 pmspr_val = get_pmspr(SPRN_PMSR);493 freq_data->pstate_id = extract_local_pstate(pmspr_val);494 freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);495 496 pr_debug("cpu %d pmsr %016lX pstate_id 0x%x frequency %d kHz\n",497 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,498 freq_data->freq);499}500 501/*502 * powernv_cpufreq_get: Returns the CPU frequency as reported by the503 * firmware for CPU 'cpu'. This value is reported through the sysfs504 * file cpuinfo_cur_freq.505 */506static unsigned int powernv_cpufreq_get(unsigned int cpu)507{508 struct powernv_smp_call_data freq_data;509 510 smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,511 &freq_data, 1);512 513 return freq_data.freq;514}515 516/*517 * set_pstate: Sets the pstate on this CPU.518 *519 * This is called via an smp_call_function.520 *521 * The caller must ensure that freq_data is of the type522 * (struct powernv_smp_call_data *) and the pstate_id which needs to be set523 * on this CPU should be present in freq_data->pstate_id.524 */525static void set_pstate(void *data)526{527 unsigned long val;528 struct powernv_smp_call_data *freq_data = data;529 unsigned long pstate_ul = freq_data->pstate_id;530 unsigned long gpstate_ul = freq_data->gpstate_id;531 532 val = get_pmspr(SPRN_PMCR);533 val = val & 0x0000FFFFFFFFFFFFULL;534 535 pstate_ul = pstate_ul & 0xFF;536 gpstate_ul = gpstate_ul & 0xFF;537 538 /* Set both global(bits 56..63) and local(bits 48..55) PStates */539 val = val | (gpstate_ul << 56) | (pstate_ul << 48);540 541 pr_debug("Setting cpu %d pmcr to %016lX\n",542 raw_smp_processor_id(), val);543 set_pmspr(SPRN_PMCR, val);544}545 546/*547 * get_nominal_index: Returns the index corresponding to the nominal548 * pstate in the cpufreq table549 */550static inline unsigned int get_nominal_index(void)551{552 return powernv_pstate_info.nominal;553}554 555static void powernv_cpufreq_throttle_check(void *data)556{557 struct chip *chip;558 unsigned int cpu = smp_processor_id();559 unsigned long pmsr;560 u8 pmsr_pmax;561 unsigned int pmsr_pmax_idx;562 563 pmsr = get_pmspr(SPRN_PMSR);564 chip = this_cpu_read(chip_info);565 566 /* Check for Pmax Capping */567 pmsr_pmax = extract_max_pstate(pmsr);568 pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);569 if (pmsr_pmax_idx != powernv_pstate_info.max) {570 if (chip->throttled)571 goto next;572 chip->throttled = true;573 if (pmsr_pmax_idx > powernv_pstate_info.nominal) {574 pr_warn_once("CPU %d on Chip %u has Pmax(0x%x) reduced below that of nominal frequency(0x%x)\n",575 cpu, chip->id, pmsr_pmax,576 idx_to_pstate(powernv_pstate_info.nominal));577 chip->throttle_sub_turbo++;578 } else {579 chip->throttle_turbo++;580 }581 trace_powernv_throttle(chip->id,582 throttle_reason[chip->throttle_reason],583 pmsr_pmax);584 } else if (chip->throttled) {585 chip->throttled = false;586 trace_powernv_throttle(chip->id,587 throttle_reason[chip->throttle_reason],588 pmsr_pmax);589 }590 591 /* Check if Psafe_mode_active is set in PMSR. */592next:593 if (pmsr & PMSR_PSAFE_ENABLE) {594 throttled = true;595 pr_info("Pstate set to safe frequency\n");596 }597 598 /* Check if SPR_EM_DISABLE is set in PMSR */599 if (pmsr & PMSR_SPR_EM_DISABLE) {600 throttled = true;601 pr_info("Frequency Control disabled from OS\n");602 }603 604 if (throttled) {605 pr_info("PMSR = %16lx\n", pmsr);606 pr_warn("CPU Frequency could be throttled\n");607 }608}609 610/**611 * calc_global_pstate - Calculate global pstate612 * @elapsed_time: Elapsed time in milliseconds613 * @local_pstate_idx: New local pstate614 * @highest_lpstate_idx: pstate from which its ramping down615 *616 * Finds the appropriate global pstate based on the pstate from which its617 * ramping down and the time elapsed in ramping down. It follows a quadratic618 * equation which ensures that it reaches ramping down to pmin in 5sec.619 */620static inline int calc_global_pstate(unsigned int elapsed_time,621 int highest_lpstate_idx,622 int local_pstate_idx)623{624 int index_diff;625 626 /*627 * Using ramp_down_percent we get the percentage of rampdown628 * that we are expecting to be dropping. Difference between629 * highest_lpstate_idx and powernv_pstate_info.min will give a absolute630 * number of how many pstates we will drop eventually by the end of631 * 5 seconds, then just scale it get the number pstates to be dropped.632 */633 index_diff = ((int)ramp_down_percent(elapsed_time) *634 (powernv_pstate_info.min - highest_lpstate_idx)) / 100;635 636 /* Ensure that global pstate is >= to local pstate */637 if (highest_lpstate_idx + index_diff >= local_pstate_idx)638 return local_pstate_idx;639 else640 return highest_lpstate_idx + index_diff;641}642 643static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)644{645 unsigned int timer_interval;646 647 /*648 * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But649 * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.650 * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME651 * seconds of ramp down time.652 */653 if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)654 > MAX_RAMP_DOWN_TIME)655 timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;656 else657 timer_interval = GPSTATE_TIMER_INTERVAL;658 659 mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));660}661 662/**663 * gpstate_timer_handler664 *665 * @t: Timer context used to fetch global pstate info struct666 *667 * This handler brings down the global pstate closer to the local pstate668 * according quadratic equation. Queues a new timer if it is still not equal669 * to local pstate670 */671static void gpstate_timer_handler(struct timer_list *t)672{673 struct global_pstate_info *gpstates = from_timer(gpstates, t, timer);674 struct cpufreq_policy *policy = gpstates->policy;675 int gpstate_idx, lpstate_idx;676 unsigned long val;677 unsigned int time_diff = jiffies_to_msecs(jiffies)678 - gpstates->last_sampled_time;679 struct powernv_smp_call_data freq_data;680 681 if (!spin_trylock(&gpstates->gpstate_lock))682 return;683 /*684 * If the timer has migrated to the different cpu then bring685 * it back to one of the policy->cpus686 */687 if (!cpumask_test_cpu(raw_smp_processor_id(), policy->cpus)) {688 gpstates->timer.expires = jiffies + msecs_to_jiffies(1);689 add_timer_on(&gpstates->timer, cpumask_first(policy->cpus));690 spin_unlock(&gpstates->gpstate_lock);691 return;692 }693 694 /*695 * If PMCR was last updated was using fast_switch then696 * We may have wrong in gpstate->last_lpstate_idx697 * value. Hence, read from PMCR to get correct data.698 */699 val = get_pmspr(SPRN_PMCR);700 freq_data.gpstate_id = extract_global_pstate(val);701 freq_data.pstate_id = extract_local_pstate(val);702 if (freq_data.gpstate_id == freq_data.pstate_id) {703 reset_gpstates(policy);704 spin_unlock(&gpstates->gpstate_lock);705 return;706 }707 708 gpstates->last_sampled_time += time_diff;709 gpstates->elapsed_time += time_diff;710 711 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {712 gpstate_idx = pstate_to_idx(freq_data.pstate_id);713 lpstate_idx = gpstate_idx;714 reset_gpstates(policy);715 gpstates->highest_lpstate_idx = gpstate_idx;716 } else {717 lpstate_idx = pstate_to_idx(freq_data.pstate_id);718 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,719 gpstates->highest_lpstate_idx,720 lpstate_idx);721 }722 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);723 gpstates->last_gpstate_idx = gpstate_idx;724 gpstates->last_lpstate_idx = lpstate_idx;725 /*726 * If local pstate is equal to global pstate, rampdown is over727 * So timer is not required to be queued.728 */729 if (gpstate_idx != gpstates->last_lpstate_idx)730 queue_gpstate_timer(gpstates);731 732 set_pstate(&freq_data);733 spin_unlock(&gpstates->gpstate_lock);734}735 736/*737 * powernv_cpufreq_target_index: Sets the frequency corresponding to738 * the cpufreq table entry indexed by new_index on the cpus in the739 * mask policy->cpus740 */741static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,742 unsigned int new_index)743{744 struct powernv_smp_call_data freq_data;745 unsigned int cur_msec, gpstate_idx;746 struct global_pstate_info *gpstates = policy->driver_data;747 748 if (unlikely(rebooting) && new_index != get_nominal_index())749 return 0;750 751 if (!throttled) {752 /* we don't want to be preempted while753 * checking if the CPU frequency has been throttled754 */755 preempt_disable();756 powernv_cpufreq_throttle_check(NULL);757 preempt_enable();758 }759 760 cur_msec = jiffies_to_msecs(get_jiffies_64());761 762 freq_data.pstate_id = idx_to_pstate(new_index);763 if (!gpstates) {764 freq_data.gpstate_id = freq_data.pstate_id;765 goto no_gpstate;766 }767 768 spin_lock(&gpstates->gpstate_lock);769 770 if (!gpstates->last_sampled_time) {771 gpstate_idx = new_index;772 gpstates->highest_lpstate_idx = new_index;773 goto gpstates_done;774 }775 776 if (gpstates->last_gpstate_idx < new_index) {777 gpstates->elapsed_time += cur_msec -778 gpstates->last_sampled_time;779 780 /*781 * If its has been ramping down for more than MAX_RAMP_DOWN_TIME782 * we should be resetting all global pstate related data. Set it783 * equal to local pstate to start fresh.784 */785 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {786 reset_gpstates(policy);787 gpstates->highest_lpstate_idx = new_index;788 gpstate_idx = new_index;789 } else {790 /* Elaspsed_time is less than 5 seconds, continue to rampdown */791 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,792 gpstates->highest_lpstate_idx,793 new_index);794 }795 } else {796 reset_gpstates(policy);797 gpstates->highest_lpstate_idx = new_index;798 gpstate_idx = new_index;799 }800 801 /*802 * If local pstate is equal to global pstate, rampdown is over803 * So timer is not required to be queued.804 */805 if (gpstate_idx != new_index)806 queue_gpstate_timer(gpstates);807 else808 del_timer_sync(&gpstates->timer);809 810gpstates_done:811 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);812 gpstates->last_sampled_time = cur_msec;813 gpstates->last_gpstate_idx = gpstate_idx;814 gpstates->last_lpstate_idx = new_index;815 816 spin_unlock(&gpstates->gpstate_lock);817 818no_gpstate:819 /*820 * Use smp_call_function to send IPI and execute the821 * mtspr on target CPU. We could do that without IPI822 * if current CPU is within policy->cpus (core)823 */824 smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);825 return 0;826}827 828static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)829{830 int base, i;831 struct kernfs_node *kn;832 struct global_pstate_info *gpstates;833 834 base = cpu_first_thread_sibling(policy->cpu);835 836 for (i = 0; i < threads_per_core; i++)837 cpumask_set_cpu(base + i, policy->cpus);838 839 kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);840 if (!kn) {841 int ret;842 843 ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);844 if (ret) {845 pr_info("Failed to create throttle stats directory for cpu %d\n",846 policy->cpu);847 return ret;848 }849 } else {850 kernfs_put(kn);851 }852 853 policy->freq_table = powernv_freqs;854 policy->fast_switch_possible = true;855 856 if (pvr_version_is(PVR_POWER9))857 return 0;858 859 /* Initialise Gpstate ramp-down timer only on POWER8 */860 gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL);861 if (!gpstates)862 return -ENOMEM;863 864 policy->driver_data = gpstates;865 866 /* initialize timer */867 gpstates->policy = policy;868 timer_setup(&gpstates->timer, gpstate_timer_handler,869 TIMER_PINNED | TIMER_DEFERRABLE);870 gpstates->timer.expires = jiffies +871 msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);872 spin_lock_init(&gpstates->gpstate_lock);873 874 return 0;875}876 877static void powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)878{879 struct powernv_smp_call_data freq_data;880 struct global_pstate_info *gpstates = policy->driver_data;881 882 freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);883 freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);884 smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);885 if (gpstates)886 del_timer_sync(&gpstates->timer);887 888 kfree(policy->driver_data);889}890 891static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,892 unsigned long action, void *unused)893{894 int cpu;895 struct cpufreq_policy *cpu_policy;896 897 rebooting = true;898 for_each_online_cpu(cpu) {899 cpu_policy = cpufreq_cpu_get(cpu);900 if (!cpu_policy)901 continue;902 powernv_cpufreq_target_index(cpu_policy, get_nominal_index());903 cpufreq_cpu_put(cpu_policy);904 }905 906 return NOTIFY_DONE;907}908 909static struct notifier_block powernv_cpufreq_reboot_nb = {910 .notifier_call = powernv_cpufreq_reboot_notifier,911};912 913static void powernv_cpufreq_work_fn(struct work_struct *work)914{915 struct chip *chip = container_of(work, struct chip, throttle);916 struct cpufreq_policy *policy;917 unsigned int cpu;918 cpumask_t mask;919 920 cpus_read_lock();921 cpumask_and(&mask, &chip->mask, cpu_online_mask);922 smp_call_function_any(&mask,923 powernv_cpufreq_throttle_check, NULL, 0);924 925 if (!chip->restore)926 goto out;927 928 chip->restore = false;929 for_each_cpu(cpu, &mask) {930 int index;931 932 policy = cpufreq_cpu_get(cpu);933 if (!policy)934 continue;935 index = cpufreq_table_find_index_c(policy, policy->cur, false);936 powernv_cpufreq_target_index(policy, index);937 cpumask_andnot(&mask, &mask, policy->cpus);938 cpufreq_cpu_put(policy);939 }940out:941 cpus_read_unlock();942}943 944static int powernv_cpufreq_occ_msg(struct notifier_block *nb,945 unsigned long msg_type, void *_msg)946{947 struct opal_msg *msg = _msg;948 struct opal_occ_msg omsg;949 int i;950 951 if (msg_type != OPAL_MSG_OCC)952 return 0;953 954 omsg.type = be64_to_cpu(msg->params[0]);955 956 switch (omsg.type) {957 case OCC_RESET:958 occ_reset = true;959 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");960 /*961 * powernv_cpufreq_throttle_check() is called in962 * target() callback which can detect the throttle state963 * for governors like ondemand.964 * But static governors will not call target() often thus965 * report throttling here.966 */967 if (!throttled) {968 throttled = true;969 pr_warn("CPU frequency is throttled for duration\n");970 }971 972 break;973 case OCC_LOAD:974 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");975 break;976 case OCC_THROTTLE:977 omsg.chip = be64_to_cpu(msg->params[1]);978 omsg.throttle_status = be64_to_cpu(msg->params[2]);979 980 if (occ_reset) {981 occ_reset = false;982 throttled = false;983 pr_info("OCC Active, CPU frequency is no longer throttled\n");984 985 for (i = 0; i < nr_chips; i++) {986 chips[i].restore = true;987 schedule_work(&chips[i].throttle);988 }989 990 return 0;991 }992 993 for (i = 0; i < nr_chips; i++)994 if (chips[i].id == omsg.chip)995 break;996 997 if (omsg.throttle_status >= 0 &&998 omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {999 chips[i].throttle_reason = omsg.throttle_status;1000 chips[i].reason[omsg.throttle_status]++;1001 }1002 1003 if (!omsg.throttle_status)1004 chips[i].restore = true;1005 1006 schedule_work(&chips[i].throttle);1007 }1008 return 0;1009}1010 1011static struct notifier_block powernv_cpufreq_opal_nb = {1012 .notifier_call = powernv_cpufreq_occ_msg,1013 .next = NULL,1014 .priority = 0,1015};1016 1017static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,1018 unsigned int target_freq)1019{1020 int index;1021 struct powernv_smp_call_data freq_data;1022 1023 index = cpufreq_table_find_index_dl(policy, target_freq, false);1024 freq_data.pstate_id = powernv_freqs[index].driver_data;1025 freq_data.gpstate_id = powernv_freqs[index].driver_data;1026 set_pstate(&freq_data);1027 1028 return powernv_freqs[index].frequency;1029}1030 1031static struct cpufreq_driver powernv_cpufreq_driver = {1032 .name = "powernv-cpufreq",1033 .flags = CPUFREQ_CONST_LOOPS,1034 .init = powernv_cpufreq_cpu_init,1035 .exit = powernv_cpufreq_cpu_exit,1036 .verify = cpufreq_generic_frequency_table_verify,1037 .target_index = powernv_cpufreq_target_index,1038 .fast_switch = powernv_fast_switch,1039 .get = powernv_cpufreq_get,1040 .attr = powernv_cpu_freq_attr,1041};1042 1043static int init_chip_info(void)1044{1045 unsigned int *chip;1046 unsigned int cpu, i;1047 unsigned int prev_chip_id = UINT_MAX;1048 cpumask_t *chip_cpu_mask;1049 int ret = 0;1050 1051 chip = kcalloc(num_possible_cpus(), sizeof(*chip), GFP_KERNEL);1052 if (!chip)1053 return -ENOMEM;1054 1055 /* Allocate a chip cpu mask large enough to fit mask for all chips */1056 chip_cpu_mask = kcalloc(MAX_NR_CHIPS, sizeof(cpumask_t), GFP_KERNEL);1057 if (!chip_cpu_mask) {1058 ret = -ENOMEM;1059 goto free_and_return;1060 }1061 1062 for_each_possible_cpu(cpu) {1063 unsigned int id = cpu_to_chip_id(cpu);1064 1065 if (prev_chip_id != id) {1066 prev_chip_id = id;1067 chip[nr_chips++] = id;1068 }1069 cpumask_set_cpu(cpu, &chip_cpu_mask[nr_chips-1]);1070 }1071 1072 chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);1073 if (!chips) {1074 ret = -ENOMEM;1075 goto out_free_chip_cpu_mask;1076 }1077 1078 for (i = 0; i < nr_chips; i++) {1079 chips[i].id = chip[i];1080 cpumask_copy(&chips[i].mask, &chip_cpu_mask[i]);1081 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);1082 for_each_cpu(cpu, &chips[i].mask)1083 per_cpu(chip_info, cpu) = &chips[i];1084 }1085 1086out_free_chip_cpu_mask:1087 kfree(chip_cpu_mask);1088free_and_return:1089 kfree(chip);1090 return ret;1091}1092 1093static inline void clean_chip_info(void)1094{1095 int i;1096 1097 /* flush any pending work items */1098 if (chips)1099 for (i = 0; i < nr_chips; i++)1100 cancel_work_sync(&chips[i].throttle);1101 kfree(chips);1102}1103 1104static inline void unregister_all_notifiers(void)1105{1106 opal_message_notifier_unregister(OPAL_MSG_OCC,1107 &powernv_cpufreq_opal_nb);1108 unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);1109}1110 1111static int __init powernv_cpufreq_init(void)1112{1113 int rc = 0;1114 1115 /* Don't probe on pseries (guest) platforms */1116 if (!firmware_has_feature(FW_FEATURE_OPAL))1117 return -ENODEV;1118 1119 /* Discover pstates from device tree and init */1120 rc = init_powernv_pstates();1121 if (rc)1122 goto out;1123 1124 /* Populate chip info */1125 rc = init_chip_info();1126 if (rc)1127 goto out;1128 1129 if (powernv_pstate_info.wof_enabled)1130 powernv_cpufreq_driver.boost_enabled = true;1131 else1132 powernv_cpu_freq_attr[SCALING_BOOST_FREQS_ATTR_INDEX] = NULL;1133 1134 rc = cpufreq_register_driver(&powernv_cpufreq_driver);1135 if (rc) {1136 pr_info("Failed to register the cpufreq driver (%d)\n", rc);1137 goto cleanup;1138 }1139 1140 if (powernv_pstate_info.wof_enabled)1141 cpufreq_enable_boost_support();1142 1143 register_reboot_notifier(&powernv_cpufreq_reboot_nb);1144 opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);1145 1146 return 0;1147cleanup:1148 clean_chip_info();1149out:1150 pr_info("Platform driver disabled. System does not support PState control\n");1151 return rc;1152}1153module_init(powernv_cpufreq_init);1154 1155static void __exit powernv_cpufreq_exit(void)1156{1157 cpufreq_unregister_driver(&powernv_cpufreq_driver);1158 unregister_all_notifiers();1159 clean_chip_info();1160}1161module_exit(powernv_cpufreq_exit);1162 1163MODULE_DESCRIPTION("cpufreq driver for IBM/OpenPOWER powernv systems");1164MODULE_LICENSE("GPL");1165MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");1166