459 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * PSCI CPU idle driver.4 *5 * Copyright (C) 2019 ARM Ltd.6 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>7 */8 9#define pr_fmt(fmt) "CPUidle PSCI: " fmt10 11#include <linux/cpuhotplug.h>12#include <linux/cpu_cooling.h>13#include <linux/cpuidle.h>14#include <linux/cpumask.h>15#include <linux/cpu_pm.h>16#include <linux/kernel.h>17#include <linux/module.h>18#include <linux/of.h>19#include <linux/platform_device.h>20#include <linux/psci.h>21#include <linux/pm_domain.h>22#include <linux/pm_runtime.h>23#include <linux/slab.h>24#include <linux/string.h>25#include <linux/syscore_ops.h>26 27#include <asm/cpuidle.h>28 29#include "cpuidle-psci.h"30#include "dt_idle_states.h"31#include "dt_idle_genpd.h"32 33struct psci_cpuidle_data {34 u32 *psci_states;35 struct device *dev;36};37 38static DEFINE_PER_CPU_READ_MOSTLY(struct psci_cpuidle_data, psci_cpuidle_data);39static DEFINE_PER_CPU(u32, domain_state);40static bool psci_cpuidle_use_syscore;41static bool psci_cpuidle_use_cpuhp;42 43void psci_set_domain_state(u32 state)44{45 __this_cpu_write(domain_state, state);46}47 48static inline u32 psci_get_domain_state(void)49{50 return __this_cpu_read(domain_state);51}52 53static __cpuidle int __psci_enter_domain_idle_state(struct cpuidle_device *dev,54 struct cpuidle_driver *drv, int idx,55 bool s2idle)56{57 struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);58 u32 *states = data->psci_states;59 struct device *pd_dev = data->dev;60 u32 state;61 int ret;62 63 ret = cpu_pm_enter();64 if (ret)65 return -1;66 67 /* Do runtime PM to manage a hierarchical CPU toplogy. */68 if (s2idle)69 dev_pm_genpd_suspend(pd_dev);70 else71 pm_runtime_put_sync_suspend(pd_dev);72 73 state = psci_get_domain_state();74 if (!state)75 state = states[idx];76 77 ret = psci_cpu_suspend_enter(state) ? -1 : idx;78 79 if (s2idle)80 dev_pm_genpd_resume(pd_dev);81 else82 pm_runtime_get_sync(pd_dev);83 84 cpu_pm_exit();85 86 /* Clear the domain state to start fresh when back from idle. */87 psci_set_domain_state(0);88 return ret;89}90 91static int psci_enter_domain_idle_state(struct cpuidle_device *dev,92 struct cpuidle_driver *drv, int idx)93{94 return __psci_enter_domain_idle_state(dev, drv, idx, false);95}96 97static int psci_enter_s2idle_domain_idle_state(struct cpuidle_device *dev,98 struct cpuidle_driver *drv,99 int idx)100{101 return __psci_enter_domain_idle_state(dev, drv, idx, true);102}103 104static int psci_idle_cpuhp_up(unsigned int cpu)105{106 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev);107 108 if (pd_dev)109 pm_runtime_get_sync(pd_dev);110 111 return 0;112}113 114static int psci_idle_cpuhp_down(unsigned int cpu)115{116 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev);117 118 if (pd_dev) {119 pm_runtime_put_sync(pd_dev);120 /* Clear domain state to start fresh at next online. */121 psci_set_domain_state(0);122 }123 124 return 0;125}126 127static void psci_idle_syscore_switch(bool suspend)128{129 bool cleared = false;130 struct device *dev;131 int cpu;132 133 for_each_possible_cpu(cpu) {134 dev = per_cpu_ptr(&psci_cpuidle_data, cpu)->dev;135 136 if (dev && suspend) {137 dev_pm_genpd_suspend(dev);138 } else if (dev) {139 dev_pm_genpd_resume(dev);140 141 /* Account for userspace having offlined a CPU. */142 if (pm_runtime_status_suspended(dev))143 pm_runtime_set_active(dev);144 145 /* Clear domain state to re-start fresh. */146 if (!cleared) {147 psci_set_domain_state(0);148 cleared = true;149 }150 }151 }152}153 154static int psci_idle_syscore_suspend(void)155{156 psci_idle_syscore_switch(true);157 return 0;158}159 160static void psci_idle_syscore_resume(void)161{162 psci_idle_syscore_switch(false);163}164 165static struct syscore_ops psci_idle_syscore_ops = {166 .suspend = psci_idle_syscore_suspend,167 .resume = psci_idle_syscore_resume,168};169 170static void psci_idle_init_syscore(void)171{172 if (psci_cpuidle_use_syscore)173 register_syscore_ops(&psci_idle_syscore_ops);174}175 176static void psci_idle_init_cpuhp(void)177{178 int err;179 180 if (!psci_cpuidle_use_cpuhp)181 return;182 183 err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING,184 "cpuidle/psci:online",185 psci_idle_cpuhp_up,186 psci_idle_cpuhp_down);187 if (err)188 pr_warn("Failed %d while setup cpuhp state\n", err);189}190 191static __cpuidle int psci_enter_idle_state(struct cpuidle_device *dev,192 struct cpuidle_driver *drv, int idx)193{194 u32 *state = __this_cpu_read(psci_cpuidle_data.psci_states);195 196 return CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(psci_cpu_suspend_enter, idx, state[idx]);197}198 199static const struct of_device_id psci_idle_state_match[] = {200 { .compatible = "arm,idle-state",201 .data = psci_enter_idle_state },202 { },203};204 205int psci_dt_parse_state_node(struct device_node *np, u32 *state)206{207 int err = of_property_read_u32(np, "arm,psci-suspend-param", state);208 209 if (err) {210 pr_warn("%pOF missing arm,psci-suspend-param property\n", np);211 return err;212 }213 214 if (!psci_power_state_is_valid(*state)) {215 pr_warn("Invalid PSCI power state %#x\n", *state);216 return -EINVAL;217 }218 219 return 0;220}221 222static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv,223 struct psci_cpuidle_data *data,224 unsigned int state_count, int cpu)225{226 /* Currently limit the hierarchical topology to be used in OSI mode. */227 if (!psci_has_osi_support())228 return 0;229 230 data->dev = dt_idle_attach_cpu(cpu, "psci");231 if (IS_ERR_OR_NULL(data->dev))232 return PTR_ERR_OR_ZERO(data->dev);233 234 psci_cpuidle_use_syscore = true;235 236 /*237 * Using the deepest state for the CPU to trigger a potential selection238 * of a shared state for the domain, assumes the domain states are all239 * deeper states. On PREEMPT_RT the hierarchical topology is limited to240 * s2ram and s2idle.241 */242 drv->states[state_count - 1].enter_s2idle = psci_enter_s2idle_domain_idle_state;243 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {244 drv->states[state_count - 1].enter = psci_enter_domain_idle_state;245 psci_cpuidle_use_cpuhp = true;246 }247 248 return 0;249}250 251static int psci_dt_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv,252 struct device_node *cpu_node,253 unsigned int state_count, int cpu)254{255 int i, ret = 0;256 u32 *psci_states;257 struct device_node *state_node;258 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu);259 260 state_count++; /* Add WFI state too */261 psci_states = devm_kcalloc(dev, state_count, sizeof(*psci_states),262 GFP_KERNEL);263 if (!psci_states)264 return -ENOMEM;265 266 for (i = 1; i < state_count; i++) {267 state_node = of_get_cpu_state_node(cpu_node, i - 1);268 if (!state_node)269 break;270 271 ret = psci_dt_parse_state_node(state_node, &psci_states[i]);272 of_node_put(state_node);273 274 if (ret)275 return ret;276 277 pr_debug("psci-power-state %#x index %d\n", psci_states[i], i);278 }279 280 if (i != state_count)281 return -ENODEV;282 283 /* Initialize optional data, used for the hierarchical topology. */284 ret = psci_dt_cpu_init_topology(drv, data, state_count, cpu);285 if (ret < 0)286 return ret;287 288 /* Idle states parsed correctly, store them in the per-cpu struct. */289 data->psci_states = psci_states;290 return 0;291}292 293static int psci_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv,294 unsigned int cpu, unsigned int state_count)295{296 struct device_node *cpu_node;297 int ret;298 299 /*300 * If the PSCI cpu_suspend function hook has not been initialized301 * idle states must not be enabled, so bail out302 */303 if (!psci_ops.cpu_suspend)304 return -EOPNOTSUPP;305 306 cpu_node = of_cpu_device_node_get(cpu);307 if (!cpu_node)308 return -ENODEV;309 310 ret = psci_dt_cpu_init_idle(dev, drv, cpu_node, state_count, cpu);311 312 of_node_put(cpu_node);313 314 return ret;315}316 317static void psci_cpu_deinit_idle(int cpu)318{319 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu);320 321 dt_idle_detach_cpu(data->dev);322 psci_cpuidle_use_syscore = false;323 psci_cpuidle_use_cpuhp = false;324}325 326static int psci_idle_init_cpu(struct device *dev, int cpu)327{328 struct cpuidle_driver *drv;329 struct device_node *cpu_node;330 const char *enable_method;331 int ret = 0;332 333 cpu_node = of_cpu_device_node_get(cpu);334 if (!cpu_node)335 return -ENODEV;336 337 /*338 * Check whether the enable-method for the cpu is PSCI, fail339 * if it is not.340 */341 enable_method = of_get_property(cpu_node, "enable-method", NULL);342 if (!enable_method || (strcmp(enable_method, "psci")))343 ret = -ENODEV;344 345 of_node_put(cpu_node);346 if (ret)347 return ret;348 349 drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);350 if (!drv)351 return -ENOMEM;352 353 drv->name = "psci_idle";354 drv->owner = THIS_MODULE;355 drv->cpumask = (struct cpumask *)cpumask_of(cpu);356 357 /*358 * PSCI idle states relies on architectural WFI to be represented as359 * state index 0.360 */361 drv->states[0].enter = psci_enter_idle_state;362 drv->states[0].exit_latency = 1;363 drv->states[0].target_residency = 1;364 drv->states[0].power_usage = UINT_MAX;365 strcpy(drv->states[0].name, "WFI");366 strcpy(drv->states[0].desc, "ARM WFI");367 368 /*369 * If no DT idle states are detected (ret == 0) let the driver370 * initialization fail accordingly since there is no reason to371 * initialize the idle driver if only wfi is supported, the372 * default archictectural back-end already executes wfi373 * on idle entry.374 */375 ret = dt_init_idle_driver(drv, psci_idle_state_match, 1);376 if (ret <= 0)377 return ret ? : -ENODEV;378 379 /*380 * Initialize PSCI idle states.381 */382 ret = psci_cpu_init_idle(dev, drv, cpu, ret);383 if (ret) {384 pr_err("CPU %d failed to PSCI idle\n", cpu);385 return ret;386 }387 388 ret = cpuidle_register(drv, NULL);389 if (ret)390 goto deinit;391 392 cpuidle_cooling_register(drv);393 394 return 0;395deinit:396 psci_cpu_deinit_idle(cpu);397 return ret;398}399 400/*401 * psci_idle_probe - Initializes PSCI cpuidle driver402 *403 * Initializes PSCI cpuidle driver for all CPUs, if any CPU fails404 * to register cpuidle driver then rollback to cancel all CPUs405 * registration.406 */407static int psci_cpuidle_probe(struct platform_device *pdev)408{409 int cpu, ret;410 struct cpuidle_driver *drv;411 struct cpuidle_device *dev;412 413 for_each_possible_cpu(cpu) {414 ret = psci_idle_init_cpu(&pdev->dev, cpu);415 if (ret)416 goto out_fail;417 }418 419 psci_idle_init_syscore();420 psci_idle_init_cpuhp();421 return 0;422 423out_fail:424 while (--cpu >= 0) {425 dev = per_cpu(cpuidle_devices, cpu);426 drv = cpuidle_get_cpu_driver(dev);427 cpuidle_unregister(drv);428 psci_cpu_deinit_idle(cpu);429 }430 431 return ret;432}433 434static struct platform_driver psci_cpuidle_driver = {435 .probe = psci_cpuidle_probe,436 .driver = {437 .name = "psci-cpuidle",438 },439};440 441static int __init psci_idle_init(void)442{443 struct platform_device *pdev;444 int ret;445 446 ret = platform_driver_register(&psci_cpuidle_driver);447 if (ret)448 return ret;449 450 pdev = platform_device_register_simple("psci-cpuidle", -1, NULL, 0);451 if (IS_ERR(pdev)) {452 platform_driver_unregister(&psci_cpuidle_driver);453 return PTR_ERR(pdev);454 }455 456 return 0;457}458device_initcall(psci_idle_init);459