403 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * CPU idle driver for Tegra CPUs4 *5 * Copyright (c) 2010-2013, NVIDIA Corporation.6 * Copyright (c) 2011 Google, Inc.7 * Author: Colin Cross <ccross@android.com>8 * Gary King <gking@nvidia.com>9 *10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com>11 *12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com>13 */14 15#define pr_fmt(fmt) "tegra-cpuidle: " fmt16 17#include <linux/atomic.h>18#include <linux/cpuidle.h>19#include <linux/cpumask.h>20#include <linux/cpu_pm.h>21#include <linux/delay.h>22#include <linux/errno.h>23#include <linux/platform_device.h>24#include <linux/types.h>25 26#include <linux/clk/tegra.h>27#include <linux/firmware/trusted_foundations.h>28 29#include <soc/tegra/cpuidle.h>30#include <soc/tegra/flowctrl.h>31#include <soc/tegra/fuse.h>32#include <soc/tegra/irq.h>33#include <soc/tegra/pm.h>34#include <soc/tegra/pmc.h>35 36#include <asm/cpuidle.h>37#include <asm/firmware.h>38#include <asm/smp_plat.h>39#include <asm/suspend.h>40 41enum tegra_state {42 TEGRA_C1,43 TEGRA_C7,44 TEGRA_CC6,45 TEGRA_STATE_COUNT,46};47 48static atomic_t tegra_idle_barrier;49static atomic_t tegra_abort_flag;50 51static void tegra_cpuidle_report_cpus_state(void)52{53 unsigned long cpu, lcpu, csr;54 55 for_each_cpu(lcpu, cpu_possible_mask) {56 cpu = cpu_logical_map(lcpu);57 csr = flowctrl_read_cpu_csr(cpu);58 59 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n",60 cpu, cpu_online(lcpu), csr);61 }62}63 64static int tegra_cpuidle_wait_for_secondary_cpus_parking(void)65{66 unsigned int retries = 3;67 68 while (retries--) {69 unsigned int delay_us = 10;70 unsigned int timeout_us = 500 * 1000 / delay_us;71 72 /*73 * The primary CPU0 core shall wait for the secondaries74 * shutdown in order to power-off CPU's cluster safely.75 * The timeout value depends on the current CPU frequency,76 * it takes about 40-150us in average and over 1000us in77 * a worst case scenario.78 */79 do {80 if (tegra_cpu_rail_off_ready())81 return 0;82 83 udelay(delay_us);84 85 } while (timeout_us--);86 87 pr_err("secondary CPU taking too long to park\n");88 89 tegra_cpuidle_report_cpus_state();90 }91 92 pr_err("timed out waiting secondaries to park\n");93 94 return -ETIMEDOUT;95}96 97static void tegra_cpuidle_unpark_secondary_cpus(void)98{99 unsigned int cpu, lcpu;100 101 for_each_cpu(lcpu, cpu_online_mask) {102 cpu = cpu_logical_map(lcpu);103 104 if (cpu > 0) {105 tegra_enable_cpu_clock(cpu);106 tegra_cpu_out_of_reset(cpu);107 flowctrl_write_cpu_halt(cpu, 0);108 }109 }110}111 112static int tegra_cpuidle_cc6_enter(unsigned int cpu)113{114 int ret;115 116 if (cpu > 0) {117 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu);118 } else {119 ret = tegra_cpuidle_wait_for_secondary_cpus_parking();120 if (!ret)121 ret = tegra_pm_enter_lp2();122 123 tegra_cpuidle_unpark_secondary_cpus();124 }125 126 return ret;127}128 129static int tegra_cpuidle_c7_enter(void)130{131 int err;132 133 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2);134 if (err && err != -ENOSYS)135 return err;136 137 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend);138}139 140static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev)141{142 if (tegra_pending_sgi()) {143 /*144 * CPU got local interrupt that will be lost after GIC's145 * shutdown because GIC driver doesn't save/restore the146 * pending SGI state across CPU cluster PM. Abort and retry147 * next time.148 */149 atomic_set(&tegra_abort_flag, 1);150 }151 152 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);153 154 if (atomic_read(&tegra_abort_flag)) {155 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);156 atomic_set(&tegra_abort_flag, 0);157 return -EINTR;158 }159 160 return 0;161}162 163static __cpuidle int tegra_cpuidle_state_enter(struct cpuidle_device *dev,164 int index, unsigned int cpu)165{166 int err;167 168 /*169 * CC6 state is the "CPU cluster power-off" state. In order to170 * enter this state, at first the secondary CPU cores need to be171 * parked into offline mode, then the last CPU should clean out172 * remaining dirty cache lines into DRAM and trigger Flow Controller173 * logic that turns off the cluster's power domain (which includes174 * CPU cores, GIC and L2 cache).175 */176 if (index == TEGRA_CC6) {177 err = tegra_cpuidle_coupled_barrier(dev);178 if (err)179 return err;180 }181 182 local_fiq_disable();183 tegra_pm_set_cpu_in_lp2();184 cpu_pm_enter();185 186 ct_cpuidle_enter();187 188 switch (index) {189 case TEGRA_C7:190 err = tegra_cpuidle_c7_enter();191 break;192 193 case TEGRA_CC6:194 err = tegra_cpuidle_cc6_enter(cpu);195 break;196 197 default:198 err = -EINVAL;199 break;200 }201 202 ct_cpuidle_exit();203 204 cpu_pm_exit();205 tegra_pm_clear_cpu_in_lp2();206 local_fiq_enable();207 208 return err ?: index;209}210 211static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu)212{213 /*214 * On Tegra30 CPU0 can't be power-gated separately from secondary215 * cores because it gates the whole CPU cluster.216 */217 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30)218 return index;219 220 /* put CPU0 into C1 if C7 is requested and secondaries are online */221 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1)222 index = TEGRA_C1;223 else224 index = TEGRA_CC6;225 226 return index;227}228 229static __cpuidle int tegra_cpuidle_enter(struct cpuidle_device *dev,230 struct cpuidle_driver *drv,231 int index)232{233 bool do_rcu = drv->states[index].flags & CPUIDLE_FLAG_RCU_IDLE;234 unsigned int cpu = cpu_logical_map(dev->cpu);235 int ret;236 237 index = tegra_cpuidle_adjust_state_index(index, cpu);238 if (dev->states_usage[index].disable)239 return -1;240 241 if (index == TEGRA_C1) {242 if (do_rcu)243 ct_cpuidle_enter();244 ret = arm_cpuidle_simple_enter(dev, drv, index);245 if (do_rcu)246 ct_cpuidle_exit();247 } else248 ret = tegra_cpuidle_state_enter(dev, index, cpu);249 250 if (ret < 0) {251 if (ret != -EINTR || index != TEGRA_CC6)252 pr_err_once("failed to enter state %d err: %d\n",253 index, ret);254 index = -1;255 } else {256 index = ret;257 }258 259 return index;260}261 262static int tegra114_enter_s2idle(struct cpuidle_device *dev,263 struct cpuidle_driver *drv,264 int index)265{266 tegra_cpuidle_enter(dev, drv, index);267 268 return 0;269}270 271/*272 * The previous versions of Tegra CPUIDLE driver used a different "legacy"273 * terminology for naming of the idling states, while this driver uses the274 * new terminology.275 *276 * Mapping of the old terms into the new ones:277 *278 * Old | New279 * ---------280 * LP3 | C1 (CPU core clock gating)281 * LP2 | C7 (CPU core power gating)282 * LP2 | CC6 (CPU cluster power gating)283 *284 * Note that that the older CPUIDLE driver versions didn't explicitly285 * differentiate the LP2 states because these states either used the same286 * code path or because CC6 wasn't supported.287 */288static struct cpuidle_driver tegra_idle_driver = {289 .name = "tegra_idle",290 .states = {291 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600),292 [TEGRA_C7] = {293 .enter = tegra_cpuidle_enter,294 .exit_latency = 2000,295 .target_residency = 2200,296 .power_usage = 100,297 .flags = CPUIDLE_FLAG_TIMER_STOP |298 CPUIDLE_FLAG_RCU_IDLE,299 .name = "C7",300 .desc = "CPU core powered off",301 },302 [TEGRA_CC6] = {303 .enter = tegra_cpuidle_enter,304 .exit_latency = 5000,305 .target_residency = 10000,306 .power_usage = 0,307 .flags = CPUIDLE_FLAG_TIMER_STOP |308 CPUIDLE_FLAG_RCU_IDLE |309 CPUIDLE_FLAG_COUPLED,310 .name = "CC6",311 .desc = "CPU cluster powered off",312 },313 },314 .state_count = TEGRA_STATE_COUNT,315 .safe_state_index = TEGRA_C1,316};317 318static inline void tegra_cpuidle_disable_state(enum tegra_state state)319{320 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true);321}322 323/*324 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether325 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around326 * this, simply disable CC6 if the PCI driver and DT node are both enabled.327 */328void tegra_cpuidle_pcie_irqs_in_use(void)329{330 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6];331 332 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) ||333 tegra_get_chip_id() != TEGRA20)334 return;335 336 pr_info("disabling CC6 state, since PCIe IRQs are in use\n");337 tegra_cpuidle_disable_state(TEGRA_CC6);338}339 340static void tegra_cpuidle_setup_tegra114_c7_state(void)341{342 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7];343 344 s->enter_s2idle = tegra114_enter_s2idle;345 s->target_residency = 1000;346 s->exit_latency = 500;347}348 349static int tegra_cpuidle_probe(struct platform_device *pdev)350{351 if (tegra_pmc_get_suspend_mode() == TEGRA_SUSPEND_NOT_READY)352 return -EPROBE_DEFER;353 354 /* LP2 could be disabled in device-tree */355 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2)356 tegra_cpuidle_disable_state(TEGRA_CC6);357 358 /*359 * Required suspend-resume functionality, which is provided by the360 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option361 * is disabled.362 */363 if (!IS_ENABLED(CONFIG_PM_SLEEP)) {364 tegra_cpuidle_disable_state(TEGRA_C7);365 tegra_cpuidle_disable_state(TEGRA_CC6);366 }367 368 /*369 * Generic WFI state (also known as C1 or LP3) and the coupled CPU370 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs.371 */372 switch (tegra_get_chip_id()) {373 case TEGRA20:374 /* Tegra20 isn't capable to power-off individual CPU cores */375 tegra_cpuidle_disable_state(TEGRA_C7);376 break;377 378 case TEGRA30:379 break;380 381 case TEGRA114:382 case TEGRA124:383 tegra_cpuidle_setup_tegra114_c7_state();384 385 /* coupled CC6 (LP2) state isn't implemented yet */386 tegra_cpuidle_disable_state(TEGRA_CC6);387 break;388 389 default:390 return -EINVAL;391 }392 393 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask);394}395 396static struct platform_driver tegra_cpuidle_driver = {397 .probe = tegra_cpuidle_probe,398 .driver = {399 .name = "tegra-cpuidle",400 },401};402builtin_platform_driver(tegra_cpuidle_driver);403