419 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * drivers/base/power/domain_governor.c - Governors for device PM domains.4 *5 * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.6 */7#include <linux/kernel.h>8#include <linux/pm_domain.h>9#include <linux/pm_qos.h>10#include <linux/hrtimer.h>11#include <linux/cpuidle.h>12#include <linux/cpumask.h>13#include <linux/ktime.h>14 15static int dev_update_qos_constraint(struct device *dev, void *data)16{17 s64 *constraint_ns_p = data;18 s64 constraint_ns;19 20 if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {21 struct gpd_timing_data *td = dev_gpd_data(dev)->td;22 23 /*24 * Only take suspend-time QoS constraints of devices into25 * account, because constraints updated after the device has26 * been suspended are not guaranteed to be taken into account27 * anyway. In order for them to take effect, the device has to28 * be resumed and suspended again.29 */30 constraint_ns = td ? td->effective_constraint_ns :31 PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;32 } else {33 /*34 * The child is not in a domain and there's no info on its35 * suspend/resume latencies, so assume them to be negligible and36 * take its current PM QoS constraint (that's the only thing37 * known at this point anyway).38 */39 constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY);40 constraint_ns *= NSEC_PER_USEC;41 }42 43 if (constraint_ns < *constraint_ns_p)44 *constraint_ns_p = constraint_ns;45 46 return 0;47}48 49/**50 * default_suspend_ok - Default PM domain governor routine to suspend devices.51 * @dev: Device to check.52 *53 * Returns: true if OK to suspend, false if not OK to suspend54 */55static bool default_suspend_ok(struct device *dev)56{57 struct gpd_timing_data *td = dev_gpd_data(dev)->td;58 unsigned long flags;59 s64 constraint_ns;60 61 dev_dbg(dev, "%s()\n", __func__);62 63 spin_lock_irqsave(&dev->power.lock, flags);64 65 if (!td->constraint_changed) {66 bool ret = td->cached_suspend_ok;67 68 spin_unlock_irqrestore(&dev->power.lock, flags);69 return ret;70 }71 td->constraint_changed = false;72 td->cached_suspend_ok = false;73 td->effective_constraint_ns = 0;74 constraint_ns = __dev_pm_qos_resume_latency(dev);75 76 spin_unlock_irqrestore(&dev->power.lock, flags);77 78 if (constraint_ns == 0)79 return false;80 81 constraint_ns *= NSEC_PER_USEC;82 /*83 * We can walk the children without any additional locking, because84 * they all have been suspended at this point and their85 * effective_constraint_ns fields won't be modified in parallel with us.86 */87 if (!dev->power.ignore_children)88 device_for_each_child(dev, &constraint_ns,89 dev_update_qos_constraint);90 91 if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) {92 /* "No restriction", so the device is allowed to suspend. */93 td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;94 td->cached_suspend_ok = true;95 } else if (constraint_ns == 0) {96 /*97 * This triggers if one of the children that don't belong to a98 * domain has a zero PM QoS constraint and it's better not to99 * suspend then. effective_constraint_ns is zero already and100 * cached_suspend_ok is false, so bail out.101 */102 return false;103 } else {104 constraint_ns -= td->suspend_latency_ns +105 td->resume_latency_ns;106 /*107 * effective_constraint_ns is zero already and cached_suspend_ok108 * is false, so if the computed value is not positive, return109 * right away.110 */111 if (constraint_ns <= 0)112 return false;113 114 td->effective_constraint_ns = constraint_ns;115 td->cached_suspend_ok = true;116 }117 118 /*119 * The children have been suspended already, so we don't need to take120 * their suspend latencies into account here.121 */122 return td->cached_suspend_ok;123}124 125static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)126{127 ktime_t domain_wakeup = KTIME_MAX;128 ktime_t next_wakeup;129 struct pm_domain_data *pdd;130 struct gpd_link *link;131 132 if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))133 return;134 135 /*136 * Devices that have a predictable wakeup pattern, may specify137 * their next wakeup. Let's find the next wakeup from all the138 * devices attached to this domain and from all the sub-domains.139 * It is possible that component's a next wakeup may have become140 * stale when we read that here. We will ignore to ensure the domain141 * is able to enter its optimal idle state.142 */143 list_for_each_entry(pdd, &genpd->dev_list, list_node) {144 next_wakeup = to_gpd_data(pdd)->td->next_wakeup;145 if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))146 if (ktime_before(next_wakeup, domain_wakeup))147 domain_wakeup = next_wakeup;148 }149 150 list_for_each_entry(link, &genpd->parent_links, parent_node) {151 struct genpd_governor_data *cgd = link->child->gd;152 153 next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX;154 if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))155 if (ktime_before(next_wakeup, domain_wakeup))156 domain_wakeup = next_wakeup;157 }158 159 genpd->gd->next_wakeup = domain_wakeup;160}161 162static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,163 unsigned int state, ktime_t now)164{165 ktime_t domain_wakeup = genpd->gd->next_wakeup;166 s64 idle_time_ns, min_sleep_ns;167 168 min_sleep_ns = genpd->states[state].power_off_latency_ns +169 genpd->states[state].residency_ns;170 171 idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));172 173 return idle_time_ns >= min_sleep_ns;174}175 176static bool __default_power_down_ok(struct dev_pm_domain *pd,177 unsigned int state)178{179 struct generic_pm_domain *genpd = pd_to_genpd(pd);180 struct gpd_link *link;181 struct pm_domain_data *pdd;182 s64 min_off_time_ns;183 s64 off_on_time_ns;184 185 off_on_time_ns = genpd->states[state].power_off_latency_ns +186 genpd->states[state].power_on_latency_ns;187 188 min_off_time_ns = -1;189 /*190 * Check if subdomains can be off for enough time.191 *192 * All subdomains have been powered off already at this point.193 */194 list_for_each_entry(link, &genpd->parent_links, parent_node) {195 struct genpd_governor_data *cgd = link->child->gd;196 197 s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1;198 199 if (sd_max_off_ns < 0)200 continue;201 202 /*203 * Check if the subdomain is allowed to be off long enough for204 * the current domain to turn off and on (that's how much time205 * it will have to wait worst case).206 */207 if (sd_max_off_ns <= off_on_time_ns)208 return false;209 210 if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0)211 min_off_time_ns = sd_max_off_ns;212 }213 214 /*215 * Check if the devices in the domain can be off enough time.216 */217 list_for_each_entry(pdd, &genpd->dev_list, list_node) {218 struct gpd_timing_data *td;219 s64 constraint_ns;220 221 /*222 * Check if the device is allowed to be off long enough for the223 * domain to turn off and on (that's how much time it will224 * have to wait worst case).225 */226 td = to_gpd_data(pdd)->td;227 constraint_ns = td->effective_constraint_ns;228 /*229 * Zero means "no suspend at all" and this runs only when all230 * devices in the domain are suspended, so it must be positive.231 */232 if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS)233 continue;234 235 if (constraint_ns <= off_on_time_ns)236 return false;237 238 if (min_off_time_ns > constraint_ns || min_off_time_ns < 0)239 min_off_time_ns = constraint_ns;240 }241 242 /*243 * If the computed minimum device off time is negative, there are no244 * latency constraints, so the domain can spend arbitrary time in the245 * "off" state.246 */247 if (min_off_time_ns < 0)248 return true;249 250 /*251 * The difference between the computed minimum subdomain or device off252 * time and the time needed to turn the domain on is the maximum253 * theoretical time this domain can spend in the "off" state.254 */255 genpd->gd->max_off_time_ns = min_off_time_ns -256 genpd->states[state].power_on_latency_ns;257 return true;258}259 260/**261 * _default_power_down_ok - Default generic PM domain power off governor routine.262 * @pd: PM domain to check.263 * @now: current ktime.264 *265 * This routine must be executed under the PM domain's lock.266 *267 * Returns: true if OK to power down, false if not OK to power down268 */269static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)270{271 struct generic_pm_domain *genpd = pd_to_genpd(pd);272 struct genpd_governor_data *gd = genpd->gd;273 int state_idx = genpd->state_count - 1;274 struct gpd_link *link;275 276 /*277 * Find the next wakeup from devices that can determine their own wakeup278 * to find when the domain would wakeup and do it for every device down279 * the hierarchy. It is not worth while to sleep if the state's residency280 * cannot be met.281 */282 update_domain_next_wakeup(genpd, now);283 if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) {284 /* Let's find out the deepest domain idle state, the devices prefer */285 while (state_idx >= 0) {286 if (next_wakeup_allows_state(genpd, state_idx, now)) {287 gd->max_off_time_changed = true;288 break;289 }290 state_idx--;291 }292 293 if (state_idx < 0) {294 state_idx = 0;295 gd->cached_power_down_ok = false;296 goto done;297 }298 }299 300 if (!gd->max_off_time_changed) {301 genpd->state_idx = gd->cached_power_down_state_idx;302 return gd->cached_power_down_ok;303 }304 305 /*306 * We have to invalidate the cached results for the parents, so307 * use the observation that default_power_down_ok() is not308 * going to be called for any parent until this instance309 * returns.310 */311 list_for_each_entry(link, &genpd->child_links, child_node) {312 struct genpd_governor_data *pgd = link->parent->gd;313 314 if (pgd)315 pgd->max_off_time_changed = true;316 }317 318 gd->max_off_time_ns = -1;319 gd->max_off_time_changed = false;320 gd->cached_power_down_ok = true;321 322 /*323 * Find a state to power down to, starting from the state324 * determined by the next wakeup.325 */326 while (!__default_power_down_ok(pd, state_idx)) {327 if (state_idx == 0) {328 gd->cached_power_down_ok = false;329 break;330 }331 state_idx--;332 }333 334done:335 genpd->state_idx = state_idx;336 gd->cached_power_down_state_idx = genpd->state_idx;337 return gd->cached_power_down_ok;338}339 340static bool default_power_down_ok(struct dev_pm_domain *pd)341{342 return _default_power_down_ok(pd, ktime_get());343}344 345#ifdef CONFIG_CPU_IDLE346static bool cpu_power_down_ok(struct dev_pm_domain *pd)347{348 struct generic_pm_domain *genpd = pd_to_genpd(pd);349 struct cpuidle_device *dev;350 ktime_t domain_wakeup, next_hrtimer;351 ktime_t now = ktime_get();352 s64 idle_duration_ns;353 int cpu, i;354 355 /* Validate dev PM QoS constraints. */356 if (!_default_power_down_ok(pd, now))357 return false;358 359 if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN))360 return true;361 362 /*363 * Find the next wakeup for any of the online CPUs within the PM domain364 * and its subdomains. Note, we only need the genpd->cpus, as it already365 * contains a mask of all CPUs from subdomains.366 */367 domain_wakeup = ktime_set(KTIME_SEC_MAX, 0);368 for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {369 dev = per_cpu(cpuidle_devices, cpu);370 if (dev) {371 next_hrtimer = READ_ONCE(dev->next_hrtimer);372 if (ktime_before(next_hrtimer, domain_wakeup))373 domain_wakeup = next_hrtimer;374 }375 }376 377 /* The minimum idle duration is from now - until the next wakeup. */378 idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));379 if (idle_duration_ns <= 0)380 return false;381 382 /* Store the next domain_wakeup to allow consumers to use it. */383 genpd->gd->next_hrtimer = domain_wakeup;384 385 /*386 * Find the deepest idle state that has its residency value satisfied387 * and by also taking into account the power off latency for the state.388 * Start at the state picked by the dev PM QoS constraint validation.389 */390 i = genpd->state_idx;391 do {392 if (idle_duration_ns >= (genpd->states[i].residency_ns +393 genpd->states[i].power_off_latency_ns)) {394 genpd->state_idx = i;395 return true;396 }397 } while (--i >= 0);398 399 return false;400}401 402struct dev_power_governor pm_domain_cpu_gov = {403 .suspend_ok = default_suspend_ok,404 .power_down_ok = cpu_power_down_ok,405};406#endif407 408struct dev_power_governor simple_qos_governor = {409 .suspend_ok = default_suspend_ok,410 .power_down_ok = default_power_down_ok,411};412 413/*414 * pm_domain_always_on_gov - A governor implementing an always-on policy415 */416struct dev_power_governor pm_domain_always_on_gov = {417 .suspend_ok = default_suspend_ok,418};419