470 lines · c
1/*2 * Copyright © 2012-2014 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 * Authors:24 * Eugeni Dodonov <eugeni.dodonov@intel.com>25 * Daniel Vetter <daniel.vetter@ffwll.ch>26 *27 */28 29#include <linux/pm_runtime.h>30 31#include <drm/drm_print.h>32 33#include "i915_drv.h"34#include "i915_trace.h"35 36/**37 * DOC: runtime pm38 *39 * The i915 driver supports dynamic enabling and disabling of entire hardware40 * blocks at runtime. This is especially important on the display side where41 * software is supposed to control many power gates manually on recent hardware,42 * since on the GT side a lot of the power management is done by the hardware.43 * But even there some manual control at the device level is required.44 *45 * Since i915 supports a diverse set of platforms with a unified codebase and46 * hardware engineers just love to shuffle functionality around between power47 * domains there's a sizeable amount of indirection required. This file provides48 * generic functions to the driver for grabbing and releasing references for49 * abstract power domains. It then maps those to the actual power wells50 * present for a given platform.51 */52 53static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm)54{55 return container_of(rpm, struct drm_i915_private, runtime_pm);56}57 58#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)59 60static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)61{62 ref_tracker_dir_init(&rpm->debug, INTEL_REFTRACK_DEAD_COUNT, dev_name(rpm->kdev));63}64 65static intel_wakeref_t66track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)67{68 if (!rpm->available || rpm->no_wakeref_tracking)69 return -1;70 71 return intel_ref_tracker_alloc(&rpm->debug);72}73 74static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,75 intel_wakeref_t wakeref)76{77 if (!rpm->available || rpm->no_wakeref_tracking)78 return;79 80 intel_ref_tracker_free(&rpm->debug, wakeref);81}82 83static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)84{85 ref_tracker_dir_exit(&rpm->debug);86}87 88static noinline void89__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)90{91 unsigned long flags;92 93 if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,94 &rpm->debug.lock,95 flags))96 return;97 98 ref_tracker_dir_print_locked(&rpm->debug, INTEL_REFTRACK_PRINT_LIMIT);99 spin_unlock_irqrestore(&rpm->debug.lock, flags);100}101 102void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,103 struct drm_printer *p)104{105 intel_ref_tracker_show(&rpm->debug, p);106}107 108#else109 110static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)111{112}113 114static intel_wakeref_t115track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)116{117 return -1;118}119 120static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,121 intel_wakeref_t wakeref)122{123}124 125static void126__intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)127{128 atomic_dec(&rpm->wakeref_count);129}130 131static void132untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)133{134}135 136#endif137 138static void139intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)140{141 if (wakelock) {142 atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);143 assert_rpm_wakelock_held(rpm);144 } else {145 atomic_inc(&rpm->wakeref_count);146 assert_rpm_raw_wakeref_held(rpm);147 }148}149 150static void151intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)152{153 if (wakelock) {154 assert_rpm_wakelock_held(rpm);155 atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);156 } else {157 assert_rpm_raw_wakeref_held(rpm);158 }159 160 __intel_wakeref_dec_and_check_tracking(rpm);161}162 163static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,164 bool wakelock)165{166 struct drm_i915_private *i915 = rpm_to_i915(rpm);167 int ret;168 169 ret = pm_runtime_get_sync(rpm->kdev);170 drm_WARN_ONCE(&i915->drm, ret < 0,171 "pm_runtime_get_sync() failed: %d\n", ret);172 173 intel_runtime_pm_acquire(rpm, wakelock);174 175 return track_intel_runtime_pm_wakeref(rpm);176}177 178/**179 * intel_runtime_pm_get_raw - grab a raw runtime pm reference180 * @rpm: the intel_runtime_pm structure181 *182 * This is the unlocked version of intel_display_power_is_enabled() and should183 * only be used from error capture and recovery code where deadlocks are184 * possible.185 * This function grabs a device-level runtime pm reference (mostly used for186 * asynchronous PM management from display code) and ensures that it is powered187 * up. Raw references are not considered during wakelock assert checks.188 *189 * Any runtime pm reference obtained by this function must have a symmetric190 * call to intel_runtime_pm_put_raw() to release the reference again.191 *192 * Returns: the wakeref cookie to pass to intel_runtime_pm_put_raw(), evaluates193 * as True if the wakeref was acquired, or False otherwise.194 */195intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)196{197 return __intel_runtime_pm_get(rpm, false);198}199 200/**201 * intel_runtime_pm_get - grab a runtime pm reference202 * @rpm: the intel_runtime_pm structure203 *204 * This function grabs a device-level runtime pm reference (mostly used for GEM205 * code to ensure the GTT or GT is on) and ensures that it is powered up.206 *207 * Any runtime pm reference obtained by this function must have a symmetric208 * call to intel_runtime_pm_put() to release the reference again.209 *210 * Returns: the wakeref cookie to pass to intel_runtime_pm_put()211 */212intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)213{214 return __intel_runtime_pm_get(rpm, true);215}216 217/**218 * __intel_runtime_pm_get_if_active - grab a runtime pm reference if device is active219 * @rpm: the intel_runtime_pm structure220 * @ignore_usecount: get a ref even if dev->power.usage_count is 0221 *222 * This function grabs a device-level runtime pm reference if the device is223 * already active and ensures that it is powered up. It is illegal to try224 * and access the HW should intel_runtime_pm_get_if_active() report failure.225 *226 * If @ignore_usecount is true, a reference will be acquired even if there is no227 * user requiring the device to be powered up (dev->power.usage_count == 0).228 * If the function returns false in this case then it's guaranteed that the229 * device's runtime suspend hook has been called already or that it will be230 * called (and hence it's also guaranteed that the device's runtime resume231 * hook will be called eventually).232 *233 * Any runtime pm reference obtained by this function must have a symmetric234 * call to intel_runtime_pm_put() to release the reference again.235 *236 * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates237 * as True if the wakeref was acquired, or False otherwise.238 */239static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,240 bool ignore_usecount)241{242 if (IS_ENABLED(CONFIG_PM)) {243 /*244 * In cases runtime PM is disabled by the RPM core and we get245 * an -EINVAL return value we are not supposed to call this246 * function, since the power state is undefined. This applies247 * atm to the late/early system suspend/resume handlers.248 */249 if ((ignore_usecount &&250 pm_runtime_get_if_active(rpm->kdev) <= 0) ||251 (!ignore_usecount &&252 pm_runtime_get_if_in_use(rpm->kdev) <= 0))253 return 0;254 }255 256 intel_runtime_pm_acquire(rpm, true);257 258 return track_intel_runtime_pm_wakeref(rpm);259}260 261intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)262{263 return __intel_runtime_pm_get_if_active(rpm, false);264}265 266intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)267{268 return __intel_runtime_pm_get_if_active(rpm, true);269}270 271/**272 * intel_runtime_pm_get_noresume - grab a runtime pm reference273 * @rpm: the intel_runtime_pm structure274 *275 * This function grabs a device-level runtime pm reference.276 *277 * It will _not_ resume the device but instead only get an extra wakeref.278 * Therefore it is only valid to call this functions from contexts where279 * the device is known to be active and with another wakeref previously hold.280 *281 * Any runtime pm reference obtained by this function must have a symmetric282 * call to intel_runtime_pm_put() to release the reference again.283 *284 * Returns: the wakeref cookie to pass to intel_runtime_pm_put()285 */286intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)287{288 assert_rpm_raw_wakeref_held(rpm);289 pm_runtime_get_noresume(rpm->kdev);290 291 intel_runtime_pm_acquire(rpm, true);292 293 return track_intel_runtime_pm_wakeref(rpm);294}295 296static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,297 intel_wakeref_t wref,298 bool wakelock)299{300 struct device *kdev = rpm->kdev;301 302 untrack_intel_runtime_pm_wakeref(rpm, wref);303 304 intel_runtime_pm_release(rpm, wakelock);305 306 pm_runtime_mark_last_busy(kdev);307 pm_runtime_put_autosuspend(kdev);308}309 310/**311 * intel_runtime_pm_put_raw - release a raw runtime pm reference312 * @rpm: the intel_runtime_pm structure313 * @wref: wakeref acquired for the reference that is being released314 *315 * This function drops the device-level runtime pm reference obtained by316 * intel_runtime_pm_get_raw() and might power down the corresponding317 * hardware block right away if this is the last reference.318 */319void320intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)321{322 __intel_runtime_pm_put(rpm, wref, false);323}324 325/**326 * intel_runtime_pm_put_unchecked - release an unchecked runtime pm reference327 * @rpm: the intel_runtime_pm structure328 *329 * This function drops the device-level runtime pm reference obtained by330 * intel_runtime_pm_get() and might power down the corresponding331 * hardware block right away if this is the last reference.332 *333 * This function exists only for historical reasons and should be avoided in334 * new code, as the correctness of its use cannot be checked. Always use335 * intel_runtime_pm_put() instead.336 */337void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)338{339 __intel_runtime_pm_put(rpm, -1, true);340}341 342#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)343/**344 * intel_runtime_pm_put - release a runtime pm reference345 * @rpm: the intel_runtime_pm structure346 * @wref: wakeref acquired for the reference that is being released347 *348 * This function drops the device-level runtime pm reference obtained by349 * intel_runtime_pm_get() and might power down the corresponding350 * hardware block right away if this is the last reference.351 */352void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)353{354 __intel_runtime_pm_put(rpm, wref, true);355}356#endif357 358/**359 * intel_runtime_pm_enable - enable runtime pm360 * @rpm: the intel_runtime_pm structure361 *362 * This function enables runtime pm at the end of the driver load sequence.363 *364 * Note that this function does currently not enable runtime pm for the365 * subordinate display power domains. That is done by366 * intel_power_domains_enable().367 */368void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)369{370 struct drm_i915_private *i915 = rpm_to_i915(rpm);371 struct device *kdev = rpm->kdev;372 373 /*374 * Disable the system suspend direct complete optimization, which can375 * leave the device suspended skipping the driver's suspend handlers376 * if the device was already runtime suspended. This is needed due to377 * the difference in our runtime and system suspend sequence and378 * becaue the HDA driver may require us to enable the audio power379 * domain during system suspend.380 */381 dev_pm_set_driver_flags(kdev, DPM_FLAG_NO_DIRECT_COMPLETE);382 383 pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */384 pm_runtime_mark_last_busy(kdev);385 386 /*387 * Take a permanent reference to disable the RPM functionality and drop388 * it only when unloading the driver. Use the low level get/put helpers,389 * so the driver's own RPM reference tracking asserts also work on390 * platforms without RPM support.391 */392 if (!rpm->available) {393 int ret;394 395 pm_runtime_dont_use_autosuspend(kdev);396 ret = pm_runtime_get_sync(kdev);397 drm_WARN(&i915->drm, ret < 0,398 "pm_runtime_get_sync() failed: %d\n", ret);399 } else {400 pm_runtime_use_autosuspend(kdev);401 }402 403 /*404 * FIXME: Temp hammer to keep autosupend disable on lmem supported platforms.405 * As per PCIe specs 5.3.1.4.1, all iomem read write request over a PCIe406 * function will be unsupported in case PCIe endpoint function is in D3.407 * Let's keep i915 autosuspend control 'on' till we fix all known issue408 * with lmem access in D3.409 */410 if (!IS_DGFX(i915))411 pm_runtime_allow(kdev);412 413 /*414 * The core calls the driver load handler with an RPM reference held.415 * We drop that here and will reacquire it during unloading in416 * intel_power_domains_fini().417 */418 pm_runtime_put_autosuspend(kdev);419}420 421void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)422{423 struct drm_i915_private *i915 = rpm_to_i915(rpm);424 struct device *kdev = rpm->kdev;425 426 /* Transfer rpm ownership back to core */427 drm_WARN(&i915->drm, pm_runtime_get_sync(kdev) < 0,428 "Failed to pass rpm ownership back to core\n");429 430 pm_runtime_dont_use_autosuspend(kdev);431 432 if (!rpm->available)433 pm_runtime_put(kdev);434}435 436void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)437{438 struct drm_i915_private *i915 = rpm_to_i915(rpm);439 int count = atomic_read(&rpm->wakeref_count);440 441 intel_wakeref_auto_fini(&rpm->userfault_wakeref);442 443 drm_WARN(&i915->drm, count,444 "i915 raw-wakerefs=%d wakelocks=%d on cleanup\n",445 intel_rpm_raw_wakeref_count(count),446 intel_rpm_wakelock_count(count));447}448 449void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm)450{451 intel_runtime_pm_driver_release(rpm);452 untrack_all_intel_runtime_pm_wakerefs(rpm);453}454 455void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)456{457 struct drm_i915_private *i915 = rpm_to_i915(rpm);458 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);459 struct device *kdev = &pdev->dev;460 461 rpm->kdev = kdev;462 rpm->available = HAS_RUNTIME_PM(i915);463 atomic_set(&rpm->wakeref_count, 0);464 465 init_intel_runtime_pm_wakeref(rpm);466 INIT_LIST_HEAD(&rpm->lmem_userfault_list);467 spin_lock_init(&rpm->lmem_userfault_lock);468 intel_wakeref_auto_init(&rpm->userfault_wakeref, i915);469}470