431 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2019 Intel Corporation4 */5 6#include <linux/string_helpers.h>7#include <linux/suspend.h>8 9#include "i915_drv.h"10#include "i915_irq.h"11#include "i915_params.h"12#include "intel_context.h"13#include "intel_engine_pm.h"14#include "intel_gt.h"15#include "intel_gt_clock_utils.h"16#include "intel_gt_mcr.h"17#include "intel_gt_pm.h"18#include "intel_gt_print.h"19#include "intel_gt_requests.h"20#include "intel_llc.h"21#include "intel_rc6.h"22#include "intel_rps.h"23#include "intel_wakeref.h"24#include "pxp/intel_pxp_pm.h"25 26#define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)27 28static void user_forcewake(struct intel_gt *gt, bool suspend)29{30 int count = atomic_read(>->user_wakeref);31 intel_wakeref_t wakeref;32 33 /* Inside suspend/resume so single threaded, no races to worry about. */34 if (likely(!count))35 return;36 37 wakeref = intel_gt_pm_get(gt);38 if (suspend) {39 GEM_BUG_ON(count > atomic_read(>->wakeref.count));40 atomic_sub(count, >->wakeref.count);41 } else {42 atomic_add(count, >->wakeref.count);43 }44 intel_gt_pm_put(gt, wakeref);45}46 47static void runtime_begin(struct intel_gt *gt)48{49 local_irq_disable();50 write_seqcount_begin(>->stats.lock);51 gt->stats.start = ktime_get();52 gt->stats.active = true;53 write_seqcount_end(>->stats.lock);54 local_irq_enable();55}56 57static void runtime_end(struct intel_gt *gt)58{59 local_irq_disable();60 write_seqcount_begin(>->stats.lock);61 gt->stats.active = false;62 gt->stats.total =63 ktime_add(gt->stats.total,64 ktime_sub(ktime_get(), gt->stats.start));65 write_seqcount_end(>->stats.lock);66 local_irq_enable();67}68 69static int __gt_unpark(struct intel_wakeref *wf)70{71 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);72 struct drm_i915_private *i915 = gt->i915;73 74 GT_TRACE(gt, "\n");75 76 /*77 * It seems that the DMC likes to transition between the DC states a lot78 * when there are no connected displays (no active power domains) during79 * command submission.80 *81 * This activity has negative impact on the performance of the chip with82 * huge latencies observed in the interrupt handler and elsewhere.83 *84 * Work around it by grabbing a GT IRQ power domain whilst there is any85 * GT activity, preventing any DC state transitions.86 */87 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);88 GEM_BUG_ON(!gt->awake);89 90 intel_rc6_unpark(>->rc6);91 intel_rps_unpark(>->rps);92 i915_pmu_gt_unparked(gt);93 intel_guc_busyness_unpark(gt);94 95 intel_gt_unpark_requests(gt);96 runtime_begin(gt);97 98 return 0;99}100 101static int __gt_park(struct intel_wakeref *wf)102{103 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);104 intel_wakeref_t wakeref = fetch_and_zero(>->awake);105 struct drm_i915_private *i915 = gt->i915;106 107 GT_TRACE(gt, "\n");108 109 runtime_end(gt);110 intel_gt_park_requests(gt);111 112 intel_guc_busyness_park(gt);113 i915_vma_parked(gt);114 i915_pmu_gt_parked(gt);115 intel_rps_park(>->rps);116 intel_rc6_park(>->rc6);117 118 /* Everything switched off, flush any residual interrupt just in case */119 intel_synchronize_irq(i915);120 121 /* Defer dropping the display power well for 100ms, it's slow! */122 GEM_BUG_ON(!wakeref);123 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);124 125 return 0;126}127 128static const struct intel_wakeref_ops wf_ops = {129 .get = __gt_unpark,130 .put = __gt_park,131};132 133void intel_gt_pm_init_early(struct intel_gt *gt)134{135 /*136 * We access the runtime_pm structure via gt->i915 here rather than137 * gt->uncore as we do elsewhere in the file because gt->uncore is not138 * yet initialized for all tiles at this point in the driver startup.139 * runtime_pm is per-device rather than per-tile, so this is still the140 * correct structure.141 */142 intel_wakeref_init(>->wakeref, gt->i915, &wf_ops, "GT");143 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex);144}145 146void intel_gt_pm_init(struct intel_gt *gt)147{148 /*149 * Enabling power-management should be "self-healing". If we cannot150 * enable a feature, simply leave it disabled with a notice to the151 * user.152 */153 intel_rc6_init(>->rc6);154 intel_rps_init(>->rps);155}156 157static bool reset_engines(struct intel_gt *gt)158{159 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)160 return false;161 162 return intel_gt_reset_all_engines(gt) == 0;163}164 165static void gt_sanitize(struct intel_gt *gt, bool force)166{167 struct intel_engine_cs *engine;168 enum intel_engine_id id;169 intel_wakeref_t wakeref;170 171 GT_TRACE(gt, "force:%s\n", str_yes_no(force));172 173 /* Use a raw wakeref to avoid calling intel_display_power_get early */174 wakeref = intel_runtime_pm_get(gt->uncore->rpm);175 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);176 177 intel_gt_check_clock_frequency(gt);178 179 /*180 * As we have just resumed the machine and woken the device up from181 * deep PCI sleep (presumably D3_cold), assume the HW has been reset182 * back to defaults, recovering from whatever wedged state we left it183 * in and so worth trying to use the device once more.184 */185 if (intel_gt_is_wedged(gt))186 intel_gt_unset_wedged(gt);187 188 /* For GuC mode, ensure submission is disabled before stopping ring */189 intel_uc_reset_prepare(>->uc);190 191 for_each_engine(engine, gt, id) {192 if (engine->reset.prepare)193 engine->reset.prepare(engine);194 195 if (engine->sanitize)196 engine->sanitize(engine);197 }198 199 if (reset_engines(gt) || force) {200 for_each_engine(engine, gt, id)201 __intel_engine_reset(engine, false);202 }203 204 intel_uc_reset(>->uc, false);205 206 for_each_engine(engine, gt, id)207 if (engine->reset.finish)208 engine->reset.finish(engine);209 210 intel_rps_sanitize(>->rps);211 212 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);213 intel_runtime_pm_put(gt->uncore->rpm, wakeref);214}215 216void intel_gt_pm_fini(struct intel_gt *gt)217{218 intel_rc6_fini(>->rc6);219}220 221void intel_gt_resume_early(struct intel_gt *gt)222{223 /*224 * Sanitize steer semaphores during driver resume. This is necessary225 * to address observed cases of steer semaphores being226 * held after a suspend operation. Confirmation from the hardware team227 * assures the safety of this operation, as no lock acquisitions228 * by other agents occur during driver load/resume process.229 */230 intel_gt_mcr_lock_sanitize(gt);231 232 intel_uncore_resume_early(gt->uncore);233 intel_gt_check_and_clear_faults(gt);234}235 236int intel_gt_resume(struct intel_gt *gt)237{238 struct intel_engine_cs *engine;239 enum intel_engine_id id;240 intel_wakeref_t wakeref;241 int err;242 243 err = intel_gt_has_unrecoverable_error(gt);244 if (err)245 return err;246 247 GT_TRACE(gt, "\n");248 249 /*250 * After resume, we may need to poke into the pinned kernel251 * contexts to paper over any damage caused by the sudden suspend.252 * Only the kernel contexts should remain pinned over suspend,253 * allowing us to fixup the user contexts on their first pin.254 */255 gt_sanitize(gt, true);256 257 wakeref = intel_gt_pm_get(gt);258 259 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);260 intel_rc6_sanitize(>->rc6);261 if (intel_gt_is_wedged(gt)) {262 err = -EIO;263 goto out_fw;264 }265 266 /* Only when the HW is re-initialised, can we replay the requests */267 err = intel_gt_init_hw(gt);268 if (err) {269 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");270 goto err_wedged;271 }272 273 intel_uc_reset_finish(>->uc);274 275 intel_rps_enable(>->rps);276 intel_llc_enable(>->llc);277 278 for_each_engine(engine, gt, id) {279 intel_engine_pm_get(engine);280 281 engine->serial++; /* kernel context lost */282 err = intel_engine_resume(engine);283 284 intel_engine_pm_put(engine);285 if (err) {286 gt_err(gt, "Failed to restart %s (%d)\n",287 engine->name, err);288 goto err_wedged;289 }290 }291 292 intel_rc6_enable(>->rc6);293 294 intel_uc_resume(>->uc);295 296 user_forcewake(gt, false);297 298out_fw:299 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);300 intel_gt_pm_put(gt, wakeref);301 intel_gt_bind_context_set_ready(gt);302 return err;303 304err_wedged:305 intel_gt_set_wedged(gt);306 goto out_fw;307}308 309static void wait_for_suspend(struct intel_gt *gt)310{311 if (!intel_gt_pm_is_awake(gt))312 return;313 314 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {315 /*316 * Forcibly cancel outstanding work and leave317 * the gpu quiet.318 */319 intel_gt_set_wedged(gt);320 intel_gt_retire_requests(gt);321 }322 323 intel_gt_pm_wait_for_idle(gt);324}325 326void intel_gt_suspend_prepare(struct intel_gt *gt)327{328 intel_gt_bind_context_set_unready(gt);329 user_forcewake(gt, true);330 wait_for_suspend(gt);331}332 333static suspend_state_t pm_suspend_target(void)334{335#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)336 return pm_suspend_target_state;337#else338 return PM_SUSPEND_TO_IDLE;339#endif340}341 342void intel_gt_suspend_late(struct intel_gt *gt)343{344 intel_wakeref_t wakeref;345 346 /* We expect to be idle already; but also want to be independent */347 wait_for_suspend(gt);348 349 if (is_mock_gt(gt))350 return;351 352 GEM_BUG_ON(gt->awake);353 354 intel_uc_suspend(>->uc);355 356 /*357 * On disabling the device, we want to turn off HW access to memory358 * that we no longer own.359 *360 * However, not all suspend-states disable the device. S0 (s2idle)361 * is effectively runtime-suspend, the device is left powered on362 * but needs to be put into a low power state. We need to keep363 * powermanagement enabled, but we also retain system state and so364 * it remains safe to keep on using our allocated memory.365 */366 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)367 return;368 369 with_intel_runtime_pm(gt->uncore->rpm, wakeref) {370 intel_rps_disable(>->rps);371 intel_rc6_disable(>->rc6);372 intel_llc_disable(>->llc);373 }374 375 gt_sanitize(gt, false);376 377 GT_TRACE(gt, "\n");378}379 380void intel_gt_runtime_suspend(struct intel_gt *gt)381{382 intel_gt_bind_context_set_unready(gt);383 intel_uc_runtime_suspend(>->uc);384 385 GT_TRACE(gt, "\n");386}387 388int intel_gt_runtime_resume(struct intel_gt *gt)389{390 int ret;391 392 GT_TRACE(gt, "\n");393 intel_gt_init_swizzling(gt);394 intel_ggtt_restore_fences(gt->ggtt);395 396 ret = intel_uc_runtime_resume(>->uc);397 if (ret)398 return ret;399 400 intel_gt_bind_context_set_ready(gt);401 return 0;402}403 404static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)405{406 ktime_t total = gt->stats.total;407 408 if (gt->stats.active)409 total = ktime_add(total,410 ktime_sub(ktime_get(), gt->stats.start));411 412 return total;413}414 415ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)416{417 unsigned int seq;418 ktime_t total;419 420 do {421 seq = read_seqcount_begin(>->stats.lock);422 total = __intel_gt_get_awake_time(gt);423 } while (read_seqcount_retry(>->stats.lock, seq));424 425 return total;426}427 428#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)429#include "selftest_gt_pm.c"430#endif431