754 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_pm.h"7 8#include <linux/pm_runtime.h>9 10#include <drm/drm_managed.h>11#include <drm/ttm/ttm_placement.h>12 13#include "display/xe_display.h"14#include "xe_bo.h"15#include "xe_bo_evict.h"16#include "xe_device.h"17#include "xe_device_sysfs.h"18#include "xe_ggtt.h"19#include "xe_gt.h"20#include "xe_guc.h"21#include "xe_irq.h"22#include "xe_pcode.h"23#include "xe_trace.h"24#include "xe_wa.h"25 26/**27 * DOC: Xe Power Management28 *29 * Xe PM implements the main routines for both system level suspend states and30 * for the opportunistic runtime suspend states.31 *32 * System Level Suspend (S-States) - In general this is OS initiated suspend33 * driven by ACPI for achieving S0ix (a.k.a. S2idle, freeze), S3 (suspend to ram),34 * S4 (disk). The main functions here are `xe_pm_suspend` and `xe_pm_resume`. They35 * are the main point for the suspend to and resume from these states.36 *37 * PCI Device Suspend (D-States) - This is the opportunistic PCIe device low power38 * state D3, controlled by the PCI subsystem and ACPI with the help from the39 * runtime_pm infrastructure.40 * PCI D3 is special and can mean D3hot, where Vcc power is on for keeping memory41 * alive and quicker low latency resume or D3Cold where Vcc power is off for42 * better power savings.43 * The Vcc control of PCI hierarchy can only be controlled at the PCI root port44 * level, while the device driver can be behind multiple bridges/switches and45 * paired with other devices. For this reason, the PCI subsystem cannot perform46 * the transition towards D3Cold. The lowest runtime PM possible from the PCI47 * subsystem is D3hot. Then, if all these paired devices in the same root port48 * are in D3hot, ACPI will assist here and run its own methods (_PR3 and _OFF)49 * to perform the transition from D3hot to D3cold. Xe may disallow this50 * transition by calling pci_d3cold_disable(root_pdev) before going to runtime51 * suspend. It will be based on runtime conditions such as VRAM usage for a52 * quick and low latency resume for instance.53 *54 * Runtime PM - This infrastructure provided by the Linux kernel allows the55 * device drivers to indicate when the can be runtime suspended, so the device56 * could be put at D3 (if supported), or allow deeper package sleep states57 * (PC-states), and/or other low level power states. Xe PM component provides58 * `xe_pm_runtime_suspend` and `xe_pm_runtime_resume` functions that PCI59 * subsystem will call before transition to/from runtime suspend.60 *61 * Also, Xe PM provides get and put functions that Xe driver will use to62 * indicate activity. In order to avoid locking complications with the memory63 * management, whenever possible, these get and put functions needs to be called64 * from the higher/outer levels.65 * The main cases that need to be protected from the outer levels are: IOCTL,66 * sysfs, debugfs, dma-buf sharing, GPU execution.67 *68 * This component is not responsible for GT idleness (RC6) nor GT frequency69 * management (RPS).70 */71 72#ifdef CONFIG_LOCKDEP73static struct lockdep_map xe_pm_runtime_d3cold_map = {74 .name = "xe_rpm_d3cold_map"75};76 77static struct lockdep_map xe_pm_runtime_nod3cold_map = {78 .name = "xe_rpm_nod3cold_map"79};80#endif81 82/**83 * xe_rpm_reclaim_safe() - Whether runtime resume can be done from reclaim context84 * @xe: The xe device.85 *86 * Return: true if it is safe to runtime resume from reclaim context.87 * false otherwise.88 */89bool xe_rpm_reclaim_safe(const struct xe_device *xe)90{91 return !xe->d3cold.capable && !xe->info.has_sriov;92}93 94static void xe_rpm_lockmap_acquire(const struct xe_device *xe)95{96 lock_map_acquire(xe_rpm_reclaim_safe(xe) ?97 &xe_pm_runtime_nod3cold_map :98 &xe_pm_runtime_d3cold_map);99}100 101static void xe_rpm_lockmap_release(const struct xe_device *xe)102{103 lock_map_release(xe_rpm_reclaim_safe(xe) ?104 &xe_pm_runtime_nod3cold_map :105 &xe_pm_runtime_d3cold_map);106}107 108/**109 * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle110 * @xe: xe device instance111 *112 * Return: 0 on success113 */114int xe_pm_suspend(struct xe_device *xe)115{116 struct xe_gt *gt;117 u8 id;118 int err;119 120 drm_dbg(&xe->drm, "Suspending device\n");121 trace_xe_pm_suspend(xe, __builtin_return_address(0));122 123 for_each_gt(gt, xe, id)124 xe_gt_suspend_prepare(gt);125 126 xe_display_pm_suspend(xe);127 128 /* FIXME: Super racey... */129 err = xe_bo_evict_all(xe);130 if (err)131 goto err;132 133 for_each_gt(gt, xe, id) {134 err = xe_gt_suspend(gt);135 if (err) {136 xe_display_pm_resume(xe);137 goto err;138 }139 }140 141 xe_irq_suspend(xe);142 143 xe_display_pm_suspend_late(xe);144 145 drm_dbg(&xe->drm, "Device suspended\n");146 return 0;147err:148 drm_dbg(&xe->drm, "Device suspend failed %d\n", err);149 return err;150}151 152/**153 * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0154 * @xe: xe device instance155 *156 * Return: 0 on success157 */158int xe_pm_resume(struct xe_device *xe)159{160 struct xe_tile *tile;161 struct xe_gt *gt;162 u8 id;163 int err;164 165 drm_dbg(&xe->drm, "Resuming device\n");166 trace_xe_pm_resume(xe, __builtin_return_address(0));167 168 for_each_tile(tile, xe, id)169 xe_wa_apply_tile_workarounds(tile);170 171 err = xe_pcode_ready(xe, true);172 if (err)173 return err;174 175 xe_display_pm_resume_early(xe);176 177 /*178 * This only restores pinned memory which is the memory required for the179 * GT(s) to resume.180 */181 err = xe_bo_restore_kernel(xe);182 if (err)183 goto err;184 185 xe_irq_resume(xe);186 187 for_each_gt(gt, xe, id)188 xe_gt_resume(gt);189 190 xe_display_pm_resume(xe);191 192 err = xe_bo_restore_user(xe);193 if (err)194 goto err;195 196 drm_dbg(&xe->drm, "Device resumed\n");197 return 0;198err:199 drm_dbg(&xe->drm, "Device resume failed %d\n", err);200 return err;201}202 203static bool xe_pm_pci_d3cold_capable(struct xe_device *xe)204{205 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);206 struct pci_dev *root_pdev;207 208 root_pdev = pcie_find_root_port(pdev);209 if (!root_pdev)210 return false;211 212 /* D3Cold requires PME capability */213 if (!pci_pme_capable(root_pdev, PCI_D3cold)) {214 drm_dbg(&xe->drm, "d3cold: PME# not supported\n");215 return false;216 }217 218 /* D3Cold requires _PR3 power resource */219 if (!pci_pr3_present(root_pdev)) {220 drm_dbg(&xe->drm, "d3cold: ACPI _PR3 not present\n");221 return false;222 }223 224 return true;225}226 227static void xe_pm_runtime_init(struct xe_device *xe)228{229 struct device *dev = xe->drm.dev;230 231 /*232 * Disable the system suspend direct complete optimization.233 * We need to ensure that the regular device suspend/resume functions234 * are called since our runtime_pm cannot guarantee local memory235 * eviction for d3cold.236 * TODO: Check HDA audio dependencies claimed by i915, and then enforce237 * this option to integrated graphics as well.238 */239 if (IS_DGFX(xe))240 dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);241 242 pm_runtime_use_autosuspend(dev);243 pm_runtime_set_autosuspend_delay(dev, 1000);244 pm_runtime_set_active(dev);245 pm_runtime_allow(dev);246 pm_runtime_mark_last_busy(dev);247 pm_runtime_put(dev);248}249 250int xe_pm_init_early(struct xe_device *xe)251{252 int err;253 254 INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list);255 256 err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock);257 if (err)258 return err;259 260 err = drmm_mutex_init(&xe->drm, &xe->d3cold.lock);261 if (err)262 return err;263 264 return 0;265}266 267/**268 * xe_pm_init - Initialize Xe Power Management269 * @xe: xe device instance270 *271 * This component is responsible for System and Device sleep states.272 *273 * Returns 0 for success, negative error code otherwise.274 */275int xe_pm_init(struct xe_device *xe)276{277 int err;278 279 /* For now suspend/resume is only allowed with GuC */280 if (!xe_device_uc_enabled(xe))281 return 0;282 283 xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe);284 285 if (xe->d3cold.capable) {286 err = xe_device_sysfs_init(xe);287 if (err)288 return err;289 290 err = xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD);291 if (err)292 return err;293 }294 295 xe_pm_runtime_init(xe);296 297 return 0;298}299 300/**301 * xe_pm_runtime_fini - Finalize Runtime PM302 * @xe: xe device instance303 */304void xe_pm_runtime_fini(struct xe_device *xe)305{306 struct device *dev = xe->drm.dev;307 308 pm_runtime_get_sync(dev);309 pm_runtime_forbid(dev);310}311 312static void xe_pm_write_callback_task(struct xe_device *xe,313 struct task_struct *task)314{315 WRITE_ONCE(xe->pm_callback_task, task);316 317 /*318 * Just in case it's somehow possible for our writes to be reordered to319 * the extent that something else re-uses the task written in320 * pm_callback_task. For example after returning from the callback, but321 * before the reordered write that resets pm_callback_task back to NULL.322 */323 smp_mb(); /* pairs with xe_pm_read_callback_task */324}325 326struct task_struct *xe_pm_read_callback_task(struct xe_device *xe)327{328 smp_mb(); /* pairs with xe_pm_write_callback_task */329 330 return READ_ONCE(xe->pm_callback_task);331}332 333/**334 * xe_pm_runtime_suspended - Check if runtime_pm state is suspended335 * @xe: xe device instance336 *337 * This does not provide any guarantee that the device is going to remain338 * suspended as it might be racing with the runtime state transitions.339 * It can be used only as a non-reliable assertion, to ensure that we are not in340 * the sleep state while trying to access some memory for instance.341 *342 * Returns true if PCI device is suspended, false otherwise.343 */344bool xe_pm_runtime_suspended(struct xe_device *xe)345{346 return pm_runtime_suspended(xe->drm.dev);347}348 349/**350 * xe_pm_runtime_suspend - Prepare our device for D3hot/D3Cold351 * @xe: xe device instance352 *353 * Returns 0 for success, negative error code otherwise.354 */355int xe_pm_runtime_suspend(struct xe_device *xe)356{357 struct xe_bo *bo, *on;358 struct xe_gt *gt;359 u8 id;360 int err = 0;361 362 trace_xe_pm_runtime_suspend(xe, __builtin_return_address(0));363 /* Disable access_ongoing asserts and prevent recursive pm calls */364 xe_pm_write_callback_task(xe, current);365 366 /*367 * The actual xe_pm_runtime_put() is always async underneath, so368 * exactly where that is called should makes no difference to us. However369 * we still need to be very careful with the locks that this callback370 * acquires and the locks that are acquired and held by any callers of371 * xe_runtime_pm_get(). We already have the matching annotation372 * on that side, but we also need it here. For example lockdep should be373 * able to tell us if the following scenario is in theory possible:374 *375 * CPU0 | CPU1 (kworker)376 * lock(A) |377 * | xe_pm_runtime_suspend()378 * | lock(A)379 * xe_pm_runtime_get() |380 *381 * This will clearly deadlock since rpm core needs to wait for382 * xe_pm_runtime_suspend() to complete, but here we are holding lock(A)383 * on CPU0 which prevents CPU1 making forward progress. With the384 * annotation here and in xe_pm_runtime_get() lockdep will see385 * the potential lock inversion and give us a nice splat.386 */387 xe_rpm_lockmap_acquire(xe);388 389 /*390 * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify391 * also checks and delets bo entry from user fault list.392 */393 mutex_lock(&xe->mem_access.vram_userfault.lock);394 list_for_each_entry_safe(bo, on,395 &xe->mem_access.vram_userfault.list, vram_userfault_link)396 xe_bo_runtime_pm_release_mmap_offset(bo);397 mutex_unlock(&xe->mem_access.vram_userfault.lock);398 399 xe_display_pm_runtime_suspend(xe);400 401 if (xe->d3cold.allowed) {402 err = xe_bo_evict_all(xe);403 if (err)404 goto out;405 }406 407 for_each_gt(gt, xe, id) {408 err = xe_gt_suspend(gt);409 if (err)410 goto out;411 }412 413 xe_irq_suspend(xe);414 415 if (xe->d3cold.allowed)416 xe_display_pm_suspend_late(xe);417out:418 if (err)419 xe_display_pm_runtime_resume(xe);420 xe_rpm_lockmap_release(xe);421 xe_pm_write_callback_task(xe, NULL);422 return err;423}424 425/**426 * xe_pm_runtime_resume - Waking up from D3hot/D3Cold427 * @xe: xe device instance428 *429 * Returns 0 for success, negative error code otherwise.430 */431int xe_pm_runtime_resume(struct xe_device *xe)432{433 struct xe_gt *gt;434 u8 id;435 int err = 0;436 437 trace_xe_pm_runtime_resume(xe, __builtin_return_address(0));438 /* Disable access_ongoing asserts and prevent recursive pm calls */439 xe_pm_write_callback_task(xe, current);440 441 xe_rpm_lockmap_acquire(xe);442 443 if (xe->d3cold.allowed) {444 err = xe_pcode_ready(xe, true);445 if (err)446 goto out;447 448 xe_display_pm_resume_early(xe);449 450 /*451 * This only restores pinned memory which is the memory452 * required for the GT(s) to resume.453 */454 err = xe_bo_restore_kernel(xe);455 if (err)456 goto out;457 }458 459 xe_irq_resume(xe);460 461 for_each_gt(gt, xe, id)462 xe_gt_resume(gt);463 464 xe_display_pm_runtime_resume(xe);465 466 if (xe->d3cold.allowed) {467 err = xe_bo_restore_user(xe);468 if (err)469 goto out;470 }471 472out:473 xe_rpm_lockmap_release(xe);474 xe_pm_write_callback_task(xe, NULL);475 return err;476}477 478/*479 * For places where resume is synchronous it can be quite easy to deadlock480 * if we are not careful. Also in practice it might be quite timing481 * sensitive to ever see the 0 -> 1 transition with the callers locks482 * held, so deadlocks might exist but are hard for lockdep to ever see.483 * With this in mind, help lockdep learn about the potentially scary484 * stuff that can happen inside the runtime_resume callback by acquiring485 * a dummy lock (it doesn't protect anything and gets compiled out on486 * non-debug builds). Lockdep then only needs to see the487 * xe_pm_runtime_xxx_map -> runtime_resume callback once, and then can488 * hopefully validate all the (callers_locks) -> xe_pm_runtime_xxx_map.489 * For example if the (callers_locks) are ever grabbed in the490 * runtime_resume callback, lockdep should give us a nice splat.491 */492static void xe_rpm_might_enter_cb(const struct xe_device *xe)493{494 xe_rpm_lockmap_acquire(xe);495 xe_rpm_lockmap_release(xe);496}497 498/*499 * Prime the lockdep maps for known locking orders that need to500 * be supported but that may not always occur on all systems.501 */502static void xe_pm_runtime_lockdep_prime(void)503{504 struct dma_resv lockdep_resv;505 506 dma_resv_init(&lockdep_resv);507 lock_map_acquire(&xe_pm_runtime_d3cold_map);508 /* D3Cold takes the dma_resv locks to evict bos */509 dma_resv_lock(&lockdep_resv, NULL);510 dma_resv_unlock(&lockdep_resv);511 lock_map_release(&xe_pm_runtime_d3cold_map);512 513 /* Shrinkers might like to wake up the device under reclaim. */514 fs_reclaim_acquire(GFP_KERNEL);515 lock_map_acquire(&xe_pm_runtime_nod3cold_map);516 lock_map_release(&xe_pm_runtime_nod3cold_map);517 fs_reclaim_release(GFP_KERNEL);518}519 520/**521 * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously522 * @xe: xe device instance523 */524void xe_pm_runtime_get(struct xe_device *xe)525{526 trace_xe_pm_runtime_get(xe, __builtin_return_address(0));527 pm_runtime_get_noresume(xe->drm.dev);528 529 if (xe_pm_read_callback_task(xe) == current)530 return;531 532 xe_rpm_might_enter_cb(xe);533 pm_runtime_resume(xe->drm.dev);534}535 536/**537 * xe_pm_runtime_put - Put the runtime_pm reference back and mark as idle538 * @xe: xe device instance539 */540void xe_pm_runtime_put(struct xe_device *xe)541{542 trace_xe_pm_runtime_put(xe, __builtin_return_address(0));543 if (xe_pm_read_callback_task(xe) == current) {544 pm_runtime_put_noidle(xe->drm.dev);545 } else {546 pm_runtime_mark_last_busy(xe->drm.dev);547 pm_runtime_put(xe->drm.dev);548 }549}550 551/**552 * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl553 * @xe: xe device instance554 *555 * Returns: Any number greater than or equal to 0 for success, negative error556 * code otherwise.557 */558int xe_pm_runtime_get_ioctl(struct xe_device *xe)559{560 trace_xe_pm_runtime_get_ioctl(xe, __builtin_return_address(0));561 if (WARN_ON(xe_pm_read_callback_task(xe) == current))562 return -ELOOP;563 564 xe_rpm_might_enter_cb(xe);565 return pm_runtime_get_sync(xe->drm.dev);566}567 568/**569 * xe_pm_runtime_get_if_active - Get a runtime_pm reference if device active570 * @xe: xe device instance571 *572 * Return: True if device is awake (regardless the previous number of references)573 * and a new reference was taken, false otherwise.574 */575bool xe_pm_runtime_get_if_active(struct xe_device *xe)576{577 return pm_runtime_get_if_active(xe->drm.dev) > 0;578}579 580/**581 * xe_pm_runtime_get_if_in_use - Get a new reference if device is active with previous ref taken582 * @xe: xe device instance583 *584 * Return: True if device is awake, a previous reference had been already taken,585 * and a new reference was now taken, false otherwise.586 */587bool xe_pm_runtime_get_if_in_use(struct xe_device *xe)588{589 if (xe_pm_read_callback_task(xe) == current) {590 /* The device is awake, grab the ref and move on */591 pm_runtime_get_noresume(xe->drm.dev);592 return true;593 }594 595 return pm_runtime_get_if_in_use(xe->drm.dev) > 0;596}597 598/*599 * Very unreliable! Should only be used to suppress the false positive case600 * in the missing outer rpm protection warning.601 */602static bool xe_pm_suspending_or_resuming(struct xe_device *xe)603{604#ifdef CONFIG_PM605 struct device *dev = xe->drm.dev;606 607 return dev->power.runtime_status == RPM_SUSPENDING ||608 dev->power.runtime_status == RPM_RESUMING;609#else610 return false;611#endif612}613 614/**615 * xe_pm_runtime_get_noresume - Bump runtime PM usage counter without resuming616 * @xe: xe device instance617 *618 * This function should be used in inner places where it is surely already619 * protected by outer-bound callers of `xe_pm_runtime_get`.620 * It will warn if not protected.621 * The reference should be put back after this function regardless, since it622 * will always bump the usage counter, regardless.623 */624void xe_pm_runtime_get_noresume(struct xe_device *xe)625{626 bool ref;627 628 ref = xe_pm_runtime_get_if_in_use(xe);629 630 if (!ref) {631 pm_runtime_get_noresume(xe->drm.dev);632 drm_WARN(&xe->drm, !xe_pm_suspending_or_resuming(xe),633 "Missing outer runtime PM protection\n");634 }635}636 637/**638 * xe_pm_runtime_resume_and_get - Resume, then get a runtime_pm ref if awake.639 * @xe: xe device instance640 *641 * Returns: True if device is awake and the reference was taken, false otherwise.642 */643bool xe_pm_runtime_resume_and_get(struct xe_device *xe)644{645 if (xe_pm_read_callback_task(xe) == current) {646 /* The device is awake, grab the ref and move on */647 pm_runtime_get_noresume(xe->drm.dev);648 return true;649 }650 651 xe_rpm_might_enter_cb(xe);652 return pm_runtime_resume_and_get(xe->drm.dev) >= 0;653}654 655/**656 * xe_pm_assert_unbounded_bridge - Disable PM on unbounded pcie parent bridge657 * @xe: xe device instance658 */659void xe_pm_assert_unbounded_bridge(struct xe_device *xe)660{661 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);662 struct pci_dev *bridge = pci_upstream_bridge(pdev);663 664 if (!bridge)665 return;666 667 if (!bridge->driver) {668 drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n");669 device_set_pm_not_required(&pdev->dev);670 }671}672 673/**674 * xe_pm_set_vram_threshold - Set a vram threshold for allowing/blocking D3Cold675 * @xe: xe device instance676 * @threshold: VRAM size in bites for the D3cold threshold677 *678 * Returns 0 for success, negative error code otherwise.679 */680int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)681{682 struct ttm_resource_manager *man;683 u32 vram_total_mb = 0;684 int i;685 686 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {687 man = ttm_manager_type(&xe->ttm, i);688 if (man)689 vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024);690 }691 692 drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb);693 694 if (threshold > vram_total_mb)695 return -EINVAL;696 697 mutex_lock(&xe->d3cold.lock);698 xe->d3cold.vram_threshold = threshold;699 mutex_unlock(&xe->d3cold.lock);700 701 return 0;702}703 704/**705 * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed706 * @xe: xe device instance707 *708 * To be called during runtime_pm idle callback.709 * Check for all the D3Cold conditions ahead of runtime suspend.710 */711void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)712{713 struct ttm_resource_manager *man;714 u32 total_vram_used_mb = 0;715 u64 vram_used;716 int i;717 718 if (!xe->d3cold.capable) {719 xe->d3cold.allowed = false;720 return;721 }722 723 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {724 man = ttm_manager_type(&xe->ttm, i);725 if (man) {726 vram_used = ttm_resource_manager_usage(man);727 total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024);728 }729 }730 731 mutex_lock(&xe->d3cold.lock);732 733 if (total_vram_used_mb < xe->d3cold.vram_threshold)734 xe->d3cold.allowed = true;735 else736 xe->d3cold.allowed = false;737 738 mutex_unlock(&xe->d3cold.lock);739 740 drm_dbg(&xe->drm,741 "d3cold: allowed=%s\n", str_yes_no(xe->d3cold.allowed));742}743 744/**745 * xe_pm_module_init() - Perform xe_pm specific module initialization.746 *747 * Return: 0 on success. Currently doesn't fail.748 */749int __init xe_pm_module_init(void)750{751 xe_pm_runtime_lockdep_prime();752 return 0;753}754