267 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2019 Intel Corporation5 */6 7#include "gem/i915_gem_pm.h"8#include "gem/i915_gem_ttm_pm.h"9#include "gt/intel_gt.h"10#include "gt/intel_gt_pm.h"11#include "gt/intel_gt_requests.h"12 13#include "i915_driver.h"14#include "i915_drv.h"15 16#if defined(CONFIG_X86)17#include <asm/smp.h>18#else19#define wbinvd_on_all_cpus() \20 pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)21#endif22 23void i915_gem_suspend(struct drm_i915_private *i915)24{25 struct intel_gt *gt;26 unsigned int i;27 28 GEM_TRACE("%s\n", dev_name(i915->drm.dev));29 30 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);31 /*32 * On rare occasions, we've observed the fence completion triggers33 * free_engines asynchronously via rcu_call. Ensure those are done.34 * This path is only called on suspend, so it's an acceptable cost.35 */36 rcu_barrier();37 38 flush_workqueue(i915->wq);39 40 /*41 * We have to flush all the executing contexts to main memory so42 * that they can saved in the hibernation image. To ensure the last43 * context image is coherent, we have to switch away from it. That44 * leaves the i915->kernel_context still active when45 * we actually suspend, and its image in memory may not match the GPU46 * state. Fortunately, the kernel_context is disposable and we do47 * not rely on its state.48 */49 for_each_gt(gt, i915, i)50 intel_gt_suspend_prepare(gt);51 52 i915_gem_drain_freed_objects(i915);53}54 55static int lmem_restore(struct drm_i915_private *i915, u32 flags)56{57 struct intel_memory_region *mr;58 int ret = 0, id;59 60 for_each_memory_region(mr, i915, id) {61 if (mr->type == INTEL_MEMORY_LOCAL) {62 ret = i915_ttm_restore_region(mr, flags);63 if (ret)64 break;65 }66 }67 68 return ret;69}70 71static int lmem_suspend(struct drm_i915_private *i915, u32 flags)72{73 struct intel_memory_region *mr;74 int ret = 0, id;75 76 for_each_memory_region(mr, i915, id) {77 if (mr->type == INTEL_MEMORY_LOCAL) {78 ret = i915_ttm_backup_region(mr, flags);79 if (ret)80 break;81 }82 }83 84 return ret;85}86 87static void lmem_recover(struct drm_i915_private *i915)88{89 struct intel_memory_region *mr;90 int id;91 92 for_each_memory_region(mr, i915, id)93 if (mr->type == INTEL_MEMORY_LOCAL)94 i915_ttm_recover_region(mr);95}96 97int i915_gem_backup_suspend(struct drm_i915_private *i915)98{99 int ret;100 101 /* Opportunistically try to evict unpinned objects */102 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);103 if (ret)104 goto out_recover;105 106 i915_gem_suspend(i915);107 108 /*109 * More objects may have become unpinned as requests were110 * retired. Now try to evict again. The gt may be wedged here111 * in which case we automatically fall back to memcpy.112 * We allow also backing up pinned objects that have not been113 * marked for early recover, and that may contain, for example,114 * page-tables for the migrate context.115 */116 ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |117 I915_TTM_BACKUP_PINNED);118 if (ret)119 goto out_recover;120 121 /*122 * Remaining objects are backed up using memcpy once we've stopped123 * using the migrate context.124 */125 ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);126 if (ret)127 goto out_recover;128 129 return 0;130 131out_recover:132 lmem_recover(i915);133 134 return ret;135}136 137void i915_gem_suspend_late(struct drm_i915_private *i915)138{139 struct drm_i915_gem_object *obj;140 struct list_head *phases[] = {141 &i915->mm.shrink_list,142 &i915->mm.purge_list,143 NULL144 }, **phase;145 struct intel_gt *gt;146 unsigned long flags;147 unsigned int i;148 bool flush = false;149 150 /*151 * Neither the BIOS, ourselves or any other kernel152 * expects the system to be in execlists mode on startup,153 * so we need to reset the GPU back to legacy mode. And the only154 * known way to disable logical contexts is through a GPU reset.155 *156 * So in order to leave the system in a known default configuration,157 * always reset the GPU upon unload and suspend. Afterwards we then158 * clean up the GEM state tracking, flushing off the requests and159 * leaving the system in a known idle state.160 *161 * Note that is of the upmost importance that the GPU is idle and162 * all stray writes are flushed *before* we dismantle the backing163 * storage for the pinned objects.164 *165 * However, since we are uncertain that resetting the GPU on older166 * machines is a good idea, we don't - just in case it leaves the167 * machine in an unusable condition.168 */169 170 /* Like i915_gem_suspend, flush tasks staged from fence triggers */171 rcu_barrier();172 173 for_each_gt(gt, i915, i)174 intel_gt_suspend_late(gt);175 176 spin_lock_irqsave(&i915->mm.obj_lock, flags);177 for (phase = phases; *phase; phase++) {178 list_for_each_entry(obj, *phase, mm.link) {179 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))180 flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;181 __start_cpu_write(obj); /* presume auto-hibernate */182 }183 }184 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);185 if (flush)186 wbinvd_on_all_cpus();187}188 189int i915_gem_freeze(struct drm_i915_private *i915)190{191 /* Discard all purgeable objects, let userspace recover those as192 * required after resuming.193 */194 i915_gem_shrink_all(i915);195 196 return 0;197}198 199int i915_gem_freeze_late(struct drm_i915_private *i915)200{201 struct drm_i915_gem_object *obj;202 intel_wakeref_t wakeref;203 204 /*205 * Called just before we write the hibernation image.206 *207 * We need to update the domain tracking to reflect that the CPU208 * will be accessing all the pages to create and restore from the209 * hibernation, and so upon restoration those pages will be in the210 * CPU domain.211 *212 * To make sure the hibernation image contains the latest state,213 * we update that state just before writing out the image.214 *215 * To try and reduce the hibernation image, we manually shrink216 * the objects as well, see i915_gem_freeze()217 */218 219 with_intel_runtime_pm(&i915->runtime_pm, wakeref)220 i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);221 i915_gem_drain_freed_objects(i915);222 223 wbinvd_on_all_cpus();224 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)225 __start_cpu_write(obj);226 227 return 0;228}229 230void i915_gem_resume(struct drm_i915_private *i915)231{232 struct intel_gt *gt;233 int ret, i, j;234 235 GEM_TRACE("%s\n", dev_name(i915->drm.dev));236 237 ret = lmem_restore(i915, 0);238 GEM_WARN_ON(ret);239 240 /*241 * As we didn't flush the kernel context before suspend, we cannot242 * guarantee that the context image is complete. So let's just reset243 * it and start again.244 */245 for_each_gt(gt, i915, i)246 if (intel_gt_resume(gt))247 goto err_wedged;248 249 ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);250 GEM_WARN_ON(ret);251 252 return;253 254err_wedged:255 for_each_gt(gt, i915, j) {256 if (!intel_gt_is_wedged(gt)) {257 dev_err(i915->drm.dev,258 "Failed to re-initialize GPU[%u], declaring it wedged!\n",259 j);260 intel_gt_set_wedged(gt);261 }262 263 if (j == i)264 break;265 }266}267