233 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_bo_evict.h"7 8#include "xe_bo.h"9#include "xe_device.h"10#include "xe_ggtt.h"11#include "xe_tile.h"12 13/**14 * xe_bo_evict_all - evict all BOs from VRAM15 *16 * @xe: xe device17 *18 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next19 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU.20 * All eviction magic done via TTM calls.21 *22 * Evict == move VRAM BOs to temporary (typically system) memory.23 *24 * This function should be called before the device goes into a suspend state25 * where the VRAM loses power.26 */27int xe_bo_evict_all(struct xe_device *xe)28{29 struct ttm_device *bdev = &xe->ttm;30 struct xe_bo *bo;31 struct xe_tile *tile;32 struct list_head still_in_list;33 u32 mem_type;34 u8 id;35 int ret;36 37 /* User memory */38 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) {39 struct ttm_resource_manager *man =40 ttm_manager_type(bdev, mem_type);41 42 /*43 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS44 * state since this state lives inside graphics stolen memory which doesn't survive45 * hibernation.46 *47 * This can be further improved by only evicting objects that we know have actually48 * used a compression enabled PAT index.49 */50 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)))51 continue;52 53 if (man) {54 ret = ttm_resource_manager_evict_all(bdev, man);55 if (ret)56 return ret;57 }58 }59 60 /* Pinned user memory in VRAM */61 INIT_LIST_HEAD(&still_in_list);62 spin_lock(&xe->pinned.lock);63 for (;;) {64 bo = list_first_entry_or_null(&xe->pinned.external_vram,65 typeof(*bo), pinned_link);66 if (!bo)67 break;68 xe_bo_get(bo);69 list_move_tail(&bo->pinned_link, &still_in_list);70 spin_unlock(&xe->pinned.lock);71 72 xe_bo_lock(bo, false);73 ret = xe_bo_evict_pinned(bo);74 xe_bo_unlock(bo);75 xe_bo_put(bo);76 if (ret) {77 spin_lock(&xe->pinned.lock);78 list_splice_tail(&still_in_list,79 &xe->pinned.external_vram);80 spin_unlock(&xe->pinned.lock);81 return ret;82 }83 84 spin_lock(&xe->pinned.lock);85 }86 list_splice_tail(&still_in_list, &xe->pinned.external_vram);87 spin_unlock(&xe->pinned.lock);88 89 /*90 * Wait for all user BO to be evicted as those evictions depend on the91 * memory moved below.92 */93 for_each_tile(tile, xe, id)94 xe_tile_migrate_wait(tile);95 96 spin_lock(&xe->pinned.lock);97 for (;;) {98 bo = list_first_entry_or_null(&xe->pinned.kernel_bo_present,99 typeof(*bo), pinned_link);100 if (!bo)101 break;102 xe_bo_get(bo);103 list_move_tail(&bo->pinned_link, &xe->pinned.evicted);104 spin_unlock(&xe->pinned.lock);105 106 xe_bo_lock(bo, false);107 ret = xe_bo_evict_pinned(bo);108 xe_bo_unlock(bo);109 xe_bo_put(bo);110 if (ret)111 return ret;112 113 spin_lock(&xe->pinned.lock);114 }115 spin_unlock(&xe->pinned.lock);116 117 return 0;118}119 120/**121 * xe_bo_restore_kernel - restore kernel BOs to VRAM122 *123 * @xe: xe device124 *125 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All126 * moves done via TTM calls.127 *128 * This function should be called early, before trying to init the GT, on device129 * resume.130 */131int xe_bo_restore_kernel(struct xe_device *xe)132{133 struct xe_bo *bo;134 int ret;135 136 spin_lock(&xe->pinned.lock);137 for (;;) {138 bo = list_first_entry_or_null(&xe->pinned.evicted,139 typeof(*bo), pinned_link);140 if (!bo)141 break;142 xe_bo_get(bo);143 list_move_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present);144 spin_unlock(&xe->pinned.lock);145 146 xe_bo_lock(bo, false);147 ret = xe_bo_restore_pinned(bo);148 xe_bo_unlock(bo);149 if (ret) {150 xe_bo_put(bo);151 return ret;152 }153 154 if (bo->flags & XE_BO_FLAG_GGTT) {155 struct xe_tile *tile = bo->tile;156 157 mutex_lock(&tile->mem.ggtt->lock);158 xe_ggtt_map_bo(tile->mem.ggtt, bo);159 mutex_unlock(&tile->mem.ggtt->lock);160 }161 162 /*163 * We expect validate to trigger a move VRAM and our move code164 * should setup the iosys map.165 */166 xe_assert(xe, !iosys_map_is_null(&bo->vmap));167 168 xe_bo_put(bo);169 170 spin_lock(&xe->pinned.lock);171 }172 spin_unlock(&xe->pinned.lock);173 174 return 0;175}176 177/**178 * xe_bo_restore_user - restore pinned user BOs to VRAM179 *180 * @xe: xe device181 *182 * Move pinned user BOs from temporary (typically system) memory to VRAM via183 * CPU. All moves done via TTM calls.184 *185 * This function should be called late, after GT init, on device resume.186 */187int xe_bo_restore_user(struct xe_device *xe)188{189 struct xe_bo *bo;190 struct xe_tile *tile;191 struct list_head still_in_list;192 u8 id;193 int ret;194 195 if (!IS_DGFX(xe))196 return 0;197 198 /* Pinned user memory in VRAM should be validated on resume */199 INIT_LIST_HEAD(&still_in_list);200 spin_lock(&xe->pinned.lock);201 for (;;) {202 bo = list_first_entry_or_null(&xe->pinned.external_vram,203 typeof(*bo), pinned_link);204 if (!bo)205 break;206 list_move_tail(&bo->pinned_link, &still_in_list);207 xe_bo_get(bo);208 spin_unlock(&xe->pinned.lock);209 210 xe_bo_lock(bo, false);211 ret = xe_bo_restore_pinned(bo);212 xe_bo_unlock(bo);213 xe_bo_put(bo);214 if (ret) {215 spin_lock(&xe->pinned.lock);216 list_splice_tail(&still_in_list,217 &xe->pinned.external_vram);218 spin_unlock(&xe->pinned.lock);219 return ret;220 }221 222 spin_lock(&xe->pinned.lock);223 }224 list_splice_tail(&still_in_list, &xe->pinned.external_vram);225 spin_unlock(&xe->pinned.lock);226 227 /* Wait for restore to complete */228 for_each_tile(tile, xe, id)229 xe_tile_migrate_wait(tile);230 231 return 0;232}233