529 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2020-2021 Intel Corporation4 */5 6#include "gt/intel_migrate.h"7#include "gt/intel_gpu_commands.h"8#include "gem/i915_gem_ttm_move.h"9 10#include "i915_deps.h"11 12#include "selftests/igt_reset.h"13#include "selftests/igt_spinner.h"14 15static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,16 struct intel_gt *gt,17 bool fill)18{19 unsigned int i, count = obj->base.size / sizeof(u32);20 enum i915_map_type map_type =21 intel_gt_coherent_map_type(gt, obj, false);22 u32 *cur;23 int err = 0;24 25 assert_object_held(obj);26 cur = i915_gem_object_pin_map(obj, map_type);27 if (IS_ERR(cur))28 return PTR_ERR(cur);29 30 if (fill)31 for (i = 0; i < count; ++i)32 *cur++ = i;33 else34 for (i = 0; i < count; ++i)35 if (*cur++ != i) {36 pr_err("Object content mismatch at location %d of %d\n", i, count);37 err = -EINVAL;38 break;39 }40 41 i915_gem_object_unpin_map(obj);42 43 return err;44}45 46static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,47 enum intel_region_id dst)48{49 struct drm_i915_private *i915 = gt->i915;50 struct intel_memory_region *src_mr = i915->mm.regions[src];51 struct intel_memory_region *dst_mr = i915->mm.regions[dst];52 struct drm_i915_gem_object *obj;53 struct i915_gem_ww_ctx ww;54 int err = 0;55 56 GEM_BUG_ON(!src_mr);57 GEM_BUG_ON(!dst_mr);58 59 /* Switch object backing-store on create */60 obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0);61 if (IS_ERR(obj))62 return PTR_ERR(obj);63 64 for_i915_gem_ww(&ww, err, true) {65 err = i915_gem_object_lock(obj, &ww);66 if (err)67 continue;68 69 err = igt_fill_check_buffer(obj, gt, true);70 if (err)71 continue;72 73 err = i915_gem_object_migrate(obj, &ww, dst);74 if (err)75 continue;76 77 err = i915_gem_object_pin_pages(obj);78 if (err)79 continue;80 81 if (i915_gem_object_can_migrate(obj, src))82 err = -EINVAL;83 84 i915_gem_object_unpin_pages(obj);85 err = i915_gem_object_wait_migration(obj, true);86 if (err)87 continue;88 89 err = igt_fill_check_buffer(obj, gt, false);90 }91 i915_gem_object_put(obj);92 93 return err;94}95 96static int igt_smem_create_migrate(void *arg)97{98 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM);99}100 101static int igt_lmem_create_migrate(void *arg)102{103 return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0);104}105 106static int igt_same_create_migrate(void *arg)107{108 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0);109}110 111static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,112 struct drm_i915_gem_object *obj,113 struct i915_vma *vma,114 bool silent_migrate)115{116 int err;117 118 err = i915_gem_object_lock(obj, ww);119 if (err)120 return err;121 122 if (vma) {123 err = i915_vma_pin_ww(vma, ww, obj->base.size, 0,124 0UL | PIN_OFFSET_FIXED |125 PIN_USER);126 if (err) {127 if (err != -EINTR && err != ERESTARTSYS &&128 err != -EDEADLK)129 pr_err("Failed to pin vma.\n");130 return err;131 }132 133 i915_vma_unpin(vma);134 }135 136 /*137 * Migration will implicitly unbind (asynchronously) any bound138 * vmas.139 */140 if (i915_gem_object_is_lmem(obj)) {141 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM);142 if (err) {143 if (!silent_migrate)144 pr_err("Object failed migration to smem\n");145 if (err)146 return err;147 }148 149 if (i915_gem_object_is_lmem(obj)) {150 pr_err("object still backed by lmem\n");151 err = -EINVAL;152 }153 154 if (!i915_gem_object_has_struct_page(obj)) {155 pr_err("object not backed by struct page\n");156 err = -EINVAL;157 }158 159 } else {160 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0);161 if (err) {162 if (!silent_migrate)163 pr_err("Object failed migration to lmem\n");164 if (err)165 return err;166 }167 168 if (i915_gem_object_has_struct_page(obj)) {169 pr_err("object still backed by struct page\n");170 err = -EINVAL;171 }172 173 if (!i915_gem_object_is_lmem(obj)) {174 pr_err("object not backed by lmem\n");175 err = -EINVAL;176 }177 }178 179 return err;180}181 182static int __igt_lmem_pages_migrate(struct intel_gt *gt,183 struct i915_address_space *vm,184 struct i915_deps *deps,185 struct igt_spinner *spin,186 struct dma_fence *spin_fence,187 bool borked_migrate)188{189 struct drm_i915_private *i915 = gt->i915;190 struct drm_i915_gem_object *obj;191 struct i915_vma *vma = NULL;192 struct i915_gem_ww_ctx ww;193 struct i915_request *rq;194 int err;195 int i;196 197 /* From LMEM to shmem and back again */198 199 obj = i915_gem_object_create_lmem(i915, SZ_2M, 0);200 if (IS_ERR(obj))201 return PTR_ERR(obj);202 203 if (vm) {204 vma = i915_vma_instance(obj, vm, NULL);205 if (IS_ERR(vma)) {206 err = PTR_ERR(vma);207 goto out_put;208 }209 }210 211 /* Initial GPU fill, sync, CPU initialization. */212 for_i915_gem_ww(&ww, err, true) {213 err = i915_gem_object_lock(obj, &ww);214 if (err)215 continue;216 217 err = ____i915_gem_object_get_pages(obj);218 if (err)219 continue;220 221 err = intel_migrate_clear(>->migrate, &ww, deps,222 obj->mm.pages->sgl, obj->pat_index,223 i915_gem_object_is_lmem(obj),224 0xdeadbeaf, &rq);225 if (rq) {226 err = dma_resv_reserve_fences(obj->base.resv, 1);227 if (!err)228 dma_resv_add_fence(obj->base.resv, &rq->fence,229 DMA_RESV_USAGE_KERNEL);230 i915_request_put(rq);231 }232 if (err)233 continue;234 235 if (!vma) {236 err = igt_fill_check_buffer(obj, gt, true);237 if (err)238 continue;239 }240 }241 if (err)242 goto out_put;243 244 /*245 * Migrate to and from smem without explicitly syncing.246 * Finalize with data in smem for fast readout.247 */248 for (i = 1; i <= 5; ++i) {249 for_i915_gem_ww(&ww, err, true)250 err = lmem_pages_migrate_one(&ww, obj, vma,251 borked_migrate);252 if (err)253 goto out_put;254 }255 256 err = i915_gem_object_lock_interruptible(obj, NULL);257 if (err)258 goto out_put;259 260 if (spin) {261 if (dma_fence_is_signaled(spin_fence)) {262 pr_err("Spinner was terminated by hangcheck.\n");263 err = -EBUSY;264 goto out_unlock;265 }266 igt_spinner_end(spin);267 }268 269 /* Finally sync migration and check content. */270 err = i915_gem_object_wait_migration(obj, true);271 if (err)272 goto out_unlock;273 274 if (vma) {275 err = i915_vma_wait_for_bind(vma);276 if (err)277 goto out_unlock;278 } else {279 err = igt_fill_check_buffer(obj, gt, false);280 }281 282out_unlock:283 i915_gem_object_unlock(obj);284out_put:285 i915_gem_object_put(obj);286 287 return err;288}289 290static int igt_lmem_pages_failsafe_migrate(void *arg)291{292 int fail_gpu, fail_alloc, ban_memcpy, ret;293 struct intel_gt *gt = arg;294 295 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {296 for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {297 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {298 pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",299 fail_gpu, fail_alloc, ban_memcpy);300 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);301 i915_ttm_migrate_set_failure_modes(fail_gpu,302 fail_alloc);303 ret = __igt_lmem_pages_migrate(gt, NULL, NULL,304 NULL, NULL,305 ban_memcpy &&306 fail_gpu);307 308 if (ban_memcpy && fail_gpu) {309 struct intel_gt *__gt;310 unsigned int id;311 312 if (ret != -EIO) {313 pr_err("expected -EIO, got (%d)\n", ret);314 ret = -EINVAL;315 } else {316 ret = 0;317 }318 319 for_each_gt(__gt, gt->i915, id) {320 intel_wakeref_t wakeref;321 bool wedged;322 323 mutex_lock(&__gt->reset.mutex);324 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);325 mutex_unlock(&__gt->reset.mutex);326 327 if (fail_gpu && !fail_alloc) {328 if (!wedged) {329 pr_err("gt(%u) not wedged\n", id);330 ret = -EINVAL;331 continue;332 }333 } else if (wedged) {334 pr_err("gt(%u) incorrectly wedged\n", id);335 ret = -EINVAL;336 } else {337 continue;338 }339 340 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);341 igt_global_reset_lock(__gt);342 intel_gt_reset(__gt, ALL_ENGINES, NULL);343 igt_global_reset_unlock(__gt);344 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);345 }346 if (ret)347 goto out_err;348 }349 }350 }351 }352 353out_err:354 i915_ttm_migrate_set_failure_modes(false, false);355 i915_ttm_migrate_set_ban_memcpy(false);356 return ret;357}358 359/*360 * This subtest tests that unbinding at migration is indeed performed361 * async. We launch a spinner and a number of migrations depending on362 * that spinner to have terminated. Before each migration we bind a363 * vma, which should then be async unbound by the migration operation.364 * If we are able to schedule migrations without blocking while the365 * spinner is still running, those unbinds are indeed async and non-366 * blocking.367 *368 * Note that each async bind operation is awaiting the previous migration369 * due to the moving fence resulting from the migration.370 */371static int igt_async_migrate(struct intel_gt *gt)372{373 struct intel_engine_cs *engine;374 enum intel_engine_id id;375 struct i915_ppgtt *ppgtt;376 struct igt_spinner spin;377 int err;378 379 ppgtt = i915_ppgtt_create(gt, 0);380 if (IS_ERR(ppgtt))381 return PTR_ERR(ppgtt);382 383 if (igt_spinner_init(&spin, gt)) {384 err = -ENOMEM;385 goto out_spin;386 }387 388 for_each_engine(engine, gt, id) {389 struct ttm_operation_ctx ctx = {390 .interruptible = true391 };392 struct dma_fence *spin_fence;393 struct intel_context *ce;394 struct i915_request *rq;395 struct i915_deps deps;396 397 ce = intel_context_create(engine);398 if (IS_ERR(ce)) {399 err = PTR_ERR(ce);400 goto out_ce;401 }402 403 /*404 * Use MI_NOOP, making the spinner non-preemptible. If there405 * is a code path where we fail async operation due to the406 * running spinner, we will block and fail to end the407 * spinner resulting in a deadlock. But with a non-408 * preemptible spinner, hangcheck will terminate the spinner409 * for us, and we will later detect that and fail the test.410 */411 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);412 intel_context_put(ce);413 if (IS_ERR(rq)) {414 err = PTR_ERR(rq);415 goto out_ce;416 }417 418 i915_deps_init(&deps, GFP_KERNEL);419 err = i915_deps_add_dependency(&deps, &rq->fence, &ctx);420 spin_fence = dma_fence_get(&rq->fence);421 i915_request_add(rq);422 if (err)423 goto out_ce;424 425 err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin,426 spin_fence, false);427 i915_deps_fini(&deps);428 dma_fence_put(spin_fence);429 if (err)430 goto out_ce;431 }432 433out_ce:434 igt_spinner_fini(&spin);435out_spin:436 i915_vm_put(&ppgtt->vm);437 438 return err;439}440 441/*442 * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while443 * arming the migration error check and block async migration. This444 * will cause us to deadlock and hangcheck will terminate the spinner445 * causing the test to fail.446 */447#define ASYNC_FAIL_ALLOC 1448static int igt_lmem_async_migrate(void *arg)449{450 int fail_gpu, fail_alloc, ban_memcpy, ret;451 struct intel_gt *gt = arg;452 453 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {454 for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {455 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {456 pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",457 fail_gpu, fail_alloc, ban_memcpy);458 i915_ttm_migrate_set_ban_memcpy(ban_memcpy);459 i915_ttm_migrate_set_failure_modes(fail_gpu,460 fail_alloc);461 ret = igt_async_migrate(gt);462 463 if (fail_gpu && ban_memcpy) {464 struct intel_gt *__gt;465 unsigned int id;466 467 if (ret != -EIO) {468 pr_err("expected -EIO, got (%d)\n", ret);469 ret = -EINVAL;470 } else {471 ret = 0;472 }473 474 for_each_gt(__gt, gt->i915, id) {475 intel_wakeref_t wakeref;476 bool wedged;477 478 mutex_lock(&__gt->reset.mutex);479 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);480 mutex_unlock(&__gt->reset.mutex);481 482 if (fail_gpu && !fail_alloc) {483 if (!wedged) {484 pr_err("gt(%u) not wedged\n", id);485 ret = -EINVAL;486 continue;487 }488 } else if (wedged) {489 pr_err("gt(%u) incorrectly wedged\n", id);490 ret = -EINVAL;491 } else {492 continue;493 }494 495 wakeref = intel_runtime_pm_get(__gt->uncore->rpm);496 igt_global_reset_lock(__gt);497 intel_gt_reset(__gt, ALL_ENGINES, NULL);498 igt_global_reset_unlock(__gt);499 intel_runtime_pm_put(__gt->uncore->rpm, wakeref);500 }501 }502 if (ret)503 goto out_err;504 }505 }506 }507 508out_err:509 i915_ttm_migrate_set_failure_modes(false, false);510 i915_ttm_migrate_set_ban_memcpy(false);511 return ret;512}513 514int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)515{516 static const struct i915_subtest tests[] = {517 SUBTEST(igt_smem_create_migrate),518 SUBTEST(igt_lmem_create_migrate),519 SUBTEST(igt_same_create_migrate),520 SUBTEST(igt_lmem_pages_failsafe_migrate),521 SUBTEST(igt_lmem_async_migrate),522 };523 524 if (!HAS_LMEM(i915))525 return 0;526 527 return intel_gt_live_subtests(tests, to_gt(i915));528}529