2037 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2016 Intel Corporation4 */5 6#include <linux/kthread.h>7 8#include "gem/i915_gem_context.h"9#include "gem/i915_gem_internal.h"10 11#include "i915_gem_evict.h"12#include "intel_gt.h"13#include "intel_engine_heartbeat.h"14#include "intel_engine_pm.h"15#include "selftest_engine_heartbeat.h"16 17#include "i915_selftest.h"18#include "selftests/i915_random.h"19#include "selftests/igt_flush_test.h"20#include "selftests/igt_reset.h"21#include "selftests/igt_atomic.h"22#include "selftests/igt_spinner.h"23#include "selftests/intel_scheduler_helpers.h"24 25#include "selftests/mock_drm.h"26 27#include "gem/selftests/mock_context.h"28#include "gem/selftests/igt_gem_utils.h"29 30#define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */31 32struct hang {33 struct intel_gt *gt;34 struct drm_i915_gem_object *hws;35 struct drm_i915_gem_object *obj;36 struct i915_gem_context *ctx;37 u32 *seqno;38 u32 *batch;39};40 41static int hang_init(struct hang *h, struct intel_gt *gt)42{43 void *vaddr;44 int err;45 46 memset(h, 0, sizeof(*h));47 h->gt = gt;48 49 h->ctx = kernel_context(gt->i915, NULL);50 if (IS_ERR(h->ctx))51 return PTR_ERR(h->ctx);52 53 GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));54 55 h->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);56 if (IS_ERR(h->hws)) {57 err = PTR_ERR(h->hws);58 goto err_ctx;59 }60 61 h->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);62 if (IS_ERR(h->obj)) {63 err = PTR_ERR(h->obj);64 goto err_hws;65 }66 67 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);68 vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);69 if (IS_ERR(vaddr)) {70 err = PTR_ERR(vaddr);71 goto err_obj;72 }73 h->seqno = memset(vaddr, 0xff, PAGE_SIZE);74 75 vaddr = i915_gem_object_pin_map_unlocked(h->obj,76 intel_gt_coherent_map_type(gt, h->obj, false));77 if (IS_ERR(vaddr)) {78 err = PTR_ERR(vaddr);79 goto err_unpin_hws;80 }81 h->batch = vaddr;82 83 return 0;84 85err_unpin_hws:86 i915_gem_object_unpin_map(h->hws);87err_obj:88 i915_gem_object_put(h->obj);89err_hws:90 i915_gem_object_put(h->hws);91err_ctx:92 kernel_context_close(h->ctx);93 return err;94}95 96static u64 hws_address(const struct i915_vma *hws,97 const struct i915_request *rq)98{99 return i915_vma_offset(hws) +100 offset_in_page(sizeof(u32) * rq->fence.context);101}102 103static struct i915_request *104hang_create_request(struct hang *h, struct intel_engine_cs *engine)105{106 struct intel_gt *gt = h->gt;107 struct i915_address_space *vm = i915_gem_context_get_eb_vm(h->ctx);108 struct drm_i915_gem_object *obj;109 struct i915_request *rq = NULL;110 struct i915_vma *hws, *vma;111 unsigned int flags;112 void *vaddr;113 u32 *batch;114 int err;115 116 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);117 if (IS_ERR(obj)) {118 i915_vm_put(vm);119 return ERR_CAST(obj);120 }121 122 vaddr = i915_gem_object_pin_map_unlocked(obj, intel_gt_coherent_map_type(gt, obj, false));123 if (IS_ERR(vaddr)) {124 i915_gem_object_put(obj);125 i915_vm_put(vm);126 return ERR_CAST(vaddr);127 }128 129 i915_gem_object_unpin_map(h->obj);130 i915_gem_object_put(h->obj);131 132 h->obj = obj;133 h->batch = vaddr;134 135 vma = i915_vma_instance(h->obj, vm, NULL);136 if (IS_ERR(vma)) {137 i915_vm_put(vm);138 return ERR_CAST(vma);139 }140 141 hws = i915_vma_instance(h->hws, vm, NULL);142 if (IS_ERR(hws)) {143 i915_vm_put(vm);144 return ERR_CAST(hws);145 }146 147 err = i915_vma_pin(vma, 0, 0, PIN_USER);148 if (err) {149 i915_vm_put(vm);150 return ERR_PTR(err);151 }152 153 err = i915_vma_pin(hws, 0, 0, PIN_USER);154 if (err)155 goto unpin_vma;156 157 rq = igt_request_alloc(h->ctx, engine);158 if (IS_ERR(rq)) {159 err = PTR_ERR(rq);160 goto unpin_hws;161 }162 163 err = igt_vma_move_to_active_unlocked(vma, rq, 0);164 if (err)165 goto cancel_rq;166 167 err = igt_vma_move_to_active_unlocked(hws, rq, 0);168 if (err)169 goto cancel_rq;170 171 batch = h->batch;172 if (GRAPHICS_VER(gt->i915) >= 8) {173 *batch++ = MI_STORE_DWORD_IMM_GEN4;174 *batch++ = lower_32_bits(hws_address(hws, rq));175 *batch++ = upper_32_bits(hws_address(hws, rq));176 *batch++ = rq->fence.seqno;177 *batch++ = MI_NOOP;178 179 memset(batch, 0, 1024);180 batch += 1024 / sizeof(*batch);181 182 *batch++ = MI_NOOP;183 *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;184 *batch++ = lower_32_bits(i915_vma_offset(vma));185 *batch++ = upper_32_bits(i915_vma_offset(vma));186 } else if (GRAPHICS_VER(gt->i915) >= 6) {187 *batch++ = MI_STORE_DWORD_IMM_GEN4;188 *batch++ = 0;189 *batch++ = lower_32_bits(hws_address(hws, rq));190 *batch++ = rq->fence.seqno;191 *batch++ = MI_NOOP;192 193 memset(batch, 0, 1024);194 batch += 1024 / sizeof(*batch);195 196 *batch++ = MI_NOOP;197 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;198 *batch++ = lower_32_bits(i915_vma_offset(vma));199 } else if (GRAPHICS_VER(gt->i915) >= 4) {200 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;201 *batch++ = 0;202 *batch++ = lower_32_bits(hws_address(hws, rq));203 *batch++ = rq->fence.seqno;204 *batch++ = MI_NOOP;205 206 memset(batch, 0, 1024);207 batch += 1024 / sizeof(*batch);208 209 *batch++ = MI_NOOP;210 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;211 *batch++ = lower_32_bits(i915_vma_offset(vma));212 } else {213 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;214 *batch++ = lower_32_bits(hws_address(hws, rq));215 *batch++ = rq->fence.seqno;216 *batch++ = MI_NOOP;217 218 memset(batch, 0, 1024);219 batch += 1024 / sizeof(*batch);220 221 *batch++ = MI_NOOP;222 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;223 *batch++ = lower_32_bits(i915_vma_offset(vma));224 }225 *batch++ = MI_BATCH_BUFFER_END; /* not reached */226 intel_gt_chipset_flush(engine->gt);227 228 if (rq->engine->emit_init_breadcrumb) {229 err = rq->engine->emit_init_breadcrumb(rq);230 if (err)231 goto cancel_rq;232 }233 234 flags = 0;235 if (GRAPHICS_VER(gt->i915) <= 5)236 flags |= I915_DISPATCH_SECURE;237 238 err = rq->engine->emit_bb_start(rq, i915_vma_offset(vma), PAGE_SIZE, flags);239 240cancel_rq:241 if (err) {242 i915_request_set_error_once(rq, err);243 i915_request_add(rq);244 }245unpin_hws:246 i915_vma_unpin(hws);247unpin_vma:248 i915_vma_unpin(vma);249 i915_vm_put(vm);250 return err ? ERR_PTR(err) : rq;251}252 253static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)254{255 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);256}257 258static void hang_fini(struct hang *h)259{260 *h->batch = MI_BATCH_BUFFER_END;261 intel_gt_chipset_flush(h->gt);262 263 i915_gem_object_unpin_map(h->obj);264 i915_gem_object_put(h->obj);265 266 i915_gem_object_unpin_map(h->hws);267 i915_gem_object_put(h->hws);268 269 kernel_context_close(h->ctx);270 271 igt_flush_test(h->gt->i915);272}273 274static bool wait_until_running(struct hang *h, struct i915_request *rq)275{276 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),277 rq->fence.seqno),278 10) &&279 wait_for(i915_seqno_passed(hws_seqno(h, rq),280 rq->fence.seqno),281 1000));282}283 284static int igt_hang_sanitycheck(void *arg)285{286 struct intel_gt *gt = arg;287 struct i915_request *rq;288 struct intel_engine_cs *engine;289 enum intel_engine_id id;290 struct hang h;291 int err;292 293 /* Basic check that we can execute our hanging batch */294 295 err = hang_init(&h, gt);296 if (err)297 return err;298 299 for_each_engine(engine, gt, id) {300 struct intel_wedge_me w;301 long timeout;302 303 if (!intel_engine_can_store_dword(engine))304 continue;305 306 rq = hang_create_request(&h, engine);307 if (IS_ERR(rq)) {308 err = PTR_ERR(rq);309 pr_err("Failed to create request for %s, err=%d\n",310 engine->name, err);311 goto fini;312 }313 314 i915_request_get(rq);315 316 *h.batch = MI_BATCH_BUFFER_END;317 intel_gt_chipset_flush(engine->gt);318 319 i915_request_add(rq);320 321 timeout = 0;322 intel_wedge_on_timeout(&w, gt, HZ / 5 /* 200ms */)323 timeout = i915_request_wait(rq, 0,324 MAX_SCHEDULE_TIMEOUT);325 if (intel_gt_is_wedged(gt))326 timeout = -EIO;327 328 i915_request_put(rq);329 330 if (timeout < 0) {331 err = timeout;332 pr_err("Wait for request failed on %s, err=%d\n",333 engine->name, err);334 goto fini;335 }336 }337 338fini:339 hang_fini(&h);340 return err;341}342 343static bool wait_for_idle(struct intel_engine_cs *engine)344{345 return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;346}347 348static int igt_reset_nop(void *arg)349{350 struct intel_gt *gt = arg;351 struct i915_gpu_error *global = >->i915->gpu_error;352 struct intel_engine_cs *engine;353 unsigned int reset_count, count;354 enum intel_engine_id id;355 IGT_TIMEOUT(end_time);356 int err = 0;357 358 /* Check that we can reset during non-user portions of requests */359 360 reset_count = i915_reset_count(global);361 count = 0;362 do {363 for_each_engine(engine, gt, id) {364 struct intel_context *ce;365 int i;366 367 ce = intel_context_create(engine);368 if (IS_ERR(ce)) {369 err = PTR_ERR(ce);370 pr_err("[%s] Create context failed: %d!\n", engine->name, err);371 break;372 }373 374 for (i = 0; i < 16; i++) {375 struct i915_request *rq;376 377 rq = intel_context_create_request(ce);378 if (IS_ERR(rq)) {379 err = PTR_ERR(rq);380 pr_err("[%s] Create request failed: %d!\n",381 engine->name, err);382 break;383 }384 385 i915_request_add(rq);386 }387 388 intel_context_put(ce);389 }390 391 igt_global_reset_lock(gt);392 intel_gt_reset(gt, ALL_ENGINES, NULL);393 igt_global_reset_unlock(gt);394 395 if (intel_gt_is_wedged(gt)) {396 pr_err("[%s] GT is wedged!\n", engine->name);397 err = -EIO;398 break;399 }400 401 if (i915_reset_count(global) != reset_count + ++count) {402 pr_err("[%s] Reset not recorded: %d vs %d + %d!\n",403 engine->name, i915_reset_count(global), reset_count, count);404 err = -EINVAL;405 break;406 }407 408 err = igt_flush_test(gt->i915);409 if (err) {410 pr_err("[%s] Flush failed: %d!\n", engine->name, err);411 break;412 }413 } while (time_before(jiffies, end_time));414 pr_info("%s: %d resets\n", __func__, count);415 416 if (igt_flush_test(gt->i915)) {417 pr_err("Post flush failed: %d!\n", err);418 err = -EIO;419 }420 421 return err;422}423 424static int igt_reset_nop_engine(void *arg)425{426 struct intel_gt *gt = arg;427 struct i915_gpu_error *global = >->i915->gpu_error;428 struct intel_engine_cs *engine;429 enum intel_engine_id id;430 431 /* Check that we can engine-reset during non-user portions */432 433 if (!intel_has_reset_engine(gt))434 return 0;435 436 for_each_engine(engine, gt, id) {437 unsigned int reset_count, reset_engine_count, count;438 struct intel_context *ce;439 IGT_TIMEOUT(end_time);440 int err;441 442 if (intel_engine_uses_guc(engine)) {443 /* Engine level resets are triggered by GuC when a hang444 * is detected. They can't be triggered by the KMD any445 * more. Thus a nop batch cannot be used as a reset test446 */447 continue;448 }449 450 ce = intel_context_create(engine);451 if (IS_ERR(ce)) {452 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);453 return PTR_ERR(ce);454 }455 456 reset_count = i915_reset_count(global);457 reset_engine_count = i915_reset_engine_count(global, engine);458 count = 0;459 460 st_engine_heartbeat_disable(engine);461 GEM_BUG_ON(test_and_set_bit(I915_RESET_ENGINE + id,462 >->reset.flags));463 do {464 int i;465 466 if (!wait_for_idle(engine)) {467 pr_err("%s failed to idle before reset\n",468 engine->name);469 err = -EIO;470 break;471 }472 473 for (i = 0; i < 16; i++) {474 struct i915_request *rq;475 476 rq = intel_context_create_request(ce);477 if (IS_ERR(rq)) {478 struct drm_printer p =479 drm_info_printer(gt->i915->drm.dev);480 intel_engine_dump(engine, &p,481 "%s(%s): failed to submit request\n",482 __func__,483 engine->name);484 485 GEM_TRACE("%s(%s): failed to submit request\n",486 __func__,487 engine->name);488 GEM_TRACE_DUMP();489 490 intel_gt_set_wedged(gt);491 492 err = PTR_ERR(rq);493 break;494 }495 496 i915_request_add(rq);497 }498 err = intel_engine_reset(engine, NULL);499 if (err) {500 pr_err("intel_engine_reset(%s) failed, err:%d\n",501 engine->name, err);502 break;503 }504 505 if (i915_reset_count(global) != reset_count) {506 pr_err("Full GPU reset recorded! (engine reset expected)\n");507 err = -EINVAL;508 break;509 }510 511 if (i915_reset_engine_count(global, engine) !=512 reset_engine_count + ++count) {513 pr_err("%s engine reset not recorded!\n",514 engine->name);515 err = -EINVAL;516 break;517 }518 } while (time_before(jiffies, end_time));519 clear_and_wake_up_bit(I915_RESET_ENGINE + id, >->reset.flags);520 st_engine_heartbeat_enable(engine);521 522 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);523 524 intel_context_put(ce);525 if (igt_flush_test(gt->i915))526 err = -EIO;527 if (err)528 return err;529 }530 531 return 0;532}533 534static void force_reset_timeout(struct intel_engine_cs *engine)535{536 engine->reset_timeout.probability = 999;537 atomic_set(&engine->reset_timeout.times, -1);538}539 540static void cancel_reset_timeout(struct intel_engine_cs *engine)541{542 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));543}544 545static int igt_reset_fail_engine(void *arg)546{547 struct intel_gt *gt = arg;548 struct intel_engine_cs *engine;549 enum intel_engine_id id;550 551 /* Check that we can recover from engine-reset failues */552 553 if (!intel_has_reset_engine(gt))554 return 0;555 556 for_each_engine(engine, gt, id) {557 unsigned int count;558 struct intel_context *ce;559 IGT_TIMEOUT(end_time);560 int err;561 562 /* Can't manually break the reset if i915 doesn't perform it */563 if (intel_engine_uses_guc(engine))564 continue;565 566 ce = intel_context_create(engine);567 if (IS_ERR(ce)) {568 pr_err("[%s] Create context failed: %pe!\n", engine->name, ce);569 return PTR_ERR(ce);570 }571 572 st_engine_heartbeat_disable(engine);573 GEM_BUG_ON(test_and_set_bit(I915_RESET_ENGINE + id,574 >->reset.flags));575 576 force_reset_timeout(engine);577 err = intel_engine_reset(engine, NULL);578 cancel_reset_timeout(engine);579 if (err == 0) /* timeouts only generated on gen8+ */580 goto skip;581 582 count = 0;583 do {584 struct i915_request *last = NULL;585 int i;586 587 if (!wait_for_idle(engine)) {588 pr_err("%s failed to idle before reset\n",589 engine->name);590 err = -EIO;591 break;592 }593 594 for (i = 0; i < count % 15; i++) {595 struct i915_request *rq;596 597 rq = intel_context_create_request(ce);598 if (IS_ERR(rq)) {599 struct drm_printer p =600 drm_info_printer(gt->i915->drm.dev);601 intel_engine_dump(engine, &p,602 "%s(%s): failed to submit request\n",603 __func__,604 engine->name);605 606 GEM_TRACE("%s(%s): failed to submit request\n",607 __func__,608 engine->name);609 GEM_TRACE_DUMP();610 611 intel_gt_set_wedged(gt);612 if (last)613 i915_request_put(last);614 615 err = PTR_ERR(rq);616 goto out;617 }618 619 if (last)620 i915_request_put(last);621 last = i915_request_get(rq);622 i915_request_add(rq);623 }624 625 if (count & 1) {626 err = intel_engine_reset(engine, NULL);627 if (err) {628 GEM_TRACE_ERR("intel_engine_reset(%s) failed, err:%d\n",629 engine->name, err);630 GEM_TRACE_DUMP();631 i915_request_put(last);632 break;633 }634 } else {635 force_reset_timeout(engine);636 err = intel_engine_reset(engine, NULL);637 cancel_reset_timeout(engine);638 if (err != -ETIMEDOUT) {639 pr_err("intel_engine_reset(%s) did not fail, err:%d\n",640 engine->name, err);641 i915_request_put(last);642 break;643 }644 }645 646 err = 0;647 if (last) {648 if (i915_request_wait(last, 0, HZ / 2) < 0) {649 struct drm_printer p =650 drm_info_printer(gt->i915->drm.dev);651 652 intel_engine_dump(engine, &p,653 "%s(%s): failed to complete request\n",654 __func__,655 engine->name);656 657 GEM_TRACE("%s(%s): failed to complete request\n",658 __func__,659 engine->name);660 GEM_TRACE_DUMP();661 662 err = -EIO;663 }664 i915_request_put(last);665 }666 count++;667 } while (err == 0 && time_before(jiffies, end_time));668out:669 pr_info("%s(%s): %d resets\n", __func__, engine->name, count);670skip:671 clear_and_wake_up_bit(I915_RESET_ENGINE + id, >->reset.flags);672 st_engine_heartbeat_enable(engine);673 intel_context_put(ce);674 675 if (igt_flush_test(gt->i915))676 err = -EIO;677 if (err)678 return err;679 }680 681 return 0;682}683 684static int __igt_reset_engine(struct intel_gt *gt, bool active)685{686 struct i915_gpu_error *global = >->i915->gpu_error;687 struct intel_engine_cs *engine;688 enum intel_engine_id id;689 struct hang h;690 int err = 0;691 692 /* Check that we can issue an engine reset on an idle engine (no-op) */693 694 if (!intel_has_reset_engine(gt))695 return 0;696 697 if (active) {698 err = hang_init(&h, gt);699 if (err)700 return err;701 }702 703 for_each_engine(engine, gt, id) {704 unsigned int reset_count, reset_engine_count;705 unsigned long count;706 bool using_guc = intel_engine_uses_guc(engine);707 IGT_TIMEOUT(end_time);708 709 if (using_guc && !active)710 continue;711 712 if (active && !intel_engine_can_store_dword(engine))713 continue;714 715 if (!wait_for_idle(engine)) {716 pr_err("%s failed to idle before reset\n",717 engine->name);718 err = -EIO;719 break;720 }721 722 reset_count = i915_reset_count(global);723 reset_engine_count = i915_reset_engine_count(global, engine);724 725 st_engine_heartbeat_disable(engine);726 GEM_BUG_ON(test_and_set_bit(I915_RESET_ENGINE + id,727 >->reset.flags));728 count = 0;729 do {730 struct i915_request *rq = NULL;731 struct intel_selftest_saved_policy saved;732 int err2;733 734 err = intel_selftest_modify_policy(engine, &saved,735 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);736 if (err) {737 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);738 break;739 }740 741 if (active) {742 rq = hang_create_request(&h, engine);743 if (IS_ERR(rq)) {744 err = PTR_ERR(rq);745 pr_err("[%s] Create hang request failed: %d!\n",746 engine->name, err);747 goto restore;748 }749 750 i915_request_get(rq);751 i915_request_add(rq);752 753 if (!wait_until_running(&h, rq)) {754 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);755 756 pr_err("%s: Failed to start request %llx, at %x\n",757 __func__, rq->fence.seqno, hws_seqno(&h, rq));758 intel_engine_dump(engine, &p,759 "%s\n", engine->name);760 761 i915_request_put(rq);762 err = -EIO;763 goto restore;764 }765 }766 767 if (!using_guc) {768 err = intel_engine_reset(engine, NULL);769 if (err) {770 pr_err("intel_engine_reset(%s) failed, err:%d\n",771 engine->name, err);772 goto skip;773 }774 }775 776 if (rq) {777 /* Ensure the reset happens and kills the engine */778 err = intel_selftest_wait_for_rq(rq);779 if (err)780 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",781 engine->name, rq->fence.context,782 rq->fence.seqno, rq->context->guc_id.id, err);783 }784 785skip:786 if (rq)787 i915_request_put(rq);788 789 if (i915_reset_count(global) != reset_count) {790 pr_err("Full GPU reset recorded! (engine reset expected)\n");791 err = -EINVAL;792 goto restore;793 }794 795 /* GuC based resets are not logged per engine */796 if (!using_guc) {797 if (i915_reset_engine_count(global, engine) !=798 ++reset_engine_count) {799 pr_err("%s engine reset not recorded!\n",800 engine->name);801 err = -EINVAL;802 goto restore;803 }804 }805 806 count++;807 808restore:809 err2 = intel_selftest_restore_policy(engine, &saved);810 if (err2)811 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err);812 if (err == 0)813 err = err2;814 if (err)815 break;816 } while (time_before(jiffies, end_time));817 clear_and_wake_up_bit(I915_RESET_ENGINE + id, >->reset.flags);818 st_engine_heartbeat_enable(engine);819 pr_info("%s: Completed %lu %s resets\n",820 engine->name, count, active ? "active" : "idle");821 822 if (err)823 break;824 825 err = igt_flush_test(gt->i915);826 if (err) {827 pr_err("[%s] Flush failed: %d!\n", engine->name, err);828 break;829 }830 }831 832 if (intel_gt_is_wedged(gt)) {833 pr_err("GT is wedged!\n");834 err = -EIO;835 }836 837 if (active)838 hang_fini(&h);839 840 return err;841}842 843static int igt_reset_idle_engine(void *arg)844{845 return __igt_reset_engine(arg, false);846}847 848static int igt_reset_active_engine(void *arg)849{850 return __igt_reset_engine(arg, true);851}852 853struct active_engine {854 struct kthread_worker *worker;855 struct kthread_work work;856 struct intel_engine_cs *engine;857 unsigned long resets;858 unsigned int flags;859 bool stop;860 int result;861};862 863#define TEST_ACTIVE BIT(0)864#define TEST_OTHERS BIT(1)865#define TEST_SELF BIT(2)866#define TEST_PRIORITY BIT(3)867 868static int active_request_put(struct i915_request *rq)869{870 int err = 0;871 872 if (!rq)873 return 0;874 875 if (i915_request_wait(rq, 0, 10 * HZ) < 0) {876 GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",877 rq->engine->name,878 rq->fence.context,879 rq->fence.seqno);880 GEM_TRACE_DUMP();881 882 intel_gt_set_wedged(rq->engine->gt);883 err = -EIO;884 }885 886 i915_request_put(rq);887 888 return err;889}890 891static void active_engine(struct kthread_work *work)892{893 I915_RND_STATE(prng);894 struct active_engine *arg = container_of(work, typeof(*arg), work);895 struct intel_engine_cs *engine = arg->engine;896 struct i915_request *rq[8] = {};897 struct intel_context *ce[ARRAY_SIZE(rq)];898 unsigned long count;899 int err = 0;900 901 for (count = 0; count < ARRAY_SIZE(ce); count++) {902 ce[count] = intel_context_create(engine);903 if (IS_ERR(ce[count])) {904 arg->result = PTR_ERR(ce[count]);905 pr_err("[%s] Create context #%ld failed: %d!\n",906 engine->name, count, arg->result);907 while (--count)908 intel_context_put(ce[count]);909 return;910 }911 }912 913 count = 0;914 while (!READ_ONCE(arg->stop)) {915 unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1);916 struct i915_request *old = rq[idx];917 struct i915_request *new;918 919 new = intel_context_create_request(ce[idx]);920 if (IS_ERR(new)) {921 err = PTR_ERR(new);922 pr_err("[%s] Create request #%d failed: %d!\n", engine->name, idx, err);923 break;924 }925 926 rq[idx] = i915_request_get(new);927 i915_request_add(new);928 929 if (engine->sched_engine->schedule && arg->flags & TEST_PRIORITY) {930 struct i915_sched_attr attr = {931 .priority =932 i915_prandom_u32_max_state(512, &prng),933 };934 engine->sched_engine->schedule(rq[idx], &attr);935 }936 937 err = active_request_put(old);938 if (err) {939 pr_err("[%s] Request put failed: %d!\n", engine->name, err);940 break;941 }942 943 cond_resched();944 }945 946 for (count = 0; count < ARRAY_SIZE(rq); count++) {947 int err__ = active_request_put(rq[count]);948 949 if (err)950 pr_err("[%s] Request put #%ld failed: %d!\n", engine->name, count, err);951 952 /* Keep the first error */953 if (!err)954 err = err__;955 956 intel_context_put(ce[count]);957 }958 959 arg->result = err;960}961 962static int __igt_reset_engines(struct intel_gt *gt,963 const char *test_name,964 unsigned int flags)965{966 struct i915_gpu_error *global = >->i915->gpu_error;967 struct intel_engine_cs *engine, *other;968 struct active_engine *threads;969 enum intel_engine_id id, tmp;970 struct hang h;971 int err = 0;972 973 /* Check that issuing a reset on one engine does not interfere974 * with any other engine.975 */976 977 if (!intel_has_reset_engine(gt))978 return 0;979 980 if (flags & TEST_ACTIVE) {981 err = hang_init(&h, gt);982 if (err)983 return err;984 985 if (flags & TEST_PRIORITY)986 h.ctx->sched.priority = 1024;987 }988 989 threads = kmalloc_array(I915_NUM_ENGINES, sizeof(*threads), GFP_KERNEL);990 if (!threads)991 return -ENOMEM;992 993 for_each_engine(engine, gt, id) {994 unsigned long device = i915_reset_count(global);995 unsigned long count = 0, reported;996 bool using_guc = intel_engine_uses_guc(engine);997 IGT_TIMEOUT(end_time);998 999 if (flags & TEST_ACTIVE) {1000 if (!intel_engine_can_store_dword(engine))1001 continue;1002 } else if (using_guc)1003 continue;1004 1005 if (!wait_for_idle(engine)) {1006 pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n",1007 engine->name, test_name);1008 err = -EIO;1009 break;1010 }1011 1012 memset(threads, 0, sizeof(*threads) * I915_NUM_ENGINES);1013 for_each_engine(other, gt, tmp) {1014 struct kthread_worker *worker;1015 1016 threads[tmp].resets =1017 i915_reset_engine_count(global, other);1018 1019 if (other == engine && !(flags & TEST_SELF))1020 continue;1021 1022 if (other != engine && !(flags & TEST_OTHERS))1023 continue;1024 1025 threads[tmp].engine = other;1026 threads[tmp].flags = flags;1027 1028 worker = kthread_create_worker(0, "igt/%s",1029 other->name);1030 if (IS_ERR(worker)) {1031 err = PTR_ERR(worker);1032 pr_err("[%s] Worker create failed: %d!\n",1033 engine->name, err);1034 goto unwind;1035 }1036 1037 threads[tmp].worker = worker;1038 1039 kthread_init_work(&threads[tmp].work, active_engine);1040 kthread_queue_work(threads[tmp].worker,1041 &threads[tmp].work);1042 }1043 1044 st_engine_heartbeat_disable_no_pm(engine);1045 GEM_BUG_ON(test_and_set_bit(I915_RESET_ENGINE + id,1046 >->reset.flags));1047 do {1048 struct i915_request *rq = NULL;1049 struct intel_selftest_saved_policy saved;1050 int err2;1051 1052 err = intel_selftest_modify_policy(engine, &saved,1053 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);1054 if (err) {1055 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);1056 break;1057 }1058 1059 if (flags & TEST_ACTIVE) {1060 rq = hang_create_request(&h, engine);1061 if (IS_ERR(rq)) {1062 err = PTR_ERR(rq);1063 pr_err("[%s] Create hang request failed: %d!\n",1064 engine->name, err);1065 goto restore;1066 }1067 1068 i915_request_get(rq);1069 i915_request_add(rq);1070 1071 if (!wait_until_running(&h, rq)) {1072 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1073 1074 pr_err("%s: Failed to start request %llx, at %x\n",1075 __func__, rq->fence.seqno, hws_seqno(&h, rq));1076 intel_engine_dump(engine, &p,1077 "%s\n", engine->name);1078 1079 i915_request_put(rq);1080 err = -EIO;1081 goto restore;1082 }1083 } else {1084 intel_engine_pm_get(engine);1085 }1086 1087 if (!using_guc) {1088 err = intel_engine_reset(engine, NULL);1089 if (err) {1090 pr_err("i915_reset_engine(%s:%s): failed, err=%d\n",1091 engine->name, test_name, err);1092 goto restore;1093 }1094 }1095 1096 if (rq) {1097 /* Ensure the reset happens and kills the engine */1098 err = intel_selftest_wait_for_rq(rq);1099 if (err)1100 pr_err("[%s] Wait for request %lld:%lld [0x%04X] failed: %d!\n",1101 engine->name, rq->fence.context,1102 rq->fence.seqno, rq->context->guc_id.id, err);1103 }1104 1105 count++;1106 1107 if (rq) {1108 if (rq->fence.error != -EIO) {1109 pr_err("i915_reset_engine(%s:%s): failed to reset request %lld:%lld [0x%04X]\n",1110 engine->name, test_name,1111 rq->fence.context,1112 rq->fence.seqno, rq->context->guc_id.id);1113 i915_request_put(rq);1114 1115 GEM_TRACE_DUMP();1116 intel_gt_set_wedged(gt);1117 err = -EIO;1118 goto restore;1119 }1120 1121 if (i915_request_wait(rq, 0, HZ / 5) < 0) {1122 struct drm_printer p =1123 drm_info_printer(gt->i915->drm.dev);1124 1125 pr_err("i915_reset_engine(%s:%s):"1126 " failed to complete request %llx:%lld after reset\n",1127 engine->name, test_name,1128 rq->fence.context,1129 rq->fence.seqno);1130 intel_engine_dump(engine, &p,1131 "%s\n", engine->name);1132 i915_request_put(rq);1133 1134 GEM_TRACE_DUMP();1135 intel_gt_set_wedged(gt);1136 err = -EIO;1137 goto restore;1138 }1139 1140 i915_request_put(rq);1141 }1142 1143 if (!(flags & TEST_ACTIVE))1144 intel_engine_pm_put(engine);1145 1146 if (!(flags & TEST_SELF) && !wait_for_idle(engine)) {1147 struct drm_printer p =1148 drm_info_printer(gt->i915->drm.dev);1149 1150 pr_err("i915_reset_engine(%s:%s):"1151 " failed to idle after reset\n",1152 engine->name, test_name);1153 intel_engine_dump(engine, &p,1154 "%s\n", engine->name);1155 1156 err = -EIO;1157 goto restore;1158 }1159 1160restore:1161 err2 = intel_selftest_restore_policy(engine, &saved);1162 if (err2)1163 pr_err("[%s] Restore policy failed: %d!\n", engine->name, err2);1164 if (err == 0)1165 err = err2;1166 if (err)1167 break;1168 } while (time_before(jiffies, end_time));1169 clear_and_wake_up_bit(I915_RESET_ENGINE + id, >->reset.flags);1170 st_engine_heartbeat_enable_no_pm(engine);1171 1172 pr_info("i915_reset_engine(%s:%s): %lu resets\n",1173 engine->name, test_name, count);1174 1175 /* GuC based resets are not logged per engine */1176 if (!using_guc) {1177 reported = i915_reset_engine_count(global, engine);1178 reported -= threads[engine->id].resets;1179 if (reported != count) {1180 pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",1181 engine->name, test_name, count, reported);1182 if (!err)1183 err = -EINVAL;1184 }1185 }1186 1187unwind:1188 for_each_engine(other, gt, tmp) {1189 int ret;1190 1191 if (!threads[tmp].worker)1192 continue;1193 1194 WRITE_ONCE(threads[tmp].stop, true);1195 kthread_flush_work(&threads[tmp].work);1196 ret = READ_ONCE(threads[tmp].result);1197 if (ret) {1198 pr_err("kthread for other engine %s failed, err=%d\n",1199 other->name, ret);1200 if (!err)1201 err = ret;1202 }1203 1204 kthread_destroy_worker(threads[tmp].worker);1205 1206 /* GuC based resets are not logged per engine */1207 if (!using_guc) {1208 if (other->uabi_class != engine->uabi_class &&1209 threads[tmp].resets !=1210 i915_reset_engine_count(global, other)) {1211 pr_err("Innocent engine %s was reset (count=%ld)\n",1212 other->name,1213 i915_reset_engine_count(global, other) -1214 threads[tmp].resets);1215 if (!err)1216 err = -EINVAL;1217 }1218 }1219 }1220 1221 if (device != i915_reset_count(global)) {1222 pr_err("Global reset (count=%ld)!\n",1223 i915_reset_count(global) - device);1224 if (!err)1225 err = -EINVAL;1226 }1227 1228 if (err)1229 break;1230 1231 err = igt_flush_test(gt->i915);1232 if (err) {1233 pr_err("[%s] Flush failed: %d!\n", engine->name, err);1234 break;1235 }1236 }1237 kfree(threads);1238 1239 if (intel_gt_is_wedged(gt))1240 err = -EIO;1241 1242 if (flags & TEST_ACTIVE)1243 hang_fini(&h);1244 1245 return err;1246}1247 1248static int igt_reset_engines(void *arg)1249{1250 static const struct {1251 const char *name;1252 unsigned int flags;1253 } phases[] = {1254 { "idle", 0 },1255 { "active", TEST_ACTIVE },1256 { "others-idle", TEST_OTHERS },1257 { "others-active", TEST_OTHERS | TEST_ACTIVE },1258 {1259 "others-priority",1260 TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY1261 },1262 {1263 "self-priority",1264 TEST_ACTIVE | TEST_PRIORITY | TEST_SELF,1265 },1266 { }1267 };1268 struct intel_gt *gt = arg;1269 typeof(*phases) *p;1270 int err;1271 1272 for (p = phases; p->name; p++) {1273 if (p->flags & TEST_PRIORITY) {1274 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))1275 continue;1276 }1277 1278 err = __igt_reset_engines(arg, p->name, p->flags);1279 if (err)1280 return err;1281 }1282 1283 return 0;1284}1285 1286static u32 fake_hangcheck(struct intel_gt *gt, intel_engine_mask_t mask)1287{1288 u32 count = i915_reset_count(>->i915->gpu_error);1289 1290 intel_gt_reset(gt, mask, NULL);1291 1292 return count;1293}1294 1295static int igt_reset_wait(void *arg)1296{1297 struct intel_gt *gt = arg;1298 struct i915_gpu_error *global = >->i915->gpu_error;1299 struct intel_engine_cs *engine;1300 struct i915_request *rq;1301 unsigned int reset_count;1302 struct hang h;1303 long timeout;1304 int err;1305 1306 engine = intel_selftest_find_any_engine(gt);1307 1308 if (!engine || !intel_engine_can_store_dword(engine))1309 return 0;1310 1311 /* Check that we detect a stuck waiter and issue a reset */1312 1313 igt_global_reset_lock(gt);1314 1315 err = hang_init(&h, gt);1316 if (err) {1317 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);1318 goto unlock;1319 }1320 1321 rq = hang_create_request(&h, engine);1322 if (IS_ERR(rq)) {1323 err = PTR_ERR(rq);1324 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);1325 goto fini;1326 }1327 1328 i915_request_get(rq);1329 i915_request_add(rq);1330 1331 if (!wait_until_running(&h, rq)) {1332 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1333 1334 pr_err("%s: Failed to start request %llx, at %x\n",1335 __func__, rq->fence.seqno, hws_seqno(&h, rq));1336 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);1337 1338 intel_gt_set_wedged(gt);1339 1340 err = -EIO;1341 goto out_rq;1342 }1343 1344 reset_count = fake_hangcheck(gt, ALL_ENGINES);1345 1346 timeout = i915_request_wait(rq, 0, 10);1347 if (timeout < 0) {1348 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",1349 timeout);1350 err = timeout;1351 goto out_rq;1352 }1353 1354 if (i915_reset_count(global) == reset_count) {1355 pr_err("No GPU reset recorded!\n");1356 err = -EINVAL;1357 goto out_rq;1358 }1359 1360out_rq:1361 i915_request_put(rq);1362fini:1363 hang_fini(&h);1364unlock:1365 igt_global_reset_unlock(gt);1366 1367 if (intel_gt_is_wedged(gt))1368 return -EIO;1369 1370 return err;1371}1372 1373struct evict_vma {1374 struct completion completion;1375 struct i915_vma *vma;1376};1377 1378static int evict_vma(void *data)1379{1380 struct evict_vma *arg = data;1381 struct i915_address_space *vm = arg->vma->vm;1382 struct drm_mm_node evict = arg->vma->node;1383 int err;1384 1385 complete(&arg->completion);1386 1387 mutex_lock(&vm->mutex);1388 err = i915_gem_evict_for_node(vm, NULL, &evict, 0);1389 mutex_unlock(&vm->mutex);1390 1391 return err;1392}1393 1394static int evict_fence(void *data)1395{1396 struct evict_vma *arg = data;1397 int err;1398 1399 complete(&arg->completion);1400 1401 /* Mark the fence register as dirty to force the mmio update. */1402 err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512);1403 if (err) {1404 pr_err("Invalid Y-tiling settings; err:%d\n", err);1405 return err;1406 }1407 1408 err = i915_vma_pin(arg->vma, 0, 0, PIN_GLOBAL | PIN_MAPPABLE);1409 if (err) {1410 pr_err("Unable to pin vma for Y-tiled fence; err:%d\n", err);1411 return err;1412 }1413 1414 err = i915_vma_pin_fence(arg->vma);1415 i915_vma_unpin(arg->vma);1416 if (err) {1417 pr_err("Unable to pin Y-tiled fence; err:%d\n", err);1418 return err;1419 }1420 1421 i915_vma_unpin_fence(arg->vma);1422 1423 return 0;1424}1425 1426static int __igt_reset_evict_vma(struct intel_gt *gt,1427 struct i915_address_space *vm,1428 int (*fn)(void *),1429 unsigned int flags)1430{1431 struct intel_engine_cs *engine;1432 struct drm_i915_gem_object *obj;1433 struct task_struct *tsk = NULL;1434 struct i915_request *rq;1435 struct evict_vma arg;1436 struct hang h;1437 unsigned int pin_flags;1438 int err;1439 1440 if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)1441 return 0;1442 1443 engine = intel_selftest_find_any_engine(gt);1444 1445 if (!engine || !intel_engine_can_store_dword(engine))1446 return 0;1447 1448 /* Check that we can recover an unbind stuck on a hanging request */1449 1450 err = hang_init(&h, gt);1451 if (err) {1452 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);1453 return err;1454 }1455 1456 obj = i915_gem_object_create_internal(gt->i915, SZ_1M);1457 if (IS_ERR(obj)) {1458 err = PTR_ERR(obj);1459 pr_err("[%s] Create object failed: %d!\n", engine->name, err);1460 goto fini;1461 }1462 1463 if (flags & EXEC_OBJECT_NEEDS_FENCE) {1464 err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512);1465 if (err) {1466 pr_err("Invalid X-tiling settings; err:%d\n", err);1467 goto out_obj;1468 }1469 }1470 1471 arg.vma = i915_vma_instance(obj, vm, NULL);1472 if (IS_ERR(arg.vma)) {1473 err = PTR_ERR(arg.vma);1474 pr_err("[%s] VMA instance failed: %d!\n", engine->name, err);1475 goto out_obj;1476 }1477 1478 rq = hang_create_request(&h, engine);1479 if (IS_ERR(rq)) {1480 err = PTR_ERR(rq);1481 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);1482 goto out_obj;1483 }1484 1485 pin_flags = i915_vma_is_ggtt(arg.vma) ? PIN_GLOBAL : PIN_USER;1486 1487 if (flags & EXEC_OBJECT_NEEDS_FENCE)1488 pin_flags |= PIN_MAPPABLE;1489 1490 err = i915_vma_pin(arg.vma, 0, 0, pin_flags);1491 if (err) {1492 i915_request_add(rq);1493 pr_err("[%s] VMA pin failed: %d!\n", engine->name, err);1494 goto out_obj;1495 }1496 1497 if (flags & EXEC_OBJECT_NEEDS_FENCE) {1498 err = i915_vma_pin_fence(arg.vma);1499 if (err) {1500 pr_err("Unable to pin X-tiled fence; err:%d\n", err);1501 i915_vma_unpin(arg.vma);1502 i915_request_add(rq);1503 goto out_obj;1504 }1505 }1506 1507 err = igt_vma_move_to_active_unlocked(arg.vma, rq, flags);1508 if (err)1509 pr_err("[%s] Move to active failed: %d!\n", engine->name, err);1510 1511 if (flags & EXEC_OBJECT_NEEDS_FENCE)1512 i915_vma_unpin_fence(arg.vma);1513 i915_vma_unpin(arg.vma);1514 1515 i915_request_get(rq);1516 i915_request_add(rq);1517 if (err)1518 goto out_rq;1519 1520 if (!wait_until_running(&h, rq)) {1521 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1522 1523 pr_err("%s: Failed to start request %llx, at %x\n",1524 __func__, rq->fence.seqno, hws_seqno(&h, rq));1525 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);1526 1527 intel_gt_set_wedged(gt);1528 goto out_reset;1529 }1530 1531 init_completion(&arg.completion);1532 1533 tsk = kthread_run(fn, &arg, "igt/evict_vma");1534 if (IS_ERR(tsk)) {1535 err = PTR_ERR(tsk);1536 pr_err("[%s] Thread spawn failed: %d!\n", engine->name, err);1537 tsk = NULL;1538 goto out_reset;1539 }1540 get_task_struct(tsk);1541 1542 wait_for_completion(&arg.completion);1543 1544 if (wait_for(!list_empty(&rq->fence.cb_list), 10)) {1545 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1546 1547 pr_err("igt/evict_vma kthread did not wait\n");1548 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);1549 1550 intel_gt_set_wedged(gt);1551 goto out_reset;1552 }1553 1554out_reset:1555 igt_global_reset_lock(gt);1556 fake_hangcheck(gt, rq->engine->mask);1557 igt_global_reset_unlock(gt);1558 1559 if (tsk) {1560 struct intel_wedge_me w;1561 1562 /* The reset, even indirectly, should take less than 10ms. */1563 intel_wedge_on_timeout(&w, gt, HZ / 10 /* 100ms */)1564 err = kthread_stop(tsk);1565 1566 put_task_struct(tsk);1567 }1568 1569out_rq:1570 i915_request_put(rq);1571out_obj:1572 i915_gem_object_put(obj);1573fini:1574 hang_fini(&h);1575 if (intel_gt_is_wedged(gt))1576 return -EIO;1577 1578 return err;1579}1580 1581static int igt_reset_evict_ggtt(void *arg)1582{1583 struct intel_gt *gt = arg;1584 1585 return __igt_reset_evict_vma(gt, >->ggtt->vm,1586 evict_vma, EXEC_OBJECT_WRITE);1587}1588 1589static int igt_reset_evict_ppgtt(void *arg)1590{1591 struct intel_gt *gt = arg;1592 struct i915_ppgtt *ppgtt;1593 int err;1594 1595 /* aliasing == global gtt locking, covered above */1596 if (INTEL_PPGTT(gt->i915) < INTEL_PPGTT_FULL)1597 return 0;1598 1599 ppgtt = i915_ppgtt_create(gt, 0);1600 if (IS_ERR(ppgtt))1601 return PTR_ERR(ppgtt);1602 1603 err = __igt_reset_evict_vma(gt, &ppgtt->vm,1604 evict_vma, EXEC_OBJECT_WRITE);1605 i915_vm_put(&ppgtt->vm);1606 1607 return err;1608}1609 1610static int igt_reset_evict_fence(void *arg)1611{1612 struct intel_gt *gt = arg;1613 1614 return __igt_reset_evict_vma(gt, >->ggtt->vm,1615 evict_fence, EXEC_OBJECT_NEEDS_FENCE);1616}1617 1618static int wait_for_others(struct intel_gt *gt,1619 struct intel_engine_cs *exclude)1620{1621 struct intel_engine_cs *engine;1622 enum intel_engine_id id;1623 1624 for_each_engine(engine, gt, id) {1625 if (engine == exclude)1626 continue;1627 1628 if (!wait_for_idle(engine))1629 return -EIO;1630 }1631 1632 return 0;1633}1634 1635static int igt_reset_queue(void *arg)1636{1637 struct intel_gt *gt = arg;1638 struct i915_gpu_error *global = >->i915->gpu_error;1639 struct intel_engine_cs *engine;1640 enum intel_engine_id id;1641 struct hang h;1642 int err;1643 1644 /* Check that we replay pending requests following a hang */1645 1646 igt_global_reset_lock(gt);1647 1648 err = hang_init(&h, gt);1649 if (err)1650 goto unlock;1651 1652 for_each_engine(engine, gt, id) {1653 struct intel_selftest_saved_policy saved;1654 struct i915_request *prev;1655 IGT_TIMEOUT(end_time);1656 unsigned int count;1657 bool using_guc = intel_engine_uses_guc(engine);1658 1659 if (!intel_engine_can_store_dword(engine))1660 continue;1661 1662 if (using_guc) {1663 err = intel_selftest_modify_policy(engine, &saved,1664 SELFTEST_SCHEDULER_MODIFY_NO_HANGCHECK);1665 if (err) {1666 pr_err("[%s] Modify policy failed: %d!\n", engine->name, err);1667 goto fini;1668 }1669 }1670 1671 prev = hang_create_request(&h, engine);1672 if (IS_ERR(prev)) {1673 err = PTR_ERR(prev);1674 pr_err("[%s] Create 'prev' hang request failed: %d!\n", engine->name, err);1675 goto restore;1676 }1677 1678 i915_request_get(prev);1679 i915_request_add(prev);1680 1681 count = 0;1682 do {1683 struct i915_request *rq;1684 unsigned int reset_count;1685 1686 rq = hang_create_request(&h, engine);1687 if (IS_ERR(rq)) {1688 err = PTR_ERR(rq);1689 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);1690 goto restore;1691 }1692 1693 i915_request_get(rq);1694 i915_request_add(rq);1695 1696 /*1697 * XXX We don't handle resetting the kernel context1698 * very well. If we trigger a device reset twice in1699 * quick succession while the kernel context is1700 * executing, we may end up skipping the breadcrumb.1701 * This is really only a problem for the selftest as1702 * normally there is a large interlude between resets1703 * (hangcheck), or we focus on resetting just one1704 * engine and so avoid repeatedly resetting innocents.1705 */1706 err = wait_for_others(gt, engine);1707 if (err) {1708 pr_err("%s(%s): Failed to idle other inactive engines after device reset\n",1709 __func__, engine->name);1710 i915_request_put(rq);1711 i915_request_put(prev);1712 1713 GEM_TRACE_DUMP();1714 intel_gt_set_wedged(gt);1715 goto restore;1716 }1717 1718 if (!wait_until_running(&h, prev)) {1719 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1720 1721 pr_err("%s(%s): Failed to start request %llx, at %x\n",1722 __func__, engine->name,1723 prev->fence.seqno, hws_seqno(&h, prev));1724 intel_engine_dump(engine, &p,1725 "%s\n", engine->name);1726 1727 i915_request_put(rq);1728 i915_request_put(prev);1729 1730 intel_gt_set_wedged(gt);1731 1732 err = -EIO;1733 goto restore;1734 }1735 1736 reset_count = fake_hangcheck(gt, BIT(id));1737 1738 if (prev->fence.error != -EIO) {1739 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",1740 prev->fence.error);1741 i915_request_put(rq);1742 i915_request_put(prev);1743 err = -EINVAL;1744 goto restore;1745 }1746 1747 if (rq->fence.error) {1748 pr_err("Fence error status not zero [%d] after unrelated reset\n",1749 rq->fence.error);1750 i915_request_put(rq);1751 i915_request_put(prev);1752 err = -EINVAL;1753 goto restore;1754 }1755 1756 if (i915_reset_count(global) == reset_count) {1757 pr_err("No GPU reset recorded!\n");1758 i915_request_put(rq);1759 i915_request_put(prev);1760 err = -EINVAL;1761 goto restore;1762 }1763 1764 i915_request_put(prev);1765 prev = rq;1766 count++;1767 } while (time_before(jiffies, end_time));1768 pr_info("%s: Completed %d queued resets\n",1769 engine->name, count);1770 1771 *h.batch = MI_BATCH_BUFFER_END;1772 intel_gt_chipset_flush(engine->gt);1773 1774 i915_request_put(prev);1775 1776restore:1777 if (using_guc) {1778 int err2 = intel_selftest_restore_policy(engine, &saved);1779 1780 if (err2)1781 pr_err("%s:%d> [%s] Restore policy failed: %d!\n",1782 __func__, __LINE__, engine->name, err2);1783 if (err == 0)1784 err = err2;1785 }1786 if (err)1787 goto fini;1788 1789 err = igt_flush_test(gt->i915);1790 if (err) {1791 pr_err("[%s] Flush failed: %d!\n", engine->name, err);1792 break;1793 }1794 }1795 1796fini:1797 hang_fini(&h);1798unlock:1799 igt_global_reset_unlock(gt);1800 1801 if (intel_gt_is_wedged(gt))1802 return -EIO;1803 1804 return err;1805}1806 1807static int igt_handle_error(void *arg)1808{1809 struct intel_gt *gt = arg;1810 struct i915_gpu_error *global = >->i915->gpu_error;1811 struct intel_engine_cs *engine;1812 struct hang h;1813 struct i915_request *rq;1814 struct i915_gpu_coredump *error;1815 int err;1816 1817 engine = intel_selftest_find_any_engine(gt);1818 1819 /* Check that we can issue a global GPU and engine reset */1820 1821 if (!intel_has_reset_engine(gt))1822 return 0;1823 1824 if (!engine || !intel_engine_can_store_dword(engine))1825 return 0;1826 1827 err = hang_init(&h, gt);1828 if (err) {1829 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);1830 return err;1831 }1832 1833 rq = hang_create_request(&h, engine);1834 if (IS_ERR(rq)) {1835 err = PTR_ERR(rq);1836 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);1837 goto err_fini;1838 }1839 1840 i915_request_get(rq);1841 i915_request_add(rq);1842 1843 if (!wait_until_running(&h, rq)) {1844 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1845 1846 pr_err("%s: Failed to start request %llx, at %x\n",1847 __func__, rq->fence.seqno, hws_seqno(&h, rq));1848 intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);1849 1850 intel_gt_set_wedged(gt);1851 1852 err = -EIO;1853 goto err_request;1854 }1855 1856 /* Temporarily disable error capture */1857 error = xchg(&global->first_error, (void *)-1);1858 1859 intel_gt_handle_error(gt, engine->mask, 0, NULL);1860 1861 xchg(&global->first_error, error);1862 1863 if (rq->fence.error != -EIO) {1864 pr_err("Guilty request not identified!\n");1865 err = -EINVAL;1866 goto err_request;1867 }1868 1869err_request:1870 i915_request_put(rq);1871err_fini:1872 hang_fini(&h);1873 return err;1874}1875 1876static int __igt_atomic_reset_engine(struct intel_engine_cs *engine,1877 const struct igt_atomic_section *p,1878 const char *mode)1879{1880 struct tasklet_struct * const t = &engine->sched_engine->tasklet;1881 int err;1882 1883 GEM_TRACE("i915_reset_engine(%s:%s) under %s\n",1884 engine->name, mode, p->name);1885 1886 if (t->func)1887 tasklet_disable(t);1888 if (strcmp(p->name, "softirq"))1889 local_bh_disable();1890 p->critical_section_begin();1891 1892 err = __intel_engine_reset_bh(engine, NULL);1893 1894 p->critical_section_end();1895 if (strcmp(p->name, "softirq"))1896 local_bh_enable();1897 if (t->func) {1898 tasklet_enable(t);1899 tasklet_hi_schedule(t);1900 }1901 1902 if (err)1903 pr_err("i915_reset_engine(%s:%s) failed under %s\n",1904 engine->name, mode, p->name);1905 1906 return err;1907}1908 1909static int igt_atomic_reset_engine(struct intel_engine_cs *engine,1910 const struct igt_atomic_section *p)1911{1912 struct i915_request *rq;1913 struct hang h;1914 int err;1915 1916 err = __igt_atomic_reset_engine(engine, p, "idle");1917 if (err)1918 return err;1919 1920 err = hang_init(&h, engine->gt);1921 if (err) {1922 pr_err("[%s] Hang init failed: %d!\n", engine->name, err);1923 return err;1924 }1925 1926 rq = hang_create_request(&h, engine);1927 if (IS_ERR(rq)) {1928 err = PTR_ERR(rq);1929 pr_err("[%s] Create hang request failed: %d!\n", engine->name, err);1930 goto out;1931 }1932 1933 i915_request_get(rq);1934 i915_request_add(rq);1935 1936 if (wait_until_running(&h, rq)) {1937 err = __igt_atomic_reset_engine(engine, p, "active");1938 } else {1939 pr_err("%s(%s): Failed to start request %llx, at %x\n",1940 __func__, engine->name,1941 rq->fence.seqno, hws_seqno(&h, rq));1942 intel_gt_set_wedged(engine->gt);1943 err = -EIO;1944 }1945 1946 if (err == 0) {1947 struct intel_wedge_me w;1948 1949 intel_wedge_on_timeout(&w, engine->gt, HZ / 20 /* 50ms */)1950 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);1951 if (intel_gt_is_wedged(engine->gt))1952 err = -EIO;1953 }1954 1955 i915_request_put(rq);1956out:1957 hang_fini(&h);1958 return err;1959}1960 1961static int igt_reset_engines_atomic(void *arg)1962{1963 struct intel_gt *gt = arg;1964 const typeof(*igt_atomic_phases) *p;1965 int err = 0;1966 1967 /* Check that the engines resets are usable from atomic context */1968 1969 if (!intel_has_reset_engine(gt))1970 return 0;1971 1972 if (intel_uc_uses_guc_submission(>->uc))1973 return 0;1974 1975 igt_global_reset_lock(gt);1976 1977 /* Flush any requests before we get started and check basics */1978 if (!igt_force_reset(gt))1979 goto unlock;1980 1981 for (p = igt_atomic_phases; p->name; p++) {1982 struct intel_engine_cs *engine;1983 enum intel_engine_id id;1984 1985 for_each_engine(engine, gt, id) {1986 err = igt_atomic_reset_engine(engine, p);1987 if (err)1988 goto out;1989 }1990 }1991 1992out:1993 /* As we poke around the guts, do a full reset before continuing. */1994 igt_force_reset(gt);1995unlock:1996 igt_global_reset_unlock(gt);1997 1998 return err;1999}2000 2001int intel_hangcheck_live_selftests(struct drm_i915_private *i915)2002{2003 static const struct i915_subtest tests[] = {2004 SUBTEST(igt_hang_sanitycheck),2005 SUBTEST(igt_reset_nop),2006 SUBTEST(igt_reset_nop_engine),2007 SUBTEST(igt_reset_idle_engine),2008 SUBTEST(igt_reset_active_engine),2009 SUBTEST(igt_reset_fail_engine),2010 SUBTEST(igt_reset_engines),2011 SUBTEST(igt_reset_engines_atomic),2012 SUBTEST(igt_reset_queue),2013 SUBTEST(igt_reset_wait),2014 SUBTEST(igt_reset_evict_ggtt),2015 SUBTEST(igt_reset_evict_ppgtt),2016 SUBTEST(igt_reset_evict_fence),2017 SUBTEST(igt_handle_error),2018 };2019 struct intel_gt *gt = to_gt(i915);2020 intel_wakeref_t wakeref;2021 int err;2022 2023 if (!intel_has_gpu_reset(gt))2024 return 0;2025 2026 if (intel_gt_is_wedged(gt))2027 return -EIO; /* we're long past hope of a successful reset */2028 2029 wakeref = intel_runtime_pm_get(gt->uncore->rpm);2030 2031 err = intel_gt_live_subtests(tests, gt);2032 2033 intel_runtime_pm_put(gt->uncore->rpm, wakeref);2034 2035 return err;2036}2037