1380 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2018 Intel Corporation4 */5 6#include "gem/i915_gem_internal.h"7#include "gem/i915_gem_pm.h"8#include "gt/intel_engine_user.h"9#include "gt/intel_gt.h"10#include "i915_selftest.h"11#include "intel_reset.h"12 13#include "selftests/igt_flush_test.h"14#include "selftests/igt_reset.h"15#include "selftests/igt_spinner.h"16#include "selftests/intel_scheduler_helpers.h"17#include "selftests/mock_drm.h"18 19#include "gem/selftests/igt_gem_utils.h"20#include "gem/selftests/mock_context.h"21 22static const struct wo_register {23 enum intel_platform platform;24 u32 reg;25} wo_registers[] = {26 { INTEL_GEMINILAKE, 0x731c }27};28 29struct wa_lists {30 struct i915_wa_list gt_wa_list;31 struct {32 struct i915_wa_list wa_list;33 struct i915_wa_list ctx_wa_list;34 } engine[I915_NUM_ENGINES];35};36 37static int request_add_sync(struct i915_request *rq, int err)38{39 i915_request_get(rq);40 i915_request_add(rq);41 if (i915_request_wait(rq, 0, HZ / 5) < 0)42 err = -EIO;43 i915_request_put(rq);44 45 return err;46}47 48static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)49{50 int err = 0;51 52 i915_request_get(rq);53 i915_request_add(rq);54 if (spin && !igt_wait_for_spinner(spin, rq))55 err = -ETIMEDOUT;56 i915_request_put(rq);57 58 return err;59}60 61static void62reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)63{64 struct intel_engine_cs *engine;65 enum intel_engine_id id;66 67 memset(lists, 0, sizeof(*lists));68 69 wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global");70 gt_init_workarounds(gt, &lists->gt_wa_list);71 wa_init_finish(&lists->gt_wa_list);72 73 for_each_engine(engine, gt, id) {74 struct i915_wa_list *wal = &lists->engine[id].wa_list;75 76 wa_init_start(wal, gt, "REF", engine->name);77 engine_init_workarounds(engine, wal);78 wa_init_finish(wal);79 80 __intel_engine_init_ctx_wa(engine,81 &lists->engine[id].ctx_wa_list,82 "CTX_REF");83 }84}85 86static void87reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)88{89 struct intel_engine_cs *engine;90 enum intel_engine_id id;91 92 for_each_engine(engine, gt, id)93 intel_wa_list_free(&lists->engine[id].wa_list);94 95 intel_wa_list_free(&lists->gt_wa_list);96}97 98static struct drm_i915_gem_object *99read_nonprivs(struct intel_context *ce)100{101 struct intel_engine_cs *engine = ce->engine;102 const u32 base = engine->mmio_base;103 struct drm_i915_gem_object *result;104 struct i915_request *rq;105 struct i915_vma *vma;106 u32 srm, *cs;107 int err;108 int i;109 110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);111 if (IS_ERR(result))112 return result;113 114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);115 116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);117 if (IS_ERR(cs)) {118 err = PTR_ERR(cs);119 goto err_obj;120 }121 memset(cs, 0xc5, PAGE_SIZE);122 i915_gem_object_flush_map(result);123 i915_gem_object_unpin_map(result);124 125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);126 if (IS_ERR(vma)) {127 err = PTR_ERR(vma);128 goto err_obj;129 }130 131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);132 if (err)133 goto err_obj;134 135 rq = intel_context_create_request(ce);136 if (IS_ERR(rq)) {137 err = PTR_ERR(rq);138 goto err_pin;139 }140 141 err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE);142 if (err)143 goto err_req;144 145 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;146 if (GRAPHICS_VER(engine->i915) >= 8)147 srm++;148 149 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);150 if (IS_ERR(cs)) {151 err = PTR_ERR(cs);152 goto err_req;153 }154 155 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {156 *cs++ = srm;157 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));158 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;159 *cs++ = 0;160 }161 intel_ring_advance(rq, cs);162 163 i915_request_add(rq);164 i915_vma_unpin(vma);165 166 return result;167 168err_req:169 i915_request_add(rq);170err_pin:171 i915_vma_unpin(vma);172err_obj:173 i915_gem_object_put(result);174 return ERR_PTR(err);175}176 177static u32178get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)179{180 i915_reg_t reg = i < engine->whitelist.count ?181 engine->whitelist.list[i].reg :182 RING_NOPID(engine->mmio_base);183 184 return i915_mmio_reg_offset(reg);185}186 187static void188print_results(const struct intel_engine_cs *engine, const u32 *results)189{190 unsigned int i;191 192 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {193 u32 expected = get_whitelist_reg(engine, i);194 u32 actual = results[i];195 196 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",197 i, expected, actual);198 }199}200 201static int check_whitelist(struct intel_context *ce)202{203 struct intel_engine_cs *engine = ce->engine;204 struct drm_i915_gem_object *results;205 struct intel_wedge_me wedge;206 u32 *vaddr;207 int err;208 int i;209 210 results = read_nonprivs(ce);211 if (IS_ERR(results))212 return PTR_ERR(results);213 214 err = 0;215 i915_gem_object_lock(results, NULL);216 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */217 err = i915_gem_object_set_to_cpu_domain(results, false);218 219 if (intel_gt_is_wedged(engine->gt))220 err = -EIO;221 if (err)222 goto out_put;223 224 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);225 if (IS_ERR(vaddr)) {226 err = PTR_ERR(vaddr);227 goto out_put;228 }229 230 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {231 u32 expected = get_whitelist_reg(engine, i);232 u32 actual = vaddr[i];233 234 if (expected != actual) {235 print_results(engine, vaddr);236 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",237 i, expected, actual);238 239 err = -EINVAL;240 break;241 }242 }243 244 i915_gem_object_unpin_map(results);245out_put:246 i915_gem_object_unlock(results);247 i915_gem_object_put(results);248 return err;249}250 251static int do_device_reset(struct intel_engine_cs *engine)252{253 intel_gt_reset(engine->gt, engine->mask, "live_workarounds");254 return 0;255}256 257static int do_engine_reset(struct intel_engine_cs *engine)258{259 return intel_engine_reset(engine, "live_workarounds");260}261 262static int do_guc_reset(struct intel_engine_cs *engine)263{264 /* Currently a no-op as the reset is handled by GuC */265 return 0;266}267 268static int269switch_to_scratch_context(struct intel_engine_cs *engine,270 struct igt_spinner *spin,271 struct i915_request **rq)272{273 struct intel_context *ce;274 int err = 0;275 276 ce = intel_context_create(engine);277 if (IS_ERR(ce))278 return PTR_ERR(ce);279 280 *rq = igt_spinner_create_request(spin, ce, MI_NOOP);281 intel_context_put(ce);282 283 if (IS_ERR(*rq)) {284 spin = NULL;285 err = PTR_ERR(*rq);286 goto err;287 }288 289 err = request_add_spin(*rq, spin);290err:291 if (err && spin)292 igt_spinner_end(spin);293 294 return err;295}296 297static int check_whitelist_across_reset(struct intel_engine_cs *engine,298 int (*reset)(struct intel_engine_cs *),299 const char *name)300{301 struct intel_context *ce, *tmp;302 struct igt_spinner spin;303 struct i915_request *rq;304 intel_wakeref_t wakeref;305 int err;306 307 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",308 engine->whitelist.count, engine->name, name);309 310 ce = intel_context_create(engine);311 if (IS_ERR(ce))312 return PTR_ERR(ce);313 314 err = igt_spinner_init(&spin, engine->gt);315 if (err)316 goto out_ctx;317 318 err = check_whitelist(ce);319 if (err) {320 pr_err("Invalid whitelist *before* %s reset!\n", name);321 goto out_spin;322 }323 324 err = switch_to_scratch_context(engine, &spin, &rq);325 if (err)326 goto out_spin;327 328 /* Ensure the spinner hasn't aborted */329 if (i915_request_completed(rq)) {330 pr_err("%s spinner failed to start\n", name);331 err = -ETIMEDOUT;332 goto out_spin;333 }334 335 with_intel_runtime_pm(engine->uncore->rpm, wakeref)336 err = reset(engine);337 338 /* Ensure the reset happens and kills the engine */339 if (err == 0)340 err = intel_selftest_wait_for_rq(rq);341 342 igt_spinner_end(&spin);343 344 if (err) {345 pr_err("%s reset failed\n", name);346 goto out_spin;347 }348 349 err = check_whitelist(ce);350 if (err) {351 pr_err("Whitelist not preserved in context across %s reset!\n",352 name);353 goto out_spin;354 }355 356 tmp = intel_context_create(engine);357 if (IS_ERR(tmp)) {358 err = PTR_ERR(tmp);359 goto out_spin;360 }361 intel_context_put(ce);362 ce = tmp;363 364 err = check_whitelist(ce);365 if (err) {366 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",367 name);368 goto out_spin;369 }370 371out_spin:372 igt_spinner_fini(&spin);373out_ctx:374 intel_context_put(ce);375 return err;376}377 378static struct i915_vma *create_batch(struct i915_address_space *vm)379{380 struct drm_i915_gem_object *obj;381 struct i915_vma *vma;382 int err;383 384 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);385 if (IS_ERR(obj))386 return ERR_CAST(obj);387 388 vma = i915_vma_instance(obj, vm, NULL);389 if (IS_ERR(vma)) {390 err = PTR_ERR(vma);391 goto err_obj;392 }393 394 err = i915_vma_pin(vma, 0, 0, PIN_USER);395 if (err)396 goto err_obj;397 398 return vma;399 400err_obj:401 i915_gem_object_put(obj);402 return ERR_PTR(err);403}404 405static u32 reg_write(u32 old, u32 new, u32 rsvd)406{407 if (rsvd == 0x0000ffff) {408 old &= ~(new >> 16);409 old |= new & (new >> 16);410 } else {411 old &= ~rsvd;412 old |= new & rsvd;413 }414 415 return old;416}417 418static bool wo_register(struct intel_engine_cs *engine, u32 reg)419{420 enum intel_platform platform = INTEL_INFO(engine->i915)->platform;421 int i;422 423 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==424 RING_FORCE_TO_NONPRIV_ACCESS_WR)425 return true;426 427 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {428 if (wo_registers[i].platform == platform &&429 wo_registers[i].reg == reg)430 return true;431 }432 433 return false;434}435 436static bool timestamp(const struct intel_engine_cs *engine, u32 reg)437{438 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;439 switch (reg) {440 case 0x358:441 case 0x35c:442 case 0x3a8:443 return true;444 445 default:446 return false;447 }448}449 450static bool ro_register(u32 reg)451{452 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==453 RING_FORCE_TO_NONPRIV_ACCESS_RD)454 return true;455 456 return false;457}458 459static int whitelist_writable_count(struct intel_engine_cs *engine)460{461 int count = engine->whitelist.count;462 int i;463 464 for (i = 0; i < engine->whitelist.count; i++) {465 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);466 467 if (ro_register(reg))468 count--;469 }470 471 return count;472}473 474static int check_dirty_whitelist(struct intel_context *ce)475{476 const u32 values[] = {477 0x00000000,478 0x01010101,479 0x10100101,480 0x03030303,481 0x30300303,482 0x05050505,483 0x50500505,484 0x0f0f0f0f,485 0xf00ff00f,486 0x10101010,487 0xf0f01010,488 0x30303030,489 0xa0a03030,490 0x50505050,491 0xc0c05050,492 0xf0f0f0f0,493 0x11111111,494 0x33333333,495 0x55555555,496 0x0000ffff,497 0x00ff00ff,498 0xff0000ff,499 0xffff00ff,500 0xffffffff,501 };502 struct intel_engine_cs *engine = ce->engine;503 struct i915_vma *scratch;504 struct i915_vma *batch;505 int err = 0, i, v, sz;506 u32 *cs, *results;507 508 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);509 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);510 if (IS_ERR(scratch))511 return PTR_ERR(scratch);512 513 batch = create_batch(ce->vm);514 if (IS_ERR(batch)) {515 err = PTR_ERR(batch);516 goto out_scratch;517 }518 519 for (i = 0; i < engine->whitelist.count; i++) {520 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);521 struct i915_gem_ww_ctx ww;522 u64 addr = i915_vma_offset(scratch);523 struct i915_request *rq;524 u32 srm, lrm, rsvd;525 u32 expect;526 int idx;527 bool ro_reg;528 529 if (wo_register(engine, reg))530 continue;531 532 if (timestamp(engine, reg))533 continue; /* timestamps are expected to autoincrement */534 535 ro_reg = ro_register(reg);536 537 i915_gem_ww_ctx_init(&ww, false);538retry:539 cs = NULL;540 err = i915_gem_object_lock(scratch->obj, &ww);541 if (!err)542 err = i915_gem_object_lock(batch->obj, &ww);543 if (!err)544 err = intel_context_pin_ww(ce, &ww);545 if (err)546 goto out;547 548 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);549 if (IS_ERR(cs)) {550 err = PTR_ERR(cs);551 goto out_ctx;552 }553 554 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);555 if (IS_ERR(results)) {556 err = PTR_ERR(results);557 goto out_unmap_batch;558 }559 560 /* Clear non priv flags */561 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;562 563 srm = MI_STORE_REGISTER_MEM;564 lrm = MI_LOAD_REGISTER_MEM;565 if (GRAPHICS_VER(engine->i915) >= 8)566 lrm++, srm++;567 568 pr_debug("%s: Writing garbage to %x\n",569 engine->name, reg);570 571 /* SRM original */572 *cs++ = srm;573 *cs++ = reg;574 *cs++ = lower_32_bits(addr);575 *cs++ = upper_32_bits(addr);576 577 idx = 1;578 for (v = 0; v < ARRAY_SIZE(values); v++) {579 /* LRI garbage */580 *cs++ = MI_LOAD_REGISTER_IMM(1);581 *cs++ = reg;582 *cs++ = values[v];583 584 /* SRM result */585 *cs++ = srm;586 *cs++ = reg;587 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);588 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);589 idx++;590 }591 for (v = 0; v < ARRAY_SIZE(values); v++) {592 /* LRI garbage */593 *cs++ = MI_LOAD_REGISTER_IMM(1);594 *cs++ = reg;595 *cs++ = ~values[v];596 597 /* SRM result */598 *cs++ = srm;599 *cs++ = reg;600 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);601 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);602 idx++;603 }604 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);605 606 /* LRM original -- don't leave garbage in the context! */607 *cs++ = lrm;608 *cs++ = reg;609 *cs++ = lower_32_bits(addr);610 *cs++ = upper_32_bits(addr);611 612 *cs++ = MI_BATCH_BUFFER_END;613 614 i915_gem_object_flush_map(batch->obj);615 i915_gem_object_unpin_map(batch->obj);616 intel_gt_chipset_flush(engine->gt);617 cs = NULL;618 619 rq = i915_request_create(ce);620 if (IS_ERR(rq)) {621 err = PTR_ERR(rq);622 goto out_unmap_scratch;623 }624 625 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */626 err = engine->emit_init_breadcrumb(rq);627 if (err)628 goto err_request;629 }630 631 err = i915_vma_move_to_active(batch, rq, 0);632 if (err)633 goto err_request;634 635 err = i915_vma_move_to_active(scratch, rq,636 EXEC_OBJECT_WRITE);637 if (err)638 goto err_request;639 640 err = engine->emit_bb_start(rq,641 i915_vma_offset(batch), PAGE_SIZE,642 0);643 if (err)644 goto err_request;645 646err_request:647 err = request_add_sync(rq, err);648 if (err) {649 pr_err("%s: Futzing %x timedout; cancelling test\n",650 engine->name, reg);651 intel_gt_set_wedged(engine->gt);652 goto out_unmap_scratch;653 }654 655 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);656 if (!ro_reg) {657 /* detect write masking */658 rsvd = results[ARRAY_SIZE(values)];659 if (!rsvd) {660 pr_err("%s: Unable to write to whitelisted register %x\n",661 engine->name, reg);662 err = -EINVAL;663 goto out_unmap_scratch;664 }665 } else {666 rsvd = 0;667 }668 669 expect = results[0];670 idx = 1;671 for (v = 0; v < ARRAY_SIZE(values); v++) {672 if (ro_reg)673 expect = results[0];674 else675 expect = reg_write(expect, values[v], rsvd);676 677 if (results[idx] != expect)678 err++;679 idx++;680 }681 for (v = 0; v < ARRAY_SIZE(values); v++) {682 if (ro_reg)683 expect = results[0];684 else685 expect = reg_write(expect, ~values[v], rsvd);686 687 if (results[idx] != expect)688 err++;689 idx++;690 }691 if (err) {692 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",693 engine->name, err, reg);694 695 if (ro_reg)696 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",697 engine->name, reg, results[0]);698 else699 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",700 engine->name, reg, results[0], rsvd);701 702 expect = results[0];703 idx = 1;704 for (v = 0; v < ARRAY_SIZE(values); v++) {705 u32 w = values[v];706 707 if (ro_reg)708 expect = results[0];709 else710 expect = reg_write(expect, w, rsvd);711 pr_info("Wrote %08x, read %08x, expect %08x\n",712 w, results[idx], expect);713 idx++;714 }715 for (v = 0; v < ARRAY_SIZE(values); v++) {716 u32 w = ~values[v];717 718 if (ro_reg)719 expect = results[0];720 else721 expect = reg_write(expect, w, rsvd);722 pr_info("Wrote %08x, read %08x, expect %08x\n",723 w, results[idx], expect);724 idx++;725 }726 727 err = -EINVAL;728 }729out_unmap_scratch:730 i915_gem_object_unpin_map(scratch->obj);731out_unmap_batch:732 if (cs)733 i915_gem_object_unpin_map(batch->obj);734out_ctx:735 intel_context_unpin(ce);736out:737 if (err == -EDEADLK) {738 err = i915_gem_ww_ctx_backoff(&ww);739 if (!err)740 goto retry;741 }742 i915_gem_ww_ctx_fini(&ww);743 if (err)744 break;745 }746 747 if (igt_flush_test(engine->i915))748 err = -EIO;749 750 i915_vma_unpin_and_release(&batch, 0);751out_scratch:752 i915_vma_unpin_and_release(&scratch, 0);753 return err;754}755 756static int live_dirty_whitelist(void *arg)757{758 struct intel_gt *gt = arg;759 struct intel_engine_cs *engine;760 enum intel_engine_id id;761 762 /* Can the user write to the whitelisted registers? */763 764 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */765 return 0;766 767 for_each_engine(engine, gt, id) {768 struct intel_context *ce;769 int err;770 771 if (engine->whitelist.count == 0)772 continue;773 774 ce = intel_context_create(engine);775 if (IS_ERR(ce))776 return PTR_ERR(ce);777 778 err = check_dirty_whitelist(ce);779 intel_context_put(ce);780 if (err)781 return err;782 }783 784 return 0;785}786 787static int live_reset_whitelist(void *arg)788{789 struct intel_gt *gt = arg;790 struct intel_engine_cs *engine;791 enum intel_engine_id id;792 int err = 0;793 794 /* If we reset the gpu, we should not lose the RING_NONPRIV */795 igt_global_reset_lock(gt);796 797 for_each_engine(engine, gt, id) {798 if (engine->whitelist.count == 0)799 continue;800 801 if (intel_has_reset_engine(gt)) {802 if (intel_engine_uses_guc(engine)) {803 struct intel_selftest_saved_policy saved;804 int err2;805 806 err = intel_selftest_modify_policy(engine, &saved,807 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);808 if (err)809 goto out;810 811 err = check_whitelist_across_reset(engine,812 do_guc_reset,813 "guc");814 815 err2 = intel_selftest_restore_policy(engine, &saved);816 if (err == 0)817 err = err2;818 } else {819 err = check_whitelist_across_reset(engine,820 do_engine_reset,821 "engine");822 }823 824 if (err)825 goto out;826 }827 828 if (intel_has_gpu_reset(gt)) {829 err = check_whitelist_across_reset(engine,830 do_device_reset,831 "device");832 if (err)833 goto out;834 }835 }836 837out:838 igt_global_reset_unlock(gt);839 return err;840}841 842static int read_whitelisted_registers(struct intel_context *ce,843 struct i915_vma *results)844{845 struct intel_engine_cs *engine = ce->engine;846 struct i915_request *rq;847 int i, err = 0;848 u32 srm, *cs;849 850 rq = intel_context_create_request(ce);851 if (IS_ERR(rq))852 return PTR_ERR(rq);853 854 err = igt_vma_move_to_active_unlocked(results, rq, EXEC_OBJECT_WRITE);855 if (err)856 goto err_req;857 858 srm = MI_STORE_REGISTER_MEM;859 if (GRAPHICS_VER(engine->i915) >= 8)860 srm++;861 862 cs = intel_ring_begin(rq, 4 * engine->whitelist.count);863 if (IS_ERR(cs)) {864 err = PTR_ERR(cs);865 goto err_req;866 }867 868 for (i = 0; i < engine->whitelist.count; i++) {869 u64 offset = i915_vma_offset(results) + sizeof(u32) * i;870 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);871 872 /* Clear non priv flags */873 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;874 875 *cs++ = srm;876 *cs++ = reg;877 *cs++ = lower_32_bits(offset);878 *cs++ = upper_32_bits(offset);879 }880 intel_ring_advance(rq, cs);881 882err_req:883 return request_add_sync(rq, err);884}885 886static int scrub_whitelisted_registers(struct intel_context *ce)887{888 struct intel_engine_cs *engine = ce->engine;889 struct i915_request *rq;890 struct i915_vma *batch;891 int i, err = 0;892 u32 *cs;893 894 batch = create_batch(ce->vm);895 if (IS_ERR(batch))896 return PTR_ERR(batch);897 898 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);899 if (IS_ERR(cs)) {900 err = PTR_ERR(cs);901 goto err_batch;902 }903 904 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));905 for (i = 0; i < engine->whitelist.count; i++) {906 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);907 908 if (ro_register(reg))909 continue;910 911 /* Clear non priv flags */912 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;913 914 *cs++ = reg;915 *cs++ = 0xffffffff;916 }917 *cs++ = MI_BATCH_BUFFER_END;918 919 i915_gem_object_flush_map(batch->obj);920 intel_gt_chipset_flush(engine->gt);921 922 rq = intel_context_create_request(ce);923 if (IS_ERR(rq)) {924 err = PTR_ERR(rq);925 goto err_unpin;926 }927 928 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */929 err = engine->emit_init_breadcrumb(rq);930 if (err)931 goto err_request;932 }933 934 err = igt_vma_move_to_active_unlocked(batch, rq, 0);935 if (err)936 goto err_request;937 938 /* Perform the writes from an unprivileged "user" batch */939 err = engine->emit_bb_start(rq, i915_vma_offset(batch), 0, 0);940 941err_request:942 err = request_add_sync(rq, err);943 944err_unpin:945 i915_gem_object_unpin_map(batch->obj);946err_batch:947 i915_vma_unpin_and_release(&batch, 0);948 return err;949}950 951struct regmask {952 i915_reg_t reg;953 u8 graphics_ver;954};955 956static bool find_reg(struct drm_i915_private *i915,957 i915_reg_t reg,958 const struct regmask *tbl,959 unsigned long count)960{961 u32 offset = i915_mmio_reg_offset(reg);962 963 while (count--) {964 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&965 i915_mmio_reg_offset(tbl->reg) == offset)966 return true;967 tbl++;968 }969 970 return false;971}972 973static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)974{975 /* Alas, we must pardon some whitelists. Mistakes already made */976 static const struct regmask pardon[] = {977 { GEN9_CTX_PREEMPT_REG, 9 },978 { _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */979 };980 981 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));982}983 984static bool result_eq(struct intel_engine_cs *engine,985 u32 a, u32 b, i915_reg_t reg)986{987 if (a != b && !pardon_reg(engine->i915, reg)) {988 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",989 i915_mmio_reg_offset(reg), a, b);990 return false;991 }992 993 return true;994}995 996static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)997{998 /* Some registers do not seem to behave and our writes unreadable */999 static const struct regmask wo[] = {1000 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },1001 };1002 1003 return find_reg(i915, reg, wo, ARRAY_SIZE(wo));1004}1005 1006static bool result_neq(struct intel_engine_cs *engine,1007 u32 a, u32 b, i915_reg_t reg)1008{1009 if (a == b && !writeonly_reg(engine->i915, reg)) {1010 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",1011 i915_mmio_reg_offset(reg), a);1012 return false;1013 }1014 1015 return true;1016}1017 1018static int1019check_whitelisted_registers(struct intel_engine_cs *engine,1020 struct i915_vma *A,1021 struct i915_vma *B,1022 bool (*fn)(struct intel_engine_cs *engine,1023 u32 a, u32 b,1024 i915_reg_t reg))1025{1026 u32 *a, *b;1027 int i, err;1028 1029 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);1030 if (IS_ERR(a))1031 return PTR_ERR(a);1032 1033 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);1034 if (IS_ERR(b)) {1035 err = PTR_ERR(b);1036 goto err_a;1037 }1038 1039 err = 0;1040 for (i = 0; i < engine->whitelist.count; i++) {1041 const struct i915_wa *wa = &engine->whitelist.list[i];1042 1043 if (i915_mmio_reg_offset(wa->reg) &1044 RING_FORCE_TO_NONPRIV_ACCESS_RD)1045 continue;1046 1047 if (!fn(engine, a[i], b[i], wa->reg))1048 err = -EINVAL;1049 }1050 1051 i915_gem_object_unpin_map(B->obj);1052err_a:1053 i915_gem_object_unpin_map(A->obj);1054 return err;1055}1056 1057static int live_isolated_whitelist(void *arg)1058{1059 struct intel_gt *gt = arg;1060 struct {1061 struct i915_vma *scratch[2];1062 } client[2] = {};1063 struct intel_engine_cs *engine;1064 enum intel_engine_id id;1065 int i, err = 0;1066 1067 /*1068 * Check that a write into a whitelist register works, but1069 * invisible to a second context.1070 */1071 1072 if (!intel_engines_has_context_isolation(gt->i915))1073 return 0;1074 1075 for (i = 0; i < ARRAY_SIZE(client); i++) {1076 client[i].scratch[0] =1077 __vm_create_scratch_for_read_pinned(gt->vm, 4096);1078 if (IS_ERR(client[i].scratch[0])) {1079 err = PTR_ERR(client[i].scratch[0]);1080 goto err;1081 }1082 1083 client[i].scratch[1] =1084 __vm_create_scratch_for_read_pinned(gt->vm, 4096);1085 if (IS_ERR(client[i].scratch[1])) {1086 err = PTR_ERR(client[i].scratch[1]);1087 i915_vma_unpin_and_release(&client[i].scratch[0], 0);1088 goto err;1089 }1090 }1091 1092 for_each_engine(engine, gt, id) {1093 struct intel_context *ce[2];1094 1095 if (!engine->kernel_context->vm)1096 continue;1097 1098 if (!whitelist_writable_count(engine))1099 continue;1100 1101 ce[0] = intel_context_create(engine);1102 if (IS_ERR(ce[0])) {1103 err = PTR_ERR(ce[0]);1104 break;1105 }1106 ce[1] = intel_context_create(engine);1107 if (IS_ERR(ce[1])) {1108 err = PTR_ERR(ce[1]);1109 intel_context_put(ce[0]);1110 break;1111 }1112 1113 /* Read default values */1114 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);1115 if (err)1116 goto err_ce;1117 1118 /* Try to overwrite registers (should only affect ctx0) */1119 err = scrub_whitelisted_registers(ce[0]);1120 if (err)1121 goto err_ce;1122 1123 /* Read values from ctx1, we expect these to be defaults */1124 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);1125 if (err)1126 goto err_ce;1127 1128 /* Verify that both reads return the same default values */1129 err = check_whitelisted_registers(engine,1130 client[0].scratch[0],1131 client[1].scratch[0],1132 result_eq);1133 if (err)1134 goto err_ce;1135 1136 /* Read back the updated values in ctx0 */1137 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);1138 if (err)1139 goto err_ce;1140 1141 /* User should be granted privilege to overwhite regs */1142 err = check_whitelisted_registers(engine,1143 client[0].scratch[0],1144 client[0].scratch[1],1145 result_neq);1146err_ce:1147 intel_context_put(ce[1]);1148 intel_context_put(ce[0]);1149 if (err)1150 break;1151 }1152 1153err:1154 for (i = 0; i < ARRAY_SIZE(client); i++) {1155 i915_vma_unpin_and_release(&client[i].scratch[1], 0);1156 i915_vma_unpin_and_release(&client[i].scratch[0], 0);1157 }1158 1159 if (igt_flush_test(gt->i915))1160 err = -EIO;1161 1162 return err;1163}1164 1165static bool1166verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,1167 const char *str)1168{1169 struct intel_engine_cs *engine;1170 enum intel_engine_id id;1171 bool ok = true;1172 1173 ok &= wa_list_verify(gt, &lists->gt_wa_list, str);1174 1175 for_each_engine(engine, gt, id) {1176 struct intel_context *ce;1177 1178 ce = intel_context_create(engine);1179 if (IS_ERR(ce))1180 return false;1181 1182 ok &= engine_wa_list_verify(ce,1183 &lists->engine[id].wa_list,1184 str) == 0;1185 1186 ok &= engine_wa_list_verify(ce,1187 &lists->engine[id].ctx_wa_list,1188 str) == 0;1189 1190 intel_context_put(ce);1191 }1192 1193 return ok;1194}1195 1196static int1197live_gpu_reset_workarounds(void *arg)1198{1199 struct intel_gt *gt = arg;1200 intel_wakeref_t wakeref;1201 struct wa_lists *lists;1202 bool ok;1203 1204 if (!intel_has_gpu_reset(gt))1205 return 0;1206 1207 lists = kzalloc(sizeof(*lists), GFP_KERNEL);1208 if (!lists)1209 return -ENOMEM;1210 1211 pr_info("Verifying after GPU reset...\n");1212 1213 igt_global_reset_lock(gt);1214 wakeref = intel_runtime_pm_get(gt->uncore->rpm);1215 1216 reference_lists_init(gt, lists);1217 1218 ok = verify_wa_lists(gt, lists, "before reset");1219 if (!ok)1220 goto out;1221 1222 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");1223 1224 ok = verify_wa_lists(gt, lists, "after reset");1225 1226out:1227 reference_lists_fini(gt, lists);1228 intel_runtime_pm_put(gt->uncore->rpm, wakeref);1229 igt_global_reset_unlock(gt);1230 kfree(lists);1231 1232 return ok ? 0 : -ESRCH;1233}1234 1235static int1236live_engine_reset_workarounds(void *arg)1237{1238 struct intel_gt *gt = arg;1239 struct intel_engine_cs *engine;1240 enum intel_engine_id id;1241 struct intel_context *ce;1242 struct igt_spinner spin;1243 struct i915_request *rq;1244 intel_wakeref_t wakeref;1245 struct wa_lists *lists;1246 int ret = 0;1247 1248 if (!intel_has_reset_engine(gt))1249 return 0;1250 1251 lists = kzalloc(sizeof(*lists), GFP_KERNEL);1252 if (!lists)1253 return -ENOMEM;1254 1255 igt_global_reset_lock(gt);1256 wakeref = intel_runtime_pm_get(gt->uncore->rpm);1257 1258 reference_lists_init(gt, lists);1259 1260 for_each_engine(engine, gt, id) {1261 struct intel_selftest_saved_policy saved;1262 bool using_guc = intel_engine_uses_guc(engine);1263 bool ok;1264 int ret2;1265 1266 pr_info("Verifying after %s reset...\n", engine->name);1267 ret = intel_selftest_modify_policy(engine, &saved,1268 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);1269 if (ret)1270 break;1271 1272 ce = intel_context_create(engine);1273 if (IS_ERR(ce)) {1274 ret = PTR_ERR(ce);1275 goto restore;1276 }1277 1278 if (!using_guc) {1279 ok = verify_wa_lists(gt, lists, "before reset");1280 if (!ok) {1281 ret = -ESRCH;1282 goto err;1283 }1284 1285 ret = intel_engine_reset(engine, "live_workarounds:idle");1286 if (ret) {1287 pr_err("%s: Reset failed while idle\n", engine->name);1288 goto err;1289 }1290 1291 ok = verify_wa_lists(gt, lists, "after idle reset");1292 if (!ok) {1293 ret = -ESRCH;1294 goto err;1295 }1296 }1297 1298 ret = igt_spinner_init(&spin, engine->gt);1299 if (ret)1300 goto err;1301 1302 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);1303 if (IS_ERR(rq)) {1304 ret = PTR_ERR(rq);1305 igt_spinner_fini(&spin);1306 goto err;1307 }1308 1309 ret = request_add_spin(rq, &spin);1310 if (ret) {1311 pr_err("%s: Spinner failed to start\n", engine->name);1312 igt_spinner_fini(&spin);1313 goto err;1314 }1315 1316 /* Ensure the spinner hasn't aborted */1317 if (i915_request_completed(rq)) {1318 ret = -ETIMEDOUT;1319 goto skip;1320 }1321 1322 if (!using_guc) {1323 ret = intel_engine_reset(engine, "live_workarounds:active");1324 if (ret) {1325 pr_err("%s: Reset failed on an active spinner\n",1326 engine->name);1327 igt_spinner_fini(&spin);1328 goto err;1329 }1330 }1331 1332 /* Ensure the reset happens and kills the engine */1333 if (ret == 0)1334 ret = intel_selftest_wait_for_rq(rq);1335 1336skip:1337 igt_spinner_end(&spin);1338 igt_spinner_fini(&spin);1339 1340 ok = verify_wa_lists(gt, lists, "after busy reset");1341 if (!ok)1342 ret = -ESRCH;1343 1344err:1345 intel_context_put(ce);1346 1347restore:1348 ret2 = intel_selftest_restore_policy(engine, &saved);1349 if (ret == 0)1350 ret = ret2;1351 if (ret)1352 break;1353 }1354 1355 reference_lists_fini(gt, lists);1356 intel_runtime_pm_put(gt->uncore->rpm, wakeref);1357 igt_global_reset_unlock(gt);1358 kfree(lists);1359 1360 igt_flush_test(gt->i915);1361 1362 return ret;1363}1364 1365int intel_workarounds_live_selftests(struct drm_i915_private *i915)1366{1367 static const struct i915_subtest tests[] = {1368 SUBTEST(live_dirty_whitelist),1369 SUBTEST(live_reset_whitelist),1370 SUBTEST(live_isolated_whitelist),1371 SUBTEST(live_gpu_reset_workarounds),1372 SUBTEST(live_engine_reset_workarounds),1373 };1374 1375 if (intel_gt_is_wedged(to_gt(i915)))1376 return 0;1377 1378 return intel_gt_live_subtests(tests, to_gt(i915));1379}1380