1986 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2018 Intel Corporation4 */5 6#include <linux/prime_numbers.h>7 8#include "gem/i915_gem_internal.h"9 10#include "i915_selftest.h"11#include "intel_engine_heartbeat.h"12#include "intel_engine_pm.h"13#include "intel_reset.h"14#include "intel_ring.h"15#include "selftest_engine_heartbeat.h"16#include "selftests/i915_random.h"17#include "selftests/igt_flush_test.h"18#include "selftests/igt_live_test.h"19#include "selftests/igt_spinner.h"20#include "selftests/lib_sw_fence.h"21#include "shmem_utils.h"22 23#include "gem/selftests/igt_gem_utils.h"24#include "gem/selftests/mock_context.h"25 26#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)27#define NUM_GPR 1628#define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */29 30#define LRI_HEADER MI_INSTR(0x22, 0)31#define LRI_LENGTH_MASK GENMASK(7, 0)32 33static struct i915_vma *create_scratch(struct intel_gt *gt)34{35 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE);36}37 38static bool is_active(struct i915_request *rq)39{40 if (i915_request_is_active(rq))41 return true;42 43 if (i915_request_on_hold(rq))44 return true;45 46 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))47 return true;48 49 return false;50}51 52static int wait_for_submit(struct intel_engine_cs *engine,53 struct i915_request *rq,54 unsigned long timeout)55{56 /* Ignore our own attempts to suppress excess tasklets */57 tasklet_hi_schedule(&engine->sched_engine->tasklet);58 59 timeout += jiffies;60 do {61 bool done = time_after(jiffies, timeout);62 63 if (i915_request_completed(rq)) /* that was quick! */64 return 0;65 66 /* Wait until the HW has acknowleged the submission (or err) */67 intel_engine_flush_submission(engine);68 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))69 return 0;70 71 if (done)72 return -ETIME;73 74 cond_resched();75 } while (1);76}77 78static int emit_semaphore_signal(struct intel_context *ce, void *slot)79{80 const u32 offset =81 i915_ggtt_offset(ce->engine->status_page.vma) +82 offset_in_page(slot);83 struct i915_request *rq;84 u32 *cs;85 86 rq = intel_context_create_request(ce);87 if (IS_ERR(rq))88 return PTR_ERR(rq);89 90 cs = intel_ring_begin(rq, 4);91 if (IS_ERR(cs)) {92 i915_request_add(rq);93 return PTR_ERR(cs);94 }95 96 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;97 *cs++ = offset;98 *cs++ = 0;99 *cs++ = 1;100 101 intel_ring_advance(rq, cs);102 103 rq->sched.attr.priority = I915_PRIORITY_BARRIER;104 i915_request_add(rq);105 return 0;106}107 108static int context_flush(struct intel_context *ce, long timeout)109{110 struct i915_request *rq;111 struct dma_fence *fence;112 int err = 0;113 114 rq = intel_engine_create_kernel_request(ce->engine);115 if (IS_ERR(rq))116 return PTR_ERR(rq);117 118 fence = i915_active_fence_get(&ce->timeline->last_request);119 if (fence) {120 i915_request_await_dma_fence(rq, fence);121 dma_fence_put(fence);122 }123 124 rq = i915_request_get(rq);125 i915_request_add(rq);126 if (i915_request_wait(rq, 0, timeout) < 0)127 err = -ETIME;128 i915_request_put(rq);129 130 rmb(); /* We know the request is written, make sure all state is too! */131 return err;132}133 134static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)135{136 if ((lri & MI_LRI_LRM_CS_MMIO) == 0)137 return ~0u;138 139 if (GRAPHICS_VER(engine->i915) < 12)140 return 0xfff;141 142 switch (engine->class) {143 default:144 case RENDER_CLASS:145 case COMPUTE_CLASS:146 return 0x07ff;147 case COPY_ENGINE_CLASS:148 return 0x0fff;149 case VIDEO_DECODE_CLASS:150 case VIDEO_ENHANCEMENT_CLASS:151 return 0x3fff;152 }153}154 155static int live_lrc_layout(void *arg)156{157 struct intel_gt *gt = arg;158 struct intel_engine_cs *engine;159 enum intel_engine_id id;160 u32 *lrc;161 int err;162 163 /*164 * Check the registers offsets we use to create the initial reg state165 * match the layout saved by HW.166 */167 168 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */169 if (!lrc)170 return -ENOMEM;171 GEM_BUG_ON(offset_in_page(lrc));172 173 err = 0;174 for_each_engine(engine, gt, id) {175 u32 *hw;176 int dw;177 178 if (!engine->default_state)179 continue;180 181 hw = shmem_pin_map(engine->default_state);182 if (!hw) {183 err = -ENOMEM;184 break;185 }186 hw += LRC_STATE_OFFSET / sizeof(*hw);187 188 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),189 engine->kernel_context, engine, true);190 191 dw = 0;192 do {193 u32 lri = READ_ONCE(hw[dw]);194 u32 lri_mask;195 196 if (lri == 0) {197 dw++;198 continue;199 }200 201 if (lrc[dw] == 0) {202 pr_debug("%s: skipped instruction %x at dword %d\n",203 engine->name, lri, dw);204 dw++;205 continue;206 }207 208 if ((lri & GENMASK(31, 23)) != LRI_HEADER) {209 pr_err("%s: Expected LRI command at dword %d, found %08x\n",210 engine->name, dw, lri);211 err = -EINVAL;212 break;213 }214 215 if (lrc[dw] != lri) {216 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",217 engine->name, dw, lri, lrc[dw]);218 err = -EINVAL;219 break;220 }221 222 /*223 * When bit 19 of MI_LOAD_REGISTER_IMM instruction224 * opcode is set on Gen12+ devices, HW does not225 * care about certain register address offsets, and226 * instead check the following for valid address227 * ranges on specific engines:228 * RCS && CCS: BITS(0 - 10)229 * BCS: BITS(0 - 11)230 * VECS && VCS: BITS(0 - 13)231 */232 lri_mask = get_lri_mask(engine, lri);233 234 lri &= 0x7f;235 lri++;236 dw++;237 238 while (lri) {239 u32 offset = READ_ONCE(hw[dw]);240 241 if ((offset ^ lrc[dw]) & lri_mask) {242 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",243 engine->name, dw, offset, lrc[dw]);244 err = -EINVAL;245 break;246 }247 248 /*249 * Skip over the actual register value as we250 * expect that to differ.251 */252 dw += 2;253 lri -= 2;254 }255 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);256 257 if (err) {258 pr_info("%s: HW register image:\n", engine->name);259 igt_hexdump(hw, PAGE_SIZE);260 261 pr_info("%s: SW register image:\n", engine->name);262 igt_hexdump(lrc, PAGE_SIZE);263 }264 265 shmem_unpin_map(engine->default_state, hw);266 if (err)267 break;268 }269 270 free_page((unsigned long)lrc);271 return err;272}273 274static int find_offset(const u32 *lri, u32 offset)275{276 int i;277 278 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)279 if (lri[i] == offset)280 return i;281 282 return -1;283}284 285static int live_lrc_fixed(void *arg)286{287 struct intel_gt *gt = arg;288 struct intel_engine_cs *engine;289 enum intel_engine_id id;290 int err = 0;291 292 /*293 * Check the assumed register offsets match the actual locations in294 * the context image.295 */296 297 for_each_engine(engine, gt, id) {298 const struct {299 u32 reg;300 u32 offset;301 const char *name;302 } tbl[] = {303 {304 i915_mmio_reg_offset(RING_START(engine->mmio_base)),305 CTX_RING_START - 1,306 "RING_START"307 },308 {309 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),310 CTX_RING_CTL - 1,311 "RING_CTL"312 },313 {314 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),315 CTX_RING_HEAD - 1,316 "RING_HEAD"317 },318 {319 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),320 CTX_RING_TAIL - 1,321 "RING_TAIL"322 },323 {324 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),325 lrc_ring_mi_mode(engine),326 "RING_MI_MODE"327 },328 {329 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),330 CTX_BB_STATE - 1,331 "BB_STATE"332 },333 {334 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),335 lrc_ring_wa_bb_per_ctx(engine),336 "RING_BB_PER_CTX_PTR"337 },338 {339 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),340 lrc_ring_indirect_ptr(engine),341 "RING_INDIRECT_CTX_PTR"342 },343 {344 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),345 lrc_ring_indirect_offset(engine),346 "RING_INDIRECT_CTX_OFFSET"347 },348 {349 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),350 CTX_TIMESTAMP - 1,351 "RING_CTX_TIMESTAMP"352 },353 {354 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),355 lrc_ring_gpr0(engine),356 "RING_CS_GPR0"357 },358 {359 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),360 lrc_ring_cmd_buf_cctl(engine),361 "RING_CMD_BUF_CCTL"362 },363 {364 i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)),365 lrc_ring_bb_offset(engine),366 "RING_BB_OFFSET"367 },368 { },369 }, *t;370 u32 *hw;371 372 if (!engine->default_state)373 continue;374 375 hw = shmem_pin_map(engine->default_state);376 if (!hw) {377 err = -ENOMEM;378 break;379 }380 hw += LRC_STATE_OFFSET / sizeof(*hw);381 382 for (t = tbl; t->name; t++) {383 int dw = find_offset(hw, t->reg);384 385 if (dw != t->offset) {386 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",387 engine->name,388 t->name,389 t->reg,390 dw,391 t->offset);392 err = -EINVAL;393 }394 }395 396 shmem_unpin_map(engine->default_state, hw);397 }398 399 return err;400}401 402static int __live_lrc_state(struct intel_engine_cs *engine,403 struct i915_vma *scratch)404{405 struct intel_context *ce;406 struct i915_request *rq;407 struct i915_gem_ww_ctx ww;408 enum {409 RING_START_IDX = 0,410 RING_TAIL_IDX,411 MAX_IDX412 };413 u32 expected[MAX_IDX];414 u32 *cs;415 int err;416 int n;417 418 ce = intel_context_create(engine);419 if (IS_ERR(ce))420 return PTR_ERR(ce);421 422 i915_gem_ww_ctx_init(&ww, false);423retry:424 err = i915_gem_object_lock(scratch->obj, &ww);425 if (!err)426 err = intel_context_pin_ww(ce, &ww);427 if (err)428 goto err_put;429 430 rq = i915_request_create(ce);431 if (IS_ERR(rq)) {432 err = PTR_ERR(rq);433 goto err_unpin;434 }435 436 cs = intel_ring_begin(rq, 4 * MAX_IDX);437 if (IS_ERR(cs)) {438 err = PTR_ERR(cs);439 i915_request_add(rq);440 goto err_unpin;441 }442 443 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;444 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));445 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);446 *cs++ = 0;447 448 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);449 450 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;451 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));452 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);453 *cs++ = 0;454 455 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);456 457 i915_request_get(rq);458 i915_request_add(rq);459 if (err)460 goto err_rq;461 462 intel_engine_flush_submission(engine);463 expected[RING_TAIL_IDX] = ce->ring->tail;464 465 if (i915_request_wait(rq, 0, HZ / 5) < 0) {466 err = -ETIME;467 goto err_rq;468 }469 470 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);471 if (IS_ERR(cs)) {472 err = PTR_ERR(cs);473 goto err_rq;474 }475 476 for (n = 0; n < MAX_IDX; n++) {477 if (cs[n] != expected[n]) {478 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",479 engine->name, n, cs[n], expected[n]);480 err = -EINVAL;481 break;482 }483 }484 485 i915_gem_object_unpin_map(scratch->obj);486 487err_rq:488 i915_request_put(rq);489err_unpin:490 intel_context_unpin(ce);491err_put:492 if (err == -EDEADLK) {493 err = i915_gem_ww_ctx_backoff(&ww);494 if (!err)495 goto retry;496 }497 i915_gem_ww_ctx_fini(&ww);498 intel_context_put(ce);499 return err;500}501 502static int live_lrc_state(void *arg)503{504 struct intel_gt *gt = arg;505 struct intel_engine_cs *engine;506 struct i915_vma *scratch;507 enum intel_engine_id id;508 int err = 0;509 510 /*511 * Check the live register state matches what we expect for this512 * intel_context.513 */514 515 scratch = create_scratch(gt);516 if (IS_ERR(scratch))517 return PTR_ERR(scratch);518 519 for_each_engine(engine, gt, id) {520 err = __live_lrc_state(engine, scratch);521 if (err)522 break;523 }524 525 if (igt_flush_test(gt->i915))526 err = -EIO;527 528 i915_vma_unpin_and_release(&scratch, 0);529 return err;530}531 532static int gpr_make_dirty(struct intel_context *ce)533{534 struct i915_request *rq;535 u32 *cs;536 int n;537 538 rq = intel_context_create_request(ce);539 if (IS_ERR(rq))540 return PTR_ERR(rq);541 542 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);543 if (IS_ERR(cs)) {544 i915_request_add(rq);545 return PTR_ERR(cs);546 }547 548 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);549 for (n = 0; n < NUM_GPR_DW; n++) {550 *cs++ = CS_GPR(ce->engine, n);551 *cs++ = STACK_MAGIC;552 }553 *cs++ = MI_NOOP;554 555 intel_ring_advance(rq, cs);556 557 rq->sched.attr.priority = I915_PRIORITY_BARRIER;558 i915_request_add(rq);559 560 return 0;561}562 563static struct i915_request *564__gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)565{566 const u32 offset =567 i915_ggtt_offset(ce->engine->status_page.vma) +568 offset_in_page(slot);569 struct i915_request *rq;570 u32 *cs;571 int err;572 int n;573 574 rq = intel_context_create_request(ce);575 if (IS_ERR(rq))576 return rq;577 578 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);579 if (IS_ERR(cs)) {580 i915_request_add(rq);581 return ERR_CAST(cs);582 }583 584 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;585 *cs++ = MI_NOOP;586 587 *cs++ = MI_SEMAPHORE_WAIT |588 MI_SEMAPHORE_GLOBAL_GTT |589 MI_SEMAPHORE_POLL |590 MI_SEMAPHORE_SAD_NEQ_SDD;591 *cs++ = 0;592 *cs++ = offset;593 *cs++ = 0;594 595 for (n = 0; n < NUM_GPR_DW; n++) {596 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;597 *cs++ = CS_GPR(ce->engine, n);598 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);599 *cs++ = 0;600 }601 602 err = igt_vma_move_to_active_unlocked(scratch, rq, EXEC_OBJECT_WRITE);603 604 i915_request_get(rq);605 i915_request_add(rq);606 if (err) {607 i915_request_put(rq);608 rq = ERR_PTR(err);609 }610 611 return rq;612}613 614static int __live_lrc_gpr(struct intel_engine_cs *engine,615 struct i915_vma *scratch,616 bool preempt)617{618 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);619 struct intel_context *ce;620 struct i915_request *rq;621 u32 *cs;622 int err;623 int n;624 625 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)626 return 0; /* GPR only on rcs0 for gen8 */627 628 err = gpr_make_dirty(engine->kernel_context);629 if (err)630 return err;631 632 ce = intel_context_create(engine);633 if (IS_ERR(ce))634 return PTR_ERR(ce);635 636 rq = __gpr_read(ce, scratch, slot);637 if (IS_ERR(rq)) {638 err = PTR_ERR(rq);639 goto err_put;640 }641 642 err = wait_for_submit(engine, rq, HZ / 2);643 if (err)644 goto err_rq;645 646 if (preempt) {647 err = gpr_make_dirty(engine->kernel_context);648 if (err)649 goto err_rq;650 651 err = emit_semaphore_signal(engine->kernel_context, slot);652 if (err)653 goto err_rq;654 655 err = wait_for_submit(engine, rq, HZ / 2);656 if (err)657 goto err_rq;658 } else {659 slot[0] = 1;660 wmb();661 }662 663 if (i915_request_wait(rq, 0, HZ / 5) < 0) {664 err = -ETIME;665 goto err_rq;666 }667 668 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);669 if (IS_ERR(cs)) {670 err = PTR_ERR(cs);671 goto err_rq;672 }673 674 for (n = 0; n < NUM_GPR_DW; n++) {675 if (cs[n]) {676 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",677 engine->name,678 n / 2, n & 1 ? "udw" : "ldw",679 cs[n]);680 err = -EINVAL;681 break;682 }683 }684 685 i915_gem_object_unpin_map(scratch->obj);686 687err_rq:688 memset32(&slot[0], -1, 4);689 wmb();690 i915_request_put(rq);691err_put:692 intel_context_put(ce);693 return err;694}695 696static int live_lrc_gpr(void *arg)697{698 struct intel_gt *gt = arg;699 struct intel_engine_cs *engine;700 struct i915_vma *scratch;701 enum intel_engine_id id;702 int err = 0;703 704 /*705 * Check that GPR registers are cleared in new contexts as we need706 * to avoid leaking any information from previous contexts.707 */708 709 scratch = create_scratch(gt);710 if (IS_ERR(scratch))711 return PTR_ERR(scratch);712 713 for_each_engine(engine, gt, id) {714 st_engine_heartbeat_disable(engine);715 716 err = __live_lrc_gpr(engine, scratch, false);717 if (err)718 goto err;719 720 err = __live_lrc_gpr(engine, scratch, true);721 if (err)722 goto err;723 724err:725 st_engine_heartbeat_enable(engine);726 if (igt_flush_test(gt->i915))727 err = -EIO;728 if (err)729 break;730 }731 732 i915_vma_unpin_and_release(&scratch, 0);733 return err;734}735 736static struct i915_request *737create_timestamp(struct intel_context *ce, void *slot, int idx)738{739 const u32 offset =740 i915_ggtt_offset(ce->engine->status_page.vma) +741 offset_in_page(slot);742 struct i915_request *rq;743 u32 *cs;744 int err;745 746 rq = intel_context_create_request(ce);747 if (IS_ERR(rq))748 return rq;749 750 cs = intel_ring_begin(rq, 10);751 if (IS_ERR(cs)) {752 err = PTR_ERR(cs);753 goto err;754 }755 756 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;757 *cs++ = MI_NOOP;758 759 *cs++ = MI_SEMAPHORE_WAIT |760 MI_SEMAPHORE_GLOBAL_GTT |761 MI_SEMAPHORE_POLL |762 MI_SEMAPHORE_SAD_NEQ_SDD;763 *cs++ = 0;764 *cs++ = offset;765 *cs++ = 0;766 767 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;768 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));769 *cs++ = offset + idx * sizeof(u32);770 *cs++ = 0;771 772 intel_ring_advance(rq, cs);773 774 err = 0;775err:776 i915_request_get(rq);777 i915_request_add(rq);778 if (err) {779 i915_request_put(rq);780 return ERR_PTR(err);781 }782 783 return rq;784}785 786struct lrc_timestamp {787 struct intel_engine_cs *engine;788 struct intel_context *ce[2];789 u32 poison;790};791 792static bool timestamp_advanced(u32 start, u32 end)793{794 return (s32)(end - start) > 0;795}796 797static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)798{799 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);800 struct i915_request *rq;801 u32 timestamp;802 int err = 0;803 804 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;805 rq = create_timestamp(arg->ce[0], slot, 1);806 if (IS_ERR(rq))807 return PTR_ERR(rq);808 809 err = wait_for_submit(rq->engine, rq, HZ / 2);810 if (err)811 goto err;812 813 if (preempt) {814 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;815 err = emit_semaphore_signal(arg->ce[1], slot);816 if (err)817 goto err;818 } else {819 slot[0] = 1;820 wmb();821 }822 823 /* And wait for switch to kernel (to save our context to memory) */824 err = context_flush(arg->ce[0], HZ / 2);825 if (err)826 goto err;827 828 if (!timestamp_advanced(arg->poison, slot[1])) {829 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",830 arg->engine->name, preempt ? "preempt" : "simple",831 arg->poison, slot[1]);832 err = -EINVAL;833 }834 835 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);836 if (!timestamp_advanced(slot[1], timestamp)) {837 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",838 arg->engine->name, preempt ? "preempt" : "simple",839 slot[1], timestamp);840 err = -EINVAL;841 }842 843err:844 memset32(slot, -1, 4);845 i915_request_put(rq);846 return err;847}848 849static int live_lrc_timestamp(void *arg)850{851 struct lrc_timestamp data = {};852 struct intel_gt *gt = arg;853 enum intel_engine_id id;854 const u32 poison[] = {855 0,856 S32_MAX,857 (u32)S32_MAX + 1,858 U32_MAX,859 };860 861 /*862 * We want to verify that the timestamp is saved and restore across863 * context switches and is monotonic.864 *865 * So we do this with a little bit of LRC poisoning to check various866 * boundary conditions, and see what happens if we preempt the context867 * with a second request (carrying more poison into the timestamp).868 */869 870 for_each_engine(data.engine, gt, id) {871 int i, err = 0;872 873 st_engine_heartbeat_disable(data.engine);874 875 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {876 struct intel_context *tmp;877 878 tmp = intel_context_create(data.engine);879 if (IS_ERR(tmp)) {880 err = PTR_ERR(tmp);881 goto err;882 }883 884 err = intel_context_pin(tmp);885 if (err) {886 intel_context_put(tmp);887 goto err;888 }889 890 data.ce[i] = tmp;891 }892 893 for (i = 0; i < ARRAY_SIZE(poison); i++) {894 data.poison = poison[i];895 896 err = __lrc_timestamp(&data, false);897 if (err)898 break;899 900 err = __lrc_timestamp(&data, true);901 if (err)902 break;903 }904 905err:906 st_engine_heartbeat_enable(data.engine);907 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {908 if (!data.ce[i])909 break;910 911 intel_context_unpin(data.ce[i]);912 intel_context_put(data.ce[i]);913 }914 915 if (igt_flush_test(gt->i915))916 err = -EIO;917 if (err)918 return err;919 }920 921 return 0;922}923 924static struct i915_vma *925create_user_vma(struct i915_address_space *vm, unsigned long size)926{927 struct drm_i915_gem_object *obj;928 struct i915_vma *vma;929 int err;930 931 obj = i915_gem_object_create_internal(vm->i915, size);932 if (IS_ERR(obj))933 return ERR_CAST(obj);934 935 vma = i915_vma_instance(obj, vm, NULL);936 if (IS_ERR(vma)) {937 i915_gem_object_put(obj);938 return vma;939 }940 941 err = i915_vma_pin(vma, 0, 0, PIN_USER);942 if (err) {943 i915_gem_object_put(obj);944 return ERR_PTR(err);945 }946 947 return vma;948}949 950static u32 safe_poison(u32 offset, u32 poison)951{952 /*953 * Do not enable predication as it will nop all subsequent commands,954 * not only disabling the tests (by preventing all the other SRM) but955 * also preventing the arbitration events at the end of the request.956 */957 if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))958 poison &= ~REG_BIT(0);959 960 return poison;961}962 963static struct i915_vma *964store_context(struct intel_context *ce, struct i915_vma *scratch)965{966 struct i915_vma *batch;967 u32 dw, x, *cs, *hw;968 u32 *defaults;969 970 batch = create_user_vma(ce->vm, SZ_64K);971 if (IS_ERR(batch))972 return batch;973 974 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);975 if (IS_ERR(cs)) {976 i915_vma_put(batch);977 return ERR_CAST(cs);978 }979 980 defaults = shmem_pin_map(ce->engine->default_state);981 if (!defaults) {982 i915_gem_object_unpin_map(batch->obj);983 i915_vma_put(batch);984 return ERR_PTR(-ENOMEM);985 }986 987 x = 0;988 dw = 0;989 hw = defaults;990 hw += LRC_STATE_OFFSET / sizeof(*hw);991 do {992 u32 len = hw[dw] & LRI_LENGTH_MASK;993 994 /*995 * Keep it simple, skip parsing complex commands996 *997 * At present, there are no more MI_LOAD_REGISTER_IMM998 * commands after the first 3D state command. Rather999 * than include a table (see i915_cmd_parser.c) of all1000 * the possible commands and their instruction lengths1001 * (or mask for variable length instructions), assume1002 * we have gathered the complete list of registers and1003 * bail out.1004 */1005 if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)1006 break;1007 1008 if (hw[dw] == 0) {1009 dw++;1010 continue;1011 }1012 1013 if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {1014 /* Assume all other MI commands match LRI length mask */1015 dw += len + 2;1016 continue;1017 }1018 1019 if (!len) {1020 pr_err("%s: invalid LRI found in context image\n",1021 ce->engine->name);1022 igt_hexdump(defaults, PAGE_SIZE);1023 break;1024 }1025 1026 dw++;1027 len = (len + 1) / 2;1028 while (len--) {1029 *cs++ = MI_STORE_REGISTER_MEM_GEN8;1030 *cs++ = hw[dw];1031 *cs++ = lower_32_bits(i915_vma_offset(scratch) + x);1032 *cs++ = upper_32_bits(i915_vma_offset(scratch) + x);1033 1034 dw += 2;1035 x += 4;1036 }1037 } while (dw < PAGE_SIZE / sizeof(u32) &&1038 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);1039 1040 *cs++ = MI_BATCH_BUFFER_END;1041 1042 shmem_unpin_map(ce->engine->default_state, defaults);1043 1044 i915_gem_object_flush_map(batch->obj);1045 i915_gem_object_unpin_map(batch->obj);1046 1047 return batch;1048}1049 1050static struct i915_request *1051record_registers(struct intel_context *ce,1052 struct i915_vma *before,1053 struct i915_vma *after,1054 u32 *sema)1055{1056 struct i915_vma *b_before, *b_after;1057 struct i915_request *rq;1058 u32 *cs;1059 int err;1060 1061 b_before = store_context(ce, before);1062 if (IS_ERR(b_before))1063 return ERR_CAST(b_before);1064 1065 b_after = store_context(ce, after);1066 if (IS_ERR(b_after)) {1067 rq = ERR_CAST(b_after);1068 goto err_before;1069 }1070 1071 rq = intel_context_create_request(ce);1072 if (IS_ERR(rq))1073 goto err_after;1074 1075 err = igt_vma_move_to_active_unlocked(before, rq, EXEC_OBJECT_WRITE);1076 if (err)1077 goto err_rq;1078 1079 err = igt_vma_move_to_active_unlocked(b_before, rq, 0);1080 if (err)1081 goto err_rq;1082 1083 err = igt_vma_move_to_active_unlocked(after, rq, EXEC_OBJECT_WRITE);1084 if (err)1085 goto err_rq;1086 1087 err = igt_vma_move_to_active_unlocked(b_after, rq, 0);1088 if (err)1089 goto err_rq;1090 1091 cs = intel_ring_begin(rq, 14);1092 if (IS_ERR(cs)) {1093 err = PTR_ERR(cs);1094 goto err_rq;1095 }1096 1097 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;1098 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);1099 *cs++ = lower_32_bits(i915_vma_offset(b_before));1100 *cs++ = upper_32_bits(i915_vma_offset(b_before));1101 1102 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;1103 *cs++ = MI_SEMAPHORE_WAIT |1104 MI_SEMAPHORE_GLOBAL_GTT |1105 MI_SEMAPHORE_POLL |1106 MI_SEMAPHORE_SAD_NEQ_SDD;1107 *cs++ = 0;1108 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +1109 offset_in_page(sema);1110 *cs++ = 0;1111 *cs++ = MI_NOOP;1112 1113 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;1114 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);1115 *cs++ = lower_32_bits(i915_vma_offset(b_after));1116 *cs++ = upper_32_bits(i915_vma_offset(b_after));1117 1118 intel_ring_advance(rq, cs);1119 1120 WRITE_ONCE(*sema, 0);1121 i915_request_get(rq);1122 i915_request_add(rq);1123err_after:1124 i915_vma_put(b_after);1125err_before:1126 i915_vma_put(b_before);1127 return rq;1128 1129err_rq:1130 i915_request_add(rq);1131 rq = ERR_PTR(err);1132 goto err_after;1133}1134 1135static struct i915_vma *load_context(struct intel_context *ce, u32 poison)1136{1137 struct i915_vma *batch;1138 u32 dw, *cs, *hw;1139 u32 *defaults;1140 1141 batch = create_user_vma(ce->vm, SZ_64K);1142 if (IS_ERR(batch))1143 return batch;1144 1145 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);1146 if (IS_ERR(cs)) {1147 i915_vma_put(batch);1148 return ERR_CAST(cs);1149 }1150 1151 defaults = shmem_pin_map(ce->engine->default_state);1152 if (!defaults) {1153 i915_gem_object_unpin_map(batch->obj);1154 i915_vma_put(batch);1155 return ERR_PTR(-ENOMEM);1156 }1157 1158 dw = 0;1159 hw = defaults;1160 hw += LRC_STATE_OFFSET / sizeof(*hw);1161 do {1162 u32 len = hw[dw] & LRI_LENGTH_MASK;1163 1164 /* For simplicity, break parsing at the first complex command */1165 if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)1166 break;1167 1168 if (hw[dw] == 0) {1169 dw++;1170 continue;1171 }1172 1173 if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {1174 dw += len + 2;1175 continue;1176 }1177 1178 if (!len) {1179 pr_err("%s: invalid LRI found in context image\n",1180 ce->engine->name);1181 igt_hexdump(defaults, PAGE_SIZE);1182 break;1183 }1184 1185 dw++;1186 len = (len + 1) / 2;1187 *cs++ = MI_LOAD_REGISTER_IMM(len);1188 while (len--) {1189 *cs++ = hw[dw];1190 *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,1191 MI_LRI_LRM_CS_MMIO),1192 poison);1193 dw += 2;1194 }1195 } while (dw < PAGE_SIZE / sizeof(u32) &&1196 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);1197 1198 *cs++ = MI_BATCH_BUFFER_END;1199 1200 shmem_unpin_map(ce->engine->default_state, defaults);1201 1202 i915_gem_object_flush_map(batch->obj);1203 i915_gem_object_unpin_map(batch->obj);1204 1205 return batch;1206}1207 1208static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)1209{1210 struct i915_request *rq;1211 struct i915_vma *batch;1212 u32 *cs;1213 int err;1214 1215 batch = load_context(ce, poison);1216 if (IS_ERR(batch))1217 return PTR_ERR(batch);1218 1219 rq = intel_context_create_request(ce);1220 if (IS_ERR(rq)) {1221 err = PTR_ERR(rq);1222 goto err_batch;1223 }1224 1225 err = igt_vma_move_to_active_unlocked(batch, rq, 0);1226 if (err)1227 goto err_rq;1228 1229 cs = intel_ring_begin(rq, 8);1230 if (IS_ERR(cs)) {1231 err = PTR_ERR(cs);1232 goto err_rq;1233 }1234 1235 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;1236 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);1237 *cs++ = lower_32_bits(i915_vma_offset(batch));1238 *cs++ = upper_32_bits(i915_vma_offset(batch));1239 1240 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;1241 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +1242 offset_in_page(sema);1243 *cs++ = 0;1244 *cs++ = 1;1245 1246 intel_ring_advance(rq, cs);1247 1248 rq->sched.attr.priority = I915_PRIORITY_BARRIER;1249err_rq:1250 i915_request_add(rq);1251err_batch:1252 i915_vma_put(batch);1253 return err;1254}1255 1256static bool is_moving(u32 a, u32 b)1257{1258 return a != b;1259}1260 1261static int compare_isolation(struct intel_engine_cs *engine,1262 struct i915_vma *ref[2],1263 struct i915_vma *result[2],1264 struct intel_context *ce,1265 u32 poison)1266{1267 u32 x, dw, *hw, *lrc;1268 u32 *A[2], *B[2];1269 u32 *defaults;1270 int err = 0;1271 1272 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);1273 if (IS_ERR(A[0]))1274 return PTR_ERR(A[0]);1275 1276 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);1277 if (IS_ERR(A[1])) {1278 err = PTR_ERR(A[1]);1279 goto err_A0;1280 }1281 1282 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);1283 if (IS_ERR(B[0])) {1284 err = PTR_ERR(B[0]);1285 goto err_A1;1286 }1287 1288 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);1289 if (IS_ERR(B[1])) {1290 err = PTR_ERR(B[1]);1291 goto err_B0;1292 }1293 1294 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,1295 intel_gt_coherent_map_type(engine->gt,1296 ce->state->obj,1297 false));1298 if (IS_ERR(lrc)) {1299 err = PTR_ERR(lrc);1300 goto err_B1;1301 }1302 lrc += LRC_STATE_OFFSET / sizeof(*hw);1303 1304 defaults = shmem_pin_map(ce->engine->default_state);1305 if (!defaults) {1306 err = -ENOMEM;1307 goto err_lrc;1308 }1309 1310 x = 0;1311 dw = 0;1312 hw = defaults;1313 hw += LRC_STATE_OFFSET / sizeof(*hw);1314 do {1315 u32 len = hw[dw] & LRI_LENGTH_MASK;1316 1317 /* For simplicity, break parsing at the first complex command */1318 if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)1319 break;1320 1321 if (hw[dw] == 0) {1322 dw++;1323 continue;1324 }1325 1326 if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {1327 dw += len + 2;1328 continue;1329 }1330 1331 if (!len) {1332 pr_err("%s: invalid LRI found in context image\n",1333 engine->name);1334 igt_hexdump(defaults, PAGE_SIZE);1335 break;1336 }1337 1338 dw++;1339 len = (len + 1) / 2;1340 while (len--) {1341 if (!is_moving(A[0][x], A[1][x]) &&1342 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {1343 switch (hw[dw] & 4095) {1344 case 0x30: /* RING_HEAD */1345 case 0x34: /* RING_TAIL */1346 break;1347 1348 default:1349 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",1350 engine->name, dw,1351 hw[dw], hw[dw + 1],1352 A[0][x], B[0][x], B[1][x],1353 poison, lrc[dw + 1]);1354 err = -EINVAL;1355 }1356 }1357 dw += 2;1358 x++;1359 }1360 } while (dw < PAGE_SIZE / sizeof(u32) &&1361 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);1362 1363 shmem_unpin_map(ce->engine->default_state, defaults);1364err_lrc:1365 i915_gem_object_unpin_map(ce->state->obj);1366err_B1:1367 i915_gem_object_unpin_map(result[1]->obj);1368err_B0:1369 i915_gem_object_unpin_map(result[0]->obj);1370err_A1:1371 i915_gem_object_unpin_map(ref[1]->obj);1372err_A0:1373 i915_gem_object_unpin_map(ref[0]->obj);1374 return err;1375}1376 1377static struct i915_vma *1378create_result_vma(struct i915_address_space *vm, unsigned long sz)1379{1380 struct i915_vma *vma;1381 void *ptr;1382 1383 vma = create_user_vma(vm, sz);1384 if (IS_ERR(vma))1385 return vma;1386 1387 /* Set the results to a known value distinct from the poison */1388 ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);1389 if (IS_ERR(ptr)) {1390 i915_vma_put(vma);1391 return ERR_CAST(ptr);1392 }1393 1394 memset(ptr, POISON_INUSE, vma->size);1395 i915_gem_object_flush_map(vma->obj);1396 i915_gem_object_unpin_map(vma->obj);1397 1398 return vma;1399}1400 1401static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)1402{1403 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);1404 struct i915_vma *ref[2], *result[2];1405 struct intel_context *A, *B;1406 struct i915_request *rq;1407 int err;1408 1409 A = intel_context_create(engine);1410 if (IS_ERR(A))1411 return PTR_ERR(A);1412 1413 B = intel_context_create(engine);1414 if (IS_ERR(B)) {1415 err = PTR_ERR(B);1416 goto err_A;1417 }1418 1419 ref[0] = create_result_vma(A->vm, SZ_64K);1420 if (IS_ERR(ref[0])) {1421 err = PTR_ERR(ref[0]);1422 goto err_B;1423 }1424 1425 ref[1] = create_result_vma(A->vm, SZ_64K);1426 if (IS_ERR(ref[1])) {1427 err = PTR_ERR(ref[1]);1428 goto err_ref0;1429 }1430 1431 rq = record_registers(A, ref[0], ref[1], sema);1432 if (IS_ERR(rq)) {1433 err = PTR_ERR(rq);1434 goto err_ref1;1435 }1436 1437 WRITE_ONCE(*sema, 1);1438 wmb();1439 1440 if (i915_request_wait(rq, 0, HZ / 2) < 0) {1441 i915_request_put(rq);1442 err = -ETIME;1443 goto err_ref1;1444 }1445 i915_request_put(rq);1446 1447 result[0] = create_result_vma(A->vm, SZ_64K);1448 if (IS_ERR(result[0])) {1449 err = PTR_ERR(result[0]);1450 goto err_ref1;1451 }1452 1453 result[1] = create_result_vma(A->vm, SZ_64K);1454 if (IS_ERR(result[1])) {1455 err = PTR_ERR(result[1]);1456 goto err_result0;1457 }1458 1459 rq = record_registers(A, result[0], result[1], sema);1460 if (IS_ERR(rq)) {1461 err = PTR_ERR(rq);1462 goto err_result1;1463 }1464 1465 err = poison_registers(B, poison, sema);1466 if (err == 0 && i915_request_wait(rq, 0, HZ / 2) < 0) {1467 pr_err("%s(%s): wait for results timed out\n",1468 __func__, engine->name);1469 err = -ETIME;1470 }1471 1472 /* Always cancel the semaphore wait, just in case the GPU gets stuck */1473 WRITE_ONCE(*sema, -1);1474 i915_request_put(rq);1475 if (err)1476 goto err_result1;1477 1478 err = compare_isolation(engine, ref, result, A, poison);1479 1480err_result1:1481 i915_vma_put(result[1]);1482err_result0:1483 i915_vma_put(result[0]);1484err_ref1:1485 i915_vma_put(ref[1]);1486err_ref0:1487 i915_vma_put(ref[0]);1488err_B:1489 intel_context_put(B);1490err_A:1491 intel_context_put(A);1492 return err;1493}1494 1495static bool skip_isolation(const struct intel_engine_cs *engine)1496{1497 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)1498 return true;1499 1500 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)1501 return true;1502 1503 return false;1504}1505 1506static int live_lrc_isolation(void *arg)1507{1508 struct intel_gt *gt = arg;1509 struct intel_engine_cs *engine;1510 enum intel_engine_id id;1511 const u32 poison[] = {1512 STACK_MAGIC,1513 0x3a3a3a3a,1514 0x5c5c5c5c,1515 0xffffffff,1516 0xffff0000,1517 };1518 int err = 0;1519 1520 /*1521 * Our goal is try and verify that per-context state cannot be1522 * tampered with by another non-privileged client.1523 *1524 * We take the list of context registers from the LRI in the default1525 * context image and attempt to modify that list from a remote context.1526 */1527 1528 for_each_engine(engine, gt, id) {1529 int i;1530 1531 /* Just don't even ask */1532 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&1533 skip_isolation(engine))1534 continue;1535 1536 intel_engine_pm_get(engine);1537 for (i = 0; i < ARRAY_SIZE(poison); i++) {1538 int result;1539 1540 result = __lrc_isolation(engine, poison[i]);1541 if (result && !err)1542 err = result;1543 1544 result = __lrc_isolation(engine, ~poison[i]);1545 if (result && !err)1546 err = result;1547 }1548 intel_engine_pm_put(engine);1549 if (igt_flush_test(gt->i915)) {1550 err = -EIO;1551 break;1552 }1553 }1554 1555 return err;1556}1557 1558static int wabb_ctx_submit_req(struct intel_context *ce)1559{1560 struct i915_request *rq;1561 int err = 0;1562 1563 rq = intel_context_create_request(ce);1564 if (IS_ERR(rq))1565 return PTR_ERR(rq);1566 1567 i915_request_get(rq);1568 i915_request_add(rq);1569 1570 if (i915_request_wait(rq, 0, HZ / 5) < 0)1571 err = -ETIME;1572 1573 i915_request_put(rq);1574 1575 return err;1576}1577 1578#define CTX_BB_CANARY_OFFSET (3 * 1024)1579#define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))1580 1581static u32 *1582emit_wabb_ctx_canary(const struct intel_context *ce,1583 u32 *cs, bool per_ctx)1584{1585 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |1586 MI_SRM_LRM_GLOBAL_GTT |1587 MI_LRI_LRM_CS_MMIO;1588 *cs++ = i915_mmio_reg_offset(RING_START(0));1589 *cs++ = i915_ggtt_offset(ce->state) +1590 context_wa_bb_offset(ce) +1591 CTX_BB_CANARY_OFFSET +1592 (per_ctx ? PAGE_SIZE : 0);1593 *cs++ = 0;1594 1595 return cs;1596}1597 1598static u32 *1599emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)1600{1601 return emit_wabb_ctx_canary(ce, cs, false);1602}1603 1604static u32 *1605emit_per_ctx_bb_canary(const struct intel_context *ce, u32 *cs)1606{1607 return emit_wabb_ctx_canary(ce, cs, true);1608}1609 1610static void1611wabb_ctx_setup(struct intel_context *ce, bool per_ctx)1612{1613 u32 *cs = context_wabb(ce, per_ctx);1614 1615 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;1616 1617 if (per_ctx)1618 setup_per_ctx_bb(ce, ce->engine, emit_per_ctx_bb_canary);1619 else1620 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);1621}1622 1623static bool check_ring_start(struct intel_context *ce, bool per_ctx)1624{1625 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -1626 LRC_STATE_OFFSET + context_wa_bb_offset(ce) +1627 (per_ctx ? PAGE_SIZE : 0);1628 1629 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])1630 return true;1631 1632 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",1633 ctx_bb[CTX_BB_CANARY_INDEX],1634 ce->lrc_reg_state[CTX_RING_START]);1635 1636 return false;1637}1638 1639static int wabb_ctx_check(struct intel_context *ce, bool per_ctx)1640{1641 int err;1642 1643 err = wabb_ctx_submit_req(ce);1644 if (err)1645 return err;1646 1647 if (!check_ring_start(ce, per_ctx))1648 return -EINVAL;1649 1650 return 0;1651}1652 1653static int __lrc_wabb_ctx(struct intel_engine_cs *engine, bool per_ctx)1654{1655 struct intel_context *a, *b;1656 int err;1657 1658 a = intel_context_create(engine);1659 if (IS_ERR(a))1660 return PTR_ERR(a);1661 err = intel_context_pin(a);1662 if (err)1663 goto put_a;1664 1665 b = intel_context_create(engine);1666 if (IS_ERR(b)) {1667 err = PTR_ERR(b);1668 goto unpin_a;1669 }1670 err = intel_context_pin(b);1671 if (err)1672 goto put_b;1673 1674 /* We use the already reserved extra page in context state */1675 if (!a->wa_bb_page) {1676 GEM_BUG_ON(b->wa_bb_page);1677 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);1678 goto unpin_b;1679 }1680 1681 /*1682 * In order to test that our per context bb is truly per context,1683 * and executes at the intended spot on context restoring process,1684 * make the batch store the ring start value to memory.1685 * As ring start is restored apriori of starting the indirect ctx bb and1686 * as it will be different for each context, it fits to this purpose.1687 */1688 wabb_ctx_setup(a, per_ctx);1689 wabb_ctx_setup(b, per_ctx);1690 1691 err = wabb_ctx_check(a, per_ctx);1692 if (err)1693 goto unpin_b;1694 1695 err = wabb_ctx_check(b, per_ctx);1696 1697unpin_b:1698 intel_context_unpin(b);1699put_b:1700 intel_context_put(b);1701unpin_a:1702 intel_context_unpin(a);1703put_a:1704 intel_context_put(a);1705 1706 return err;1707}1708 1709static int lrc_wabb_ctx(void *arg, bool per_ctx)1710{1711 struct intel_gt *gt = arg;1712 struct intel_engine_cs *engine;1713 enum intel_engine_id id;1714 int err = 0;1715 1716 for_each_engine(engine, gt, id) {1717 intel_engine_pm_get(engine);1718 err = __lrc_wabb_ctx(engine, per_ctx);1719 intel_engine_pm_put(engine);1720 1721 if (igt_flush_test(gt->i915))1722 err = -EIO;1723 1724 if (err)1725 break;1726 }1727 1728 return err;1729}1730 1731static int live_lrc_indirect_ctx_bb(void *arg)1732{1733 return lrc_wabb_ctx(arg, false);1734}1735 1736static int live_lrc_per_ctx_bb(void *arg)1737{1738 return lrc_wabb_ctx(arg, true);1739}1740 1741static void garbage_reset(struct intel_engine_cs *engine,1742 struct i915_request *rq)1743{1744 const unsigned int bit = I915_RESET_ENGINE + engine->id;1745 unsigned long *lock = &engine->gt->reset.flags;1746 1747 local_bh_disable();1748 if (!test_and_set_bit(bit, lock)) {1749 tasklet_disable(&engine->sched_engine->tasklet);1750 1751 if (!rq->fence.error)1752 __intel_engine_reset_bh(engine, NULL);1753 1754 tasklet_enable(&engine->sched_engine->tasklet);1755 clear_and_wake_up_bit(bit, lock);1756 }1757 local_bh_enable();1758}1759 1760static struct i915_request *garbage(struct intel_context *ce,1761 struct rnd_state *prng)1762{1763 struct i915_request *rq;1764 int err;1765 1766 err = intel_context_pin(ce);1767 if (err)1768 return ERR_PTR(err);1769 1770 prandom_bytes_state(prng,1771 ce->lrc_reg_state,1772 ce->engine->context_size -1773 LRC_STATE_OFFSET);1774 1775 rq = intel_context_create_request(ce);1776 if (IS_ERR(rq)) {1777 err = PTR_ERR(rq);1778 goto err_unpin;1779 }1780 1781 i915_request_get(rq);1782 i915_request_add(rq);1783 return rq;1784 1785err_unpin:1786 intel_context_unpin(ce);1787 return ERR_PTR(err);1788}1789 1790static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)1791{1792 struct intel_context *ce;1793 struct i915_request *hang;1794 int err = 0;1795 1796 ce = intel_context_create(engine);1797 if (IS_ERR(ce))1798 return PTR_ERR(ce);1799 1800 hang = garbage(ce, prng);1801 if (IS_ERR(hang)) {1802 err = PTR_ERR(hang);1803 goto err_ce;1804 }1805 1806 if (wait_for_submit(engine, hang, HZ / 2)) {1807 i915_request_put(hang);1808 err = -ETIME;1809 goto err_ce;1810 }1811 1812 intel_context_set_banned(ce);1813 garbage_reset(engine, hang);1814 1815 intel_engine_flush_submission(engine);1816 if (!hang->fence.error) {1817 i915_request_put(hang);1818 pr_err("%s: corrupted context was not reset\n",1819 engine->name);1820 err = -EINVAL;1821 goto err_ce;1822 }1823 1824 if (i915_request_wait(hang, 0, HZ / 2) < 0) {1825 pr_err("%s: corrupted context did not recover\n",1826 engine->name);1827 i915_request_put(hang);1828 err = -EIO;1829 goto err_ce;1830 }1831 i915_request_put(hang);1832 1833err_ce:1834 intel_context_put(ce);1835 return err;1836}1837 1838static int live_lrc_garbage(void *arg)1839{1840 struct intel_gt *gt = arg;1841 struct intel_engine_cs *engine;1842 enum intel_engine_id id;1843 1844 /*1845 * Verify that we can recover if one context state is completely1846 * corrupted.1847 */1848 1849 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))1850 return 0;1851 1852 for_each_engine(engine, gt, id) {1853 I915_RND_STATE(prng);1854 int err = 0, i;1855 1856 if (!intel_has_reset_engine(engine->gt))1857 continue;1858 1859 intel_engine_pm_get(engine);1860 for (i = 0; i < 3; i++) {1861 err = __lrc_garbage(engine, &prng);1862 if (err)1863 break;1864 }1865 intel_engine_pm_put(engine);1866 1867 if (igt_flush_test(gt->i915))1868 err = -EIO;1869 if (err)1870 return err;1871 }1872 1873 return 0;1874}1875 1876static int __live_pphwsp_runtime(struct intel_engine_cs *engine)1877{1878 struct intel_context *ce;1879 struct i915_request *rq;1880 IGT_TIMEOUT(end_time);1881 int err;1882 1883 ce = intel_context_create(engine);1884 if (IS_ERR(ce))1885 return PTR_ERR(ce);1886 1887 ce->stats.runtime.num_underflow = 0;1888 ce->stats.runtime.max_underflow = 0;1889 1890 do {1891 unsigned int loop = 1024;1892 1893 while (loop) {1894 rq = intel_context_create_request(ce);1895 if (IS_ERR(rq)) {1896 err = PTR_ERR(rq);1897 goto err_rq;1898 }1899 1900 if (--loop == 0)1901 i915_request_get(rq);1902 1903 i915_request_add(rq);1904 }1905 1906 if (__igt_timeout(end_time, NULL))1907 break;1908 1909 i915_request_put(rq);1910 } while (1);1911 1912 err = i915_request_wait(rq, 0, HZ / 5);1913 if (err < 0) {1914 pr_err("%s: request not completed!\n", engine->name);1915 goto err_wait;1916 }1917 1918 igt_flush_test(engine->i915);1919 1920 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",1921 engine->name,1922 intel_context_get_total_runtime_ns(ce),1923 intel_context_get_avg_runtime_ns(ce));1924 1925 err = 0;1926 if (ce->stats.runtime.num_underflow) {1927 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",1928 engine->name,1929 ce->stats.runtime.num_underflow,1930 ce->stats.runtime.max_underflow);1931 GEM_TRACE_DUMP();1932 err = -EOVERFLOW;1933 }1934 1935err_wait:1936 i915_request_put(rq);1937err_rq:1938 intel_context_put(ce);1939 return err;1940}1941 1942static int live_pphwsp_runtime(void *arg)1943{1944 struct intel_gt *gt = arg;1945 struct intel_engine_cs *engine;1946 enum intel_engine_id id;1947 int err = 0;1948 1949 /*1950 * Check that cumulative context runtime as stored in the pphwsp[16]1951 * is monotonic.1952 */1953 1954 for_each_engine(engine, gt, id) {1955 err = __live_pphwsp_runtime(engine);1956 if (err)1957 break;1958 }1959 1960 if (igt_flush_test(gt->i915))1961 err = -EIO;1962 1963 return err;1964}1965 1966int intel_lrc_live_selftests(struct drm_i915_private *i915)1967{1968 static const struct i915_subtest tests[] = {1969 SUBTEST(live_lrc_layout),1970 SUBTEST(live_lrc_fixed),1971 SUBTEST(live_lrc_state),1972 SUBTEST(live_lrc_gpr),1973 SUBTEST(live_lrc_isolation),1974 SUBTEST(live_lrc_timestamp),1975 SUBTEST(live_lrc_garbage),1976 SUBTEST(live_pphwsp_runtime),1977 SUBTEST(live_lrc_indirect_ctx_bb),1978 SUBTEST(live_lrc_per_ctx_bb),1979 };1980 1981 if (!HAS_LOGICAL_RING_CONTEXTS(i915))1982 return 0;1983 1984 return intel_gt_live_subtests(tests, to_gt(i915));1985}1986