1434 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2017-2018 Intel Corporation4 */5 6#include <linux/prime_numbers.h>7#include <linux/string_helpers.h>8 9#include "intel_context.h"10#include "intel_engine_heartbeat.h"11#include "intel_engine_pm.h"12#include "intel_engine_regs.h"13#include "intel_gpu_commands.h"14#include "intel_gt.h"15#include "intel_gt_requests.h"16#include "intel_ring.h"17#include "selftest_engine_heartbeat.h"18 19#include "../selftests/i915_random.h"20#include "../i915_selftest.h"21 22#include "selftests/igt_flush_test.h"23#include "selftests/lib_sw_fence.h"24#include "selftests/mock_gem_device.h"25#include "selftests/mock_timeline.h"26 27static struct page *hwsp_page(struct intel_timeline *tl)28{29 struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj;30 31 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));32 return sg_page(obj->mm.pages->sgl);33}34 35static unsigned long hwsp_cacheline(struct intel_timeline *tl)36{37 unsigned long address = (unsigned long)page_address(hwsp_page(tl));38 39 return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES;40}41 42static int selftest_tl_pin(struct intel_timeline *tl)43{44 struct i915_gem_ww_ctx ww;45 int err;46 47 i915_gem_ww_ctx_init(&ww, false);48retry:49 err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww);50 if (!err)51 err = intel_timeline_pin(tl, &ww);52 53 if (err == -EDEADLK) {54 err = i915_gem_ww_ctx_backoff(&ww);55 if (!err)56 goto retry;57 }58 i915_gem_ww_ctx_fini(&ww);59 return err;60}61 62/* Only half of seqno's are usable, see __intel_timeline_get_seqno() */63#define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2)64 65struct mock_hwsp_freelist {66 struct intel_gt *gt;67 struct radix_tree_root cachelines;68 struct intel_timeline **history;69 unsigned long count, max;70 struct rnd_state prng;71};72 73enum {74 SHUFFLE = BIT(0),75};76 77static void __mock_hwsp_record(struct mock_hwsp_freelist *state,78 unsigned int idx,79 struct intel_timeline *tl)80{81 tl = xchg(&state->history[idx], tl);82 if (tl) {83 radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));84 intel_timeline_unpin(tl);85 intel_timeline_put(tl);86 }87}88 89static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state,90 unsigned int count,91 unsigned int flags)92{93 struct intel_timeline *tl;94 unsigned int idx;95 96 while (count--) {97 unsigned long cacheline;98 int err;99 100 tl = intel_timeline_create(state->gt);101 if (IS_ERR(tl))102 return PTR_ERR(tl);103 104 err = selftest_tl_pin(tl);105 if (err) {106 intel_timeline_put(tl);107 return err;108 }109 110 cacheline = hwsp_cacheline(tl);111 err = radix_tree_insert(&state->cachelines, cacheline, tl);112 if (err) {113 if (err == -EEXIST) {114 pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",115 cacheline);116 }117 intel_timeline_unpin(tl);118 intel_timeline_put(tl);119 return err;120 }121 122 idx = state->count++ % state->max;123 __mock_hwsp_record(state, idx, tl);124 }125 126 if (flags & SHUFFLE)127 i915_prandom_shuffle(state->history,128 sizeof(*state->history),129 min(state->count, state->max),130 &state->prng);131 132 count = i915_prandom_u32_max_state(min(state->count, state->max),133 &state->prng);134 while (count--) {135 idx = --state->count % state->max;136 __mock_hwsp_record(state, idx, NULL);137 }138 139 return 0;140}141 142static int mock_hwsp_freelist(void *arg)143{144 struct mock_hwsp_freelist state;145 struct drm_i915_private *i915;146 const struct {147 const char *name;148 unsigned int flags;149 } phases[] = {150 { "linear", 0 },151 { "shuffled", SHUFFLE },152 { },153 }, *p;154 unsigned int na;155 int err = 0;156 157 i915 = mock_gem_device();158 if (!i915)159 return -ENOMEM;160 161 INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL);162 state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed);163 164 state.gt = to_gt(i915);165 166 /*167 * Create a bunch of timelines and check that their HWSP do not overlap.168 * Free some, and try again.169 */170 171 state.max = PAGE_SIZE / sizeof(*state.history);172 state.count = 0;173 state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL);174 if (!state.history) {175 err = -ENOMEM;176 goto err_put;177 }178 179 for (p = phases; p->name; p++) {180 pr_debug("%s(%s)\n", __func__, p->name);181 for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) {182 err = __mock_hwsp_timeline(&state, na, p->flags);183 if (err)184 goto out;185 }186 }187 188out:189 for (na = 0; na < state.max; na++)190 __mock_hwsp_record(&state, na, NULL);191 kfree(state.history);192err_put:193 mock_destroy_device(i915);194 return err;195}196 197struct __igt_sync {198 const char *name;199 u32 seqno;200 bool expected;201 bool set;202};203 204static int __igt_sync(struct intel_timeline *tl,205 u64 ctx,206 const struct __igt_sync *p,207 const char *name)208{209 int ret;210 211 if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {212 pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n",213 name, p->name, ctx, p->seqno, str_yes_no(p->expected));214 return -EINVAL;215 }216 217 if (p->set) {218 ret = __intel_timeline_sync_set(tl, ctx, p->seqno);219 if (ret)220 return ret;221 }222 223 return 0;224}225 226static int igt_sync(void *arg)227{228 const struct __igt_sync pass[] = {229 { "unset", 0, false, false },230 { "new", 0, false, true },231 { "0a", 0, true, true },232 { "1a", 1, false, true },233 { "1b", 1, true, true },234 { "0b", 0, true, false },235 { "2a", 2, false, true },236 { "4", 4, false, true },237 { "INT_MAX", INT_MAX, false, true },238 { "INT_MAX-1", INT_MAX-1, true, false },239 { "INT_MAX+1", (u32)INT_MAX+1, false, true },240 { "INT_MAX", INT_MAX, true, false },241 { "UINT_MAX", UINT_MAX, false, true },242 { "wrap", 0, false, true },243 { "unwrap", UINT_MAX, true, false },244 {},245 }, *p;246 struct intel_timeline tl;247 int order, offset;248 int ret = -ENODEV;249 250 mock_timeline_init(&tl, 0);251 for (p = pass; p->name; p++) {252 for (order = 1; order < 64; order++) {253 for (offset = -1; offset <= (order > 1); offset++) {254 u64 ctx = BIT_ULL(order) + offset;255 256 ret = __igt_sync(&tl, ctx, p, "1");257 if (ret)258 goto out;259 }260 }261 }262 mock_timeline_fini(&tl);263 264 mock_timeline_init(&tl, 0);265 for (order = 1; order < 64; order++) {266 for (offset = -1; offset <= (order > 1); offset++) {267 u64 ctx = BIT_ULL(order) + offset;268 269 for (p = pass; p->name; p++) {270 ret = __igt_sync(&tl, ctx, p, "2");271 if (ret)272 goto out;273 }274 }275 }276 277out:278 mock_timeline_fini(&tl);279 return ret;280}281 282static unsigned int random_engine(struct rnd_state *rnd)283{284 return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd);285}286 287static int bench_sync(void *arg)288{289 struct rnd_state prng;290 struct intel_timeline tl;291 unsigned long end_time, count;292 u64 prng32_1M;293 ktime_t kt;294 int order, last_order;295 296 mock_timeline_init(&tl, 0);297 298 /* Lookups from cache are very fast and so the random number generation299 * and the loop itself becomes a significant factor in the per-iteration300 * timings. We try to compensate the results by measuring the overhead301 * of the prng and subtract it from the reported results.302 */303 prandom_seed_state(&prng, i915_selftest.random_seed);304 count = 0;305 kt = ktime_get();306 end_time = jiffies + HZ/10;307 do {308 u32 x;309 310 /* Make sure the compiler doesn't optimise away the prng call */311 WRITE_ONCE(x, prandom_u32_state(&prng));312 313 count++;314 } while (!time_after(jiffies, end_time));315 kt = ktime_sub(ktime_get(), kt);316 pr_debug("%s: %lu random evaluations, %lluns/prng\n",317 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));318 prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count);319 320 /* Benchmark (only) setting random context ids */321 prandom_seed_state(&prng, i915_selftest.random_seed);322 count = 0;323 kt = ktime_get();324 end_time = jiffies + HZ/10;325 do {326 u64 id = i915_prandom_u64_state(&prng);327 328 __intel_timeline_sync_set(&tl, id, 0);329 count++;330 } while (!time_after(jiffies, end_time));331 kt = ktime_sub(ktime_get(), kt);332 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);333 pr_info("%s: %lu random insertions, %lluns/insert\n",334 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));335 336 /* Benchmark looking up the exact same context ids as we just set */337 prandom_seed_state(&prng, i915_selftest.random_seed);338 end_time = count;339 kt = ktime_get();340 while (end_time--) {341 u64 id = i915_prandom_u64_state(&prng);342 343 if (!__intel_timeline_sync_is_later(&tl, id, 0)) {344 mock_timeline_fini(&tl);345 pr_err("Lookup of %llu failed\n", id);346 return -EINVAL;347 }348 }349 kt = ktime_sub(ktime_get(), kt);350 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);351 pr_info("%s: %lu random lookups, %lluns/lookup\n",352 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));353 354 mock_timeline_fini(&tl);355 cond_resched();356 357 mock_timeline_init(&tl, 0);358 359 /* Benchmark setting the first N (in order) contexts */360 count = 0;361 kt = ktime_get();362 end_time = jiffies + HZ/10;363 do {364 __intel_timeline_sync_set(&tl, count++, 0);365 } while (!time_after(jiffies, end_time));366 kt = ktime_sub(ktime_get(), kt);367 pr_info("%s: %lu in-order insertions, %lluns/insert\n",368 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));369 370 /* Benchmark looking up the exact same context ids as we just set */371 end_time = count;372 kt = ktime_get();373 while (end_time--) {374 if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) {375 pr_err("Lookup of %lu failed\n", end_time);376 mock_timeline_fini(&tl);377 return -EINVAL;378 }379 }380 kt = ktime_sub(ktime_get(), kt);381 pr_info("%s: %lu in-order lookups, %lluns/lookup\n",382 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));383 384 mock_timeline_fini(&tl);385 cond_resched();386 387 mock_timeline_init(&tl, 0);388 389 /* Benchmark searching for a random context id and maybe changing it */390 prandom_seed_state(&prng, i915_selftest.random_seed);391 count = 0;392 kt = ktime_get();393 end_time = jiffies + HZ/10;394 do {395 u32 id = random_engine(&prng);396 u32 seqno = prandom_u32_state(&prng);397 398 if (!__intel_timeline_sync_is_later(&tl, id, seqno))399 __intel_timeline_sync_set(&tl, id, seqno);400 401 count++;402 } while (!time_after(jiffies, end_time));403 kt = ktime_sub(ktime_get(), kt);404 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);405 pr_info("%s: %lu repeated insert/lookups, %lluns/op\n",406 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));407 mock_timeline_fini(&tl);408 cond_resched();409 410 /* Benchmark searching for a known context id and changing the seqno */411 for (last_order = 1, order = 1; order < 32;412 ({ int tmp = last_order; last_order = order; order += tmp; })) {413 unsigned int mask = BIT(order) - 1;414 415 mock_timeline_init(&tl, 0);416 417 count = 0;418 kt = ktime_get();419 end_time = jiffies + HZ/10;420 do {421 /* Without assuming too many details of the underlying422 * implementation, try to identify its phase-changes423 * (if any)!424 */425 u64 id = (u64)(count & mask) << order;426 427 __intel_timeline_sync_is_later(&tl, id, 0);428 __intel_timeline_sync_set(&tl, id, 0);429 430 count++;431 } while (!time_after(jiffies, end_time));432 kt = ktime_sub(ktime_get(), kt);433 pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n",434 __func__, count, order,435 (long long)div64_ul(ktime_to_ns(kt), count));436 mock_timeline_fini(&tl);437 cond_resched();438 }439 440 return 0;441}442 443int intel_timeline_mock_selftests(void)444{445 static const struct i915_subtest tests[] = {446 SUBTEST(mock_hwsp_freelist),447 SUBTEST(igt_sync),448 SUBTEST(bench_sync),449 };450 451 return i915_subtests(tests, NULL);452}453 454static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value)455{456 u32 *cs;457 458 cs = intel_ring_begin(rq, 4);459 if (IS_ERR(cs))460 return PTR_ERR(cs);461 462 if (GRAPHICS_VER(rq->i915) >= 8) {463 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;464 *cs++ = addr;465 *cs++ = 0;466 *cs++ = value;467 } else if (GRAPHICS_VER(rq->i915) >= 4) {468 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;469 *cs++ = 0;470 *cs++ = addr;471 *cs++ = value;472 } else {473 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;474 *cs++ = addr;475 *cs++ = value;476 *cs++ = MI_NOOP;477 }478 479 intel_ring_advance(rq, cs);480 481 return 0;482}483 484static struct i915_request *485checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)486{487 struct i915_request *rq;488 int err;489 490 err = selftest_tl_pin(tl);491 if (err) {492 rq = ERR_PTR(err);493 goto out;494 }495 496 if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {497 pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",498 *tl->hwsp_seqno, tl->seqno);499 intel_timeline_unpin(tl);500 return ERR_PTR(-EINVAL);501 }502 503 rq = intel_engine_create_kernel_request(engine);504 if (IS_ERR(rq))505 goto out_unpin;506 507 i915_request_get(rq);508 509 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value);510 i915_request_add(rq);511 if (err) {512 i915_request_put(rq);513 rq = ERR_PTR(err);514 }515 516out_unpin:517 intel_timeline_unpin(tl);518out:519 if (IS_ERR(rq))520 pr_err("Failed to write to timeline!\n");521 return rq;522}523 524static int live_hwsp_engine(void *arg)525{526#define NUM_TIMELINES 4096527 struct intel_gt *gt = arg;528 struct intel_timeline **timelines;529 struct intel_engine_cs *engine;530 enum intel_engine_id id;531 unsigned long count, n;532 int err = 0;533 534 /*535 * Create a bunch of timelines and check we can write536 * independently to each of their breadcrumb slots.537 */538 539 timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES,540 sizeof(*timelines),541 GFP_KERNEL);542 if (!timelines)543 return -ENOMEM;544 545 count = 0;546 for_each_engine(engine, gt, id) {547 if (!intel_engine_can_store_dword(engine))548 continue;549 550 intel_engine_pm_get(engine);551 552 for (n = 0; n < NUM_TIMELINES; n++) {553 struct intel_timeline *tl;554 struct i915_request *rq;555 556 tl = intel_timeline_create(gt);557 if (IS_ERR(tl)) {558 err = PTR_ERR(tl);559 break;560 }561 562 rq = checked_tl_write(tl, engine, count);563 if (IS_ERR(rq)) {564 intel_timeline_put(tl);565 err = PTR_ERR(rq);566 break;567 }568 569 timelines[count++] = tl;570 i915_request_put(rq);571 }572 573 intel_engine_pm_put(engine);574 if (err)575 break;576 }577 578 if (igt_flush_test(gt->i915))579 err = -EIO;580 581 for (n = 0; n < count; n++) {582 struct intel_timeline *tl = timelines[n];583 584 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {585 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n",586 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);587 GEM_TRACE_DUMP();588 err = -EINVAL;589 }590 intel_timeline_put(tl);591 }592 593 kvfree(timelines);594 return err;595#undef NUM_TIMELINES596}597 598static int live_hwsp_alternate(void *arg)599{600#define NUM_TIMELINES 4096601 struct intel_gt *gt = arg;602 struct intel_timeline **timelines;603 struct intel_engine_cs *engine;604 enum intel_engine_id id;605 unsigned long count, n;606 int err = 0;607 608 /*609 * Create a bunch of timelines and check we can write610 * independently to each of their breadcrumb slots with adjacent611 * engines.612 */613 614 timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES,615 sizeof(*timelines),616 GFP_KERNEL);617 if (!timelines)618 return -ENOMEM;619 620 count = 0;621 for (n = 0; n < NUM_TIMELINES; n++) {622 for_each_engine(engine, gt, id) {623 struct intel_timeline *tl;624 struct i915_request *rq;625 626 if (!intel_engine_can_store_dword(engine))627 continue;628 629 tl = intel_timeline_create(gt);630 if (IS_ERR(tl)) {631 err = PTR_ERR(tl);632 goto out;633 }634 635 intel_engine_pm_get(engine);636 rq = checked_tl_write(tl, engine, count);637 intel_engine_pm_put(engine);638 if (IS_ERR(rq)) {639 intel_timeline_put(tl);640 err = PTR_ERR(rq);641 goto out;642 }643 644 timelines[count++] = tl;645 i915_request_put(rq);646 }647 }648 649out:650 if (igt_flush_test(gt->i915))651 err = -EIO;652 653 for (n = 0; n < count; n++) {654 struct intel_timeline *tl = timelines[n];655 656 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {657 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n",658 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);659 GEM_TRACE_DUMP();660 err = -EINVAL;661 }662 intel_timeline_put(tl);663 }664 665 kvfree(timelines);666 return err;667#undef NUM_TIMELINES668}669 670static int live_hwsp_wrap(void *arg)671{672 struct intel_gt *gt = arg;673 struct intel_engine_cs *engine;674 struct intel_timeline *tl;675 enum intel_engine_id id;676 int err = 0;677 678 /*679 * Across a seqno wrap, we need to keep the old cacheline alive for680 * foreign GPU references.681 */682 683 tl = intel_timeline_create(gt);684 if (IS_ERR(tl))685 return PTR_ERR(tl);686 687 if (!tl->has_initial_breadcrumb)688 goto out_free;689 690 err = selftest_tl_pin(tl);691 if (err)692 goto out_free;693 694 for_each_engine(engine, gt, id) {695 const u32 *hwsp_seqno[2];696 struct i915_request *rq;697 u32 seqno[2];698 699 if (!intel_engine_can_store_dword(engine))700 continue;701 702 rq = intel_engine_create_kernel_request(engine);703 if (IS_ERR(rq)) {704 err = PTR_ERR(rq);705 goto out;706 }707 708 tl->seqno = -4u;709 710 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);711 err = intel_timeline_get_seqno(tl, rq, &seqno[0]);712 mutex_unlock(&tl->mutex);713 if (err) {714 i915_request_add(rq);715 goto out;716 }717 pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n",718 seqno[0], tl->hwsp_offset);719 720 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]);721 if (err) {722 i915_request_add(rq);723 goto out;724 }725 hwsp_seqno[0] = tl->hwsp_seqno;726 727 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);728 err = intel_timeline_get_seqno(tl, rq, &seqno[1]);729 mutex_unlock(&tl->mutex);730 if (err) {731 i915_request_add(rq);732 goto out;733 }734 pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n",735 seqno[1], tl->hwsp_offset);736 737 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]);738 if (err) {739 i915_request_add(rq);740 goto out;741 }742 hwsp_seqno[1] = tl->hwsp_seqno;743 744 /* With wrap should come a new hwsp */745 GEM_BUG_ON(seqno[1] >= seqno[0]);746 GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]);747 748 i915_request_add(rq);749 750 if (i915_request_wait(rq, 0, HZ / 5) < 0) {751 pr_err("Wait for timeline writes timed out!\n");752 err = -EIO;753 goto out;754 }755 756 if (READ_ONCE(*hwsp_seqno[0]) != seqno[0] ||757 READ_ONCE(*hwsp_seqno[1]) != seqno[1]) {758 pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n",759 *hwsp_seqno[0], *hwsp_seqno[1],760 seqno[0], seqno[1]);761 err = -EINVAL;762 goto out;763 }764 765 intel_gt_retire_requests(gt); /* recycle HWSP */766 }767 768out:769 if (igt_flush_test(gt->i915))770 err = -EIO;771 772 intel_timeline_unpin(tl);773out_free:774 intel_timeline_put(tl);775 return err;776}777 778static int emit_read_hwsp(struct i915_request *rq,779 u32 seqno, u32 hwsp,780 u32 *addr)781{782 const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0));783 u32 *cs;784 785 cs = intel_ring_begin(rq, 12);786 if (IS_ERR(cs))787 return PTR_ERR(cs);788 789 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;790 *cs++ = *addr;791 *cs++ = 0;792 *cs++ = seqno;793 *addr += 4;794 795 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT;796 *cs++ = gpr;797 *cs++ = hwsp;798 *cs++ = 0;799 800 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;801 *cs++ = gpr;802 *cs++ = *addr;803 *cs++ = 0;804 *addr += 4;805 806 intel_ring_advance(rq, cs);807 808 return 0;809}810 811struct hwsp_watcher {812 struct i915_vma *vma;813 struct i915_request *rq;814 u32 addr;815 u32 *map;816};817 818static bool cmp_lt(u32 a, u32 b)819{820 return a < b;821}822 823static bool cmp_gte(u32 a, u32 b)824{825 return a >= b;826}827 828static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt,829 struct intel_timeline *tl)830{831 struct drm_i915_gem_object *obj;832 struct i915_vma *vma;833 834 obj = i915_gem_object_create_internal(gt->i915, SZ_2M);835 if (IS_ERR(obj))836 return PTR_ERR(obj);837 838 /* keep the same cache settings as timeline */839 i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index);840 w->map = i915_gem_object_pin_map_unlocked(obj,841 page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping));842 if (IS_ERR(w->map)) {843 i915_gem_object_put(obj);844 return PTR_ERR(w->map);845 }846 847 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);848 if (IS_ERR(vma)) {849 i915_gem_object_put(obj);850 return PTR_ERR(vma);851 }852 853 w->vma = vma;854 w->addr = i915_ggtt_offset(vma);855 return 0;856}857 858static void switch_tl_lock(struct i915_request *from, struct i915_request *to)859{860 /* some light mutex juggling required; think co-routines */861 862 if (from) {863 lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie);864 mutex_unlock(&from->context->timeline->mutex);865 }866 867 if (to) {868 mutex_lock(&to->context->timeline->mutex);869 to->cookie = lockdep_pin_lock(&to->context->timeline->mutex);870 }871}872 873static int create_watcher(struct hwsp_watcher *w,874 struct intel_engine_cs *engine,875 int ringsz)876{877 struct intel_context *ce;878 879 ce = intel_context_create(engine);880 if (IS_ERR(ce))881 return PTR_ERR(ce);882 883 ce->ring_size = ringsz;884 w->rq = intel_context_create_request(ce);885 intel_context_put(ce);886 if (IS_ERR(w->rq))887 return PTR_ERR(w->rq);888 889 w->addr = i915_ggtt_offset(w->vma);890 891 switch_tl_lock(w->rq, NULL);892 893 return 0;894}895 896static int check_watcher(struct hwsp_watcher *w, const char *name,897 bool (*op)(u32 hwsp, u32 seqno))898{899 struct i915_request *rq = fetch_and_zero(&w->rq);900 u32 offset, end;901 int err;902 903 GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size);904 905 i915_request_get(rq);906 switch_tl_lock(NULL, rq);907 i915_request_add(rq);908 909 if (i915_request_wait(rq, 0, HZ) < 0) {910 err = -ETIME;911 goto out;912 }913 914 err = 0;915 offset = 0;916 end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map);917 while (offset < end) {918 if (!op(w->map[offset + 1], w->map[offset])) {919 pr_err("Watcher '%s' found HWSP value %x for seqno %x\n",920 name, w->map[offset + 1], w->map[offset]);921 err = -EINVAL;922 }923 924 offset += 2;925 }926 927out:928 i915_request_put(rq);929 return err;930}931 932static void cleanup_watcher(struct hwsp_watcher *w)933{934 if (w->rq) {935 switch_tl_lock(NULL, w->rq);936 937 i915_request_add(w->rq);938 }939 940 i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP);941}942 943static bool retire_requests(struct intel_timeline *tl)944{945 struct i915_request *rq, *rn;946 947 mutex_lock(&tl->mutex);948 list_for_each_entry_safe(rq, rn, &tl->requests, link)949 if (!i915_request_retire(rq))950 break;951 mutex_unlock(&tl->mutex);952 953 return !i915_active_fence_isset(&tl->last_request);954}955 956static struct i915_request *wrap_timeline(struct i915_request *rq)957{958 struct intel_context *ce = rq->context;959 struct intel_timeline *tl = ce->timeline;960 u32 seqno = rq->fence.seqno;961 962 while (tl->seqno >= seqno) { /* Cause a wrap */963 i915_request_put(rq);964 rq = intel_context_create_request(ce);965 if (IS_ERR(rq))966 return rq;967 968 i915_request_get(rq);969 i915_request_add(rq);970 }971 972 i915_request_put(rq);973 rq = i915_request_create(ce);974 if (IS_ERR(rq))975 return rq;976 977 i915_request_get(rq);978 i915_request_add(rq);979 980 return rq;981}982 983static int live_hwsp_read(void *arg)984{985 struct intel_gt *gt = arg;986 struct hwsp_watcher watcher[2] = {};987 struct intel_engine_cs *engine;988 struct intel_timeline *tl;989 enum intel_engine_id id;990 int err = 0;991 int i;992 993 /*994 * If we take a reference to the HWSP for reading on the GPU, that995 * read may be arbitrarily delayed (either by foreign fence or996 * priority saturation) and a wrap can happen within 30 minutes.997 * When the GPU read is finally submitted it should be correct,998 * even across multiple wraps.999 */1000 1001 if (GRAPHICS_VER(gt->i915) < 8) /* CS convenience [SRM/LRM] */1002 return 0;1003 1004 tl = intel_timeline_create(gt);1005 if (IS_ERR(tl))1006 return PTR_ERR(tl);1007 1008 if (!tl->has_initial_breadcrumb)1009 goto out_free;1010 1011 selftest_tl_pin(tl);1012 1013 for (i = 0; i < ARRAY_SIZE(watcher); i++) {1014 err = setup_watcher(&watcher[i], gt, tl);1015 if (err)1016 goto out;1017 }1018 1019 for_each_engine(engine, gt, id) {1020 struct intel_context *ce;1021 unsigned long count = 0;1022 IGT_TIMEOUT(end_time);1023 1024 /* Create a request we can use for remote reading of the HWSP */1025 err = create_watcher(&watcher[1], engine, SZ_512K);1026 if (err)1027 goto out;1028 1029 do {1030 struct i915_sw_fence *submit;1031 struct i915_request *rq;1032 u32 hwsp, dummy;1033 1034 submit = heap_fence_create(GFP_KERNEL);1035 if (!submit) {1036 err = -ENOMEM;1037 goto out;1038 }1039 1040 err = create_watcher(&watcher[0], engine, SZ_4K);1041 if (err)1042 goto out;1043 1044 ce = intel_context_create(engine);1045 if (IS_ERR(ce)) {1046 err = PTR_ERR(ce);1047 goto out;1048 }1049 1050 ce->timeline = intel_timeline_get(tl);1051 1052 /* Ensure timeline is mapped, done during first pin */1053 err = intel_context_pin(ce);1054 if (err) {1055 intel_context_put(ce);1056 goto out;1057 }1058 1059 /*1060 * Start at a new wrap, and set seqno right before another wrap,1061 * saving 30 minutes of nops1062 */1063 tl->seqno = -12u + 2 * (count & 3);1064 __intel_timeline_get_seqno(tl, &dummy);1065 1066 rq = i915_request_create(ce);1067 if (IS_ERR(rq)) {1068 err = PTR_ERR(rq);1069 intel_context_unpin(ce);1070 intel_context_put(ce);1071 goto out;1072 }1073 1074 err = i915_sw_fence_await_dma_fence(&rq->submit,1075 &watcher[0].rq->fence, 0,1076 GFP_KERNEL);1077 if (err < 0) {1078 i915_request_add(rq);1079 intel_context_unpin(ce);1080 intel_context_put(ce);1081 goto out;1082 }1083 1084 switch_tl_lock(rq, watcher[0].rq);1085 err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp);1086 if (err == 0)1087 err = emit_read_hwsp(watcher[0].rq, /* before */1088 rq->fence.seqno, hwsp,1089 &watcher[0].addr);1090 switch_tl_lock(watcher[0].rq, rq);1091 if (err) {1092 i915_request_add(rq);1093 intel_context_unpin(ce);1094 intel_context_put(ce);1095 goto out;1096 }1097 1098 switch_tl_lock(rq, watcher[1].rq);1099 err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp);1100 if (err == 0)1101 err = emit_read_hwsp(watcher[1].rq, /* after */1102 rq->fence.seqno, hwsp,1103 &watcher[1].addr);1104 switch_tl_lock(watcher[1].rq, rq);1105 if (err) {1106 i915_request_add(rq);1107 intel_context_unpin(ce);1108 intel_context_put(ce);1109 goto out;1110 }1111 1112 i915_request_get(rq);1113 i915_request_add(rq);1114 1115 rq = wrap_timeline(rq);1116 intel_context_unpin(ce);1117 intel_context_put(ce);1118 if (IS_ERR(rq)) {1119 err = PTR_ERR(rq);1120 goto out;1121 }1122 1123 err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit,1124 &rq->fence, 0,1125 GFP_KERNEL);1126 if (err < 0) {1127 i915_request_put(rq);1128 goto out;1129 }1130 1131 err = check_watcher(&watcher[0], "before", cmp_lt);1132 i915_sw_fence_commit(submit);1133 heap_fence_put(submit);1134 if (err) {1135 i915_request_put(rq);1136 goto out;1137 }1138 count++;1139 1140 /* Flush the timeline before manually wrapping again */1141 if (i915_request_wait(rq,1142 I915_WAIT_INTERRUPTIBLE,1143 HZ) < 0) {1144 err = -ETIME;1145 i915_request_put(rq);1146 goto out;1147 }1148 retire_requests(tl);1149 i915_request_put(rq);1150 1151 /* Single requests are limited to half a ring at most */1152 if (8 * watcher[1].rq->ring->emit >1153 3 * watcher[1].rq->ring->size)1154 break;1155 1156 } while (!__igt_timeout(end_time, NULL) &&1157 count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2);1158 1159 pr_info("%s: simulated %lu wraps\n", engine->name, count);1160 err = check_watcher(&watcher[1], "after", cmp_gte);1161 if (err)1162 goto out;1163 }1164 1165out:1166 for (i = 0; i < ARRAY_SIZE(watcher); i++)1167 cleanup_watcher(&watcher[i]);1168 1169 intel_timeline_unpin(tl);1170 1171 if (igt_flush_test(gt->i915))1172 err = -EIO;1173 1174out_free:1175 intel_timeline_put(tl);1176 return err;1177}1178 1179static int live_hwsp_rollover_kernel(void *arg)1180{1181 struct intel_gt *gt = arg;1182 struct intel_engine_cs *engine;1183 enum intel_engine_id id;1184 int err = 0;1185 1186 /*1187 * Run the host for long enough, and even the kernel context will1188 * see a seqno rollover.1189 */1190 1191 for_each_engine(engine, gt, id) {1192 struct intel_context *ce = engine->kernel_context;1193 struct intel_timeline *tl = ce->timeline;1194 struct i915_request *rq[3] = {};1195 int i;1196 1197 st_engine_heartbeat_disable(engine);1198 if (intel_gt_wait_for_idle(gt, HZ / 2)) {1199 err = -EIO;1200 goto out;1201 }1202 1203 GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));1204 tl->seqno = -2u;1205 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);1206 1207 for (i = 0; i < ARRAY_SIZE(rq); i++) {1208 struct i915_request *this;1209 1210 this = i915_request_create(ce);1211 if (IS_ERR(this)) {1212 err = PTR_ERR(this);1213 goto out;1214 }1215 1216 pr_debug("%s: create fence.seqnp:%d\n",1217 engine->name,1218 lower_32_bits(this->fence.seqno));1219 1220 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);1221 1222 rq[i] = i915_request_get(this);1223 i915_request_add(this);1224 }1225 1226 /* We expected a wrap! */1227 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno);1228 1229 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {1230 pr_err("Wait for timeline wrap timed out!\n");1231 err = -EIO;1232 goto out;1233 }1234 1235 for (i = 0; i < ARRAY_SIZE(rq); i++) {1236 if (!i915_request_completed(rq[i])) {1237 pr_err("Pre-wrap request not completed!\n");1238 err = -EINVAL;1239 goto out;1240 }1241 }1242 1243out:1244 for (i = 0; i < ARRAY_SIZE(rq); i++)1245 i915_request_put(rq[i]);1246 st_engine_heartbeat_enable(engine);1247 if (err)1248 break;1249 }1250 1251 if (igt_flush_test(gt->i915))1252 err = -EIO;1253 1254 return err;1255}1256 1257static int live_hwsp_rollover_user(void *arg)1258{1259 struct intel_gt *gt = arg;1260 struct intel_engine_cs *engine;1261 enum intel_engine_id id;1262 int err = 0;1263 1264 /*1265 * Simulate a long running user context, and force the seqno wrap1266 * on the user's timeline.1267 */1268 1269 for_each_engine(engine, gt, id) {1270 struct i915_request *rq[3] = {};1271 struct intel_timeline *tl;1272 struct intel_context *ce;1273 int i;1274 1275 ce = intel_context_create(engine);1276 if (IS_ERR(ce))1277 return PTR_ERR(ce);1278 1279 err = intel_context_alloc_state(ce);1280 if (err)1281 goto out;1282 1283 tl = ce->timeline;1284 if (!tl->has_initial_breadcrumb)1285 goto out;1286 1287 err = intel_context_pin(ce);1288 if (err)1289 goto out;1290 1291 tl->seqno = -4u;1292 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);1293 1294 for (i = 0; i < ARRAY_SIZE(rq); i++) {1295 struct i915_request *this;1296 1297 this = intel_context_create_request(ce);1298 if (IS_ERR(this)) {1299 err = PTR_ERR(this);1300 goto out_unpin;1301 }1302 1303 pr_debug("%s: create fence.seqnp:%d\n",1304 engine->name,1305 lower_32_bits(this->fence.seqno));1306 1307 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);1308 1309 rq[i] = i915_request_get(this);1310 i915_request_add(this);1311 }1312 1313 /* We expected a wrap! */1314 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno);1315 1316 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {1317 pr_err("Wait for timeline wrap timed out!\n");1318 err = -EIO;1319 goto out_unpin;1320 }1321 1322 for (i = 0; i < ARRAY_SIZE(rq); i++) {1323 if (!i915_request_completed(rq[i])) {1324 pr_err("Pre-wrap request not completed!\n");1325 err = -EINVAL;1326 goto out_unpin;1327 }1328 }1329out_unpin:1330 intel_context_unpin(ce);1331out:1332 for (i = 0; i < ARRAY_SIZE(rq); i++)1333 i915_request_put(rq[i]);1334 intel_context_put(ce);1335 if (err)1336 break;1337 }1338 1339 if (igt_flush_test(gt->i915))1340 err = -EIO;1341 1342 return err;1343}1344 1345static int live_hwsp_recycle(void *arg)1346{1347 struct intel_gt *gt = arg;1348 struct intel_engine_cs *engine;1349 enum intel_engine_id id;1350 unsigned long count;1351 int err = 0;1352 1353 /*1354 * Check seqno writes into one timeline at a time. We expect to1355 * recycle the breadcrumb slot between iterations and neither1356 * want to confuse ourselves or the GPU.1357 */1358 1359 count = 0;1360 for_each_engine(engine, gt, id) {1361 IGT_TIMEOUT(end_time);1362 1363 if (!intel_engine_can_store_dword(engine))1364 continue;1365 1366 intel_engine_pm_get(engine);1367 1368 do {1369 struct intel_timeline *tl;1370 struct i915_request *rq;1371 1372 tl = intel_timeline_create(gt);1373 if (IS_ERR(tl)) {1374 err = PTR_ERR(tl);1375 break;1376 }1377 1378 rq = checked_tl_write(tl, engine, count);1379 if (IS_ERR(rq)) {1380 intel_timeline_put(tl);1381 err = PTR_ERR(rq);1382 break;1383 }1384 1385 if (i915_request_wait(rq, 0, HZ / 5) < 0) {1386 pr_err("Wait for timeline writes timed out!\n");1387 i915_request_put(rq);1388 intel_timeline_put(tl);1389 err = -EIO;1390 break;1391 }1392 1393 if (READ_ONCE(*tl->hwsp_seqno) != count) {1394 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n",1395 count, tl->fence_context,1396 tl->hwsp_offset, *tl->hwsp_seqno);1397 GEM_TRACE_DUMP();1398 err = -EINVAL;1399 }1400 1401 i915_request_put(rq);1402 intel_timeline_put(tl);1403 count++;1404 1405 if (err)1406 break;1407 } while (!__igt_timeout(end_time, NULL));1408 1409 intel_engine_pm_put(engine);1410 if (err)1411 break;1412 }1413 1414 return err;1415}1416 1417int intel_timeline_live_selftests(struct drm_i915_private *i915)1418{1419 static const struct i915_subtest tests[] = {1420 SUBTEST(live_hwsp_recycle),1421 SUBTEST(live_hwsp_engine),1422 SUBTEST(live_hwsp_alternate),1423 SUBTEST(live_hwsp_wrap),1424 SUBTEST(live_hwsp_read),1425 SUBTEST(live_hwsp_rollover_kernel),1426 SUBTEST(live_hwsp_rollover_user),1427 };1428 1429 if (intel_gt_is_wedged(to_gt(i915)))1430 return 0;1431 1432 return intel_gt_live_subtests(tests, to_gt(i915));1433}1434