1086 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2019 Intel Corporation4 */5 6#include <drm/drm_managed.h>7#include <drm/intel/intel-gtt.h>8 9#include "gem/i915_gem_internal.h"10#include "gem/i915_gem_lmem.h"11 12#include "i915_drv.h"13#include "i915_perf_oa_regs.h"14#include "i915_reg.h"15#include "intel_context.h"16#include "intel_engine_pm.h"17#include "intel_engine_regs.h"18#include "intel_ggtt_gmch.h"19#include "intel_gt.h"20#include "intel_gt_buffer_pool.h"21#include "intel_gt_clock_utils.h"22#include "intel_gt_debugfs.h"23#include "intel_gt_mcr.h"24#include "intel_gt_pm.h"25#include "intel_gt_print.h"26#include "intel_gt_regs.h"27#include "intel_gt_requests.h"28#include "intel_migrate.h"29#include "intel_mocs.h"30#include "intel_pci_config.h"31#include "intel_rc6.h"32#include "intel_renderstate.h"33#include "intel_rps.h"34#include "intel_sa_media.h"35#include "intel_gt_sysfs.h"36#include "intel_tlb.h"37#include "intel_uncore.h"38#include "shmem_utils.h"39 40void intel_gt_common_init_early(struct intel_gt *gt)41{42 spin_lock_init(gt->irq_lock);43 44 INIT_LIST_HEAD(>->closed_vma);45 spin_lock_init(>->closed_lock);46 47 init_llist_head(>->watchdog.list);48 INIT_WORK(>->watchdog.work, intel_gt_watchdog_work);49 50 intel_gt_init_buffer_pool(gt);51 intel_gt_init_reset(gt);52 intel_gt_init_requests(gt);53 intel_gt_init_timelines(gt);54 intel_gt_init_tlb(gt);55 intel_gt_pm_init_early(gt);56 57 intel_wopcm_init_early(>->wopcm);58 intel_uc_init_early(>->uc);59 intel_rps_init_early(>->rps);60}61 62/* Preliminary initialization of Tile 0 */63int intel_root_gt_init_early(struct drm_i915_private *i915)64{65 struct intel_gt *gt;66 67 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL);68 if (!gt)69 return -ENOMEM;70 71 i915->gt[0] = gt;72 73 gt->i915 = i915;74 gt->uncore = &i915->uncore;75 gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL);76 if (!gt->irq_lock)77 return -ENOMEM;78 79 intel_gt_common_init_early(gt);80 81 return 0;82}83 84static int intel_gt_probe_lmem(struct intel_gt *gt)85{86 struct drm_i915_private *i915 = gt->i915;87 unsigned int instance = gt->info.id;88 int id = INTEL_REGION_LMEM_0 + instance;89 struct intel_memory_region *mem;90 int err;91 92 mem = intel_gt_setup_lmem(gt);93 if (IS_ERR(mem)) {94 err = PTR_ERR(mem);95 if (err == -ENODEV)96 return 0;97 98 gt_err(gt, "Failed to setup region(%d) type=%d\n",99 err, INTEL_MEMORY_LOCAL);100 return err;101 }102 103 mem->id = id;104 mem->instance = instance;105 106 intel_memory_region_set_name(mem, "local%u", mem->instance);107 108 GEM_BUG_ON(!HAS_REGION(i915, id));109 GEM_BUG_ON(i915->mm.regions[id]);110 i915->mm.regions[id] = mem;111 112 return 0;113}114 115int intel_gt_assign_ggtt(struct intel_gt *gt)116{117 /* Media GT shares primary GT's GGTT */118 if (gt->type == GT_MEDIA) {119 gt->ggtt = to_gt(gt->i915)->ggtt;120 } else {121 gt->ggtt = i915_ggtt_create(gt->i915);122 if (IS_ERR(gt->ggtt))123 return PTR_ERR(gt->ggtt);124 }125 126 list_add_tail(>->ggtt_link, >->ggtt->gt_list);127 128 return 0;129}130 131int intel_gt_init_mmio(struct intel_gt *gt)132{133 intel_gt_init_clock_frequency(gt);134 135 intel_uc_init_mmio(>->uc);136 intel_sseu_info_init(gt);137 intel_gt_mcr_init(gt);138 139 return intel_engines_init_mmio(gt);140}141 142static void init_unused_ring(struct intel_gt *gt, u32 base)143{144 struct intel_uncore *uncore = gt->uncore;145 146 intel_uncore_write(uncore, RING_CTL(base), 0);147 intel_uncore_write(uncore, RING_HEAD(base), 0);148 intel_uncore_write(uncore, RING_TAIL(base), 0);149 intel_uncore_write(uncore, RING_START(base), 0);150}151 152static void init_unused_rings(struct intel_gt *gt)153{154 struct drm_i915_private *i915 = gt->i915;155 156 if (IS_I830(i915)) {157 init_unused_ring(gt, PRB1_BASE);158 init_unused_ring(gt, SRB0_BASE);159 init_unused_ring(gt, SRB1_BASE);160 init_unused_ring(gt, SRB2_BASE);161 init_unused_ring(gt, SRB3_BASE);162 } else if (GRAPHICS_VER(i915) == 2) {163 init_unused_ring(gt, SRB0_BASE);164 init_unused_ring(gt, SRB1_BASE);165 } else if (GRAPHICS_VER(i915) == 3) {166 init_unused_ring(gt, PRB1_BASE);167 init_unused_ring(gt, PRB2_BASE);168 }169}170 171int intel_gt_init_hw(struct intel_gt *gt)172{173 struct drm_i915_private *i915 = gt->i915;174 struct intel_uncore *uncore = gt->uncore;175 int ret;176 177 gt->last_init_time = ktime_get();178 179 /* Double layer security blanket, see i915_gem_init() */180 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);181 182 if (HAS_EDRAM(i915) && GRAPHICS_VER(i915) < 9)183 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));184 185 if (IS_HASWELL(i915))186 intel_uncore_write(uncore,187 HSW_MI_PREDICATE_RESULT_2,188 IS_HASWELL_GT3(i915) ?189 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);190 191 /* Apply the GT workarounds... */192 intel_gt_apply_workarounds(gt);193 /* ...and determine whether they are sticking. */194 intel_gt_verify_workarounds(gt, "init");195 196 intel_gt_init_swizzling(gt);197 198 /*199 * At least 830 can leave some of the unused rings200 * "active" (ie. head != tail) after resume which201 * will prevent c3 entry. Makes sure all unused rings202 * are totally idle.203 */204 init_unused_rings(gt);205 206 ret = i915_ppgtt_init_hw(gt);207 if (ret) {208 gt_err(gt, "Enabling PPGTT failed (%d)\n", ret);209 goto out;210 }211 212 /* We can't enable contexts until all firmware is loaded */213 ret = intel_uc_init_hw(>->uc);214 if (ret) {215 gt_probe_error(gt, "Enabling uc failed (%d)\n", ret);216 goto out;217 }218 219 intel_mocs_init(gt);220 221out:222 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);223 return ret;224}225 226static void gen6_clear_engine_error_register(struct intel_engine_cs *engine)227{228 GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);229 GEN6_RING_FAULT_REG_POSTING_READ(engine);230}231 232i915_reg_t intel_gt_perf_limit_reasons_reg(struct intel_gt *gt)233{234 /* GT0_PERF_LIMIT_REASONS is available only for Gen11+ */235 if (GRAPHICS_VER(gt->i915) < 11)236 return INVALID_MMIO_REG;237 238 return gt->type == GT_MEDIA ?239 MTL_MEDIA_PERF_LIMIT_REASONS : GT0_PERF_LIMIT_REASONS;240}241 242void243intel_gt_clear_error_registers(struct intel_gt *gt,244 intel_engine_mask_t engine_mask)245{246 struct drm_i915_private *i915 = gt->i915;247 struct intel_uncore *uncore = gt->uncore;248 u32 eir;249 250 if (GRAPHICS_VER(i915) != 2)251 intel_uncore_write(uncore, PGTBL_ER, 0);252 253 if (GRAPHICS_VER(i915) < 4)254 intel_uncore_write(uncore, IPEIR(RENDER_RING_BASE), 0);255 else256 intel_uncore_write(uncore, IPEIR_I965, 0);257 258 intel_uncore_write(uncore, EIR, 0);259 eir = intel_uncore_read(uncore, EIR);260 if (eir) {261 /*262 * some errors might have become stuck,263 * mask them.264 */265 gt_dbg(gt, "EIR stuck: 0x%08x, masking\n", eir);266 intel_uncore_rmw(uncore, EMR, 0, eir);267 intel_uncore_write(uncore, GEN2_IIR,268 I915_MASTER_ERROR_INTERRUPT);269 }270 271 /*272 * For the media GT, this ring fault register is not replicated,273 * so don't do multicast/replicated register read/write operation on it.274 */275 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) {276 intel_uncore_rmw(uncore, XELPMP_RING_FAULT_REG,277 RING_FAULT_VALID, 0);278 intel_uncore_posting_read(uncore,279 XELPMP_RING_FAULT_REG);280 281 } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {282 intel_gt_mcr_multicast_rmw(gt, XEHP_RING_FAULT_REG,283 RING_FAULT_VALID, 0);284 intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG);285 286 } else if (GRAPHICS_VER(i915) >= 12) {287 intel_uncore_rmw(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID, 0);288 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);289 } else if (GRAPHICS_VER(i915) >= 8) {290 intel_uncore_rmw(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID, 0);291 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);292 } else if (GRAPHICS_VER(i915) >= 6) {293 struct intel_engine_cs *engine;294 enum intel_engine_id id;295 296 for_each_engine_masked(engine, gt, engine_mask, id)297 gen6_clear_engine_error_register(engine);298 }299}300 301static void gen6_check_faults(struct intel_gt *gt)302{303 struct intel_engine_cs *engine;304 enum intel_engine_id id;305 u32 fault;306 307 for_each_engine(engine, gt, id) {308 fault = GEN6_RING_FAULT_REG_READ(engine);309 if (fault & RING_FAULT_VALID) {310 gt_dbg(gt, "Unexpected fault\n"311 "\tAddr: 0x%08lx\n"312 "\tAddress space: %s\n"313 "\tSource ID: %d\n"314 "\tType: %d\n",315 fault & PAGE_MASK,316 fault & RING_FAULT_GTTSEL_MASK ?317 "GGTT" : "PPGTT",318 RING_FAULT_SRCID(fault),319 RING_FAULT_FAULT_TYPE(fault));320 }321 }322}323 324static void xehp_check_faults(struct intel_gt *gt)325{326 u32 fault;327 328 /*329 * Although the fault register now lives in an MCR register range,330 * the GAM registers are special and we only truly need to read331 * the "primary" GAM instance rather than handling each instance332 * individually. intel_gt_mcr_read_any() will automatically steer333 * toward the primary instance.334 */335 fault = intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG);336 if (fault & RING_FAULT_VALID) {337 u32 fault_data0, fault_data1;338 u64 fault_addr;339 340 fault_data0 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA0);341 fault_data1 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA1);342 343 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |344 ((u64)fault_data0 << 12);345 346 gt_dbg(gt, "Unexpected fault\n"347 "\tAddr: 0x%08x_%08x\n"348 "\tAddress space: %s\n"349 "\tEngine ID: %d\n"350 "\tSource ID: %d\n"351 "\tType: %d\n",352 upper_32_bits(fault_addr), lower_32_bits(fault_addr),353 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",354 GEN8_RING_FAULT_ENGINE_ID(fault),355 RING_FAULT_SRCID(fault),356 RING_FAULT_FAULT_TYPE(fault));357 }358}359 360static void gen8_check_faults(struct intel_gt *gt)361{362 struct intel_uncore *uncore = gt->uncore;363 i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;364 u32 fault;365 366 if (GRAPHICS_VER(gt->i915) >= 12) {367 fault_reg = GEN12_RING_FAULT_REG;368 fault_data0_reg = GEN12_FAULT_TLB_DATA0;369 fault_data1_reg = GEN12_FAULT_TLB_DATA1;370 } else {371 fault_reg = GEN8_RING_FAULT_REG;372 fault_data0_reg = GEN8_FAULT_TLB_DATA0;373 fault_data1_reg = GEN8_FAULT_TLB_DATA1;374 }375 376 fault = intel_uncore_read(uncore, fault_reg);377 if (fault & RING_FAULT_VALID) {378 u32 fault_data0, fault_data1;379 u64 fault_addr;380 381 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);382 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);383 384 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |385 ((u64)fault_data0 << 12);386 387 gt_dbg(gt, "Unexpected fault\n"388 "\tAddr: 0x%08x_%08x\n"389 "\tAddress space: %s\n"390 "\tEngine ID: %d\n"391 "\tSource ID: %d\n"392 "\tType: %d\n",393 upper_32_bits(fault_addr), lower_32_bits(fault_addr),394 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",395 GEN8_RING_FAULT_ENGINE_ID(fault),396 RING_FAULT_SRCID(fault),397 RING_FAULT_FAULT_TYPE(fault));398 }399}400 401void intel_gt_check_and_clear_faults(struct intel_gt *gt)402{403 struct drm_i915_private *i915 = gt->i915;404 405 /* From GEN8 onwards we only have one 'All Engine Fault Register' */406 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))407 xehp_check_faults(gt);408 else if (GRAPHICS_VER(i915) >= 8)409 gen8_check_faults(gt);410 else if (GRAPHICS_VER(i915) >= 6)411 gen6_check_faults(gt);412 else413 return;414 415 intel_gt_clear_error_registers(gt, ALL_ENGINES);416}417 418void intel_gt_flush_ggtt_writes(struct intel_gt *gt)419{420 struct intel_uncore *uncore = gt->uncore;421 intel_wakeref_t wakeref;422 423 /*424 * No actual flushing is required for the GTT write domain for reads425 * from the GTT domain. Writes to it "immediately" go to main memory426 * as far as we know, so there's no chipset flush. It also doesn't427 * land in the GPU render cache.428 *429 * However, we do have to enforce the order so that all writes through430 * the GTT land before any writes to the device, such as updates to431 * the GATT itself.432 *433 * We also have to wait a bit for the writes to land from the GTT.434 * An uncached read (i.e. mmio) seems to be ideal for the round-trip435 * timing. This issue has only been observed when switching quickly436 * between GTT writes and CPU reads from inside the kernel on recent hw,437 * and it appears to only affect discrete GTT blocks (i.e. on LLC438 * system agents we cannot reproduce this behaviour, until Cannonlake439 * that was!).440 */441 442 wmb();443 444 if (INTEL_INFO(gt->i915)->has_coherent_ggtt)445 return;446 447 intel_gt_chipset_flush(gt);448 449 with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {450 unsigned long flags;451 452 spin_lock_irqsave(&uncore->lock, flags);453 intel_uncore_posting_read_fw(uncore,454 RING_TAIL(RENDER_RING_BASE));455 spin_unlock_irqrestore(&uncore->lock, flags);456 }457}458 459void intel_gt_chipset_flush(struct intel_gt *gt)460{461 wmb();462 if (GRAPHICS_VER(gt->i915) < 6)463 intel_ggtt_gmch_flush();464}465 466void intel_gt_driver_register(struct intel_gt *gt)467{468 intel_gsc_init(>->gsc, gt->i915);469 470 intel_rps_driver_register(>->rps);471 472 intel_gt_debugfs_register(gt);473 intel_gt_sysfs_register(gt);474}475 476static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)477{478 struct drm_i915_private *i915 = gt->i915;479 struct drm_i915_gem_object *obj;480 struct i915_vma *vma;481 int ret;482 483 obj = i915_gem_object_create_lmem(i915, size,484 I915_BO_ALLOC_VOLATILE |485 I915_BO_ALLOC_GPU_ONLY);486 if (IS_ERR(obj) && !IS_METEORLAKE(i915)) /* Wa_22018444074 */487 obj = i915_gem_object_create_stolen(i915, size);488 if (IS_ERR(obj))489 obj = i915_gem_object_create_internal(i915, size);490 if (IS_ERR(obj)) {491 gt_err(gt, "Failed to allocate scratch page\n");492 return PTR_ERR(obj);493 }494 495 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);496 if (IS_ERR(vma)) {497 ret = PTR_ERR(vma);498 goto err_unref;499 }500 501 ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);502 if (ret)503 goto err_unref;504 505 gt->scratch = i915_vma_make_unshrinkable(vma);506 507 return 0;508 509err_unref:510 i915_gem_object_put(obj);511 return ret;512}513 514static void intel_gt_fini_scratch(struct intel_gt *gt)515{516 i915_vma_unpin_and_release(>->scratch, 0);517}518 519static struct i915_address_space *kernel_vm(struct intel_gt *gt)520{521 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)522 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm;523 else524 return i915_vm_get(>->ggtt->vm);525}526 527static int __engines_record_defaults(struct intel_gt *gt)528{529 struct i915_request *requests[I915_NUM_ENGINES] = {};530 struct intel_engine_cs *engine;531 enum intel_engine_id id;532 int err = 0;533 534 /*535 * As we reset the gpu during very early sanitisation, the current536 * register state on the GPU should reflect its defaults values.537 * We load a context onto the hw (with restore-inhibit), then switch538 * over to a second context to save that default register state. We539 * can then prime every new context with that state so they all start540 * from the same default HW values.541 */542 543 for_each_engine(engine, gt, id) {544 struct intel_renderstate so;545 struct intel_context *ce;546 struct i915_request *rq;547 548 /* We must be able to switch to something! */549 GEM_BUG_ON(!engine->kernel_context);550 551 ce = intel_context_create(engine);552 if (IS_ERR(ce)) {553 err = PTR_ERR(ce);554 goto out;555 }556 557 err = intel_renderstate_init(&so, ce);558 if (err)559 goto err;560 561 rq = i915_request_create(ce);562 if (IS_ERR(rq)) {563 err = PTR_ERR(rq);564 goto err_fini;565 }566 567 err = intel_engine_emit_ctx_wa(rq);568 if (err)569 goto err_rq;570 571 err = intel_renderstate_emit(&so, rq);572 if (err)573 goto err_rq;574 575err_rq:576 requests[id] = i915_request_get(rq);577 i915_request_add(rq);578err_fini:579 intel_renderstate_fini(&so, ce);580err:581 if (err) {582 intel_context_put(ce);583 goto out;584 }585 }586 587 /* Flush the default context image to memory, and enable powersaving. */588 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {589 err = -EIO;590 goto out;591 }592 593 for (id = 0; id < ARRAY_SIZE(requests); id++) {594 struct i915_request *rq;595 struct file *state;596 597 rq = requests[id];598 if (!rq)599 continue;600 601 if (rq->fence.error) {602 err = -EIO;603 goto out;604 }605 606 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));607 if (!rq->context->state)608 continue;609 610 /* Keep a copy of the state's backing pages; free the obj */611 state = shmem_create_from_object(rq->context->state->obj);612 if (IS_ERR(state)) {613 err = PTR_ERR(state);614 goto out;615 }616 rq->engine->default_state = state;617 }618 619out:620 /*621 * If we have to abandon now, we expect the engines to be idle622 * and ready to be torn-down. The quickest way we can accomplish623 * this is by declaring ourselves wedged.624 */625 if (err)626 intel_gt_set_wedged(gt);627 628 for (id = 0; id < ARRAY_SIZE(requests); id++) {629 struct intel_context *ce;630 struct i915_request *rq;631 632 rq = requests[id];633 if (!rq)634 continue;635 636 ce = rq->context;637 i915_request_put(rq);638 intel_context_put(ce);639 }640 return err;641}642 643static int __engines_verify_workarounds(struct intel_gt *gt)644{645 struct intel_engine_cs *engine;646 enum intel_engine_id id;647 int err = 0;648 649 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))650 return 0;651 652 for_each_engine(engine, gt, id) {653 if (intel_engine_verify_workarounds(engine, "load"))654 err = -EIO;655 }656 657 /* Flush and restore the kernel context for safety */658 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)659 err = -EIO;660 661 return err;662}663 664static void __intel_gt_disable(struct intel_gt *gt)665{666 intel_gt_set_wedged_on_fini(gt);667 668 intel_gt_suspend_prepare(gt);669 intel_gt_suspend_late(gt);670 671 GEM_BUG_ON(intel_gt_pm_is_awake(gt));672}673 674int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)675{676 long remaining_timeout;677 678 /* If the device is asleep, we have no requests outstanding */679 if (!intel_gt_pm_is_awake(gt))680 return 0;681 682 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout,683 &remaining_timeout)) > 0) {684 cond_resched();685 if (signal_pending(current))686 return -EINTR;687 }688 689 if (timeout)690 return timeout;691 692 if (remaining_timeout < 0)693 remaining_timeout = 0;694 695 return intel_uc_wait_for_idle(>->uc, remaining_timeout);696}697 698int intel_gt_init(struct intel_gt *gt)699{700 int err;701 702 err = i915_inject_probe_error(gt->i915, -ENODEV);703 if (err)704 return err;705 706 intel_gt_init_workarounds(gt);707 708 /*709 * This is just a security blanket to placate dragons.710 * On some systems, we very sporadically observe that the first TLBs711 * used by the CS may be stale, despite us poking the TLB reset. If712 * we hold the forcewake during initialisation these problems713 * just magically go away.714 */715 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);716 717 err = intel_gt_init_scratch(gt,718 GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K);719 if (err)720 goto out_fw;721 722 intel_gt_pm_init(gt);723 724 gt->vm = kernel_vm(gt);725 if (!gt->vm) {726 err = -ENOMEM;727 goto err_pm;728 }729 730 intel_set_mocs_index(gt);731 732 err = intel_engines_init(gt);733 if (err)734 goto err_engines;735 736 err = intel_uc_init(>->uc);737 if (err)738 goto err_engines;739 740 err = intel_gt_resume(gt);741 if (err)742 goto err_uc_init;743 744 err = intel_gt_init_hwconfig(gt);745 if (err)746 gt_err(gt, "Failed to retrieve hwconfig table: %pe\n", ERR_PTR(err));747 748 err = __engines_record_defaults(gt);749 if (err)750 goto err_gt;751 752 err = __engines_verify_workarounds(gt);753 if (err)754 goto err_gt;755 756 err = i915_inject_probe_error(gt->i915, -EIO);757 if (err)758 goto err_gt;759 760 intel_uc_init_late(>->uc);761 762 intel_migrate_init(>->migrate, gt);763 764 goto out_fw;765err_gt:766 __intel_gt_disable(gt);767 intel_uc_fini_hw(>->uc);768err_uc_init:769 intel_uc_fini(>->uc);770err_engines:771 intel_engines_release(gt);772 i915_vm_put(fetch_and_zero(>->vm));773err_pm:774 intel_gt_pm_fini(gt);775 intel_gt_fini_scratch(gt);776out_fw:777 if (err)778 intel_gt_set_wedged_on_init(gt);779 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);780 return err;781}782 783void intel_gt_driver_remove(struct intel_gt *gt)784{785 __intel_gt_disable(gt);786 787 intel_migrate_fini(>->migrate);788 intel_uc_driver_remove(>->uc);789 790 intel_engines_release(gt);791 792 intel_gt_flush_buffer_pool(gt);793}794 795void intel_gt_driver_unregister(struct intel_gt *gt)796{797 intel_wakeref_t wakeref;798 799 intel_gt_sysfs_unregister(gt);800 intel_rps_driver_unregister(>->rps);801 intel_gsc_fini(>->gsc);802 803 /*804 * If we unload the driver and wedge before the GSC worker is complete,805 * the worker will hit an error on its submission to the GSC engine and806 * then exit. This is hard to hit for a user, but it is reproducible807 * with skipping selftests. The error is handled gracefully by the808 * worker, so there are no functional issues, but we still end up with809 * an error message in dmesg, which is something we want to avoid as810 * this is a supported scenario. We could modify the worker to better811 * handle a wedging occurring during its execution, but that gets812 * complicated for a couple of reasons:813 * - We do want the error on runtime wedging, because there are814 * implications for subsystems outside of GT (i.e., PXP, HDCP), it's815 * only the error on driver unload that we want to silence.816 * - The worker is responsible for multiple submissions (GSC FW load,817 * HuC auth, SW proxy), so all of those will have to be adapted to818 * handle the wedged_on_fini scenario.819 * Therefore, it's much simpler to just wait for the worker to be done820 * before wedging on driver removal, also considering that the worker821 * will likely already be idle in the great majority of non-selftest822 * scenarios.823 */824 intel_gsc_uc_flush_work(>->uc.gsc);825 826 /*827 * Upon unregistering the device to prevent any new users, cancel828 * all in-flight requests so that we can quickly unbind the active829 * resources.830 */831 intel_gt_set_wedged_on_fini(gt);832 833 /* Scrub all HW state upon release */834 with_intel_runtime_pm(gt->uncore->rpm, wakeref)835 intel_gt_reset_all_engines(gt);836}837 838void intel_gt_driver_release(struct intel_gt *gt)839{840 struct i915_address_space *vm;841 842 vm = fetch_and_zero(>->vm);843 if (vm) /* FIXME being called twice on error paths :( */844 i915_vm_put(vm);845 846 intel_wa_list_free(>->wa_list);847 intel_gt_pm_fini(gt);848 intel_gt_fini_scratch(gt);849 intel_gt_fini_buffer_pool(gt);850 intel_gt_fini_hwconfig(gt);851}852 853void intel_gt_driver_late_release_all(struct drm_i915_private *i915)854{855 struct intel_gt *gt;856 unsigned int id;857 858 /* We need to wait for inflight RCU frees to release their grip */859 rcu_barrier();860 861 for_each_gt(gt, i915, id) {862 intel_uc_driver_late_release(>->uc);863 intel_gt_fini_requests(gt);864 intel_gt_fini_reset(gt);865 intel_gt_fini_timelines(gt);866 intel_gt_fini_tlb(gt);867 intel_engines_free(gt);868 }869}870 871static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)872{873 int ret;874 875 if (!gt_is_root(gt)) {876 struct intel_uncore *uncore;877 spinlock_t *irq_lock;878 879 uncore = drmm_kzalloc(>->i915->drm, sizeof(*uncore), GFP_KERNEL);880 if (!uncore)881 return -ENOMEM;882 883 irq_lock = drmm_kzalloc(>->i915->drm, sizeof(*irq_lock), GFP_KERNEL);884 if (!irq_lock)885 return -ENOMEM;886 887 gt->uncore = uncore;888 gt->irq_lock = irq_lock;889 890 intel_gt_common_init_early(gt);891 }892 893 intel_uncore_init_early(gt->uncore, gt);894 895 ret = intel_uncore_setup_mmio(gt->uncore, phys_addr);896 if (ret)897 return ret;898 899 gt->phys_addr = phys_addr;900 901 return 0;902}903 904int intel_gt_probe_all(struct drm_i915_private *i915)905{906 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);907 struct intel_gt *gt = to_gt(i915);908 const struct intel_gt_definition *gtdef;909 phys_addr_t phys_addr;910 unsigned int mmio_bar;911 unsigned int i;912 int ret;913 914 mmio_bar = intel_mmio_bar(GRAPHICS_VER(i915));915 phys_addr = pci_resource_start(pdev, mmio_bar);916 917 /*918 * We always have at least one primary GT on any device919 * and it has been already initialized early during probe920 * in i915_driver_probe()921 */922 gt->i915 = i915;923 gt->name = "Primary GT";924 gt->info.engine_mask = INTEL_INFO(i915)->platform_engine_mask;925 926 gt_dbg(gt, "Setting up %s\n", gt->name);927 ret = intel_gt_tile_setup(gt, phys_addr);928 if (ret)929 return ret;930 931 if (!HAS_EXTRA_GT_LIST(i915))932 return 0;933 934 for (i = 1, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1];935 gtdef->name != NULL;936 i++, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]) {937 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL);938 if (!gt) {939 ret = -ENOMEM;940 goto err;941 }942 943 gt->i915 = i915;944 gt->name = gtdef->name;945 gt->type = gtdef->type;946 gt->info.engine_mask = gtdef->engine_mask;947 gt->info.id = i;948 949 gt_dbg(gt, "Setting up %s\n", gt->name);950 if (GEM_WARN_ON(range_overflows_t(resource_size_t,951 gtdef->mapping_base,952 SZ_16M,953 pci_resource_len(pdev, mmio_bar)))) {954 ret = -ENODEV;955 goto err;956 }957 958 switch (gtdef->type) {959 case GT_TILE:960 ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base);961 break;962 963 case GT_MEDIA:964 ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base,965 gtdef->gsi_offset);966 break;967 968 case GT_PRIMARY:969 /* Primary GT should not appear in extra GT list */970 default:971 MISSING_CASE(gtdef->type);972 ret = -ENODEV;973 }974 975 if (ret)976 goto err;977 978 i915->gt[i] = gt;979 }980 981 return 0;982 983err:984 i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret);985 return ret;986}987 988int intel_gt_tiles_init(struct drm_i915_private *i915)989{990 struct intel_gt *gt;991 unsigned int id;992 int ret;993 994 for_each_gt(gt, i915, id) {995 ret = intel_gt_probe_lmem(gt);996 if (ret)997 return ret;998 }999 1000 return 0;1001}1002 1003void intel_gt_info_print(const struct intel_gt_info *info,1004 struct drm_printer *p)1005{1006 drm_printf(p, "available engines: %x\n", info->engine_mask);1007 1008 intel_sseu_dump(&info->sseu, p);1009}1010 1011enum i915_map_type intel_gt_coherent_map_type(struct intel_gt *gt,1012 struct drm_i915_gem_object *obj,1013 bool always_coherent)1014{1015 /*1016 * Wa_22016122933: always return I915_MAP_WC for Media1017 * version 13.0 when the object is on the Media GT1018 */1019 if (i915_gem_object_is_lmem(obj) || intel_gt_needs_wa_22016122933(gt))1020 return I915_MAP_WC;1021 if (HAS_LLC(gt->i915) || always_coherent)1022 return I915_MAP_WB;1023 else1024 return I915_MAP_WC;1025}1026 1027bool intel_gt_needs_wa_16018031267(struct intel_gt *gt)1028{1029 /* Wa_16018031267, Wa_16018063123 */1030 return IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 55), IP_VER(12, 71));1031}1032 1033bool intel_gt_needs_wa_22016122933(struct intel_gt *gt)1034{1035 return MEDIA_VER_FULL(gt->i915) == IP_VER(13, 0) && gt->type == GT_MEDIA;1036}1037 1038static void __intel_gt_bind_context_set_ready(struct intel_gt *gt, bool ready)1039{1040 struct intel_engine_cs *engine = gt->engine[BCS0];1041 1042 if (engine && engine->bind_context)1043 engine->bind_context_ready = ready;1044}1045 1046/**1047 * intel_gt_bind_context_set_ready - Set the context binding as ready1048 *1049 * @gt: GT structure1050 *1051 * This function marks the binder context as ready.1052 */1053void intel_gt_bind_context_set_ready(struct intel_gt *gt)1054{1055 __intel_gt_bind_context_set_ready(gt, true);1056}1057 1058/**1059 * intel_gt_bind_context_set_unready - Set the context binding as ready1060 * @gt: GT structure1061 *1062 * This function marks the binder context as not ready.1063 */1064 1065void intel_gt_bind_context_set_unready(struct intel_gt *gt)1066{1067 __intel_gt_bind_context_set_ready(gt, false);1068}1069 1070/**1071 * intel_gt_is_bind_context_ready - Check if context binding is ready1072 *1073 * @gt: GT structure1074 *1075 * This function returns binder context's ready status.1076 */1077bool intel_gt_is_bind_context_ready(struct intel_gt *gt)1078{1079 struct intel_engine_cs *engine = gt->engine[BCS0];1080 1081 if (engine)1082 return engine->bind_context_ready;1083 1084 return false;1085}1086