976 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_gt.h"7 8#include <linux/minmax.h>9 10#include <drm/drm_managed.h>11#include <uapi/drm/xe_drm.h>12 13#include <generated/xe_wa_oob.h>14 15#include "instructions/xe_gfxpipe_commands.h"16#include "instructions/xe_mi_commands.h"17#include "regs/xe_gt_regs.h"18#include "xe_assert.h"19#include "xe_bb.h"20#include "xe_bo.h"21#include "xe_device.h"22#include "xe_exec_queue.h"23#include "xe_execlist.h"24#include "xe_force_wake.h"25#include "xe_ggtt.h"26#include "xe_gsc.h"27#include "xe_gt_ccs_mode.h"28#include "xe_gt_clock.h"29#include "xe_gt_freq.h"30#include "xe_gt_idle.h"31#include "xe_gt_mcr.h"32#include "xe_gt_pagefault.h"33#include "xe_gt_printk.h"34#include "xe_gt_sriov_pf.h"35#include "xe_gt_sysfs.h"36#include "xe_gt_tlb_invalidation.h"37#include "xe_gt_topology.h"38#include "xe_guc_exec_queue_types.h"39#include "xe_guc_pc.h"40#include "xe_hw_fence.h"41#include "xe_hw_engine_class_sysfs.h"42#include "xe_irq.h"43#include "xe_lmtt.h"44#include "xe_lrc.h"45#include "xe_map.h"46#include "xe_migrate.h"47#include "xe_mmio.h"48#include "xe_pat.h"49#include "xe_pm.h"50#include "xe_mocs.h"51#include "xe_reg_sr.h"52#include "xe_ring_ops.h"53#include "xe_sa.h"54#include "xe_sched_job.h"55#include "xe_sriov.h"56#include "xe_tuning.h"57#include "xe_uc.h"58#include "xe_uc_fw.h"59#include "xe_vm.h"60#include "xe_wa.h"61#include "xe_wopcm.h"62 63static void gt_fini(struct drm_device *drm, void *arg)64{65 struct xe_gt *gt = arg;66 67 destroy_workqueue(gt->ordered_wq);68}69 70struct xe_gt *xe_gt_alloc(struct xe_tile *tile)71{72 struct xe_gt *gt;73 int err;74 75 gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL);76 if (!gt)77 return ERR_PTR(-ENOMEM);78 79 gt->tile = tile;80 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0);81 82 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt);83 if (err)84 return ERR_PTR(err);85 86 return gt;87}88 89void xe_gt_sanitize(struct xe_gt *gt)90{91 /*92 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not93 * reload94 */95 gt->uc.guc.submission_state.enabled = false;96}97 98static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)99{100 u32 reg;101 int err;102 103 if (!XE_WA(gt, 16023588340))104 return;105 106 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);107 if (WARN_ON(err))108 return;109 110 if (!xe_gt_is_media_type(gt)) {111 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);112 reg |= CG_DIS_CNTLBUS;113 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);114 }115 116 xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0x3);117 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);118}119 120static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)121{122 u32 reg;123 int err;124 125 if (!XE_WA(gt, 16023588340))126 return;127 128 if (xe_gt_is_media_type(gt))129 return;130 131 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);132 if (WARN_ON(err))133 return;134 135 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);136 reg &= ~CG_DIS_CNTLBUS;137 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);138 139 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);140}141 142/**143 * xe_gt_remove() - Clean up the GT structures before driver removal144 * @gt: the GT object145 *146 * This function should only act on objects/structures that must be cleaned147 * before the driver removal callback is complete and therefore can't be148 * deferred to a drmm action.149 */150void xe_gt_remove(struct xe_gt *gt)151{152 int i;153 154 xe_uc_remove(>->uc);155 156 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)157 xe_hw_fence_irq_finish(>->fence_irq[i]);158 159 xe_gt_disable_host_l2_vram(gt);160}161 162static void gt_reset_worker(struct work_struct *w);163 164static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)165{166 struct xe_sched_job *job;167 struct xe_bb *bb;168 struct dma_fence *fence;169 long timeout;170 171 bb = xe_bb_new(gt, 4, false);172 if (IS_ERR(bb))173 return PTR_ERR(bb);174 175 job = xe_bb_create_job(q, bb);176 if (IS_ERR(job)) {177 xe_bb_free(bb, NULL);178 return PTR_ERR(job);179 }180 181 xe_sched_job_arm(job);182 fence = dma_fence_get(&job->drm.s_fence->finished);183 xe_sched_job_push(job);184 185 timeout = dma_fence_wait_timeout(fence, false, HZ);186 dma_fence_put(fence);187 xe_bb_free(bb, NULL);188 if (timeout < 0)189 return timeout;190 else if (!timeout)191 return -ETIME;192 193 return 0;194}195 196/*197 * Convert back from encoded value to type-safe, only to be used when reg.mcr198 * is true199 */200static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)201{202 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };203}204 205static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)206{207 struct xe_reg_sr *sr = &q->hwe->reg_lrc;208 struct xe_reg_sr_entry *entry;209 unsigned long idx;210 struct xe_sched_job *job;211 struct xe_bb *bb;212 struct dma_fence *fence;213 long timeout;214 int count = 0;215 216 if (q->hwe->class == XE_ENGINE_CLASS_RENDER)217 /* Big enough to emit all of the context's 3DSTATE */218 bb = xe_bb_new(gt, xe_gt_lrc_size(gt, q->hwe->class), false);219 else220 /* Just pick a large BB size */221 bb = xe_bb_new(gt, SZ_4K, false);222 223 if (IS_ERR(bb))224 return PTR_ERR(bb);225 226 xa_for_each(&sr->xa, idx, entry)227 ++count;228 229 if (count) {230 xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name);231 232 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);233 234 xa_for_each(&sr->xa, idx, entry) {235 struct xe_reg reg = entry->reg;236 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);237 u32 val;238 239 /*240 * Skip reading the register if it's not really needed241 */242 if (reg.masked)243 val = entry->clr_bits << 16;244 else if (entry->clr_bits + 1)245 val = (reg.mcr ?246 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :247 xe_mmio_read32(gt, reg)) & (~entry->clr_bits);248 else249 val = 0;250 251 val |= entry->set_bits;252 253 bb->cs[bb->len++] = reg.addr;254 bb->cs[bb->len++] = val;255 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);256 }257 }258 259 xe_lrc_emit_hwe_state_instructions(q, bb);260 261 job = xe_bb_create_job(q, bb);262 if (IS_ERR(job)) {263 xe_bb_free(bb, NULL);264 return PTR_ERR(job);265 }266 267 xe_sched_job_arm(job);268 fence = dma_fence_get(&job->drm.s_fence->finished);269 xe_sched_job_push(job);270 271 timeout = dma_fence_wait_timeout(fence, false, HZ);272 dma_fence_put(fence);273 xe_bb_free(bb, NULL);274 if (timeout < 0)275 return timeout;276 else if (!timeout)277 return -ETIME;278 279 return 0;280}281 282int xe_gt_record_default_lrcs(struct xe_gt *gt)283{284 struct xe_device *xe = gt_to_xe(gt);285 struct xe_hw_engine *hwe;286 enum xe_hw_engine_id id;287 int err = 0;288 289 for_each_hw_engine(hwe, gt, id) {290 struct xe_exec_queue *q, *nop_q;291 void *default_lrc;292 293 if (gt->default_lrc[hwe->class])294 continue;295 296 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);297 xe_wa_process_lrc(hwe);298 xe_hw_engine_setup_default_lrc_state(hwe);299 xe_tuning_process_lrc(hwe);300 301 default_lrc = drmm_kzalloc(&xe->drm,302 xe_gt_lrc_size(gt, hwe->class),303 GFP_KERNEL);304 if (!default_lrc)305 return -ENOMEM;306 307 q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1,308 hwe, EXEC_QUEUE_FLAG_KERNEL, 0);309 if (IS_ERR(q)) {310 err = PTR_ERR(q);311 xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n",312 hwe->name, q);313 return err;314 }315 316 /* Prime golden LRC with known good state */317 err = emit_wa_job(gt, q);318 if (err) {319 xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n",320 hwe->name, ERR_PTR(err), q->guc->id);321 goto put_exec_queue;322 }323 324 nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance),325 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0);326 if (IS_ERR(nop_q)) {327 err = PTR_ERR(nop_q);328 xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n",329 hwe->name, nop_q);330 goto put_exec_queue;331 }332 333 /* Switch to different LRC */334 err = emit_nop_job(gt, nop_q);335 if (err) {336 xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n",337 hwe->name, ERR_PTR(err), nop_q->guc->id);338 goto put_nop_q;339 }340 341 /* Reload golden LRC to record the effect of any indirect W/A */342 err = emit_nop_job(gt, q);343 if (err) {344 xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n",345 hwe->name, ERR_PTR(err), q->guc->id);346 goto put_nop_q;347 }348 349 xe_map_memcpy_from(xe, default_lrc,350 &q->lrc[0]->bo->vmap,351 xe_lrc_pphwsp_offset(q->lrc[0]),352 xe_gt_lrc_size(gt, hwe->class));353 354 gt->default_lrc[hwe->class] = default_lrc;355put_nop_q:356 xe_exec_queue_put(nop_q);357put_exec_queue:358 xe_exec_queue_put(q);359 if (err)360 break;361 }362 363 return err;364}365 366int xe_gt_init_early(struct xe_gt *gt)367{368 int err;369 370 if (IS_SRIOV_PF(gt_to_xe(gt))) {371 err = xe_gt_sriov_pf_init_early(gt);372 if (err)373 return err;374 }375 376 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt));377 378 err = xe_wa_init(gt);379 if (err)380 return err;381 382 xe_wa_process_gt(gt);383 xe_wa_process_oob(gt);384 xe_tuning_process_gt(gt);385 386 xe_force_wake_init_gt(gt, gt_to_fw(gt));387 spin_lock_init(>->global_invl_lock);388 389 return 0;390}391 392static void dump_pat_on_error(struct xe_gt *gt)393{394 struct drm_printer p;395 char prefix[32];396 397 snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id);398 p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix);399 400 xe_pat_dump(gt, &p);401}402 403static int gt_fw_domain_init(struct xe_gt *gt)404{405 int err, i;406 407 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);408 if (err)409 goto err_hw_fence_irq;410 411 if (!xe_gt_is_media_type(gt)) {412 err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);413 if (err)414 goto err_force_wake;415 if (IS_SRIOV_PF(gt_to_xe(gt)))416 xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt);417 }418 419 /* Enable per hw engine IRQs */420 xe_irq_enable_hwe(gt);421 422 /* Rerun MCR init as we now have hw engine list */423 xe_gt_mcr_init(gt);424 425 err = xe_hw_engines_init_early(gt);426 if (err)427 goto err_force_wake;428 429 err = xe_hw_engine_class_sysfs_init(gt);430 if (err)431 goto err_force_wake;432 433 /* Initialize CCS mode sysfs after early initialization of HW engines */434 err = xe_gt_ccs_mode_sysfs_init(gt);435 if (err)436 goto err_force_wake;437 438 /*439 * Stash hardware-reported version. Since this register does not exist440 * on pre-MTL platforms, reading it there will (correctly) return 0.441 */442 gt->info.gmdid = xe_mmio_read32(gt, GMD_ID);443 444 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);445 XE_WARN_ON(err);446 447 return 0;448 449err_force_wake:450 dump_pat_on_error(gt);451 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);452err_hw_fence_irq:453 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)454 xe_hw_fence_irq_finish(>->fence_irq[i]);455 456 return err;457}458 459static int all_fw_domain_init(struct xe_gt *gt)460{461 int err, i;462 463 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);464 if (err)465 goto err_hw_fence_irq;466 467 xe_gt_mcr_set_implicit_defaults(gt);468 xe_reg_sr_apply_mmio(>->reg_sr, gt);469 470 err = xe_gt_clock_init(gt);471 if (err)472 goto err_force_wake;473 474 xe_mocs_init(gt);475 err = xe_execlist_init(gt);476 if (err)477 goto err_force_wake;478 479 err = xe_hw_engines_init(gt);480 if (err)481 goto err_force_wake;482 483 err = xe_uc_init_post_hwconfig(>->uc);484 if (err)485 goto err_force_wake;486 487 if (!xe_gt_is_media_type(gt)) {488 /*489 * USM has its only SA pool to non-block behind user operations490 */491 if (gt_to_xe(gt)->info.has_usm) {492 struct xe_device *xe = gt_to_xe(gt);493 494 gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),495 IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);496 if (IS_ERR(gt->usm.bb_pool)) {497 err = PTR_ERR(gt->usm.bb_pool);498 goto err_force_wake;499 }500 }501 }502 503 if (!xe_gt_is_media_type(gt)) {504 struct xe_tile *tile = gt_to_tile(gt);505 506 tile->migrate = xe_migrate_init(tile);507 if (IS_ERR(tile->migrate)) {508 err = PTR_ERR(tile->migrate);509 goto err_force_wake;510 }511 }512 513 err = xe_uc_init_hw(>->uc);514 if (err)515 goto err_force_wake;516 517 /* Configure default CCS mode of 1 engine with all resources */518 if (xe_gt_ccs_mode_enabled(gt)) {519 gt->ccs_mode = 1;520 xe_gt_apply_ccs_mode(gt);521 }522 523 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))524 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);525 526 if (IS_SRIOV_PF(gt_to_xe(gt)))527 xe_gt_sriov_pf_init_hw(gt);528 529 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);530 XE_WARN_ON(err);531 532 return 0;533 534err_force_wake:535 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);536err_hw_fence_irq:537 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)538 xe_hw_fence_irq_finish(>->fence_irq[i]);539 540 return err;541}542 543/*544 * Initialize enough GT to be able to load GuC in order to obtain hwconfig and545 * enable CTB communication.546 */547int xe_gt_init_hwconfig(struct xe_gt *gt)548{549 int err;550 551 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);552 if (err)553 goto out;554 555 xe_gt_mcr_init_early(gt);556 xe_pat_init(gt);557 558 err = xe_uc_init(>->uc);559 if (err)560 goto out_fw;561 562 err = xe_uc_init_hwconfig(>->uc);563 if (err)564 goto out_fw;565 566 xe_gt_topology_init(gt);567 xe_gt_mcr_init(gt);568 xe_gt_enable_host_l2_vram(gt);569 570out_fw:571 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);572out:573 return err;574}575 576int xe_gt_init(struct xe_gt *gt)577{578 int err;579 int i;580 581 INIT_WORK(>->reset.worker, gt_reset_worker);582 583 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {584 gt->ring_ops[i] = xe_ring_ops_get(gt, i);585 xe_hw_fence_irq_init(>->fence_irq[i]);586 }587 588 err = xe_gt_tlb_invalidation_init(gt);589 if (err)590 return err;591 592 err = xe_gt_pagefault_init(gt);593 if (err)594 return err;595 596 xe_mocs_init_early(gt);597 598 err = xe_gt_sysfs_init(gt);599 if (err)600 return err;601 602 err = gt_fw_domain_init(gt);603 if (err)604 return err;605 606 err = xe_gt_idle_init(>->gtidle);607 if (err)608 return err;609 610 err = xe_gt_freq_init(gt);611 if (err)612 return err;613 614 xe_force_wake_init_engines(gt, gt_to_fw(gt));615 616 err = all_fw_domain_init(gt);617 if (err)618 return err;619 620 xe_gt_record_user_engines(gt);621 622 return 0;623}624 625void xe_gt_record_user_engines(struct xe_gt *gt)626{627 struct xe_hw_engine *hwe;628 enum xe_hw_engine_id id;629 630 gt->user_engines.mask = 0;631 memset(gt->user_engines.instances_per_class, 0,632 sizeof(gt->user_engines.instances_per_class));633 634 for_each_hw_engine(hwe, gt, id) {635 if (xe_hw_engine_is_reserved(hwe))636 continue;637 638 gt->user_engines.mask |= BIT_ULL(id);639 gt->user_engines.instances_per_class[hwe->class]++;640 }641 642 xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask)643 == gt->info.engine_mask);644}645 646static int do_gt_reset(struct xe_gt *gt)647{648 int err;649 650 xe_gsc_wa_14015076503(gt, true);651 652 xe_mmio_write32(gt, GDRST, GRDOM_FULL);653 err = xe_mmio_wait32(gt, GDRST, GRDOM_FULL, 0, 5000, NULL, false);654 if (err)655 xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",656 ERR_PTR(err));657 658 xe_gsc_wa_14015076503(gt, false);659 660 return err;661}662 663static int vf_gt_restart(struct xe_gt *gt)664{665 int err;666 667 err = xe_uc_sanitize_reset(>->uc);668 if (err)669 return err;670 671 err = xe_uc_init_hw(>->uc);672 if (err)673 return err;674 675 err = xe_uc_start(>->uc);676 if (err)677 return err;678 679 return 0;680}681 682static int do_gt_restart(struct xe_gt *gt)683{684 struct xe_hw_engine *hwe;685 enum xe_hw_engine_id id;686 int err;687 688 if (IS_SRIOV_VF(gt_to_xe(gt)))689 return vf_gt_restart(gt);690 691 xe_pat_init(gt);692 693 xe_gt_enable_host_l2_vram(gt);694 695 xe_gt_mcr_set_implicit_defaults(gt);696 xe_reg_sr_apply_mmio(>->reg_sr, gt);697 698 err = xe_wopcm_init(>->uc.wopcm);699 if (err)700 return err;701 702 for_each_hw_engine(hwe, gt, id)703 xe_hw_engine_enable_ring(hwe);704 705 err = xe_uc_sanitize_reset(>->uc);706 if (err)707 return err;708 709 err = xe_uc_init_hw(>->uc);710 if (err)711 return err;712 713 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))714 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);715 716 if (IS_SRIOV_PF(gt_to_xe(gt)))717 xe_gt_sriov_pf_init_hw(gt);718 719 xe_mocs_init(gt);720 err = xe_uc_start(>->uc);721 if (err)722 return err;723 724 for_each_hw_engine(hwe, gt, id) {725 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);726 xe_reg_sr_apply_whitelist(hwe);727 }728 729 /* Get CCS mode in sync between sw/hw */730 xe_gt_apply_ccs_mode(gt);731 732 /* Restore GT freq to expected values */733 xe_gt_sanitize_freq(gt);734 735 if (IS_SRIOV_PF(gt_to_xe(gt)))736 xe_gt_sriov_pf_restart(gt);737 738 return 0;739}740 741static int gt_reset(struct xe_gt *gt)742{743 int err;744 745 if (xe_device_wedged(gt_to_xe(gt)))746 return -ECANCELED;747 748 /* We only support GT resets with GuC submission */749 if (!xe_device_uc_enabled(gt_to_xe(gt)))750 return -ENODEV;751 752 xe_gt_info(gt, "reset started\n");753 754 xe_pm_runtime_get(gt_to_xe(gt));755 756 if (xe_fault_inject_gt_reset()) {757 err = -ECANCELED;758 goto err_fail;759 }760 761 xe_gt_sanitize(gt);762 763 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);764 if (err)765 goto err_msg;766 767 xe_uc_gucrc_disable(>->uc);768 xe_uc_stop_prepare(>->uc);769 xe_gt_pagefault_reset(gt);770 771 xe_uc_stop(>->uc);772 773 xe_gt_tlb_invalidation_reset(gt);774 775 err = do_gt_reset(gt);776 if (err)777 goto err_out;778 779 err = do_gt_restart(gt);780 if (err)781 goto err_out;782 783 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);784 XE_WARN_ON(err);785 xe_pm_runtime_put(gt_to_xe(gt));786 787 xe_gt_info(gt, "reset done\n");788 789 return 0;790 791err_out:792 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));793err_msg:794 XE_WARN_ON(xe_uc_start(>->uc));795err_fail:796 xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));797 798 xe_device_declare_wedged(gt_to_xe(gt));799 xe_pm_runtime_put(gt_to_xe(gt));800 801 return err;802}803 804static void gt_reset_worker(struct work_struct *w)805{806 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);807 808 gt_reset(gt);809}810 811void xe_gt_reset_async(struct xe_gt *gt)812{813 xe_gt_info(gt, "trying reset\n");814 815 /* Don't do a reset while one is already in flight */816 if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc))817 return;818 819 xe_gt_info(gt, "reset queued\n");820 queue_work(gt->ordered_wq, >->reset.worker);821}822 823void xe_gt_suspend_prepare(struct xe_gt *gt)824{825 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));826 827 xe_uc_stop_prepare(>->uc);828 829 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));830}831 832int xe_gt_suspend(struct xe_gt *gt)833{834 int err;835 836 xe_gt_dbg(gt, "suspending\n");837 xe_gt_sanitize(gt);838 839 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);840 if (err)841 goto err_msg;842 843 err = xe_uc_suspend(>->uc);844 if (err)845 goto err_force_wake;846 847 xe_gt_idle_disable_pg(gt);848 849 xe_gt_disable_host_l2_vram(gt);850 851 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));852 xe_gt_dbg(gt, "suspended\n");853 854 return 0;855 856err_force_wake:857 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));858err_msg:859 xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));860 861 return err;862}863 864/**865 * xe_gt_sanitize_freq() - Restore saved frequencies if necessary.866 * @gt: the GT object867 *868 * Called after driver init/GSC load completes to restore GT frequencies if we869 * limited them for any WAs.870 */871int xe_gt_sanitize_freq(struct xe_gt *gt)872{873 int ret = 0;874 875 if ((!xe_uc_fw_is_available(>->uc.gsc.fw) ||876 xe_uc_fw_is_loaded(>->uc.gsc.fw) ||877 xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) &&878 XE_WA(gt, 22019338487))879 ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc);880 881 return ret;882}883 884int xe_gt_resume(struct xe_gt *gt)885{886 int err;887 888 xe_gt_dbg(gt, "resuming\n");889 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);890 if (err)891 goto err_msg;892 893 err = do_gt_restart(gt);894 if (err)895 goto err_force_wake;896 897 xe_gt_idle_enable_pg(gt);898 899 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));900 xe_gt_dbg(gt, "resumed\n");901 902 return 0;903 904err_force_wake:905 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));906err_msg:907 xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));908 909 return err;910}911 912struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,913 enum xe_engine_class class,914 u16 instance, bool logical)915{916 struct xe_hw_engine *hwe;917 enum xe_hw_engine_id id;918 919 for_each_hw_engine(hwe, gt, id)920 if (hwe->class == class &&921 ((!logical && hwe->instance == instance) ||922 (logical && hwe->logical_instance == instance)))923 return hwe;924 925 return NULL;926}927 928struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,929 enum xe_engine_class class)930{931 struct xe_hw_engine *hwe;932 enum xe_hw_engine_id id;933 934 for_each_hw_engine(hwe, gt, id) {935 switch (class) {936 case XE_ENGINE_CLASS_RENDER:937 case XE_ENGINE_CLASS_COMPUTE:938 if (hwe->class == XE_ENGINE_CLASS_RENDER ||939 hwe->class == XE_ENGINE_CLASS_COMPUTE)940 return hwe;941 break;942 default:943 if (hwe->class == class)944 return hwe;945 }946 }947 948 return NULL;949}950 951struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt)952{953 struct xe_hw_engine *hwe;954 enum xe_hw_engine_id id;955 956 for_each_hw_engine(hwe, gt, id)957 return hwe;958 959 return NULL;960}961 962/**963 * xe_gt_declare_wedged() - Declare GT wedged964 * @gt: the GT object965 *966 * Wedge the GT which stops all submission, saves desired debug state, and967 * cleans up anything which could timeout.968 */969void xe_gt_declare_wedged(struct xe_gt *gt)970{971 xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);972 973 xe_uc_declare_wedged(>->uc);974 xe_gt_tlb_invalidation_reset(gt);975}976