2265 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_guc_submit.h"7 8#include <linux/bitfield.h>9#include <linux/bitmap.h>10#include <linux/circ_buf.h>11#include <linux/delay.h>12#include <linux/dma-fence-array.h>13#include <linux/math64.h>14 15#include <drm/drm_managed.h>16 17#include "abi/guc_actions_abi.h"18#include "abi/guc_klvs_abi.h"19#include "regs/xe_lrc_layout.h"20#include "xe_assert.h"21#include "xe_devcoredump.h"22#include "xe_device.h"23#include "xe_exec_queue.h"24#include "xe_force_wake.h"25#include "xe_gpu_scheduler.h"26#include "xe_gt.h"27#include "xe_gt_clock.h"28#include "xe_gt_printk.h"29#include "xe_guc.h"30#include "xe_guc_ct.h"31#include "xe_guc_exec_queue_types.h"32#include "xe_guc_id_mgr.h"33#include "xe_guc_submit_types.h"34#include "xe_hw_engine.h"35#include "xe_hw_fence.h"36#include "xe_lrc.h"37#include "xe_macros.h"38#include "xe_map.h"39#include "xe_mocs.h"40#include "xe_pm.h"41#include "xe_ring_ops_types.h"42#include "xe_sched_job.h"43#include "xe_trace.h"44#include "xe_vm.h"45 46static struct xe_guc *47exec_queue_to_guc(struct xe_exec_queue *q)48{49 return &q->gt->uc.guc;50}51 52/*53 * Helpers for engine state, using an atomic as some of the bits can transition54 * as the same time (e.g. a suspend can be happning at the same time as schedule55 * engine done being processed).56 */57#define EXEC_QUEUE_STATE_REGISTERED (1 << 0)58#define EXEC_QUEUE_STATE_ENABLED (1 << 1)59#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2)60#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3)61#define EXEC_QUEUE_STATE_DESTROYED (1 << 4)62#define EXEC_QUEUE_STATE_SUSPENDED (1 << 5)63#define EXEC_QUEUE_STATE_RESET (1 << 6)64#define EXEC_QUEUE_STATE_KILLED (1 << 7)65#define EXEC_QUEUE_STATE_WEDGED (1 << 8)66#define EXEC_QUEUE_STATE_BANNED (1 << 9)67#define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10)68#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11)69 70static bool exec_queue_registered(struct xe_exec_queue *q)71{72 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED;73}74 75static void set_exec_queue_registered(struct xe_exec_queue *q)76{77 atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);78}79 80static void clear_exec_queue_registered(struct xe_exec_queue *q)81{82 atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);83}84 85static bool exec_queue_enabled(struct xe_exec_queue *q)86{87 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED;88}89 90static void set_exec_queue_enabled(struct xe_exec_queue *q)91{92 atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state);93}94 95static void clear_exec_queue_enabled(struct xe_exec_queue *q)96{97 atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state);98}99 100static bool exec_queue_pending_enable(struct xe_exec_queue *q)101{102 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE;103}104 105static void set_exec_queue_pending_enable(struct xe_exec_queue *q)106{107 atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);108}109 110static void clear_exec_queue_pending_enable(struct xe_exec_queue *q)111{112 atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);113}114 115static bool exec_queue_pending_disable(struct xe_exec_queue *q)116{117 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE;118}119 120static void set_exec_queue_pending_disable(struct xe_exec_queue *q)121{122 atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);123}124 125static void clear_exec_queue_pending_disable(struct xe_exec_queue *q)126{127 atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);128}129 130static bool exec_queue_destroyed(struct xe_exec_queue *q)131{132 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED;133}134 135static void set_exec_queue_destroyed(struct xe_exec_queue *q)136{137 atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);138}139 140static bool exec_queue_banned(struct xe_exec_queue *q)141{142 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED;143}144 145static void set_exec_queue_banned(struct xe_exec_queue *q)146{147 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);148}149 150static bool exec_queue_suspended(struct xe_exec_queue *q)151{152 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;153}154 155static void set_exec_queue_suspended(struct xe_exec_queue *q)156{157 atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);158}159 160static void clear_exec_queue_suspended(struct xe_exec_queue *q)161{162 atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state);163}164 165static bool exec_queue_reset(struct xe_exec_queue *q)166{167 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET;168}169 170static void set_exec_queue_reset(struct xe_exec_queue *q)171{172 atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state);173}174 175static bool exec_queue_killed(struct xe_exec_queue *q)176{177 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED;178}179 180static void set_exec_queue_killed(struct xe_exec_queue *q)181{182 atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state);183}184 185static bool exec_queue_wedged(struct xe_exec_queue *q)186{187 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED;188}189 190static void set_exec_queue_wedged(struct xe_exec_queue *q)191{192 atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state);193}194 195static bool exec_queue_check_timeout(struct xe_exec_queue *q)196{197 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT;198}199 200static void set_exec_queue_check_timeout(struct xe_exec_queue *q)201{202 atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state);203}204 205static void clear_exec_queue_check_timeout(struct xe_exec_queue *q)206{207 atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state);208}209 210static bool exec_queue_extra_ref(struct xe_exec_queue *q)211{212 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF;213}214 215static void set_exec_queue_extra_ref(struct xe_exec_queue *q)216{217 atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state);218}219 220static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)221{222 return (atomic_read(&q->guc->state) &223 (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED |224 EXEC_QUEUE_STATE_BANNED));225}226 227static void guc_submit_fini(struct drm_device *drm, void *arg)228{229 struct xe_guc *guc = arg;230 231 xa_destroy(&guc->submission_state.exec_queue_lookup);232}233 234static void guc_submit_wedged_fini(void *arg)235{236 struct xe_guc *guc = arg;237 struct xe_exec_queue *q;238 unsigned long index;239 240 mutex_lock(&guc->submission_state.lock);241 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {242 if (exec_queue_wedged(q)) {243 mutex_unlock(&guc->submission_state.lock);244 xe_exec_queue_put(q);245 mutex_lock(&guc->submission_state.lock);246 }247 }248 mutex_unlock(&guc->submission_state.lock);249}250 251static const struct xe_exec_queue_ops guc_exec_queue_ops;252 253static void primelockdep(struct xe_guc *guc)254{255 if (!IS_ENABLED(CONFIG_LOCKDEP))256 return;257 258 fs_reclaim_acquire(GFP_KERNEL);259 260 mutex_lock(&guc->submission_state.lock);261 mutex_unlock(&guc->submission_state.lock);262 263 fs_reclaim_release(GFP_KERNEL);264}265 266/**267 * xe_guc_submit_init() - Initialize GuC submission.268 * @guc: the &xe_guc to initialize269 * @num_ids: number of GuC context IDs to use270 *271 * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all272 * GuC context IDs supported by the GuC firmware should be used for submission.273 *274 * Only VF drivers will have to provide explicit number of GuC context IDs275 * that they can use for submission.276 *277 * Return: 0 on success or a negative error code on failure.278 */279int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids)280{281 struct xe_device *xe = guc_to_xe(guc);282 struct xe_gt *gt = guc_to_gt(guc);283 int err;284 285 err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock);286 if (err)287 return err;288 289 err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids);290 if (err)291 return err;292 293 gt->exec_queue_ops = &guc_exec_queue_ops;294 295 xa_init(&guc->submission_state.exec_queue_lookup);296 297 init_waitqueue_head(&guc->submission_state.fini_wq);298 299 primelockdep(guc);300 301 return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);302}303 304static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count)305{306 int i;307 308 lockdep_assert_held(&guc->submission_state.lock);309 310 for (i = 0; i < xa_count; ++i)311 xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i);312 313 xe_guc_id_mgr_release_locked(&guc->submission_state.idm,314 q->guc->id, q->width);315 316 if (xa_empty(&guc->submission_state.exec_queue_lookup))317 wake_up(&guc->submission_state.fini_wq);318}319 320static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)321{322 int ret;323 int i;324 325 /*326 * Must use GFP_NOWAIT as this lock is in the dma fence signalling path,327 * worse case user gets -ENOMEM on engine create and has to try again.328 *329 * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent330 * failure.331 */332 lockdep_assert_held(&guc->submission_state.lock);333 334 ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm,335 q->width);336 if (ret < 0)337 return ret;338 339 q->guc->id = ret;340 341 for (i = 0; i < q->width; ++i) {342 ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup,343 q->guc->id + i, q, GFP_NOWAIT));344 if (ret)345 goto err_release;346 }347 348 return 0;349 350err_release:351 __release_guc_id(guc, q, i);352 353 return ret;354}355 356static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)357{358 mutex_lock(&guc->submission_state.lock);359 __release_guc_id(guc, q, q->width);360 mutex_unlock(&guc->submission_state.lock);361}362 363struct exec_queue_policy {364 u32 count;365 struct guc_update_exec_queue_policy h2g;366};367 368static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy)369{370 size_t bytes = sizeof(policy->h2g.header) +371 (sizeof(policy->h2g.klv[0]) * policy->count);372 373 return bytes / sizeof(u32);374}375 376static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy,377 u16 guc_id)378{379 policy->h2g.header.action =380 XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;381 policy->h2g.header.guc_id = guc_id;382 policy->count = 0;383}384 385#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \386static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \387 u32 data) \388{ \389 XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \390\391 policy->h2g.klv[policy->count].kl = \392 FIELD_PREP(GUC_KLV_0_KEY, \393 GUC_CONTEXT_POLICIES_KLV_ID_##id) | \394 FIELD_PREP(GUC_KLV_0_LEN, 1); \395 policy->h2g.klv[policy->count].value = data; \396 policy->count++; \397}398 399MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)400MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)401MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY)402#undef MAKE_EXEC_QUEUE_POLICY_ADD403 404static const int xe_exec_queue_prio_to_guc[] = {405 [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL,406 [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL,407 [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH,408 [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH,409};410 411static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)412{413 struct exec_queue_policy policy;414 struct xe_device *xe = guc_to_xe(guc);415 enum xe_exec_queue_priority prio = q->sched_props.priority;416 u32 timeslice_us = q->sched_props.timeslice_us;417 u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;418 419 xe_assert(xe, exec_queue_registered(q));420 421 __guc_exec_queue_policy_start_klv(&policy, q->guc->id);422 __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]);423 __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us);424 __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us);425 426 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,427 __guc_exec_queue_policy_action_size(&policy), 0, 0);428}429 430static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q)431{432 struct exec_queue_policy policy;433 434 __guc_exec_queue_policy_start_klv(&policy, q->guc->id);435 __guc_exec_queue_policy_add_preemption_timeout(&policy, 1);436 437 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,438 __guc_exec_queue_policy_action_size(&policy), 0, 0);439}440 441#define parallel_read(xe_, map_, field_) \442 xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \443 field_)444#define parallel_write(xe_, map_, field_, val_) \445 xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \446 field_, val_)447 448static void __register_mlrc_exec_queue(struct xe_guc *guc,449 struct xe_exec_queue *q,450 struct guc_ctxt_registration_info *info)451{452#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2)453 struct xe_device *xe = guc_to_xe(guc);454 u32 action[MAX_MLRC_REG_SIZE];455 int len = 0;456 int i;457 458 xe_assert(xe, xe_exec_queue_is_parallel(q));459 460 action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;461 action[len++] = info->flags;462 action[len++] = info->context_idx;463 action[len++] = info->engine_class;464 action[len++] = info->engine_submit_mask;465 action[len++] = info->wq_desc_lo;466 action[len++] = info->wq_desc_hi;467 action[len++] = info->wq_base_lo;468 action[len++] = info->wq_base_hi;469 action[len++] = info->wq_size;470 action[len++] = q->width;471 action[len++] = info->hwlrca_lo;472 action[len++] = info->hwlrca_hi;473 474 for (i = 1; i < q->width; ++i) {475 struct xe_lrc *lrc = q->lrc[i];476 477 action[len++] = lower_32_bits(xe_lrc_descriptor(lrc));478 action[len++] = upper_32_bits(xe_lrc_descriptor(lrc));479 }480 481 xe_assert(xe, len <= MAX_MLRC_REG_SIZE);482#undef MAX_MLRC_REG_SIZE483 484 xe_guc_ct_send(&guc->ct, action, len, 0, 0);485}486 487static void __register_exec_queue(struct xe_guc *guc,488 struct guc_ctxt_registration_info *info)489{490 u32 action[] = {491 XE_GUC_ACTION_REGISTER_CONTEXT,492 info->flags,493 info->context_idx,494 info->engine_class,495 info->engine_submit_mask,496 info->wq_desc_lo,497 info->wq_desc_hi,498 info->wq_base_lo,499 info->wq_base_hi,500 info->wq_size,501 info->hwlrca_lo,502 info->hwlrca_hi,503 };504 505 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);506}507 508static void register_exec_queue(struct xe_exec_queue *q)509{510 struct xe_guc *guc = exec_queue_to_guc(q);511 struct xe_device *xe = guc_to_xe(guc);512 struct xe_lrc *lrc = q->lrc[0];513 struct guc_ctxt_registration_info info;514 515 xe_assert(xe, !exec_queue_registered(q));516 517 memset(&info, 0, sizeof(info));518 info.context_idx = q->guc->id;519 info.engine_class = xe_engine_class_to_guc_class(q->class);520 info.engine_submit_mask = q->logical_mask;521 info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc));522 info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc));523 info.flags = CONTEXT_REGISTRATION_FLAG_KMD;524 525 if (xe_exec_queue_is_parallel(q)) {526 u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);527 struct iosys_map map = xe_lrc_parallel_map(lrc);528 529 info.wq_desc_lo = lower_32_bits(ggtt_addr +530 offsetof(struct guc_submit_parallel_scratch, wq_desc));531 info.wq_desc_hi = upper_32_bits(ggtt_addr +532 offsetof(struct guc_submit_parallel_scratch, wq_desc));533 info.wq_base_lo = lower_32_bits(ggtt_addr +534 offsetof(struct guc_submit_parallel_scratch, wq[0]));535 info.wq_base_hi = upper_32_bits(ggtt_addr +536 offsetof(struct guc_submit_parallel_scratch, wq[0]));537 info.wq_size = WQ_SIZE;538 539 q->guc->wqi_head = 0;540 q->guc->wqi_tail = 0;541 xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE);542 parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE);543 }544 545 /*546 * We must keep a reference for LR engines if engine is registered with547 * the GuC as jobs signal immediately and can't destroy an engine if the548 * GuC has a reference to it.549 */550 if (xe_exec_queue_is_lr(q))551 xe_exec_queue_get(q);552 553 set_exec_queue_registered(q);554 trace_xe_exec_queue_register(q);555 if (xe_exec_queue_is_parallel(q))556 __register_mlrc_exec_queue(guc, q, &info);557 else558 __register_exec_queue(guc, &info);559 init_policies(guc, q);560}561 562static u32 wq_space_until_wrap(struct xe_exec_queue *q)563{564 return (WQ_SIZE - q->guc->wqi_tail);565}566 567static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size)568{569 struct xe_guc *guc = exec_queue_to_guc(q);570 struct xe_device *xe = guc_to_xe(guc);571 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);572 unsigned int sleep_period_ms = 1;573 574#define AVAILABLE_SPACE \575 CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)576 if (wqi_size > AVAILABLE_SPACE) {577try_again:578 q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);579 if (wqi_size > AVAILABLE_SPACE) {580 if (sleep_period_ms == 1024) {581 xe_gt_reset_async(q->gt);582 return -ENODEV;583 }584 585 msleep(sleep_period_ms);586 sleep_period_ms <<= 1;587 goto try_again;588 }589 }590#undef AVAILABLE_SPACE591 592 return 0;593}594 595static int wq_noop_append(struct xe_exec_queue *q)596{597 struct xe_guc *guc = exec_queue_to_guc(q);598 struct xe_device *xe = guc_to_xe(guc);599 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);600 u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1;601 602 if (wq_wait_for_space(q, wq_space_until_wrap(q)))603 return -ENODEV;604 605 xe_assert(xe, FIELD_FIT(WQ_LEN_MASK, len_dw));606 607 parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)],608 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |609 FIELD_PREP(WQ_LEN_MASK, len_dw));610 q->guc->wqi_tail = 0;611 612 return 0;613}614 615static void wq_item_append(struct xe_exec_queue *q)616{617 struct xe_guc *guc = exec_queue_to_guc(q);618 struct xe_device *xe = guc_to_xe(guc);619 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);620#define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */621 u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)];622 u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32);623 u32 len_dw = (wqi_size / sizeof(u32)) - 1;624 int i = 0, j;625 626 if (wqi_size > wq_space_until_wrap(q)) {627 if (wq_noop_append(q))628 return;629 }630 if (wq_wait_for_space(q, wqi_size))631 return;632 633 wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |634 FIELD_PREP(WQ_LEN_MASK, len_dw);635 wqi[i++] = xe_lrc_descriptor(q->lrc[0]);636 wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) |637 FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64));638 wqi[i++] = 0;639 for (j = 1; j < q->width; ++j) {640 struct xe_lrc *lrc = q->lrc[j];641 642 wqi[i++] = lrc->ring.tail / sizeof(u64);643 }644 645 xe_assert(xe, i == wqi_size / sizeof(u32));646 647 iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch,648 wq[q->guc->wqi_tail / sizeof(u32)]));649 xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size);650 q->guc->wqi_tail += wqi_size;651 xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE);652 653 xe_device_wmb(xe);654 655 map = xe_lrc_parallel_map(q->lrc[0]);656 parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail);657}658 659#define RESUME_PENDING ~0x0ull660static void submit_exec_queue(struct xe_exec_queue *q)661{662 struct xe_guc *guc = exec_queue_to_guc(q);663 struct xe_device *xe = guc_to_xe(guc);664 struct xe_lrc *lrc = q->lrc[0];665 u32 action[3];666 u32 g2h_len = 0;667 u32 num_g2h = 0;668 int len = 0;669 bool extra_submit = false;670 671 xe_assert(xe, exec_queue_registered(q));672 673 if (xe_exec_queue_is_parallel(q))674 wq_item_append(q);675 else676 xe_lrc_set_ring_tail(lrc, lrc->ring.tail);677 678 if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))679 return;680 681 if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {682 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;683 action[len++] = q->guc->id;684 action[len++] = GUC_CONTEXT_ENABLE;685 g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;686 num_g2h = 1;687 if (xe_exec_queue_is_parallel(q))688 extra_submit = true;689 690 q->guc->resume_time = RESUME_PENDING;691 set_exec_queue_pending_enable(q);692 set_exec_queue_enabled(q);693 trace_xe_exec_queue_scheduling_enable(q);694 } else {695 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;696 action[len++] = q->guc->id;697 trace_xe_exec_queue_submit(q);698 }699 700 xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);701 702 if (extra_submit) {703 len = 0;704 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;705 action[len++] = q->guc->id;706 trace_xe_exec_queue_submit(q);707 708 xe_guc_ct_send(&guc->ct, action, len, 0, 0);709 }710}711 712static struct dma_fence *713guc_exec_queue_run_job(struct drm_sched_job *drm_job)714{715 struct xe_sched_job *job = to_xe_sched_job(drm_job);716 struct xe_exec_queue *q = job->q;717 struct xe_guc *guc = exec_queue_to_guc(q);718 struct xe_device *xe = guc_to_xe(guc);719 bool lr = xe_exec_queue_is_lr(q);720 721 xe_assert(xe, !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||722 exec_queue_banned(q) || exec_queue_suspended(q));723 724 trace_xe_sched_job_run(job);725 726 if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) {727 if (!exec_queue_registered(q))728 register_exec_queue(q);729 if (!lr) /* LR jobs are emitted in the exec IOCTL */730 q->ring_ops->emit_job(job);731 submit_exec_queue(q);732 }733 734 if (lr) {735 xe_sched_job_set_error(job, -EOPNOTSUPP);736 return NULL;737 } else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) {738 return job->fence;739 } else {740 return dma_fence_get(job->fence);741 }742}743 744static void guc_exec_queue_free_job(struct drm_sched_job *drm_job)745{746 struct xe_sched_job *job = to_xe_sched_job(drm_job);747 748 trace_xe_sched_job_free(job);749 xe_sched_job_put(job);750}751 752static int guc_read_stopped(struct xe_guc *guc)753{754 return atomic_read(&guc->submission_state.stopped);755}756 757#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \758 u32 action[] = { \759 XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \760 q->guc->id, \761 GUC_CONTEXT_##enable_disable, \762 }763 764static void disable_scheduling_deregister(struct xe_guc *guc,765 struct xe_exec_queue *q)766{767 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);768 struct xe_device *xe = guc_to_xe(guc);769 int ret;770 771 set_min_preemption_timeout(guc, q);772 smp_rmb();773 ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) ||774 guc_read_stopped(guc), HZ * 5);775 if (!ret) {776 struct xe_gpu_scheduler *sched = &q->guc->sched;777 778 drm_warn(&xe->drm, "Pending enable failed to respond");779 xe_sched_submission_start(sched);780 xe_gt_reset_async(q->gt);781 xe_sched_tdr_queue_imm(sched);782 return;783 }784 785 clear_exec_queue_enabled(q);786 set_exec_queue_pending_disable(q);787 set_exec_queue_destroyed(q);788 trace_xe_exec_queue_scheduling_disable(q);789 790 /*791 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H792 * handler and we are not allowed to reserved G2H space in handlers.793 */794 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),795 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +796 G2H_LEN_DW_DEREGISTER_CONTEXT, 2);797}798 799static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)800{801 struct xe_guc *guc = exec_queue_to_guc(q);802 struct xe_device *xe = guc_to_xe(guc);803 804 /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */805 wake_up_all(&xe->ufence_wq);806 807 if (xe_exec_queue_is_lr(q))808 queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);809 else810 xe_sched_tdr_queue_imm(&q->guc->sched);811}812 813/**814 * xe_guc_submit_wedge() - Wedge GuC submission815 * @guc: the GuC object816 *817 * Save exec queue's registered with GuC state by taking a ref to each queue.818 * Register a DRMM handler to drop refs upon driver unload.819 */820void xe_guc_submit_wedge(struct xe_guc *guc)821{822 struct xe_device *xe = guc_to_xe(guc);823 struct xe_exec_queue *q;824 unsigned long index;825 int err;826 827 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode);828 829 err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,830 guc_submit_wedged_fini, guc);831 if (err) {832 drm_err(&xe->drm, "Failed to register xe_guc_submit clean-up on wedged.mode=2. Although device is wedged.\n");833 return;834 }835 836 mutex_lock(&guc->submission_state.lock);837 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)838 if (xe_exec_queue_get_unless_zero(q))839 set_exec_queue_wedged(q);840 mutex_unlock(&guc->submission_state.lock);841}842 843static bool guc_submit_hint_wedged(struct xe_guc *guc)844{845 struct xe_device *xe = guc_to_xe(guc);846 847 if (xe->wedged.mode != 2)848 return false;849 850 if (xe_device_wedged(xe))851 return true;852 853 xe_device_declare_wedged(xe);854 855 return true;856}857 858static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w)859{860 struct xe_guc_exec_queue *ge =861 container_of(w, struct xe_guc_exec_queue, lr_tdr);862 struct xe_exec_queue *q = ge->q;863 struct xe_guc *guc = exec_queue_to_guc(q);864 struct xe_device *xe = guc_to_xe(guc);865 struct xe_gpu_scheduler *sched = &ge->sched;866 bool wedged;867 868 xe_assert(xe, xe_exec_queue_is_lr(q));869 trace_xe_exec_queue_lr_cleanup(q);870 871 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q));872 873 /* Kill the run_job / process_msg entry points */874 xe_sched_submission_stop(sched);875 876 /*877 * Engine state now mostly stable, disable scheduling / deregister if878 * needed. This cleanup routine might be called multiple times, where879 * the actual async engine deregister drops the final engine ref.880 * Calling disable_scheduling_deregister will mark the engine as881 * destroyed and fire off the CT requests to disable scheduling /882 * deregister, which we only want to do once. We also don't want to mark883 * the engine as pending_disable again as this may race with the884 * xe_guc_deregister_done_handler() which treats it as an unexpected885 * state.886 */887 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) {888 struct xe_guc *guc = exec_queue_to_guc(q);889 int ret;890 891 set_exec_queue_banned(q);892 disable_scheduling_deregister(guc, q);893 894 /*895 * Must wait for scheduling to be disabled before signalling896 * any fences, if GT broken the GT reset code should signal us.897 */898 ret = wait_event_timeout(guc->ct.wq,899 !exec_queue_pending_disable(q) ||900 guc_read_stopped(guc), HZ * 5);901 if (!ret) {902 drm_warn(&xe->drm, "Schedule disable failed to respond");903 xe_sched_submission_start(sched);904 xe_gt_reset_async(q->gt);905 return;906 }907 }908 909 xe_sched_submission_start(sched);910}911 912#define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100)913 914static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job)915{916 struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q));917 u32 ctx_timestamp, ctx_job_timestamp;918 u32 timeout_ms = q->sched_props.job_timeout_ms;919 u32 diff;920 u64 running_time_ms;921 922 if (!xe_sched_job_started(job)) {923 xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started",924 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),925 q->guc->id);926 927 return xe_sched_invalidate_job(job, 2);928 }929 930 ctx_timestamp = xe_lrc_ctx_timestamp(q->lrc[0]);931 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]);932 933 /*934 * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch935 * possible overflows with a high timeout.936 */937 xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC);938 939 if (ctx_timestamp < ctx_job_timestamp)940 diff = ctx_timestamp + U32_MAX - ctx_job_timestamp;941 else942 diff = ctx_timestamp - ctx_job_timestamp;943 944 /*945 * Ensure timeout is within 5% to account for an GuC scheduling latency946 */947 running_time_ms =948 ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff));949 950 xe_gt_dbg(gt,951 "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x",952 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),953 q->guc->id, running_time_ms, timeout_ms, diff);954 955 return running_time_ms >= timeout_ms;956}957 958static void enable_scheduling(struct xe_exec_queue *q)959{960 MAKE_SCHED_CONTEXT_ACTION(q, ENABLE);961 struct xe_guc *guc = exec_queue_to_guc(q);962 int ret;963 964 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));965 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));966 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));967 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));968 969 set_exec_queue_pending_enable(q);970 set_exec_queue_enabled(q);971 trace_xe_exec_queue_scheduling_enable(q);972 973 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),974 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);975 976 ret = wait_event_timeout(guc->ct.wq,977 !exec_queue_pending_enable(q) ||978 guc_read_stopped(guc), HZ * 5);979 if (!ret || guc_read_stopped(guc)) {980 xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond");981 set_exec_queue_banned(q);982 xe_gt_reset_async(q->gt);983 xe_sched_tdr_queue_imm(&q->guc->sched);984 }985}986 987static void disable_scheduling(struct xe_exec_queue *q, bool immediate)988{989 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);990 struct xe_guc *guc = exec_queue_to_guc(q);991 992 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));993 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));994 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));995 996 if (immediate)997 set_min_preemption_timeout(guc, q);998 clear_exec_queue_enabled(q);999 set_exec_queue_pending_disable(q);1000 trace_xe_exec_queue_scheduling_disable(q);1001 1002 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),1003 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);1004}1005 1006static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)1007{1008 u32 action[] = {1009 XE_GUC_ACTION_DEREGISTER_CONTEXT,1010 q->guc->id,1011 };1012 1013 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));1014 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));1015 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));1016 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));1017 1018 set_exec_queue_destroyed(q);1019 trace_xe_exec_queue_deregister(q);1020 1021 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),1022 G2H_LEN_DW_DEREGISTER_CONTEXT, 1);1023}1024 1025static enum drm_gpu_sched_stat1026guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)1027{1028 struct xe_sched_job *job = to_xe_sched_job(drm_job);1029 struct xe_sched_job *tmp_job;1030 struct xe_exec_queue *q = job->q;1031 struct xe_gpu_scheduler *sched = &q->guc->sched;1032 struct xe_guc *guc = exec_queue_to_guc(q);1033 const char *process_name = "no process";1034 int err = -ETIME;1035 pid_t pid = -1;1036 int i = 0;1037 bool wedged, skip_timeout_check;1038 1039 /*1040 * TDR has fired before free job worker. Common if exec queue1041 * immediately closed after last fence signaled. Add back to pending1042 * list so job can be freed and kick scheduler ensuring free job is not1043 * lost.1044 */1045 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) {1046 xe_sched_add_pending_job(sched, job);1047 xe_sched_submission_start(sched);1048 1049 return DRM_GPU_SCHED_STAT_NOMINAL;1050 }1051 1052 /* Kill the run_job entry point */1053 xe_sched_submission_stop(sched);1054 1055 /* Must check all state after stopping scheduler */1056 skip_timeout_check = exec_queue_reset(q) ||1057 exec_queue_killed_or_banned_or_wedged(q) ||1058 exec_queue_destroyed(q);1059 1060 /*1061 * XXX: Sampling timeout doesn't work in wedged mode as we have to1062 * modify scheduling state to read timestamp. We could read the1063 * timestamp from a register to accumulate current running time but this1064 * doesn't work for SRIOV. For now assuming timeouts in wedged mode are1065 * genuine timeouts.1066 */1067 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q));1068 1069 /* Engine state now stable, disable scheduling to check timestamp */1070 if (!wedged && exec_queue_registered(q)) {1071 int ret;1072 1073 if (exec_queue_reset(q))1074 err = -EIO;1075 1076 if (!exec_queue_destroyed(q)) {1077 /*1078 * Wait for any pending G2H to flush out before1079 * modifying state1080 */1081 ret = wait_event_timeout(guc->ct.wq,1082 !exec_queue_pending_enable(q) ||1083 guc_read_stopped(guc), HZ * 5);1084 if (!ret || guc_read_stopped(guc))1085 goto trigger_reset;1086 1087 /*1088 * Flag communicates to G2H handler that schedule1089 * disable originated from a timeout check. The G2H then1090 * avoid triggering cleanup or deregistering the exec1091 * queue.1092 */1093 set_exec_queue_check_timeout(q);1094 disable_scheduling(q, skip_timeout_check);1095 }1096 1097 /*1098 * Must wait for scheduling to be disabled before signalling1099 * any fences, if GT broken the GT reset code should signal us.1100 *1101 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault1102 * error) messages which can cause the schedule disable to get1103 * lost. If this occurs, trigger a GT reset to recover.1104 */1105 smp_rmb();1106 ret = wait_event_timeout(guc->ct.wq,1107 !exec_queue_pending_disable(q) ||1108 guc_read_stopped(guc), HZ * 5);1109 if (!ret || guc_read_stopped(guc)) {1110trigger_reset:1111 if (!ret)1112 xe_gt_warn(guc_to_gt(guc), "Schedule disable failed to respond");1113 set_exec_queue_extra_ref(q);1114 xe_exec_queue_get(q); /* GT reset owns this */1115 set_exec_queue_banned(q);1116 xe_gt_reset_async(q->gt);1117 xe_sched_tdr_queue_imm(sched);1118 goto rearm;1119 }1120 }1121 1122 /*1123 * Check if job is actually timed out, if so restart job execution and TDR1124 */1125 if (!wedged && !skip_timeout_check && !check_timeout(q, job) &&1126 !exec_queue_reset(q) && exec_queue_registered(q)) {1127 clear_exec_queue_check_timeout(q);1128 goto sched_enable;1129 }1130 1131 if (q->vm && q->vm->xef) {1132 process_name = q->vm->xef->process_name;1133 pid = q->vm->xef->pid;1134 }1135 xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]",1136 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),1137 q->guc->id, q->flags, process_name, pid);1138 1139 trace_xe_sched_job_timedout(job);1140 1141 if (!exec_queue_killed(q))1142 xe_devcoredump(job);1143 1144 /*1145 * Kernel jobs should never fail, nor should VM jobs if they do1146 * somethings has gone wrong and the GT needs a reset1147 */1148 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,1149 "Kernel-submitted job timed out\n");1150 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),1151 "VM job timed out on non-killed execqueue\n");1152 if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL ||1153 (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) {1154 if (!xe_sched_invalidate_job(job, 2)) {1155 clear_exec_queue_check_timeout(q);1156 xe_gt_reset_async(q->gt);1157 goto rearm;1158 }1159 }1160 1161 /* Finish cleaning up exec queue via deregister */1162 set_exec_queue_banned(q);1163 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) {1164 set_exec_queue_extra_ref(q);1165 xe_exec_queue_get(q);1166 __deregister_exec_queue(guc, q);1167 }1168 1169 /* Stop fence signaling */1170 xe_hw_fence_irq_stop(q->fence_irq);1171 1172 /*1173 * Fence state now stable, stop / start scheduler which cleans up any1174 * fences that are complete1175 */1176 xe_sched_add_pending_job(sched, job);1177 xe_sched_submission_start(sched);1178 1179 xe_guc_exec_queue_trigger_cleanup(q);1180 1181 /* Mark all outstanding jobs as bad, thus completing them */1182 spin_lock(&sched->base.job_list_lock);1183 list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list)1184 xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED);1185 spin_unlock(&sched->base.job_list_lock);1186 1187 /* Start fence signaling */1188 xe_hw_fence_irq_start(q->fence_irq);1189 1190 return DRM_GPU_SCHED_STAT_NOMINAL;1191 1192sched_enable:1193 enable_scheduling(q);1194rearm:1195 /*1196 * XXX: Ideally want to adjust timeout based on current exection time1197 * but there is not currently an easy way to do in DRM scheduler. With1198 * some thought, do this in a follow up.1199 */1200 xe_sched_add_pending_job(sched, job);1201 xe_sched_submission_start(sched);1202 1203 return DRM_GPU_SCHED_STAT_NOMINAL;1204}1205 1206static void __guc_exec_queue_fini_async(struct work_struct *w)1207{1208 struct xe_guc_exec_queue *ge =1209 container_of(w, struct xe_guc_exec_queue, fini_async);1210 struct xe_exec_queue *q = ge->q;1211 struct xe_guc *guc = exec_queue_to_guc(q);1212 1213 xe_pm_runtime_get(guc_to_xe(guc));1214 trace_xe_exec_queue_destroy(q);1215 1216 if (xe_exec_queue_is_lr(q))1217 cancel_work_sync(&ge->lr_tdr);1218 release_guc_id(guc, q);1219 xe_sched_entity_fini(&ge->entity);1220 xe_sched_fini(&ge->sched);1221 1222 kfree(ge);1223 xe_exec_queue_fini(q);1224 xe_pm_runtime_put(guc_to_xe(guc));1225}1226 1227static void guc_exec_queue_fini_async(struct xe_exec_queue *q)1228{1229 struct xe_guc *guc = exec_queue_to_guc(q);1230 struct xe_device *xe = guc_to_xe(guc);1231 1232 INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async);1233 1234 /* We must block on kernel engines so slabs are empty on driver unload */1235 if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q))1236 __guc_exec_queue_fini_async(&q->guc->fini_async);1237 else1238 queue_work(xe->destroy_wq, &q->guc->fini_async);1239}1240 1241static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q)1242{1243 /*1244 * Might be done from within the GPU scheduler, need to do async as we1245 * fini the scheduler when the engine is fini'd, the scheduler can't1246 * complete fini within itself (circular dependency). Async resolves1247 * this we and don't really care when everything is fini'd, just that it1248 * is.1249 */1250 guc_exec_queue_fini_async(q);1251}1252 1253static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg)1254{1255 struct xe_exec_queue *q = msg->private_data;1256 struct xe_guc *guc = exec_queue_to_guc(q);1257 struct xe_device *xe = guc_to_xe(guc);1258 1259 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));1260 trace_xe_exec_queue_cleanup_entity(q);1261 1262 if (exec_queue_registered(q))1263 disable_scheduling_deregister(guc, q);1264 else1265 __guc_exec_queue_fini(guc, q);1266}1267 1268static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)1269{1270 return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q);1271}1272 1273static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg)1274{1275 struct xe_exec_queue *q = msg->private_data;1276 struct xe_guc *guc = exec_queue_to_guc(q);1277 1278 if (guc_exec_queue_allowed_to_change_state(q))1279 init_policies(guc, q);1280 kfree(msg);1281}1282 1283static void __suspend_fence_signal(struct xe_exec_queue *q)1284{1285 if (!q->guc->suspend_pending)1286 return;1287 1288 WRITE_ONCE(q->guc->suspend_pending, false);1289 wake_up(&q->guc->suspend_wait);1290}1291 1292static void suspend_fence_signal(struct xe_exec_queue *q)1293{1294 struct xe_guc *guc = exec_queue_to_guc(q);1295 struct xe_device *xe = guc_to_xe(guc);1296 1297 xe_assert(xe, exec_queue_suspended(q) || exec_queue_killed(q) ||1298 guc_read_stopped(guc));1299 xe_assert(xe, q->guc->suspend_pending);1300 1301 __suspend_fence_signal(q);1302}1303 1304static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)1305{1306 struct xe_exec_queue *q = msg->private_data;1307 struct xe_guc *guc = exec_queue_to_guc(q);1308 1309 if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&1310 exec_queue_enabled(q)) {1311 wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING ||1312 guc_read_stopped(guc));1313 1314 if (!guc_read_stopped(guc)) {1315 s64 since_resume_ms =1316 ktime_ms_delta(ktime_get(),1317 q->guc->resume_time);1318 s64 wait_ms = q->vm->preempt.min_run_period_ms -1319 since_resume_ms;1320 1321 if (wait_ms > 0 && q->guc->resume_time)1322 msleep(wait_ms);1323 1324 set_exec_queue_suspended(q);1325 disable_scheduling(q, false);1326 }1327 } else if (q->guc->suspend_pending) {1328 set_exec_queue_suspended(q);1329 suspend_fence_signal(q);1330 }1331}1332 1333static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)1334{1335 struct xe_exec_queue *q = msg->private_data;1336 1337 if (guc_exec_queue_allowed_to_change_state(q)) {1338 clear_exec_queue_suspended(q);1339 if (!exec_queue_enabled(q)) {1340 q->guc->resume_time = RESUME_PENDING;1341 enable_scheduling(q);1342 }1343 } else {1344 clear_exec_queue_suspended(q);1345 }1346}1347 1348#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */1349#define SET_SCHED_PROPS 21350#define SUSPEND 31351#define RESUME 41352#define OPCODE_MASK 0xf1353#define MSG_LOCKED BIT(8)1354 1355static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)1356{1357 struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data));1358 1359 trace_xe_sched_msg_recv(msg);1360 1361 switch (msg->opcode) {1362 case CLEANUP:1363 __guc_exec_queue_process_msg_cleanup(msg);1364 break;1365 case SET_SCHED_PROPS:1366 __guc_exec_queue_process_msg_set_sched_props(msg);1367 break;1368 case SUSPEND:1369 __guc_exec_queue_process_msg_suspend(msg);1370 break;1371 case RESUME:1372 __guc_exec_queue_process_msg_resume(msg);1373 break;1374 default:1375 XE_WARN_ON("Unknown message type");1376 }1377 1378 xe_pm_runtime_put(xe);1379}1380 1381static const struct drm_sched_backend_ops drm_sched_ops = {1382 .run_job = guc_exec_queue_run_job,1383 .free_job = guc_exec_queue_free_job,1384 .timedout_job = guc_exec_queue_timedout_job,1385};1386 1387static const struct xe_sched_backend_ops xe_sched_ops = {1388 .process_msg = guc_exec_queue_process_msg,1389};1390 1391static int guc_exec_queue_init(struct xe_exec_queue *q)1392{1393 struct xe_gpu_scheduler *sched;1394 struct xe_guc *guc = exec_queue_to_guc(q);1395 struct xe_device *xe = guc_to_xe(guc);1396 struct xe_guc_exec_queue *ge;1397 long timeout;1398 int err, i;1399 1400 xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc)));1401 1402 ge = kzalloc(sizeof(*ge), GFP_KERNEL);1403 if (!ge)1404 return -ENOMEM;1405 1406 q->guc = ge;1407 ge->q = q;1408 init_waitqueue_head(&ge->suspend_wait);1409 1410 for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i)1411 INIT_LIST_HEAD(&ge->static_msgs[i].link);1412 1413 timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :1414 msecs_to_jiffies(q->sched_props.job_timeout_ms);1415 err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,1416 NULL, q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 64,1417 timeout, guc_to_gt(guc)->ordered_wq, NULL,1418 q->name, gt_to_xe(q->gt)->drm.dev);1419 if (err)1420 goto err_free;1421 1422 sched = &ge->sched;1423 err = xe_sched_entity_init(&ge->entity, sched);1424 if (err)1425 goto err_sched;1426 1427 if (xe_exec_queue_is_lr(q))1428 INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup);1429 1430 mutex_lock(&guc->submission_state.lock);1431 1432 err = alloc_guc_id(guc, q);1433 if (err)1434 goto err_entity;1435 1436 q->entity = &ge->entity;1437 1438 if (guc_read_stopped(guc))1439 xe_sched_stop(sched);1440 1441 mutex_unlock(&guc->submission_state.lock);1442 1443 xe_exec_queue_assign_name(q, q->guc->id);1444 1445 trace_xe_exec_queue_create(q);1446 1447 return 0;1448 1449err_entity:1450 mutex_unlock(&guc->submission_state.lock);1451 xe_sched_entity_fini(&ge->entity);1452err_sched:1453 xe_sched_fini(&ge->sched);1454err_free:1455 kfree(ge);1456 1457 return err;1458}1459 1460static void guc_exec_queue_kill(struct xe_exec_queue *q)1461{1462 trace_xe_exec_queue_kill(q);1463 set_exec_queue_killed(q);1464 __suspend_fence_signal(q);1465 xe_guc_exec_queue_trigger_cleanup(q);1466}1467 1468static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,1469 u32 opcode)1470{1471 xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q)));1472 1473 INIT_LIST_HEAD(&msg->link);1474 msg->opcode = opcode & OPCODE_MASK;1475 msg->private_data = q;1476 1477 trace_xe_sched_msg_add(msg);1478 if (opcode & MSG_LOCKED)1479 xe_sched_add_msg_locked(&q->guc->sched, msg);1480 else1481 xe_sched_add_msg(&q->guc->sched, msg);1482}1483 1484static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q,1485 struct xe_sched_msg *msg,1486 u32 opcode)1487{1488 if (!list_empty(&msg->link))1489 return false;1490 1491 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED);1492 1493 return true;1494}1495 1496#define STATIC_MSG_CLEANUP 01497#define STATIC_MSG_SUSPEND 11498#define STATIC_MSG_RESUME 21499static void guc_exec_queue_fini(struct xe_exec_queue *q)1500{1501 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;1502 1503 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q))1504 guc_exec_queue_add_msg(q, msg, CLEANUP);1505 else1506 __guc_exec_queue_fini(exec_queue_to_guc(q), q);1507}1508 1509static int guc_exec_queue_set_priority(struct xe_exec_queue *q,1510 enum xe_exec_queue_priority priority)1511{1512 struct xe_sched_msg *msg;1513 1514 if (q->sched_props.priority == priority ||1515 exec_queue_killed_or_banned_or_wedged(q))1516 return 0;1517 1518 msg = kmalloc(sizeof(*msg), GFP_KERNEL);1519 if (!msg)1520 return -ENOMEM;1521 1522 q->sched_props.priority = priority;1523 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);1524 1525 return 0;1526}1527 1528static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us)1529{1530 struct xe_sched_msg *msg;1531 1532 if (q->sched_props.timeslice_us == timeslice_us ||1533 exec_queue_killed_or_banned_or_wedged(q))1534 return 0;1535 1536 msg = kmalloc(sizeof(*msg), GFP_KERNEL);1537 if (!msg)1538 return -ENOMEM;1539 1540 q->sched_props.timeslice_us = timeslice_us;1541 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);1542 1543 return 0;1544}1545 1546static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q,1547 u32 preempt_timeout_us)1548{1549 struct xe_sched_msg *msg;1550 1551 if (q->sched_props.preempt_timeout_us == preempt_timeout_us ||1552 exec_queue_killed_or_banned_or_wedged(q))1553 return 0;1554 1555 msg = kmalloc(sizeof(*msg), GFP_KERNEL);1556 if (!msg)1557 return -ENOMEM;1558 1559 q->sched_props.preempt_timeout_us = preempt_timeout_us;1560 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS);1561 1562 return 0;1563}1564 1565static int guc_exec_queue_suspend(struct xe_exec_queue *q)1566{1567 struct xe_gpu_scheduler *sched = &q->guc->sched;1568 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;1569 1570 if (exec_queue_killed_or_banned_or_wedged(q))1571 return -EINVAL;1572 1573 xe_sched_msg_lock(sched);1574 if (guc_exec_queue_try_add_msg(q, msg, SUSPEND))1575 q->guc->suspend_pending = true;1576 xe_sched_msg_unlock(sched);1577 1578 return 0;1579}1580 1581static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)1582{1583 struct xe_guc *guc = exec_queue_to_guc(q);1584 int ret;1585 1586 /*1587 * Likely don't need to check exec_queue_killed() as we clear1588 * suspend_pending upon kill but to be paranoid but races in which1589 * suspend_pending is set after kill also check kill here.1590 */1591 ret = wait_event_interruptible_timeout(q->guc->suspend_wait,1592 !READ_ONCE(q->guc->suspend_pending) ||1593 exec_queue_killed(q) ||1594 guc_read_stopped(guc),1595 HZ * 5);1596 1597 if (!ret) {1598 xe_gt_warn(guc_to_gt(guc),1599 "Suspend fence, guc_id=%d, failed to respond",1600 q->guc->id);1601 /* XXX: Trigger GT reset? */1602 return -ETIME;1603 }1604 1605 return ret < 0 ? ret : 0;1606}1607 1608static void guc_exec_queue_resume(struct xe_exec_queue *q)1609{1610 struct xe_gpu_scheduler *sched = &q->guc->sched;1611 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME;1612 struct xe_guc *guc = exec_queue_to_guc(q);1613 struct xe_device *xe = guc_to_xe(guc);1614 1615 xe_assert(xe, !q->guc->suspend_pending);1616 1617 xe_sched_msg_lock(sched);1618 guc_exec_queue_try_add_msg(q, msg, RESUME);1619 xe_sched_msg_unlock(sched);1620}1621 1622static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)1623{1624 return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);1625}1626 1627/*1628 * All of these functions are an abstraction layer which other parts of XE can1629 * use to trap into the GuC backend. All of these functions, aside from init,1630 * really shouldn't do much other than trap into the DRM scheduler which1631 * synchronizes these operations.1632 */1633static const struct xe_exec_queue_ops guc_exec_queue_ops = {1634 .init = guc_exec_queue_init,1635 .kill = guc_exec_queue_kill,1636 .fini = guc_exec_queue_fini,1637 .set_priority = guc_exec_queue_set_priority,1638 .set_timeslice = guc_exec_queue_set_timeslice,1639 .set_preempt_timeout = guc_exec_queue_set_preempt_timeout,1640 .suspend = guc_exec_queue_suspend,1641 .suspend_wait = guc_exec_queue_suspend_wait,1642 .resume = guc_exec_queue_resume,1643 .reset_status = guc_exec_queue_reset_status,1644};1645 1646static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)1647{1648 struct xe_gpu_scheduler *sched = &q->guc->sched;1649 1650 /* Stop scheduling + flush any DRM scheduler operations */1651 xe_sched_submission_stop(sched);1652 1653 /* Clean up lost G2H + reset engine state */1654 if (exec_queue_registered(q)) {1655 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q))1656 xe_exec_queue_put(q);1657 else if (exec_queue_destroyed(q))1658 __guc_exec_queue_fini(guc, q);1659 }1660 if (q->guc->suspend_pending) {1661 set_exec_queue_suspended(q);1662 suspend_fence_signal(q);1663 }1664 atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED |1665 EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED |1666 EXEC_QUEUE_STATE_SUSPENDED,1667 &q->guc->state);1668 q->guc->resume_time = 0;1669 trace_xe_exec_queue_stop(q);1670 1671 /*1672 * Ban any engine (aside from kernel and engines used for VM ops) with a1673 * started but not complete job or if a job has gone through a GT reset1674 * more than twice.1675 */1676 if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) {1677 struct xe_sched_job *job = xe_sched_first_pending_job(sched);1678 bool ban = false;1679 1680 if (job) {1681 if ((xe_sched_job_started(job) &&1682 !xe_sched_job_completed(job)) ||1683 xe_sched_invalidate_job(job, 2)) {1684 trace_xe_sched_job_ban(job);1685 ban = true;1686 }1687 } else if (xe_exec_queue_is_lr(q) &&1688 (xe_lrc_ring_head(q->lrc[0]) != xe_lrc_ring_tail(q->lrc[0]))) {1689 ban = true;1690 }1691 1692 if (ban) {1693 set_exec_queue_banned(q);1694 xe_guc_exec_queue_trigger_cleanup(q);1695 }1696 }1697}1698 1699int xe_guc_submit_reset_prepare(struct xe_guc *guc)1700{1701 int ret;1702 1703 /*1704 * Using an atomic here rather than submission_state.lock as this1705 * function can be called while holding the CT lock (engine reset1706 * failure). submission_state.lock needs the CT lock to resubmit jobs.1707 * Atomic is not ideal, but it works to prevent against concurrent reset1708 * and releasing any TDRs waiting on guc->submission_state.stopped.1709 */1710 ret = atomic_fetch_or(1, &guc->submission_state.stopped);1711 smp_wmb();1712 wake_up_all(&guc->ct.wq);1713 1714 return ret;1715}1716 1717void xe_guc_submit_reset_wait(struct xe_guc *guc)1718{1719 wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) ||1720 !guc_read_stopped(guc));1721}1722 1723void xe_guc_submit_stop(struct xe_guc *guc)1724{1725 struct xe_exec_queue *q;1726 unsigned long index;1727 struct xe_device *xe = guc_to_xe(guc);1728 1729 xe_assert(xe, guc_read_stopped(guc) == 1);1730 1731 mutex_lock(&guc->submission_state.lock);1732 1733 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {1734 /* Prevent redundant attempts to stop parallel queues */1735 if (q->guc->id != index)1736 continue;1737 1738 guc_exec_queue_stop(guc, q);1739 }1740 1741 mutex_unlock(&guc->submission_state.lock);1742 1743 /*1744 * No one can enter the backend at this point, aside from new engine1745 * creation which is protected by guc->submission_state.lock.1746 */1747 1748}1749 1750static void guc_exec_queue_start(struct xe_exec_queue *q)1751{1752 struct xe_gpu_scheduler *sched = &q->guc->sched;1753 1754 if (!exec_queue_killed_or_banned_or_wedged(q)) {1755 int i;1756 1757 trace_xe_exec_queue_resubmit(q);1758 for (i = 0; i < q->width; ++i)1759 xe_lrc_set_ring_head(q->lrc[i], q->lrc[i]->ring.tail);1760 xe_sched_resubmit_jobs(sched);1761 }1762 1763 xe_sched_submission_start(sched);1764 xe_sched_submission_resume_tdr(sched);1765}1766 1767int xe_guc_submit_start(struct xe_guc *guc)1768{1769 struct xe_exec_queue *q;1770 unsigned long index;1771 struct xe_device *xe = guc_to_xe(guc);1772 1773 xe_assert(xe, guc_read_stopped(guc) == 1);1774 1775 mutex_lock(&guc->submission_state.lock);1776 atomic_dec(&guc->submission_state.stopped);1777 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {1778 /* Prevent redundant attempts to start parallel queues */1779 if (q->guc->id != index)1780 continue;1781 1782 guc_exec_queue_start(q);1783 }1784 mutex_unlock(&guc->submission_state.lock);1785 1786 wake_up_all(&guc->ct.wq);1787 1788 return 0;1789}1790 1791static struct xe_exec_queue *1792g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id)1793{1794 struct xe_device *xe = guc_to_xe(guc);1795 struct xe_exec_queue *q;1796 1797 if (unlikely(guc_id >= GUC_ID_MAX)) {1798 drm_err(&xe->drm, "Invalid guc_id %u", guc_id);1799 return NULL;1800 }1801 1802 q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id);1803 if (unlikely(!q)) {1804 drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id);1805 return NULL;1806 }1807 1808 xe_assert(xe, guc_id >= q->guc->id);1809 xe_assert(xe, guc_id < (q->guc->id + q->width));1810 1811 return q;1812}1813 1814static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)1815{1816 u32 action[] = {1817 XE_GUC_ACTION_DEREGISTER_CONTEXT,1818 q->guc->id,1819 };1820 1821 xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q));1822 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));1823 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));1824 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q));1825 1826 trace_xe_exec_queue_deregister(q);1827 1828 xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));1829}1830 1831static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q,1832 u32 runnable_state)1833{1834 trace_xe_exec_queue_scheduling_done(q);1835 1836 if (runnable_state == 1) {1837 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q));1838 1839 q->guc->resume_time = ktime_get();1840 clear_exec_queue_pending_enable(q);1841 smp_wmb();1842 wake_up_all(&guc->ct.wq);1843 } else {1844 bool check_timeout = exec_queue_check_timeout(q);1845 1846 xe_gt_assert(guc_to_gt(guc), runnable_state == 0);1847 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q));1848 1849 clear_exec_queue_pending_disable(q);1850 if (q->guc->suspend_pending) {1851 suspend_fence_signal(q);1852 } else {1853 if (exec_queue_banned(q) || check_timeout) {1854 smp_wmb();1855 wake_up_all(&guc->ct.wq);1856 }1857 if (!check_timeout)1858 deregister_exec_queue(guc, q);1859 }1860 }1861}1862 1863int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)1864{1865 struct xe_device *xe = guc_to_xe(guc);1866 struct xe_exec_queue *q;1867 u32 guc_id = msg[0];1868 u32 runnable_state = msg[1];1869 1870 if (unlikely(len < 2)) {1871 drm_err(&xe->drm, "Invalid length %u", len);1872 return -EPROTO;1873 }1874 1875 q = g2h_exec_queue_lookup(guc, guc_id);1876 if (unlikely(!q))1877 return -EPROTO;1878 1879 if (unlikely(!exec_queue_pending_enable(q) &&1880 !exec_queue_pending_disable(q))) {1881 xe_gt_err(guc_to_gt(guc),1882 "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u",1883 atomic_read(&q->guc->state), q->guc->id,1884 runnable_state);1885 return -EPROTO;1886 }1887 1888 handle_sched_done(guc, q, runnable_state);1889 1890 return 0;1891}1892 1893static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)1894{1895 trace_xe_exec_queue_deregister_done(q);1896 1897 clear_exec_queue_registered(q);1898 1899 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q))1900 xe_exec_queue_put(q);1901 else1902 __guc_exec_queue_fini(guc, q);1903}1904 1905int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)1906{1907 struct xe_device *xe = guc_to_xe(guc);1908 struct xe_exec_queue *q;1909 u32 guc_id = msg[0];1910 1911 if (unlikely(len < 1)) {1912 drm_err(&xe->drm, "Invalid length %u", len);1913 return -EPROTO;1914 }1915 1916 q = g2h_exec_queue_lookup(guc, guc_id);1917 if (unlikely(!q))1918 return -EPROTO;1919 1920 if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) ||1921 exec_queue_pending_enable(q) || exec_queue_enabled(q)) {1922 xe_gt_err(guc_to_gt(guc),1923 "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d",1924 atomic_read(&q->guc->state), q->guc->id);1925 return -EPROTO;1926 }1927 1928 handle_deregister_done(guc, q);1929 1930 return 0;1931}1932 1933int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)1934{1935 struct xe_gt *gt = guc_to_gt(guc);1936 struct xe_device *xe = guc_to_xe(guc);1937 struct xe_exec_queue *q;1938 u32 guc_id = msg[0];1939 1940 if (unlikely(len < 1)) {1941 drm_err(&xe->drm, "Invalid length %u", len);1942 return -EPROTO;1943 }1944 1945 q = g2h_exec_queue_lookup(guc, guc_id);1946 if (unlikely(!q))1947 return -EPROTO;1948 1949 xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d",1950 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);1951 1952 /* FIXME: Do error capture, most likely async */1953 1954 trace_xe_exec_queue_reset(q);1955 1956 /*1957 * A banned engine is a NOP at this point (came from1958 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel1959 * jobs by setting timeout of the job to the minimum value kicking1960 * guc_exec_queue_timedout_job.1961 */1962 set_exec_queue_reset(q);1963 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))1964 xe_guc_exec_queue_trigger_cleanup(q);1965 1966 return 0;1967}1968 1969int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,1970 u32 len)1971{1972 struct xe_gt *gt = guc_to_gt(guc);1973 struct xe_device *xe = guc_to_xe(guc);1974 struct xe_exec_queue *q;1975 u32 guc_id = msg[0];1976 1977 if (unlikely(len < 1)) {1978 drm_err(&xe->drm, "Invalid length %u", len);1979 return -EPROTO;1980 }1981 1982 q = g2h_exec_queue_lookup(guc, guc_id);1983 if (unlikely(!q))1984 return -EPROTO;1985 1986 xe_gt_dbg(gt, "Engine memory cat error: engine_class=%s, logical_mask: 0x%x, guc_id=%d",1987 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);1988 1989 trace_xe_exec_queue_memory_cat_error(q);1990 1991 /* Treat the same as engine reset */1992 set_exec_queue_reset(q);1993 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))1994 xe_guc_exec_queue_trigger_cleanup(q);1995 1996 return 0;1997}1998 1999int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)2000{2001 struct xe_device *xe = guc_to_xe(guc);2002 u8 guc_class, instance;2003 u32 reason;2004 2005 if (unlikely(len != 3)) {2006 drm_err(&xe->drm, "Invalid length %u", len);2007 return -EPROTO;2008 }2009 2010 guc_class = msg[0];2011 instance = msg[1];2012 reason = msg[2];2013 2014 /* Unexpected failure of a hardware feature, log an actual error */2015 drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X",2016 guc_class, instance, reason);2017 2018 xe_gt_reset_async(guc_to_gt(guc));2019 2020 return 0;2021}2022 2023static void2024guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,2025 struct xe_guc_submit_exec_queue_snapshot *snapshot)2026{2027 struct xe_guc *guc = exec_queue_to_guc(q);2028 struct xe_device *xe = guc_to_xe(guc);2029 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]);2030 int i;2031 2032 snapshot->guc.wqi_head = q->guc->wqi_head;2033 snapshot->guc.wqi_tail = q->guc->wqi_tail;2034 snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head);2035 snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail);2036 snapshot->parallel.wq_desc.status = parallel_read(xe, map,2037 wq_desc.wq_status);2038 2039 if (snapshot->parallel.wq_desc.head !=2040 snapshot->parallel.wq_desc.tail) {2041 for (i = snapshot->parallel.wq_desc.head;2042 i != snapshot->parallel.wq_desc.tail;2043 i = (i + sizeof(u32)) % WQ_SIZE)2044 snapshot->parallel.wq[i / sizeof(u32)] =2045 parallel_read(xe, map, wq[i / sizeof(u32)]);2046 }2047}2048 2049static void2050guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,2051 struct drm_printer *p)2052{2053 int i;2054 2055 drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n",2056 snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head);2057 drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n",2058 snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail);2059 drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status);2060 2061 if (snapshot->parallel.wq_desc.head !=2062 snapshot->parallel.wq_desc.tail) {2063 for (i = snapshot->parallel.wq_desc.head;2064 i != snapshot->parallel.wq_desc.tail;2065 i = (i + sizeof(u32)) % WQ_SIZE)2066 drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32),2067 snapshot->parallel.wq[i / sizeof(u32)]);2068 }2069}2070 2071/**2072 * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine.2073 * @q: faulty exec queue2074 *2075 * This can be printed out in a later stage like during dev_coredump2076 * analysis.2077 *2078 * Returns: a GuC Submit Engine snapshot object that must be freed by the2079 * caller, using `xe_guc_exec_queue_snapshot_free`.2080 */2081struct xe_guc_submit_exec_queue_snapshot *2082xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)2083{2084 struct xe_gpu_scheduler *sched = &q->guc->sched;2085 struct xe_guc_submit_exec_queue_snapshot *snapshot;2086 int i;2087 2088 snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC);2089 2090 if (!snapshot)2091 return NULL;2092 2093 snapshot->guc.id = q->guc->id;2094 memcpy(&snapshot->name, &q->name, sizeof(snapshot->name));2095 snapshot->class = q->class;2096 snapshot->logical_mask = q->logical_mask;2097 snapshot->width = q->width;2098 snapshot->refcount = kref_read(&q->refcount);2099 snapshot->sched_timeout = sched->base.timeout;2100 snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;2101 snapshot->sched_props.preempt_timeout_us =2102 q->sched_props.preempt_timeout_us;2103 2104 snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *),2105 GFP_ATOMIC);2106 2107 if (snapshot->lrc) {2108 for (i = 0; i < q->width; ++i) {2109 struct xe_lrc *lrc = q->lrc[i];2110 2111 snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc);2112 }2113 }2114 2115 snapshot->schedule_state = atomic_read(&q->guc->state);2116 snapshot->exec_queue_flags = q->flags;2117 2118 snapshot->parallel_execution = xe_exec_queue_is_parallel(q);2119 if (snapshot->parallel_execution)2120 guc_exec_queue_wq_snapshot_capture(q, snapshot);2121 2122 spin_lock(&sched->base.job_list_lock);2123 snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);2124 snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,2125 sizeof(struct pending_list_snapshot),2126 GFP_ATOMIC);2127 2128 if (snapshot->pending_list) {2129 struct xe_sched_job *job_iter;2130 2131 i = 0;2132 list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) {2133 snapshot->pending_list[i].seqno =2134 xe_sched_job_seqno(job_iter);2135 snapshot->pending_list[i].fence =2136 dma_fence_is_signaled(job_iter->fence) ? 1 : 0;2137 snapshot->pending_list[i].finished =2138 dma_fence_is_signaled(&job_iter->drm.s_fence->finished)2139 ? 1 : 0;2140 i++;2141 }2142 }2143 2144 spin_unlock(&sched->base.job_list_lock);2145 2146 return snapshot;2147}2148 2149/**2150 * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine.2151 * @snapshot: Previously captured snapshot of job.2152 *2153 * This captures some data that requires taking some locks, so it cannot be done in signaling path.2154 */2155void2156xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot)2157{2158 int i;2159 2160 if (!snapshot || !snapshot->lrc)2161 return;2162 2163 for (i = 0; i < snapshot->width; ++i)2164 xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]);2165}2166 2167/**2168 * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot.2169 * @snapshot: GuC Submit Engine snapshot object.2170 * @p: drm_printer where it will be printed out.2171 *2172 * This function prints out a given GuC Submit Engine snapshot object.2173 */2174void2175xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot,2176 struct drm_printer *p)2177{2178 int i;2179 2180 if (!snapshot)2181 return;2182 2183 drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id);2184 drm_printf(p, "\tName: %s\n", snapshot->name);2185 drm_printf(p, "\tClass: %d\n", snapshot->class);2186 drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);2187 drm_printf(p, "\tWidth: %d\n", snapshot->width);2188 drm_printf(p, "\tRef: %d\n", snapshot->refcount);2189 drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);2190 drm_printf(p, "\tTimeslice: %u (us)\n",2191 snapshot->sched_props.timeslice_us);2192 drm_printf(p, "\tPreempt timeout: %u (us)\n",2193 snapshot->sched_props.preempt_timeout_us);2194 2195 for (i = 0; snapshot->lrc && i < snapshot->width; ++i)2196 xe_lrc_snapshot_print(snapshot->lrc[i], p);2197 2198 drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state);2199 drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags);2200 2201 if (snapshot->parallel_execution)2202 guc_exec_queue_wq_snapshot_print(snapshot, p);2203 2204 for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;2205 i++)2206 drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",2207 snapshot->pending_list[i].seqno,2208 snapshot->pending_list[i].fence,2209 snapshot->pending_list[i].finished);2210}2211 2212/**2213 * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given2214 * snapshot.2215 * @snapshot: GuC Submit Engine snapshot object.2216 *2217 * This function free all the memory that needed to be allocated at capture2218 * time.2219 */2220void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot)2221{2222 int i;2223 2224 if (!snapshot)2225 return;2226 2227 if (snapshot->lrc) {2228 for (i = 0; i < snapshot->width; i++)2229 xe_lrc_snapshot_free(snapshot->lrc[i]);2230 kfree(snapshot->lrc);2231 }2232 kfree(snapshot->pending_list);2233 kfree(snapshot);2234}2235 2236static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p)2237{2238 struct xe_guc_submit_exec_queue_snapshot *snapshot;2239 2240 snapshot = xe_guc_exec_queue_snapshot_capture(q);2241 xe_guc_exec_queue_snapshot_print(snapshot, p);2242 xe_guc_exec_queue_snapshot_free(snapshot);2243}2244 2245/**2246 * xe_guc_submit_print - GuC Submit Print.2247 * @guc: GuC.2248 * @p: drm_printer where it will be printed out.2249 *2250 * This function capture and prints snapshots of **all** GuC Engines.2251 */2252void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p)2253{2254 struct xe_exec_queue *q;2255 unsigned long index;2256 2257 if (!xe_device_uc_enabled(guc_to_xe(guc)))2258 return;2259 2260 mutex_lock(&guc->submission_state.lock);2261 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)2262 guc_exec_queue_print(q, p);2263 mutex_unlock(&guc->submission_state.lock);2264}2265