5940 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2014 Intel Corporation4 */5 6#include <linux/circ_buf.h>7 8#include "gem/i915_gem_context.h"9#include "gem/i915_gem_lmem.h"10#include "gt/gen8_engine_cs.h"11#include "gt/intel_breadcrumbs.h"12#include "gt/intel_context.h"13#include "gt/intel_engine_heartbeat.h"14#include "gt/intel_engine_pm.h"15#include "gt/intel_engine_regs.h"16#include "gt/intel_gpu_commands.h"17#include "gt/intel_gt.h"18#include "gt/intel_gt_clock_utils.h"19#include "gt/intel_gt_irq.h"20#include "gt/intel_gt_pm.h"21#include "gt/intel_gt_regs.h"22#include "gt/intel_gt_requests.h"23#include "gt/intel_lrc.h"24#include "gt/intel_lrc_reg.h"25#include "gt/intel_mocs.h"26#include "gt/intel_ring.h"27 28#include "intel_guc_ads.h"29#include "intel_guc_capture.h"30#include "intel_guc_print.h"31#include "intel_guc_submission.h"32 33#include "i915_drv.h"34#include "i915_reg.h"35#include "i915_irq.h"36#include "i915_trace.h"37 38/**39 * DOC: GuC-based command submission40 *41 * The Scratch registers:42 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes43 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then44 * triggers an interrupt on the GuC via another register write (0xC4C8).45 * Firmware writes a success/fail code back to the action register after46 * processes the request. The kernel driver polls waiting for this update and47 * then proceeds.48 *49 * Command Transport buffers (CTBs):50 * Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host51 * - G2H) are a message interface between the i915 and GuC.52 *53 * Context registration:54 * Before a context can be submitted it must be registered with the GuC via a55 * H2G. A unique guc_id is associated with each context. The context is either56 * registered at request creation time (normal operation) or at submission time57 * (abnormal operation, e.g. after a reset).58 *59 * Context submission:60 * The i915 updates the LRC tail value in memory. The i915 must enable the61 * scheduling of the context within the GuC for the GuC to actually consider it.62 * Therefore, the first time a disabled context is submitted we use a schedule63 * enable H2G, while follow up submissions are done via the context submit H2G,64 * which informs the GuC that a previously enabled context has new work65 * available.66 *67 * Context unpin:68 * To unpin a context a H2G is used to disable scheduling. When the69 * corresponding G2H returns indicating the scheduling disable operation has70 * completed it is safe to unpin the context. While a disable is in flight it71 * isn't safe to resubmit the context so a fence is used to stall all future72 * requests of that context until the G2H is returned. Because this interaction73 * with the GuC takes a non-zero amount of time we delay the disabling of74 * scheduling after the pin count goes to zero by a configurable period of time75 * (see SCHED_DISABLE_DELAY_MS). The thought is this gives the user a window of76 * time to resubmit something on the context before doing this costly operation.77 * This delay is only done if the context isn't closed and the guc_id usage is78 * less than a threshold (see NUM_SCHED_DISABLE_GUC_IDS_THRESHOLD).79 *80 * Context deregistration:81 * Before a context can be destroyed or if we steal its guc_id we must82 * deregister the context with the GuC via H2G. If stealing the guc_id it isn't83 * safe to submit anything to this guc_id until the deregister completes so a84 * fence is used to stall all requests associated with this guc_id until the85 * corresponding G2H returns indicating the guc_id has been deregistered.86 *87 * submission_state.guc_ids:88 * Unique number associated with private GuC context data passed in during89 * context registration / submission / deregistration. 64k available. Simple ida90 * is used for allocation.91 *92 * Stealing guc_ids:93 * If no guc_ids are available they can be stolen from another context at94 * request creation time if that context is unpinned. If a guc_id can't be found95 * we punt this problem to the user as we believe this is near impossible to hit96 * during normal use cases.97 *98 * Locking:99 * In the GuC submission code we have 3 basic spin locks which protect100 * everything. Details about each below.101 *102 * sched_engine->lock103 * This is the submission lock for all contexts that share an i915 schedule104 * engine (sched_engine), thus only one of the contexts which share a105 * sched_engine can be submitting at a time. Currently only one sched_engine is106 * used for all of GuC submission but that could change in the future.107 *108 * guc->submission_state.lock109 * Global lock for GuC submission state. Protects guc_ids and destroyed contexts110 * list.111 *112 * ce->guc_state.lock113 * Protects everything under ce->guc_state. Ensures that a context is in the114 * correct state before issuing a H2G. e.g. We don't issue a schedule disable115 * on a disabled context (bad idea), we don't issue a schedule enable when a116 * schedule disable is in flight, etc... Also protects list of inflight requests117 * on the context and the priority management state. Lock is individual to each118 * context.119 *120 * Lock ordering rules:121 * sched_engine->lock -> ce->guc_state.lock122 * guc->submission_state.lock -> ce->guc_state.lock123 *124 * Reset races:125 * When a full GT reset is triggered it is assumed that some G2H responses to126 * H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be127 * fatal as we do certain operations upon receiving a G2H (e.g. destroy128 * contexts, release guc_ids, etc...). When this occurs we can scrub the129 * context state and cleanup appropriately, however this is quite racey.130 * To avoid races, the reset code must disable submission before scrubbing for131 * the missing G2H, while the submission code must check for submission being132 * disabled and skip sending H2Gs and updating context states when it is. Both133 * sides must also make sure to hold the relevant locks.134 */135 136/* GuC Virtual Engine */137struct guc_virtual_engine {138 struct intel_engine_cs base;139 struct intel_context context;140};141 142static struct intel_context *143guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,144 unsigned long flags);145 146static struct intel_context *147guc_create_parallel(struct intel_engine_cs **engines,148 unsigned int num_siblings,149 unsigned int width);150 151#define GUC_REQUEST_SIZE 64 /* bytes */152 153/*154 * We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous155 * per the GuC submission interface. A different allocation algorithm is used156 * (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to157 * partition the guc_id space. We believe the number of multi-lrc contexts in158 * use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for159 * multi-lrc.160 */161#define NUMBER_MULTI_LRC_GUC_ID(guc) \162 ((guc)->submission_state.num_guc_ids / 16)163 164/*165 * Below is a set of functions which control the GuC scheduling state which166 * require a lock.167 */168#define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0)169#define SCHED_STATE_DESTROYED BIT(1)170#define SCHED_STATE_PENDING_DISABLE BIT(2)171#define SCHED_STATE_BANNED BIT(3)172#define SCHED_STATE_ENABLED BIT(4)173#define SCHED_STATE_PENDING_ENABLE BIT(5)174#define SCHED_STATE_REGISTERED BIT(6)175#define SCHED_STATE_POLICY_REQUIRED BIT(7)176#define SCHED_STATE_CLOSED BIT(8)177#define SCHED_STATE_BLOCKED_SHIFT 9178#define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT)179#define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT)180 181static inline void init_sched_state(struct intel_context *ce)182{183 lockdep_assert_held(&ce->guc_state.lock);184 ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK;185}186 187/*188 * Kernel contexts can have SCHED_STATE_REGISTERED after suspend.189 * A context close can race with the submission path, so SCHED_STATE_CLOSED190 * can be set immediately before we try to register.191 */192#define SCHED_STATE_VALID_INIT \193 (SCHED_STATE_BLOCKED_MASK | \194 SCHED_STATE_CLOSED | \195 SCHED_STATE_REGISTERED)196 197__maybe_unused198static bool sched_state_is_init(struct intel_context *ce)199{200 return !(ce->guc_state.sched_state & ~SCHED_STATE_VALID_INIT);201}202 203static inline bool204context_wait_for_deregister_to_register(struct intel_context *ce)205{206 return ce->guc_state.sched_state &207 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;208}209 210static inline void211set_context_wait_for_deregister_to_register(struct intel_context *ce)212{213 lockdep_assert_held(&ce->guc_state.lock);214 ce->guc_state.sched_state |=215 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;216}217 218static inline void219clr_context_wait_for_deregister_to_register(struct intel_context *ce)220{221 lockdep_assert_held(&ce->guc_state.lock);222 ce->guc_state.sched_state &=223 ~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;224}225 226static inline bool227context_destroyed(struct intel_context *ce)228{229 return ce->guc_state.sched_state & SCHED_STATE_DESTROYED;230}231 232static inline void233set_context_destroyed(struct intel_context *ce)234{235 lockdep_assert_held(&ce->guc_state.lock);236 ce->guc_state.sched_state |= SCHED_STATE_DESTROYED;237}238 239static inline void240clr_context_destroyed(struct intel_context *ce)241{242 lockdep_assert_held(&ce->guc_state.lock);243 ce->guc_state.sched_state &= ~SCHED_STATE_DESTROYED;244}245 246static inline bool context_pending_disable(struct intel_context *ce)247{248 return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE;249}250 251static inline void set_context_pending_disable(struct intel_context *ce)252{253 lockdep_assert_held(&ce->guc_state.lock);254 ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE;255}256 257static inline void clr_context_pending_disable(struct intel_context *ce)258{259 lockdep_assert_held(&ce->guc_state.lock);260 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE;261}262 263static inline bool context_banned(struct intel_context *ce)264{265 return ce->guc_state.sched_state & SCHED_STATE_BANNED;266}267 268static inline void set_context_banned(struct intel_context *ce)269{270 lockdep_assert_held(&ce->guc_state.lock);271 ce->guc_state.sched_state |= SCHED_STATE_BANNED;272}273 274static inline void clr_context_banned(struct intel_context *ce)275{276 lockdep_assert_held(&ce->guc_state.lock);277 ce->guc_state.sched_state &= ~SCHED_STATE_BANNED;278}279 280static inline bool context_enabled(struct intel_context *ce)281{282 return ce->guc_state.sched_state & SCHED_STATE_ENABLED;283}284 285static inline void set_context_enabled(struct intel_context *ce)286{287 lockdep_assert_held(&ce->guc_state.lock);288 ce->guc_state.sched_state |= SCHED_STATE_ENABLED;289}290 291static inline void clr_context_enabled(struct intel_context *ce)292{293 lockdep_assert_held(&ce->guc_state.lock);294 ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED;295}296 297static inline bool context_pending_enable(struct intel_context *ce)298{299 return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE;300}301 302static inline void set_context_pending_enable(struct intel_context *ce)303{304 lockdep_assert_held(&ce->guc_state.lock);305 ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE;306}307 308static inline void clr_context_pending_enable(struct intel_context *ce)309{310 lockdep_assert_held(&ce->guc_state.lock);311 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE;312}313 314static inline bool context_registered(struct intel_context *ce)315{316 return ce->guc_state.sched_state & SCHED_STATE_REGISTERED;317}318 319static inline void set_context_registered(struct intel_context *ce)320{321 lockdep_assert_held(&ce->guc_state.lock);322 ce->guc_state.sched_state |= SCHED_STATE_REGISTERED;323}324 325static inline void clr_context_registered(struct intel_context *ce)326{327 lockdep_assert_held(&ce->guc_state.lock);328 ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED;329}330 331static inline bool context_policy_required(struct intel_context *ce)332{333 return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED;334}335 336static inline void set_context_policy_required(struct intel_context *ce)337{338 lockdep_assert_held(&ce->guc_state.lock);339 ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED;340}341 342static inline void clr_context_policy_required(struct intel_context *ce)343{344 lockdep_assert_held(&ce->guc_state.lock);345 ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED;346}347 348static inline bool context_close_done(struct intel_context *ce)349{350 return ce->guc_state.sched_state & SCHED_STATE_CLOSED;351}352 353static inline void set_context_close_done(struct intel_context *ce)354{355 lockdep_assert_held(&ce->guc_state.lock);356 ce->guc_state.sched_state |= SCHED_STATE_CLOSED;357}358 359static inline u32 context_blocked(struct intel_context *ce)360{361 return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >>362 SCHED_STATE_BLOCKED_SHIFT;363}364 365static inline void incr_context_blocked(struct intel_context *ce)366{367 lockdep_assert_held(&ce->guc_state.lock);368 369 ce->guc_state.sched_state += SCHED_STATE_BLOCKED;370 371 GEM_BUG_ON(!context_blocked(ce)); /* Overflow check */372}373 374static inline void decr_context_blocked(struct intel_context *ce)375{376 lockdep_assert_held(&ce->guc_state.lock);377 378 GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */379 380 ce->guc_state.sched_state -= SCHED_STATE_BLOCKED;381}382 383static struct intel_context *384request_to_scheduling_context(struct i915_request *rq)385{386 return intel_context_to_parent(rq->context);387}388 389static inline bool context_guc_id_invalid(struct intel_context *ce)390{391 return ce->guc_id.id == GUC_INVALID_CONTEXT_ID;392}393 394static inline void set_context_guc_id_invalid(struct intel_context *ce)395{396 ce->guc_id.id = GUC_INVALID_CONTEXT_ID;397}398 399static inline struct intel_guc *ce_to_guc(struct intel_context *ce)400{401 return gt_to_guc(ce->engine->gt);402}403 404static inline struct i915_priolist *to_priolist(struct rb_node *rb)405{406 return rb_entry(rb, struct i915_priolist, node);407}408 409/*410 * When using multi-lrc submission a scratch memory area is reserved in the411 * parent's context state for the process descriptor, work queue, and handshake412 * between the parent + children contexts to insert safe preemption points413 * between each of the BBs. Currently the scratch area is sized to a page.414 *415 * The layout of this scratch area is below:416 * 0 guc_process_desc417 * + sizeof(struct guc_process_desc) child go418 * + CACHELINE_BYTES child join[0]419 * ...420 * + CACHELINE_BYTES child join[n - 1]421 * ... unused422 * PARENT_SCRATCH_SIZE / 2 work queue start423 * ... work queue424 * PARENT_SCRATCH_SIZE - 1 work queue end425 */426#define WQ_SIZE (PARENT_SCRATCH_SIZE / 2)427#define WQ_OFFSET (PARENT_SCRATCH_SIZE - WQ_SIZE)428 429struct sync_semaphore {430 u32 semaphore;431 u8 unused[CACHELINE_BYTES - sizeof(u32)];432};433 434struct parent_scratch {435 union guc_descs {436 struct guc_sched_wq_desc wq_desc;437 struct guc_process_desc_v69 pdesc;438 } descs;439 440 struct sync_semaphore go;441 struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1];442 443 u8 unused[WQ_OFFSET - sizeof(union guc_descs) -444 sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)];445 446 u32 wq[WQ_SIZE / sizeof(u32)];447};448 449static u32 __get_parent_scratch_offset(struct intel_context *ce)450{451 GEM_BUG_ON(!ce->parallel.guc.parent_page);452 453 return ce->parallel.guc.parent_page * PAGE_SIZE;454}455 456static u32 __get_wq_offset(struct intel_context *ce)457{458 BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET);459 460 return __get_parent_scratch_offset(ce) + WQ_OFFSET;461}462 463static struct parent_scratch *464__get_parent_scratch(struct intel_context *ce)465{466 BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE);467 BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES);468 469 /*470 * Need to subtract LRC_STATE_OFFSET here as the471 * parallel.guc.parent_page is the offset into ce->state while472 * ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET.473 */474 return (struct parent_scratch *)475 (ce->lrc_reg_state +476 ((__get_parent_scratch_offset(ce) -477 LRC_STATE_OFFSET) / sizeof(u32)));478}479 480static struct guc_process_desc_v69 *481__get_process_desc_v69(struct intel_context *ce)482{483 struct parent_scratch *ps = __get_parent_scratch(ce);484 485 return &ps->descs.pdesc;486}487 488static struct guc_sched_wq_desc *489__get_wq_desc_v70(struct intel_context *ce)490{491 struct parent_scratch *ps = __get_parent_scratch(ce);492 493 return &ps->descs.wq_desc;494}495 496static u32 *get_wq_pointer(struct intel_context *ce, u32 wqi_size)497{498 /*499 * Check for space in work queue. Caching a value of head pointer in500 * intel_context structure in order reduce the number accesses to shared501 * GPU memory which may be across a PCIe bus.502 */503#define AVAILABLE_SPACE \504 CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)505 if (wqi_size > AVAILABLE_SPACE) {506 ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head);507 508 if (wqi_size > AVAILABLE_SPACE)509 return NULL;510 }511#undef AVAILABLE_SPACE512 513 return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];514}515 516static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)517{518 struct intel_context *ce = xa_load(&guc->context_lookup, id);519 520 GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID);521 522 return ce;523}524 525static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)526{527 struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;528 529 if (!base)530 return NULL;531 532 GEM_BUG_ON(index >= GUC_MAX_CONTEXT_ID);533 534 return &base[index];535}536 537static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)538{539 u32 size;540 int ret;541 542 size = PAGE_ALIGN(sizeof(struct guc_lrc_desc_v69) *543 GUC_MAX_CONTEXT_ID);544 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool_v69,545 (void **)&guc->lrc_desc_pool_vaddr_v69);546 if (ret)547 return ret;548 549 return 0;550}551 552static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)553{554 if (!guc->lrc_desc_pool_vaddr_v69)555 return;556 557 guc->lrc_desc_pool_vaddr_v69 = NULL;558 i915_vma_unpin_and_release(&guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP);559}560 561static inline bool guc_submission_initialized(struct intel_guc *guc)562{563 return guc->submission_initialized;564}565 566static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)567{568 struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, id);569 570 if (desc)571 memset(desc, 0, sizeof(*desc));572}573 574static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id)575{576 return __get_context(guc, id);577}578 579static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,580 struct intel_context *ce)581{582 unsigned long flags;583 584 /*585 * xarray API doesn't have xa_save_irqsave wrapper, so calling the586 * lower level functions directly.587 */588 xa_lock_irqsave(&guc->context_lookup, flags);589 __xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC);590 xa_unlock_irqrestore(&guc->context_lookup, flags);591}592 593static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)594{595 unsigned long flags;596 597 if (unlikely(!guc_submission_initialized(guc)))598 return;599 600 _reset_lrc_desc_v69(guc, id);601 602 /*603 * xarray API doesn't have xa_erase_irqsave wrapper, so calling604 * the lower level functions directly.605 */606 xa_lock_irqsave(&guc->context_lookup, flags);607 __xa_erase(&guc->context_lookup, id);608 xa_unlock_irqrestore(&guc->context_lookup, flags);609}610 611static void decr_outstanding_submission_g2h(struct intel_guc *guc)612{613 if (atomic_dec_and_test(&guc->outstanding_submission_g2h))614 wake_up_all(&guc->ct.wq);615}616 617static int guc_submission_send_busy_loop(struct intel_guc *guc,618 const u32 *action,619 u32 len,620 u32 g2h_len_dw,621 bool loop)622{623 int ret;624 625 /*626 * We always loop when a send requires a reply (i.e. g2h_len_dw > 0),627 * so we don't handle the case where we don't get a reply because we628 * aborted the send due to the channel being busy.629 */630 GEM_BUG_ON(g2h_len_dw && !loop);631 632 if (g2h_len_dw)633 atomic_inc(&guc->outstanding_submission_g2h);634 635 ret = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);636 if (ret)637 atomic_dec(&guc->outstanding_submission_g2h);638 639 return ret;640}641 642int intel_guc_wait_for_pending_msg(struct intel_guc *guc,643 atomic_t *wait_var,644 bool interruptible,645 long timeout)646{647 const int state = interruptible ?648 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;649 DEFINE_WAIT(wait);650 651 might_sleep();652 GEM_BUG_ON(timeout < 0);653 654 if (!atomic_read(wait_var))655 return 0;656 657 if (!timeout)658 return -ETIME;659 660 for (;;) {661 prepare_to_wait(&guc->ct.wq, &wait, state);662 663 if (!atomic_read(wait_var))664 break;665 666 if (signal_pending_state(state, current)) {667 timeout = -EINTR;668 break;669 }670 671 if (!timeout) {672 timeout = -ETIME;673 break;674 }675 676 timeout = io_schedule_timeout(timeout);677 }678 finish_wait(&guc->ct.wq, &wait);679 680 return (timeout < 0) ? timeout : 0;681}682 683int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)684{685 if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc))686 return 0;687 688 return intel_guc_wait_for_pending_msg(guc,689 &guc->outstanding_submission_g2h,690 true, timeout);691}692 693static int guc_context_policy_init_v70(struct intel_context *ce, bool loop);694static int try_context_registration(struct intel_context *ce, bool loop);695 696static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)697{698 int err = 0;699 struct intel_context *ce = request_to_scheduling_context(rq);700 u32 action[3];701 int len = 0;702 u32 g2h_len_dw = 0;703 bool enabled;704 705 lockdep_assert_held(&rq->engine->sched_engine->lock);706 707 /*708 * Corner case where requests were sitting in the priority list or a709 * request resubmitted after the context was banned.710 */711 if (unlikely(!intel_context_is_schedulable(ce))) {712 i915_request_put(i915_request_mark_eio(rq));713 intel_engine_signal_breadcrumbs(ce->engine);714 return 0;715 }716 717 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));718 GEM_BUG_ON(context_guc_id_invalid(ce));719 720 if (context_policy_required(ce)) {721 err = guc_context_policy_init_v70(ce, false);722 if (err)723 return err;724 }725 726 spin_lock(&ce->guc_state.lock);727 728 /*729 * The request / context will be run on the hardware when scheduling730 * gets enabled in the unblock. For multi-lrc we still submit the731 * context to move the LRC tails.732 */733 if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce)))734 goto out;735 736 enabled = context_enabled(ce) || context_blocked(ce);737 738 if (!enabled) {739 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET;740 action[len++] = ce->guc_id.id;741 action[len++] = GUC_CONTEXT_ENABLE;742 set_context_pending_enable(ce);743 intel_context_get(ce);744 g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;745 } else {746 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT;747 action[len++] = ce->guc_id.id;748 }749 750 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);751 if (!enabled && !err) {752 trace_intel_context_sched_enable(ce);753 atomic_inc(&guc->outstanding_submission_g2h);754 set_context_enabled(ce);755 756 /*757 * Without multi-lrc KMD does the submission step (moving the758 * lrc tail) so enabling scheduling is sufficient to submit the759 * context. This isn't the case in multi-lrc submission as the760 * GuC needs to move the tails, hence the need for another H2G761 * to submit a multi-lrc context after enabling scheduling.762 */763 if (intel_context_is_parent(ce)) {764 action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT;765 err = intel_guc_send_nb(guc, action, len - 1, 0);766 }767 } else if (!enabled) {768 clr_context_pending_enable(ce);769 intel_context_put(ce);770 }771 if (likely(!err))772 trace_i915_request_guc_submit(rq);773 774out:775 spin_unlock(&ce->guc_state.lock);776 return err;777}778 779static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)780{781 int ret = __guc_add_request(guc, rq);782 783 if (unlikely(ret == -EBUSY)) {784 guc->stalled_request = rq;785 guc->submission_stall_reason = STALL_ADD_REQUEST;786 }787 788 return ret;789}790 791static inline void guc_set_lrc_tail(struct i915_request *rq)792{793 rq->context->lrc_reg_state[CTX_RING_TAIL] =794 intel_ring_set_tail(rq->ring, rq->tail);795}796 797static inline int rq_prio(const struct i915_request *rq)798{799 return rq->sched.attr.priority;800}801 802static bool is_multi_lrc_rq(struct i915_request *rq)803{804 return intel_context_is_parallel(rq->context);805}806 807static bool can_merge_rq(struct i915_request *rq,808 struct i915_request *last)809{810 return request_to_scheduling_context(rq) ==811 request_to_scheduling_context(last);812}813 814static u32 wq_space_until_wrap(struct intel_context *ce)815{816 return (WQ_SIZE - ce->parallel.guc.wqi_tail);817}818 819static void write_wqi(struct intel_context *ce, u32 wqi_size)820{821 BUILD_BUG_ON(!is_power_of_2(WQ_SIZE));822 823 /*824 * Ensure WQI are visible before updating tail825 */826 intel_guc_write_barrier(ce_to_guc(ce));827 828 ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &829 (WQ_SIZE - 1);830 WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail);831}832 833static int guc_wq_noop_append(struct intel_context *ce)834{835 u32 *wqi = get_wq_pointer(ce, wq_space_until_wrap(ce));836 u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1;837 838 if (!wqi)839 return -EBUSY;840 841 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));842 843 *wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |844 FIELD_PREP(WQ_LEN_MASK, len_dw);845 ce->parallel.guc.wqi_tail = 0;846 847 return 0;848}849 850static int __guc_wq_item_append(struct i915_request *rq)851{852 struct intel_context *ce = request_to_scheduling_context(rq);853 struct intel_context *child;854 unsigned int wqi_size = (ce->parallel.number_children + 4) *855 sizeof(u32);856 u32 *wqi;857 u32 len_dw = (wqi_size / sizeof(u32)) - 1;858 int ret;859 860 /* Ensure context is in correct state updating work queue */861 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));862 GEM_BUG_ON(context_guc_id_invalid(ce));863 GEM_BUG_ON(context_wait_for_deregister_to_register(ce));864 GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id));865 866 /* Insert NOOP if this work queue item will wrap the tail pointer. */867 if (wqi_size > wq_space_until_wrap(ce)) {868 ret = guc_wq_noop_append(ce);869 if (ret)870 return ret;871 }872 873 wqi = get_wq_pointer(ce, wqi_size);874 if (!wqi)875 return -EBUSY;876 877 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));878 879 *wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |880 FIELD_PREP(WQ_LEN_MASK, len_dw);881 *wqi++ = ce->lrc.lrca;882 *wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id) |883 FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64));884 *wqi++ = 0; /* fence_id */885 for_each_child(ce, child)886 *wqi++ = child->ring->tail / sizeof(u64);887 888 write_wqi(ce, wqi_size);889 890 return 0;891}892 893static int guc_wq_item_append(struct intel_guc *guc,894 struct i915_request *rq)895{896 struct intel_context *ce = request_to_scheduling_context(rq);897 int ret;898 899 if (unlikely(!intel_context_is_schedulable(ce)))900 return 0;901 902 ret = __guc_wq_item_append(rq);903 if (unlikely(ret == -EBUSY)) {904 guc->stalled_request = rq;905 guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;906 }907 908 return ret;909}910 911static bool multi_lrc_submit(struct i915_request *rq)912{913 struct intel_context *ce = request_to_scheduling_context(rq);914 915 intel_ring_set_tail(rq->ring, rq->tail);916 917 /*918 * We expect the front end (execbuf IOCTL) to set this flag on the last919 * request generated from a multi-BB submission. This indicates to the920 * backend (GuC interface) that we should submit this context thus921 * submitting all the requests generated in parallel.922 */923 return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||924 !intel_context_is_schedulable(ce);925}926 927static int guc_dequeue_one_context(struct intel_guc *guc)928{929 struct i915_sched_engine * const sched_engine = guc->sched_engine;930 struct i915_request *last = NULL;931 bool submit = false;932 struct rb_node *rb;933 int ret;934 935 lockdep_assert_held(&sched_engine->lock);936 937 if (guc->stalled_request) {938 submit = true;939 last = guc->stalled_request;940 941 switch (guc->submission_stall_reason) {942 case STALL_REGISTER_CONTEXT:943 goto register_context;944 case STALL_MOVE_LRC_TAIL:945 goto move_lrc_tail;946 case STALL_ADD_REQUEST:947 goto add_request;948 default:949 MISSING_CASE(guc->submission_stall_reason);950 }951 }952 953 while ((rb = rb_first_cached(&sched_engine->queue))) {954 struct i915_priolist *p = to_priolist(rb);955 struct i915_request *rq, *rn;956 957 priolist_for_each_request_consume(rq, rn, p) {958 if (last && !can_merge_rq(rq, last))959 goto register_context;960 961 list_del_init(&rq->sched.link);962 963 __i915_request_submit(rq);964 965 trace_i915_request_in(rq, 0);966 last = rq;967 968 if (is_multi_lrc_rq(rq)) {969 /*970 * We need to coalesce all multi-lrc requests in971 * a relationship into a single H2G. We are972 * guaranteed that all of these requests will be973 * submitted sequentially.974 */975 if (multi_lrc_submit(rq)) {976 submit = true;977 goto register_context;978 }979 } else {980 submit = true;981 }982 }983 984 rb_erase_cached(&p->node, &sched_engine->queue);985 i915_priolist_free(p);986 }987 988register_context:989 if (submit) {990 struct intel_context *ce = request_to_scheduling_context(last);991 992 if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&993 intel_context_is_schedulable(ce))) {994 ret = try_context_registration(ce, false);995 if (unlikely(ret == -EPIPE)) {996 goto deadlk;997 } else if (ret == -EBUSY) {998 guc->stalled_request = last;999 guc->submission_stall_reason =1000 STALL_REGISTER_CONTEXT;1001 goto schedule_tasklet;1002 } else if (ret != 0) {1003 GEM_WARN_ON(ret); /* Unexpected */1004 goto deadlk;1005 }1006 }1007 1008move_lrc_tail:1009 if (is_multi_lrc_rq(last)) {1010 ret = guc_wq_item_append(guc, last);1011 if (ret == -EBUSY) {1012 goto schedule_tasklet;1013 } else if (ret != 0) {1014 GEM_WARN_ON(ret); /* Unexpected */1015 goto deadlk;1016 }1017 } else {1018 guc_set_lrc_tail(last);1019 }1020 1021add_request:1022 ret = guc_add_request(guc, last);1023 if (unlikely(ret == -EPIPE)) {1024 goto deadlk;1025 } else if (ret == -EBUSY) {1026 goto schedule_tasklet;1027 } else if (ret != 0) {1028 GEM_WARN_ON(ret); /* Unexpected */1029 goto deadlk;1030 }1031 }1032 1033 guc->stalled_request = NULL;1034 guc->submission_stall_reason = STALL_NONE;1035 return submit;1036 1037deadlk:1038 sched_engine->tasklet.callback = NULL;1039 tasklet_disable_nosync(&sched_engine->tasklet);1040 return false;1041 1042schedule_tasklet:1043 tasklet_schedule(&sched_engine->tasklet);1044 return false;1045}1046 1047static void guc_submission_tasklet(struct tasklet_struct *t)1048{1049 struct i915_sched_engine *sched_engine =1050 from_tasklet(sched_engine, t, tasklet);1051 unsigned long flags;1052 bool loop;1053 1054 spin_lock_irqsave(&sched_engine->lock, flags);1055 1056 do {1057 loop = guc_dequeue_one_context(sched_engine->private_data);1058 } while (loop);1059 1060 i915_sched_engine_reset_on_empty(sched_engine);1061 1062 spin_unlock_irqrestore(&sched_engine->lock, flags);1063}1064 1065static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir)1066{1067 if (iir & GT_RENDER_USER_INTERRUPT)1068 intel_engine_signal_breadcrumbs(engine);1069}1070 1071static void __guc_context_destroy(struct intel_context *ce);1072static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);1073static void guc_signal_context_fence(struct intel_context *ce);1074static void guc_cancel_context_requests(struct intel_context *ce);1075static void guc_blocked_fence_complete(struct intel_context *ce);1076 1077static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)1078{1079 struct intel_context *ce;1080 unsigned long index, flags;1081 bool pending_disable, pending_enable, deregister, destroyed, banned;1082 1083 xa_lock_irqsave(&guc->context_lookup, flags);1084 xa_for_each(&guc->context_lookup, index, ce) {1085 /*1086 * Corner case where the ref count on the object is zero but and1087 * deregister G2H was lost. In this case we don't touch the ref1088 * count and finish the destroy of the context.1089 */1090 bool do_put = kref_get_unless_zero(&ce->ref);1091 1092 xa_unlock(&guc->context_lookup);1093 1094 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&1095 (cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))) {1096 /* successful cancel so jump straight to close it */1097 intel_context_sched_disable_unpin(ce);1098 }1099 1100 spin_lock(&ce->guc_state.lock);1101 1102 /*1103 * Once we are at this point submission_disabled() is guaranteed1104 * to be visible to all callers who set the below flags (see above1105 * flush and flushes in reset_prepare). If submission_disabled()1106 * is set, the caller shouldn't set these flags.1107 */1108 1109 destroyed = context_destroyed(ce);1110 pending_enable = context_pending_enable(ce);1111 pending_disable = context_pending_disable(ce);1112 deregister = context_wait_for_deregister_to_register(ce);1113 banned = context_banned(ce);1114 init_sched_state(ce);1115 1116 spin_unlock(&ce->guc_state.lock);1117 1118 if (pending_enable || destroyed || deregister) {1119 decr_outstanding_submission_g2h(guc);1120 if (deregister)1121 guc_signal_context_fence(ce);1122 if (destroyed) {1123 intel_gt_pm_put_async_untracked(guc_to_gt(guc));1124 release_guc_id(guc, ce);1125 __guc_context_destroy(ce);1126 }1127 if (pending_enable || deregister)1128 intel_context_put(ce);1129 }1130 1131 /* Not mutualy exclusive with above if statement. */1132 if (pending_disable) {1133 guc_signal_context_fence(ce);1134 if (banned) {1135 guc_cancel_context_requests(ce);1136 intel_engine_signal_breadcrumbs(ce->engine);1137 }1138 intel_context_sched_disable_unpin(ce);1139 decr_outstanding_submission_g2h(guc);1140 1141 spin_lock(&ce->guc_state.lock);1142 guc_blocked_fence_complete(ce);1143 spin_unlock(&ce->guc_state.lock);1144 1145 intel_context_put(ce);1146 }1147 1148 if (do_put)1149 intel_context_put(ce);1150 xa_lock(&guc->context_lookup);1151 }1152 xa_unlock_irqrestore(&guc->context_lookup, flags);1153}1154 1155/*1156 * GuC stores busyness stats for each engine at context in/out boundaries. A1157 * context 'in' logs execution start time, 'out' adds in -> out delta to total.1158 * i915/kmd accesses 'start', 'total' and 'context id' from memory shared with1159 * GuC.1160 *1161 * __i915_pmu_event_read samples engine busyness. When sampling, if context id1162 * is valid (!= ~0) and start is non-zero, the engine is considered to be1163 * active. For an active engine total busyness = total + (now - start), where1164 * 'now' is the time at which the busyness is sampled. For inactive engine,1165 * total busyness = total.1166 *1167 * All times are captured from GUCPMTIMESTAMP reg and are in gt clock domain.1168 *1169 * The start and total values provided by GuC are 32 bits and wrap around in a1170 * few minutes. Since perf pmu provides busyness as 64 bit monotonically1171 * increasing ns values, there is a need for this implementation to account for1172 * overflows and extend the GuC provided values to 64 bits before returning1173 * busyness to the user. In order to do that, a worker runs periodically at1174 * frequency = 1/8th the time it takes for the timestamp to wrap (i.e. once in1175 * 27 seconds for a gt clock frequency of 19.2 MHz).1176 */1177 1178#define WRAP_TIME_CLKS U32_MAX1179#define POLL_TIME_CLKS (WRAP_TIME_CLKS >> 3)1180 1181static void1182__extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)1183{1184 u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);1185 u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp);1186 1187 if (new_start == lower_32_bits(*prev_start))1188 return;1189 1190 /*1191 * When gt is unparked, we update the gt timestamp and start the ping1192 * worker that updates the gt_stamp every POLL_TIME_CLKS. As long as gt1193 * is unparked, all switched in contexts will have a start time that is1194 * within +/- POLL_TIME_CLKS of the most recent gt_stamp.1195 *1196 * If neither gt_stamp nor new_start has rolled over, then the1197 * gt_stamp_hi does not need to be adjusted, however if one of them has1198 * rolled over, we need to adjust gt_stamp_hi accordingly.1199 *1200 * The below conditions address the cases of new_start rollover and1201 * gt_stamp_last rollover respectively.1202 */1203 if (new_start < gt_stamp_last &&1204 (new_start - gt_stamp_last) <= POLL_TIME_CLKS)1205 gt_stamp_hi++;1206 1207 if (new_start > gt_stamp_last &&1208 (gt_stamp_last - new_start) <= POLL_TIME_CLKS && gt_stamp_hi)1209 gt_stamp_hi--;1210 1211 *prev_start = ((u64)gt_stamp_hi << 32) | new_start;1212}1213 1214#define record_read(map_, field_) \1215 iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_)1216 1217/*1218 * GuC updates shared memory and KMD reads it. Since this is not synchronized,1219 * we run into a race where the value read is inconsistent. Sometimes the1220 * inconsistency is in reading the upper MSB bytes of the last_in value when1221 * this race occurs. 2 types of cases are seen - upper 8 bits are zero and upper1222 * 24 bits are zero. Since these are non-zero values, it is non-trivial to1223 * determine validity of these values. Instead we read the values multiple times1224 * until they are consistent. In test runs, 3 attempts results in consistent1225 * values. The upper bound is set to 6 attempts and may need to be tuned as per1226 * any new occurences.1227 */1228static void __get_engine_usage_record(struct intel_engine_cs *engine,1229 u32 *last_in, u32 *id, u32 *total)1230{1231 struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);1232 int i = 0;1233 1234 do {1235 *last_in = record_read(&rec_map, last_switch_in_stamp);1236 *id = record_read(&rec_map, current_context_index);1237 *total = record_read(&rec_map, total_runtime);1238 1239 if (record_read(&rec_map, last_switch_in_stamp) == *last_in &&1240 record_read(&rec_map, current_context_index) == *id &&1241 record_read(&rec_map, total_runtime) == *total)1242 break;1243 } while (++i < 6);1244}1245 1246static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)1247{1248 struct intel_engine_guc_stats *stats = &engine->stats.guc;1249 struct intel_guc *guc = gt_to_guc(engine->gt);1250 u32 last_switch, ctx_id, total;1251 1252 lockdep_assert_held(&guc->timestamp.lock);1253 1254 __get_engine_usage_record(engine, &last_switch, &ctx_id, &total);1255 1256 stats->running = ctx_id != ~0U && last_switch;1257 if (stats->running)1258 __extend_last_switch(guc, &stats->start_gt_clk, last_switch);1259 1260 /*1261 * Instead of adjusting the total for overflow, just add the1262 * difference from previous sample stats->total_gt_clks1263 */1264 if (total && total != ~0U) {1265 stats->total_gt_clks += (u32)(total - stats->prev_total);1266 stats->prev_total = total;1267 }1268}1269 1270static u32 gpm_timestamp_shift(struct intel_gt *gt)1271{1272 intel_wakeref_t wakeref;1273 u32 reg, shift;1274 1275 with_intel_runtime_pm(gt->uncore->rpm, wakeref)1276 reg = intel_uncore_read(gt->uncore, RPM_CONFIG0);1277 1278 shift = (reg & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>1279 GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT;1280 1281 return 3 - shift;1282}1283 1284static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)1285{1286 struct intel_gt *gt = guc_to_gt(guc);1287 u32 gt_stamp_lo, gt_stamp_hi;1288 u64 gpm_ts;1289 1290 lockdep_assert_held(&guc->timestamp.lock);1291 1292 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);1293 gpm_ts = intel_uncore_read64_2x32(gt->uncore, MISC_STATUS0,1294 MISC_STATUS1) >> guc->timestamp.shift;1295 gt_stamp_lo = lower_32_bits(gpm_ts);1296 *now = ktime_get();1297 1298 if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp))1299 gt_stamp_hi++;1300 1301 guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;1302}1303 1304/*1305 * Unlike the execlist mode of submission total and active times are in terms of1306 * gt clocks. The *now parameter is retained to return the cpu time at which the1307 * busyness was sampled.1308 */1309static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)1310{1311 struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;1312 struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;1313 struct intel_gt *gt = engine->gt;1314 struct intel_guc *guc = gt_to_guc(gt);1315 u64 total, gt_stamp_saved;1316 unsigned long flags;1317 u32 reset_count;1318 bool in_reset;1319 intel_wakeref_t wakeref;1320 1321 spin_lock_irqsave(&guc->timestamp.lock, flags);1322 1323 /*1324 * If a reset happened, we risk reading partially updated engine1325 * busyness from GuC, so we just use the driver stored copy of busyness.1326 * Synchronize with gt reset using reset_count and the1327 * I915_RESET_BACKOFF flag. Note that reset flow updates the reset_count1328 * after I915_RESET_BACKOFF flag, so ensure that the reset_count is1329 * usable by checking the flag afterwards.1330 */1331 reset_count = i915_reset_count(gpu_error);1332 in_reset = test_bit(I915_RESET_BACKOFF, >->reset.flags);1333 1334 *now = ktime_get();1335 1336 /*1337 * The active busyness depends on start_gt_clk and gt_stamp.1338 * gt_stamp is updated by i915 only when gt is awake and the1339 * start_gt_clk is derived from GuC state. To get a consistent1340 * view of activity, we query the GuC state only if gt is awake.1341 */1342 wakeref = in_reset ? 0 : intel_gt_pm_get_if_awake(gt);1343 if (wakeref) {1344 stats_saved = *stats;1345 gt_stamp_saved = guc->timestamp.gt_stamp;1346 /*1347 * Update gt_clks, then gt timestamp to simplify the 'gt_stamp -1348 * start_gt_clk' calculation below for active engines.1349 */1350 guc_update_engine_gt_clks(engine);1351 guc_update_pm_timestamp(guc, now);1352 intel_gt_pm_put_async(gt, wakeref);1353 if (i915_reset_count(gpu_error) != reset_count) {1354 *stats = stats_saved;1355 guc->timestamp.gt_stamp = gt_stamp_saved;1356 }1357 }1358 1359 total = intel_gt_clock_interval_to_ns(gt, stats->total_gt_clks);1360 if (stats->running) {1361 u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;1362 1363 total += intel_gt_clock_interval_to_ns(gt, clk);1364 }1365 1366 spin_unlock_irqrestore(&guc->timestamp.lock, flags);1367 1368 return ns_to_ktime(total);1369}1370 1371static void guc_enable_busyness_worker(struct intel_guc *guc)1372{1373 mod_delayed_work(system_highpri_wq, &guc->timestamp.work, guc->timestamp.ping_delay);1374}1375 1376static void guc_cancel_busyness_worker(struct intel_guc *guc)1377{1378 /*1379 * There are many different call stacks that can get here. Some of them1380 * hold the reset mutex. The busyness worker also attempts to acquire the1381 * reset mutex. Synchronously flushing a worker thread requires acquiring1382 * the worker mutex. Lockdep sees this as a conflict. It thinks that the1383 * flush can deadlock because it holds the worker mutex while waiting for1384 * the reset mutex, but another thread is holding the reset mutex and might1385 * attempt to use other worker functions.1386 *1387 * In practice, this scenario does not exist because the busyness worker1388 * does not block waiting for the reset mutex. It does a try-lock on it and1389 * immediately exits if the lock is already held. Unfortunately, the mutex1390 * in question (I915_RESET_BACKOFF) is an i915 implementation which has lockdep1391 * annotation but not to the extent of explaining the 'might lock' is also a1392 * 'does not need to lock'. So one option would be to add more complex lockdep1393 * annotations to ignore the issue (if at all possible). A simpler option is to1394 * just not flush synchronously when a rest in progress. Given that the worker1395 * will just early exit and re-schedule itself anyway, there is no advantage1396 * to running it immediately.1397 *1398 * If a reset is not in progress, then the synchronous flush may be required.1399 * As noted many call stacks lead here, some during suspend and driver unload1400 * which do require a synchronous flush to make sure the worker is stopped1401 * before memory is freed.1402 *1403 * Trying to pass a 'need_sync' or 'in_reset' flag all the way down through1404 * every possible call stack is unfeasible. It would be too intrusive to many1405 * areas that really don't care about the GuC backend. However, there is the1406 * I915_RESET_BACKOFF flag and the gt->reset.mutex can be tested for is_locked.1407 * So just use those. Note that testing both is required due to the hideously1408 * complex nature of the i915 driver's reset code paths.1409 *1410 * And note that in the case of a reset occurring during driver unload1411 * (wedged_on_fini), skipping the cancel in reset_prepare/reset_fini (when the1412 * reset flag/mutex are set) is fine because there is another explicit cancel in1413 * intel_guc_submission_fini (when the reset flag/mutex are not).1414 */1415 if (mutex_is_locked(&guc_to_gt(guc)->reset.mutex) ||1416 test_bit(I915_RESET_BACKOFF, &guc_to_gt(guc)->reset.flags))1417 cancel_delayed_work(&guc->timestamp.work);1418 else1419 cancel_delayed_work_sync(&guc->timestamp.work);1420}1421 1422static void __reset_guc_busyness_stats(struct intel_guc *guc)1423{1424 struct intel_gt *gt = guc_to_gt(guc);1425 struct intel_engine_cs *engine;1426 enum intel_engine_id id;1427 unsigned long flags;1428 ktime_t unused;1429 1430 spin_lock_irqsave(&guc->timestamp.lock, flags);1431 1432 guc_update_pm_timestamp(guc, &unused);1433 for_each_engine(engine, gt, id) {1434 guc_update_engine_gt_clks(engine);1435 engine->stats.guc.prev_total = 0;1436 }1437 1438 spin_unlock_irqrestore(&guc->timestamp.lock, flags);1439}1440 1441static void __update_guc_busyness_stats(struct intel_guc *guc)1442{1443 struct intel_gt *gt = guc_to_gt(guc);1444 struct intel_engine_cs *engine;1445 enum intel_engine_id id;1446 unsigned long flags;1447 ktime_t unused;1448 1449 guc->timestamp.last_stat_jiffies = jiffies;1450 1451 spin_lock_irqsave(&guc->timestamp.lock, flags);1452 1453 guc_update_pm_timestamp(guc, &unused);1454 for_each_engine(engine, gt, id)1455 guc_update_engine_gt_clks(engine);1456 1457 spin_unlock_irqrestore(&guc->timestamp.lock, flags);1458}1459 1460static void __guc_context_update_stats(struct intel_context *ce)1461{1462 struct intel_guc *guc = ce_to_guc(ce);1463 unsigned long flags;1464 1465 spin_lock_irqsave(&guc->timestamp.lock, flags);1466 lrc_update_runtime(ce);1467 spin_unlock_irqrestore(&guc->timestamp.lock, flags);1468}1469 1470static void guc_context_update_stats(struct intel_context *ce)1471{1472 if (!intel_context_pin_if_active(ce))1473 return;1474 1475 __guc_context_update_stats(ce);1476 intel_context_unpin(ce);1477}1478 1479static void guc_timestamp_ping(struct work_struct *wrk)1480{1481 struct intel_guc *guc = container_of(wrk, typeof(*guc),1482 timestamp.work.work);1483 struct intel_uc *uc = container_of(guc, typeof(*uc), guc);1484 struct intel_gt *gt = guc_to_gt(guc);1485 struct intel_context *ce;1486 intel_wakeref_t wakeref;1487 unsigned long index;1488 int srcu, ret;1489 1490 /*1491 * Ideally the busyness worker should take a gt pm wakeref because the1492 * worker only needs to be active while gt is awake. However, the1493 * gt_park path cancels the worker synchronously and this complicates1494 * the flow if the worker is also running at the same time. The cancel1495 * waits for the worker and when the worker releases the wakeref, that1496 * would call gt_park and would lead to a deadlock.1497 *1498 * The resolution is to take the global pm wakeref if runtime pm is1499 * already active. If not, we don't need to update the busyness stats as1500 * the stats would already be updated when the gt was parked.1501 *1502 * Note:1503 * - We do not requeue the worker if we cannot take a reference to runtime1504 * pm since intel_guc_busyness_unpark would requeue the worker in the1505 * resume path.1506 *1507 * - If the gt was parked longer than time taken for GT timestamp to roll1508 * over, we ignore those rollovers since we don't care about tracking1509 * the exact GT time. We only care about roll overs when the gt is1510 * active and running workloads.1511 *1512 * - There is a window of time between gt_park and runtime suspend,1513 * where the worker may run. This is acceptable since the worker will1514 * not find any new data to update busyness.1515 */1516 wakeref = intel_runtime_pm_get_if_active(>->i915->runtime_pm);1517 if (!wakeref)1518 return;1519 1520 /*1521 * Synchronize with gt reset to make sure the worker does not1522 * corrupt the engine/guc stats. NB: can't actually block waiting1523 * for a reset to complete as the reset requires flushing out1524 * this worker thread if started. So waiting would deadlock.1525 */1526 ret = intel_gt_reset_trylock(gt, &srcu);1527 if (ret)1528 goto err_trylock;1529 1530 __update_guc_busyness_stats(guc);1531 1532 /* adjust context stats for overflow */1533 xa_for_each(&guc->context_lookup, index, ce)1534 guc_context_update_stats(ce);1535 1536 intel_gt_reset_unlock(gt, srcu);1537 1538 guc_enable_busyness_worker(guc);1539 1540err_trylock:1541 intel_runtime_pm_put(>->i915->runtime_pm, wakeref);1542}1543 1544static int guc_action_enable_usage_stats(struct intel_guc *guc)1545{1546 u32 offset = intel_guc_engine_usage_offset(guc);1547 u32 action[] = {1548 INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF,1549 offset,1550 0,1551 };1552 1553 return intel_guc_send(guc, action, ARRAY_SIZE(action));1554}1555 1556static int guc_init_engine_stats(struct intel_guc *guc)1557{1558 struct intel_gt *gt = guc_to_gt(guc);1559 intel_wakeref_t wakeref;1560 int ret;1561 1562 with_intel_runtime_pm(>->i915->runtime_pm, wakeref)1563 ret = guc_action_enable_usage_stats(guc);1564 1565 if (ret)1566 guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));1567 else1568 guc_enable_busyness_worker(guc);1569 1570 return ret;1571}1572 1573static void guc_fini_engine_stats(struct intel_guc *guc)1574{1575 guc_cancel_busyness_worker(guc);1576}1577 1578void intel_guc_busyness_park(struct intel_gt *gt)1579{1580 struct intel_guc *guc = gt_to_guc(gt);1581 1582 if (!guc_submission_initialized(guc))1583 return;1584 1585 /*1586 * There is a race with suspend flow where the worker runs after suspend1587 * and causes an unclaimed register access warning. Cancel the worker1588 * synchronously here.1589 */1590 guc_cancel_busyness_worker(guc);1591 1592 /*1593 * Before parking, we should sample engine busyness stats if we need to.1594 * We can skip it if we are less than half a ping from the last time we1595 * sampled the busyness stats.1596 */1597 if (guc->timestamp.last_stat_jiffies &&1598 !time_after(jiffies, guc->timestamp.last_stat_jiffies +1599 (guc->timestamp.ping_delay / 2)))1600 return;1601 1602 __update_guc_busyness_stats(guc);1603}1604 1605void intel_guc_busyness_unpark(struct intel_gt *gt)1606{1607 struct intel_guc *guc = gt_to_guc(gt);1608 unsigned long flags;1609 ktime_t unused;1610 1611 if (!guc_submission_initialized(guc))1612 return;1613 1614 spin_lock_irqsave(&guc->timestamp.lock, flags);1615 guc_update_pm_timestamp(guc, &unused);1616 spin_unlock_irqrestore(&guc->timestamp.lock, flags);1617 guc_enable_busyness_worker(guc);1618}1619 1620static inline bool1621submission_disabled(struct intel_guc *guc)1622{1623 struct i915_sched_engine * const sched_engine = guc->sched_engine;1624 1625 return unlikely(!sched_engine ||1626 !__tasklet_is_enabled(&sched_engine->tasklet) ||1627 intel_gt_is_wedged(guc_to_gt(guc)));1628}1629 1630static void disable_submission(struct intel_guc *guc)1631{1632 struct i915_sched_engine * const sched_engine = guc->sched_engine;1633 1634 if (__tasklet_is_enabled(&sched_engine->tasklet)) {1635 GEM_BUG_ON(!guc->ct.enabled);1636 __tasklet_disable_sync_once(&sched_engine->tasklet);1637 sched_engine->tasklet.callback = NULL;1638 }1639}1640 1641static void enable_submission(struct intel_guc *guc)1642{1643 struct i915_sched_engine * const sched_engine = guc->sched_engine;1644 unsigned long flags;1645 1646 spin_lock_irqsave(&guc->sched_engine->lock, flags);1647 sched_engine->tasklet.callback = guc_submission_tasklet;1648 wmb(); /* Make sure callback visible */1649 if (!__tasklet_is_enabled(&sched_engine->tasklet) &&1650 __tasklet_enable(&sched_engine->tasklet)) {1651 GEM_BUG_ON(!guc->ct.enabled);1652 1653 /* And kick in case we missed a new request submission. */1654 tasklet_hi_schedule(&sched_engine->tasklet);1655 }1656 spin_unlock_irqrestore(&guc->sched_engine->lock, flags);1657}1658 1659static void guc_flush_submissions(struct intel_guc *guc)1660{1661 struct i915_sched_engine * const sched_engine = guc->sched_engine;1662 unsigned long flags;1663 1664 spin_lock_irqsave(&sched_engine->lock, flags);1665 spin_unlock_irqrestore(&sched_engine->lock, flags);1666}1667 1668void intel_guc_submission_flush_work(struct intel_guc *guc)1669{1670 flush_work(&guc->submission_state.destroyed_worker);1671}1672 1673static void guc_flush_destroyed_contexts(struct intel_guc *guc);1674 1675void intel_guc_submission_reset_prepare(struct intel_guc *guc)1676{1677 if (unlikely(!guc_submission_initialized(guc))) {1678 /* Reset called during driver load? GuC not yet initialised! */1679 return;1680 }1681 1682 intel_gt_park_heartbeats(guc_to_gt(guc));1683 disable_submission(guc);1684 guc->interrupts.disable(guc);1685 __reset_guc_busyness_stats(guc);1686 1687 /* Flush IRQ handler */1688 spin_lock_irq(guc_to_gt(guc)->irq_lock);1689 spin_unlock_irq(guc_to_gt(guc)->irq_lock);1690 1691 guc_flush_submissions(guc);1692 guc_flush_destroyed_contexts(guc);1693 flush_work(&guc->ct.requests.worker);1694 1695 scrub_guc_desc_for_outstanding_g2h(guc);1696}1697 1698static struct intel_engine_cs *1699guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling)1700{1701 struct intel_engine_cs *engine;1702 intel_engine_mask_t tmp, mask = ve->mask;1703 unsigned int num_siblings = 0;1704 1705 for_each_engine_masked(engine, ve->gt, mask, tmp)1706 if (num_siblings++ == sibling)1707 return engine;1708 1709 return NULL;1710}1711 1712static inline struct intel_engine_cs *1713__context_to_physical_engine(struct intel_context *ce)1714{1715 struct intel_engine_cs *engine = ce->engine;1716 1717 if (intel_engine_is_virtual(engine))1718 engine = guc_virtual_get_sibling(engine, 0);1719 1720 return engine;1721}1722 1723static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub)1724{1725 struct intel_engine_cs *engine = __context_to_physical_engine(ce);1726 1727 if (!intel_context_is_schedulable(ce))1728 return;1729 1730 GEM_BUG_ON(!intel_context_is_pinned(ce));1731 1732 /*1733 * We want a simple context + ring to execute the breadcrumb update.1734 * We cannot rely on the context being intact across the GPU hang,1735 * so clear it and rebuild just what we need for the breadcrumb.1736 * All pending requests for this context will be zapped, and any1737 * future request will be after userspace has had the opportunity1738 * to recreate its own state.1739 */1740 if (scrub)1741 lrc_init_regs(ce, engine, true);1742 1743 /* Rerun the request; its payload has been neutered (if guilty). */1744 lrc_update_regs(ce, engine, head);1745}1746 1747static void guc_engine_reset_prepare(struct intel_engine_cs *engine)1748{1749 /*1750 * Wa_22011802037: In addition to stopping the cs, we need1751 * to wait for any pending mi force wakeups1752 */1753 if (intel_engine_reset_needs_wa_22011802037(engine->gt)) {1754 intel_engine_stop_cs(engine);1755 intel_engine_wait_for_pending_mi_fw(engine);1756 }1757}1758 1759static void guc_reset_nop(struct intel_engine_cs *engine)1760{1761}1762 1763static void guc_rewind_nop(struct intel_engine_cs *engine, bool stalled)1764{1765}1766 1767static void1768__unwind_incomplete_requests(struct intel_context *ce)1769{1770 struct i915_request *rq, *rn;1771 struct list_head *pl;1772 int prio = I915_PRIORITY_INVALID;1773 struct i915_sched_engine * const sched_engine =1774 ce->engine->sched_engine;1775 unsigned long flags;1776 1777 spin_lock_irqsave(&sched_engine->lock, flags);1778 spin_lock(&ce->guc_state.lock);1779 list_for_each_entry_safe_reverse(rq, rn,1780 &ce->guc_state.requests,1781 sched.link) {1782 if (i915_request_completed(rq))1783 continue;1784 1785 list_del_init(&rq->sched.link);1786 __i915_request_unsubmit(rq);1787 1788 /* Push the request back into the queue for later resubmission. */1789 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);1790 if (rq_prio(rq) != prio) {1791 prio = rq_prio(rq);1792 pl = i915_sched_lookup_priolist(sched_engine, prio);1793 }1794 GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine));1795 1796 list_add(&rq->sched.link, pl);1797 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);1798 }1799 spin_unlock(&ce->guc_state.lock);1800 spin_unlock_irqrestore(&sched_engine->lock, flags);1801}1802 1803static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stalled)1804{1805 bool guilty;1806 struct i915_request *rq;1807 unsigned long flags;1808 u32 head;1809 int i, number_children = ce->parallel.number_children;1810 struct intel_context *parent = ce;1811 1812 GEM_BUG_ON(intel_context_is_child(ce));1813 1814 intel_context_get(ce);1815 1816 /*1817 * GuC will implicitly mark the context as non-schedulable when it sends1818 * the reset notification. Make sure our state reflects this change. The1819 * context will be marked enabled on resubmission.1820 */1821 spin_lock_irqsave(&ce->guc_state.lock, flags);1822 clr_context_enabled(ce);1823 spin_unlock_irqrestore(&ce->guc_state.lock, flags);1824 1825 /*1826 * For each context in the relationship find the hanging request1827 * resetting each context / request as needed1828 */1829 for (i = 0; i < number_children + 1; ++i) {1830 if (!intel_context_is_pinned(ce))1831 goto next_context;1832 1833 guilty = false;1834 rq = intel_context_get_active_request(ce);1835 if (!rq) {1836 head = ce->ring->tail;1837 goto out_replay;1838 }1839 1840 if (i915_request_started(rq))1841 guilty = stalled & ce->engine->mask;1842 1843 GEM_BUG_ON(i915_active_is_idle(&ce->active));1844 head = intel_ring_wrap(ce->ring, rq->head);1845 1846 __i915_request_reset(rq, guilty);1847 i915_request_put(rq);1848out_replay:1849 guc_reset_state(ce, head, guilty);1850next_context:1851 if (i != number_children)1852 ce = list_next_entry(ce, parallel.child_link);1853 }1854 1855 __unwind_incomplete_requests(parent);1856 intel_context_put(parent);1857}1858 1859void wake_up_all_tlb_invalidate(struct intel_guc *guc)1860{1861 struct intel_guc_tlb_wait *wait;1862 unsigned long i;1863 1864 if (!intel_guc_tlb_invalidation_is_available(guc))1865 return;1866 1867 xa_lock_irq(&guc->tlb_lookup);1868 xa_for_each(&guc->tlb_lookup, i, wait)1869 wake_up(&wait->wq);1870 xa_unlock_irq(&guc->tlb_lookup);1871}1872 1873void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)1874{1875 struct intel_context *ce;1876 unsigned long index;1877 unsigned long flags;1878 1879 if (unlikely(!guc_submission_initialized(guc))) {1880 /* Reset called during driver load? GuC not yet initialised! */1881 return;1882 }1883 1884 xa_lock_irqsave(&guc->context_lookup, flags);1885 xa_for_each(&guc->context_lookup, index, ce) {1886 if (!kref_get_unless_zero(&ce->ref))1887 continue;1888 1889 xa_unlock(&guc->context_lookup);1890 1891 if (intel_context_is_pinned(ce) &&1892 !intel_context_is_child(ce))1893 __guc_reset_context(ce, stalled);1894 1895 intel_context_put(ce);1896 1897 xa_lock(&guc->context_lookup);1898 }1899 xa_unlock_irqrestore(&guc->context_lookup, flags);1900 1901 /* GuC is blown away, drop all references to contexts */1902 xa_destroy(&guc->context_lookup);1903}1904 1905static void guc_cancel_context_requests(struct intel_context *ce)1906{1907 struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine;1908 struct i915_request *rq;1909 unsigned long flags;1910 1911 /* Mark all executing requests as skipped. */1912 spin_lock_irqsave(&sched_engine->lock, flags);1913 spin_lock(&ce->guc_state.lock);1914 list_for_each_entry(rq, &ce->guc_state.requests, sched.link)1915 i915_request_put(i915_request_mark_eio(rq));1916 spin_unlock(&ce->guc_state.lock);1917 spin_unlock_irqrestore(&sched_engine->lock, flags);1918}1919 1920static void1921guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine)1922{1923 struct i915_request *rq, *rn;1924 struct rb_node *rb;1925 unsigned long flags;1926 1927 /* Can be called during boot if GuC fails to load */1928 if (!sched_engine)1929 return;1930 1931 /*1932 * Before we call engine->cancel_requests(), we should have exclusive1933 * access to the submission state. This is arranged for us by the1934 * caller disabling the interrupt generation, the tasklet and other1935 * threads that may then access the same state, giving us a free hand1936 * to reset state. However, we still need to let lockdep be aware that1937 * we know this state may be accessed in hardirq context, so we1938 * disable the irq around this manipulation and we want to keep1939 * the spinlock focused on its duties and not accidentally conflate1940 * coverage to the submission's irq state. (Similarly, although we1941 * shouldn't need to disable irq around the manipulation of the1942 * submission's irq state, we also wish to remind ourselves that1943 * it is irq state.)1944 */1945 spin_lock_irqsave(&sched_engine->lock, flags);1946 1947 /* Flush the queued requests to the timeline list (for retiring). */1948 while ((rb = rb_first_cached(&sched_engine->queue))) {1949 struct i915_priolist *p = to_priolist(rb);1950 1951 priolist_for_each_request_consume(rq, rn, p) {1952 list_del_init(&rq->sched.link);1953 1954 __i915_request_submit(rq);1955 1956 i915_request_put(i915_request_mark_eio(rq));1957 }1958 1959 rb_erase_cached(&p->node, &sched_engine->queue);1960 i915_priolist_free(p);1961 }1962 1963 /* Remaining _unready_ requests will be nop'ed when submitted */1964 1965 sched_engine->queue_priority_hint = INT_MIN;1966 sched_engine->queue = RB_ROOT_CACHED;1967 1968 spin_unlock_irqrestore(&sched_engine->lock, flags);1969}1970 1971void intel_guc_submission_cancel_requests(struct intel_guc *guc)1972{1973 struct intel_context *ce;1974 unsigned long index;1975 unsigned long flags;1976 1977 xa_lock_irqsave(&guc->context_lookup, flags);1978 xa_for_each(&guc->context_lookup, index, ce) {1979 if (!kref_get_unless_zero(&ce->ref))1980 continue;1981 1982 xa_unlock(&guc->context_lookup);1983 1984 if (intel_context_is_pinned(ce) &&1985 !intel_context_is_child(ce))1986 guc_cancel_context_requests(ce);1987 1988 intel_context_put(ce);1989 1990 xa_lock(&guc->context_lookup);1991 }1992 xa_unlock_irqrestore(&guc->context_lookup, flags);1993 1994 guc_cancel_sched_engine_requests(guc->sched_engine);1995 1996 /* GuC is blown away, drop all references to contexts */1997 xa_destroy(&guc->context_lookup);1998 1999 /*2000 * Wedged GT won't respond to any TLB invalidation request. Simply2001 * release all the blocked waiters.2002 */2003 wake_up_all_tlb_invalidate(guc);2004}2005 2006void intel_guc_submission_reset_finish(struct intel_guc *guc)2007{2008 /* Reset called during driver load or during wedge? */2009 if (unlikely(!guc_submission_initialized(guc) ||2010 !intel_guc_is_fw_running(guc) ||2011 intel_gt_is_wedged(guc_to_gt(guc)))) {2012 return;2013 }2014 2015 /*2016 * Technically possible for either of these values to be non-zero here,2017 * but very unlikely + harmless. Regardless let's add an error so we can2018 * see in CI if this happens frequently / a precursor to taking down the2019 * machine.2020 */2021 if (atomic_read(&guc->outstanding_submission_g2h))2022 guc_err(guc, "Unexpected outstanding GuC to Host in reset finish\n");2023 atomic_set(&guc->outstanding_submission_g2h, 0);2024 2025 intel_guc_global_policies_update(guc);2026 enable_submission(guc);2027 intel_gt_unpark_heartbeats(guc_to_gt(guc));2028 2029 /*2030 * The full GT reset will have cleared the TLB caches and flushed the2031 * G2H message queue; we can release all the blocked waiters.2032 */2033 wake_up_all_tlb_invalidate(guc);2034}2035 2036static void destroyed_worker_func(struct work_struct *w);2037static void reset_fail_worker_func(struct work_struct *w);2038 2039bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc)2040{2041 return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) &&2042 intel_guc_is_ready(guc);2043}2044 2045static int init_tlb_lookup(struct intel_guc *guc)2046{2047 struct intel_guc_tlb_wait *wait;2048 int err;2049 2050 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))2051 return 0;2052 2053 xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC);2054 2055 wait = kzalloc(sizeof(*wait), GFP_KERNEL);2056 if (!wait)2057 return -ENOMEM;2058 2059 init_waitqueue_head(&wait->wq);2060 2061 /* Preallocate a shared id for use under memory pressure. */2062 err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait,2063 xa_limit_32b, &guc->next_seqno, GFP_KERNEL);2064 if (err < 0) {2065 kfree(wait);2066 return err;2067 }2068 2069 return 0;2070}2071 2072static void fini_tlb_lookup(struct intel_guc *guc)2073{2074 struct intel_guc_tlb_wait *wait;2075 2076 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))2077 return;2078 2079 wait = xa_load(&guc->tlb_lookup, guc->serial_slot);2080 if (wait && wait->busy)2081 guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n");2082 kfree(wait);2083 2084 xa_destroy(&guc->tlb_lookup);2085}2086 2087/*2088 * Set up the memory resources to be shared with the GuC (via the GGTT)2089 * at firmware loading time.2090 */2091int intel_guc_submission_init(struct intel_guc *guc)2092{2093 struct intel_gt *gt = guc_to_gt(guc);2094 int ret;2095 2096 if (guc->submission_initialized)2097 return 0;2098 2099 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {2100 ret = guc_lrc_desc_pool_create_v69(guc);2101 if (ret)2102 return ret;2103 }2104 2105 ret = init_tlb_lookup(guc);2106 if (ret)2107 goto destroy_pool;2108 2109 guc->submission_state.guc_ids_bitmap =2110 bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);2111 if (!guc->submission_state.guc_ids_bitmap) {2112 ret = -ENOMEM;2113 goto destroy_tlb;2114 }2115 2116 guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ;2117 guc->timestamp.shift = gpm_timestamp_shift(gt);2118 guc->submission_initialized = true;2119 2120 return 0;2121 2122destroy_tlb:2123 fini_tlb_lookup(guc);2124destroy_pool:2125 guc_lrc_desc_pool_destroy_v69(guc);2126 return ret;2127}2128 2129void intel_guc_submission_fini(struct intel_guc *guc)2130{2131 if (!guc->submission_initialized)2132 return;2133 2134 guc_fini_engine_stats(guc);2135 guc_flush_destroyed_contexts(guc);2136 guc_lrc_desc_pool_destroy_v69(guc);2137 i915_sched_engine_put(guc->sched_engine);2138 bitmap_free(guc->submission_state.guc_ids_bitmap);2139 fini_tlb_lookup(guc);2140 guc->submission_initialized = false;2141}2142 2143static inline void queue_request(struct i915_sched_engine *sched_engine,2144 struct i915_request *rq,2145 int prio)2146{2147 GEM_BUG_ON(!list_empty(&rq->sched.link));2148 list_add_tail(&rq->sched.link,2149 i915_sched_lookup_priolist(sched_engine, prio));2150 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);2151 tasklet_hi_schedule(&sched_engine->tasklet);2152}2153 2154static int guc_bypass_tasklet_submit(struct intel_guc *guc,2155 struct i915_request *rq)2156{2157 int ret = 0;2158 2159 __i915_request_submit(rq);2160 2161 trace_i915_request_in(rq, 0);2162 2163 if (is_multi_lrc_rq(rq)) {2164 if (multi_lrc_submit(rq)) {2165 ret = guc_wq_item_append(guc, rq);2166 if (!ret)2167 ret = guc_add_request(guc, rq);2168 }2169 } else {2170 guc_set_lrc_tail(rq);2171 ret = guc_add_request(guc, rq);2172 }2173 2174 if (unlikely(ret == -EPIPE))2175 disable_submission(guc);2176 2177 return ret;2178}2179 2180static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)2181{2182 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;2183 struct intel_context *ce = request_to_scheduling_context(rq);2184 2185 return submission_disabled(guc) || guc->stalled_request ||2186 !i915_sched_engine_is_empty(sched_engine) ||2187 !ctx_id_mapped(guc, ce->guc_id.id);2188}2189 2190static void guc_submit_request(struct i915_request *rq)2191{2192 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;2193 struct intel_guc *guc = gt_to_guc(rq->engine->gt);2194 unsigned long flags;2195 2196 /* Will be called from irq-context when using foreign fences. */2197 spin_lock_irqsave(&sched_engine->lock, flags);2198 2199 if (need_tasklet(guc, rq))2200 queue_request(sched_engine, rq, rq_prio(rq));2201 else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY)2202 tasklet_hi_schedule(&sched_engine->tasklet);2203 2204 spin_unlock_irqrestore(&sched_engine->lock, flags);2205}2206 2207static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)2208{2209 int ret;2210 2211 GEM_BUG_ON(intel_context_is_child(ce));2212 2213 if (intel_context_is_parent(ce))2214 ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap,2215 NUMBER_MULTI_LRC_GUC_ID(guc),2216 order_base_2(ce->parallel.number_children2217 + 1));2218 else2219 ret = ida_alloc_range(&guc->submission_state.guc_ids,2220 NUMBER_MULTI_LRC_GUC_ID(guc),2221 guc->submission_state.num_guc_ids - 1,2222 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);2223 if (unlikely(ret < 0))2224 return ret;2225 2226 if (!intel_context_is_parent(ce))2227 ++guc->submission_state.guc_ids_in_use;2228 2229 ce->guc_id.id = ret;2230 return 0;2231}2232 2233static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)2234{2235 GEM_BUG_ON(intel_context_is_child(ce));2236 2237 if (!context_guc_id_invalid(ce)) {2238 if (intel_context_is_parent(ce)) {2239 bitmap_release_region(guc->submission_state.guc_ids_bitmap,2240 ce->guc_id.id,2241 order_base_2(ce->parallel.number_children2242 + 1));2243 } else {2244 --guc->submission_state.guc_ids_in_use;2245 ida_free(&guc->submission_state.guc_ids,2246 ce->guc_id.id);2247 }2248 clr_ctx_id_mapping(guc, ce->guc_id.id);2249 set_context_guc_id_invalid(ce);2250 }2251 if (!list_empty(&ce->guc_id.link))2252 list_del_init(&ce->guc_id.link);2253}2254 2255static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)2256{2257 unsigned long flags;2258 2259 spin_lock_irqsave(&guc->submission_state.lock, flags);2260 __release_guc_id(guc, ce);2261 spin_unlock_irqrestore(&guc->submission_state.lock, flags);2262}2263 2264static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)2265{2266 struct intel_context *cn;2267 2268 lockdep_assert_held(&guc->submission_state.lock);2269 GEM_BUG_ON(intel_context_is_child(ce));2270 GEM_BUG_ON(intel_context_is_parent(ce));2271 2272 if (!list_empty(&guc->submission_state.guc_id_list)) {2273 cn = list_first_entry(&guc->submission_state.guc_id_list,2274 struct intel_context,2275 guc_id.link);2276 2277 GEM_BUG_ON(atomic_read(&cn->guc_id.ref));2278 GEM_BUG_ON(context_guc_id_invalid(cn));2279 GEM_BUG_ON(intel_context_is_child(cn));2280 GEM_BUG_ON(intel_context_is_parent(cn));2281 2282 list_del_init(&cn->guc_id.link);2283 ce->guc_id.id = cn->guc_id.id;2284 2285 spin_lock(&cn->guc_state.lock);2286 clr_context_registered(cn);2287 spin_unlock(&cn->guc_state.lock);2288 2289 set_context_guc_id_invalid(cn);2290 2291#ifdef CONFIG_DRM_I915_SELFTEST2292 guc->number_guc_id_stolen++;2293#endif2294 2295 return 0;2296 } else {2297 return -EAGAIN;2298 }2299}2300 2301static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)2302{2303 int ret;2304 2305 lockdep_assert_held(&guc->submission_state.lock);2306 GEM_BUG_ON(intel_context_is_child(ce));2307 2308 ret = new_guc_id(guc, ce);2309 if (unlikely(ret < 0)) {2310 if (intel_context_is_parent(ce))2311 return -ENOSPC;2312 2313 ret = steal_guc_id(guc, ce);2314 if (ret < 0)2315 return ret;2316 }2317 2318 if (intel_context_is_parent(ce)) {2319 struct intel_context *child;2320 int i = 1;2321 2322 for_each_child(ce, child)2323 child->guc_id.id = ce->guc_id.id + i++;2324 }2325 2326 return 0;2327}2328 2329#define PIN_GUC_ID_TRIES 42330static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)2331{2332 int ret = 0;2333 unsigned long flags, tries = PIN_GUC_ID_TRIES;2334 2335 GEM_BUG_ON(atomic_read(&ce->guc_id.ref));2336 2337try_again:2338 spin_lock_irqsave(&guc->submission_state.lock, flags);2339 2340 might_lock(&ce->guc_state.lock);2341 2342 if (context_guc_id_invalid(ce)) {2343 ret = assign_guc_id(guc, ce);2344 if (ret)2345 goto out_unlock;2346 ret = 1; /* Indidcates newly assigned guc_id */2347 }2348 if (!list_empty(&ce->guc_id.link))2349 list_del_init(&ce->guc_id.link);2350 atomic_inc(&ce->guc_id.ref);2351 2352out_unlock:2353 spin_unlock_irqrestore(&guc->submission_state.lock, flags);2354 2355 /*2356 * -EAGAIN indicates no guc_id are available, let's retire any2357 * outstanding requests to see if that frees up a guc_id. If the first2358 * retire didn't help, insert a sleep with the timeslice duration before2359 * attempting to retire more requests. Double the sleep period each2360 * subsequent pass before finally giving up. The sleep period has max of2361 * 100ms and minimum of 1ms.2362 */2363 if (ret == -EAGAIN && --tries) {2364 if (PIN_GUC_ID_TRIES - tries > 1) {2365 unsigned int timeslice_shifted =2366 ce->engine->props.timeslice_duration_ms <<2367 (PIN_GUC_ID_TRIES - tries - 2);2368 unsigned int max = min_t(unsigned int, 100,2369 timeslice_shifted);2370 2371 msleep(max_t(unsigned int, max, 1));2372 }2373 intel_gt_retire_requests(guc_to_gt(guc));2374 goto try_again;2375 }2376 2377 return ret;2378}2379 2380static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)2381{2382 unsigned long flags;2383 2384 GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0);2385 GEM_BUG_ON(intel_context_is_child(ce));2386 2387 if (unlikely(context_guc_id_invalid(ce) ||2388 intel_context_is_parent(ce)))2389 return;2390 2391 spin_lock_irqsave(&guc->submission_state.lock, flags);2392 if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id.link) &&2393 !atomic_read(&ce->guc_id.ref))2394 list_add_tail(&ce->guc_id.link,2395 &guc->submission_state.guc_id_list);2396 spin_unlock_irqrestore(&guc->submission_state.lock, flags);2397}2398 2399static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,2400 struct intel_context *ce,2401 u32 guc_id,2402 u32 offset,2403 bool loop)2404{2405 struct intel_context *child;2406 u32 action[4 + MAX_ENGINE_INSTANCE];2407 int len = 0;2408 2409 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);2410 2411 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;2412 action[len++] = guc_id;2413 action[len++] = ce->parallel.number_children + 1;2414 action[len++] = offset;2415 for_each_child(ce, child) {2416 offset += sizeof(struct guc_lrc_desc_v69);2417 action[len++] = offset;2418 }2419 2420 return guc_submission_send_busy_loop(guc, action, len, 0, loop);2421}2422 2423static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,2424 struct intel_context *ce,2425 struct guc_ctxt_registration_info *info,2426 bool loop)2427{2428 struct intel_context *child;2429 u32 action[13 + (MAX_ENGINE_INSTANCE * 2)];2430 int len = 0;2431 u32 next_id;2432 2433 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);2434 2435 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;2436 action[len++] = info->flags;2437 action[len++] = info->context_idx;2438 action[len++] = info->engine_class;2439 action[len++] = info->engine_submit_mask;2440 action[len++] = info->wq_desc_lo;2441 action[len++] = info->wq_desc_hi;2442 action[len++] = info->wq_base_lo;2443 action[len++] = info->wq_base_hi;2444 action[len++] = info->wq_size;2445 action[len++] = ce->parallel.number_children + 1;2446 action[len++] = info->hwlrca_lo;2447 action[len++] = info->hwlrca_hi;2448 2449 next_id = info->context_idx + 1;2450 for_each_child(ce, child) {2451 GEM_BUG_ON(next_id++ != child->guc_id.id);2452 2453 /*2454 * NB: GuC interface supports 64 bit LRCA even though i915/HW2455 * only supports 32 bit currently.2456 */2457 action[len++] = lower_32_bits(child->lrc.lrca);2458 action[len++] = upper_32_bits(child->lrc.lrca);2459 }2460 2461 GEM_BUG_ON(len > ARRAY_SIZE(action));2462 2463 return guc_submission_send_busy_loop(guc, action, len, 0, loop);2464}2465 2466static int __guc_action_register_context_v69(struct intel_guc *guc,2467 u32 guc_id,2468 u32 offset,2469 bool loop)2470{2471 u32 action[] = {2472 INTEL_GUC_ACTION_REGISTER_CONTEXT,2473 guc_id,2474 offset,2475 };2476 2477 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),2478 0, loop);2479}2480 2481static int __guc_action_register_context_v70(struct intel_guc *guc,2482 struct guc_ctxt_registration_info *info,2483 bool loop)2484{2485 u32 action[] = {2486 INTEL_GUC_ACTION_REGISTER_CONTEXT,2487 info->flags,2488 info->context_idx,2489 info->engine_class,2490 info->engine_submit_mask,2491 info->wq_desc_lo,2492 info->wq_desc_hi,2493 info->wq_base_lo,2494 info->wq_base_hi,2495 info->wq_size,2496 info->hwlrca_lo,2497 info->hwlrca_hi,2498 };2499 2500 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),2501 0, loop);2502}2503 2504static void prepare_context_registration_info_v69(struct intel_context *ce);2505static void prepare_context_registration_info_v70(struct intel_context *ce,2506 struct guc_ctxt_registration_info *info);2507 2508static int2509register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool loop)2510{2511 u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool_v69) +2512 ce->guc_id.id * sizeof(struct guc_lrc_desc_v69);2513 2514 prepare_context_registration_info_v69(ce);2515 2516 if (intel_context_is_parent(ce))2517 return __guc_action_register_multi_lrc_v69(guc, ce, ce->guc_id.id,2518 offset, loop);2519 else2520 return __guc_action_register_context_v69(guc, ce->guc_id.id,2521 offset, loop);2522}2523 2524static int2525register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool loop)2526{2527 struct guc_ctxt_registration_info info;2528 2529 prepare_context_registration_info_v70(ce, &info);2530 2531 if (intel_context_is_parent(ce))2532 return __guc_action_register_multi_lrc_v70(guc, ce, &info, loop);2533 else2534 return __guc_action_register_context_v70(guc, &info, loop);2535}2536 2537static int register_context(struct intel_context *ce, bool loop)2538{2539 struct intel_guc *guc = ce_to_guc(ce);2540 int ret;2541 2542 GEM_BUG_ON(intel_context_is_child(ce));2543 trace_intel_context_register(ce);2544 2545 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))2546 ret = register_context_v70(guc, ce, loop);2547 else2548 ret = register_context_v69(guc, ce, loop);2549 2550 if (likely(!ret)) {2551 unsigned long flags;2552 2553 spin_lock_irqsave(&ce->guc_state.lock, flags);2554 set_context_registered(ce);2555 spin_unlock_irqrestore(&ce->guc_state.lock, flags);2556 2557 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))2558 guc_context_policy_init_v70(ce, loop);2559 }2560 2561 return ret;2562}2563 2564static int __guc_action_deregister_context(struct intel_guc *guc,2565 u32 guc_id)2566{2567 u32 action[] = {2568 INTEL_GUC_ACTION_DEREGISTER_CONTEXT,2569 guc_id,2570 };2571 2572 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),2573 G2H_LEN_DW_DEREGISTER_CONTEXT,2574 true);2575}2576 2577static int deregister_context(struct intel_context *ce, u32 guc_id)2578{2579 struct intel_guc *guc = ce_to_guc(ce);2580 2581 GEM_BUG_ON(intel_context_is_child(ce));2582 trace_intel_context_deregister(ce);2583 2584 return __guc_action_deregister_context(guc, guc_id);2585}2586 2587static inline void clear_children_join_go_memory(struct intel_context *ce)2588{2589 struct parent_scratch *ps = __get_parent_scratch(ce);2590 int i;2591 2592 ps->go.semaphore = 0;2593 for (i = 0; i < ce->parallel.number_children + 1; ++i)2594 ps->join[i].semaphore = 0;2595}2596 2597static inline u32 get_children_go_value(struct intel_context *ce)2598{2599 return __get_parent_scratch(ce)->go.semaphore;2600}2601 2602static inline u32 get_children_join_value(struct intel_context *ce,2603 u8 child_index)2604{2605 return __get_parent_scratch(ce)->join[child_index].semaphore;2606}2607 2608struct context_policy {2609 u32 count;2610 struct guc_update_context_policy h2g;2611};2612 2613static u32 __guc_context_policy_action_size(struct context_policy *policy)2614{2615 size_t bytes = sizeof(policy->h2g.header) +2616 (sizeof(policy->h2g.klv[0]) * policy->count);2617 2618 return bytes / sizeof(u32);2619}2620 2621static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id)2622{2623 policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;2624 policy->h2g.header.ctx_id = guc_id;2625 policy->count = 0;2626}2627 2628#define MAKE_CONTEXT_POLICY_ADD(func, id) \2629static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \2630{ \2631 GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \2632 policy->h2g.klv[policy->count].kl = \2633 FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id) | \2634 FIELD_PREP(GUC_KLV_0_LEN, 1); \2635 policy->h2g.klv[policy->count].value = data; \2636 policy->count++; \2637}2638 2639MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)2640MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)2641MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)2642MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)2643MAKE_CONTEXT_POLICY_ADD(slpc_ctx_freq_req, SLPM_GT_FREQUENCY)2644 2645#undef MAKE_CONTEXT_POLICY_ADD2646 2647static int __guc_context_set_context_policies(struct intel_guc *guc,2648 struct context_policy *policy,2649 bool loop)2650{2651 return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g,2652 __guc_context_policy_action_size(policy),2653 0, loop);2654}2655 2656static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)2657{2658 struct intel_engine_cs *engine = ce->engine;2659 struct intel_guc *guc = gt_to_guc(engine->gt);2660 struct context_policy policy;2661 u32 execution_quantum;2662 u32 preemption_timeout;2663 u32 slpc_ctx_freq_req = 0;2664 unsigned long flags;2665 int ret;2666 2667 /* NB: For both of these, zero means disabled. */2668 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,2669 execution_quantum));2670 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,2671 preemption_timeout));2672 execution_quantum = engine->props.timeslice_duration_ms * 1000;2673 preemption_timeout = engine->props.preempt_timeout_ms * 1000;2674 2675 if (ce->flags & BIT(CONTEXT_LOW_LATENCY))2676 slpc_ctx_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE;2677 2678 __guc_context_policy_start_klv(&policy, ce->guc_id.id);2679 2680 __guc_context_policy_add_priority(&policy, ce->guc_state.prio);2681 __guc_context_policy_add_execution_quantum(&policy, execution_quantum);2682 __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);2683 __guc_context_policy_add_slpc_ctx_freq_req(&policy, slpc_ctx_freq_req);2684 2685 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)2686 __guc_context_policy_add_preempt_to_idle(&policy, 1);2687 2688 ret = __guc_context_set_context_policies(guc, &policy, loop);2689 2690 spin_lock_irqsave(&ce->guc_state.lock, flags);2691 if (ret != 0)2692 set_context_policy_required(ce);2693 else2694 clr_context_policy_required(ce);2695 spin_unlock_irqrestore(&ce->guc_state.lock, flags);2696 2697 return ret;2698}2699 2700static void guc_context_policy_init_v69(struct intel_engine_cs *engine,2701 struct guc_lrc_desc_v69 *desc)2702{2703 desc->policy_flags = 0;2704 2705 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)2706 desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE_V69;2707 2708 /* NB: For both of these, zero means disabled. */2709 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,2710 desc->execution_quantum));2711 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,2712 desc->preemption_timeout));2713 desc->execution_quantum = engine->props.timeslice_duration_ms * 1000;2714 desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000;2715}2716 2717static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)2718{2719 /*2720 * this matches the mapping we do in map_i915_prio_to_guc_prio()2721 * (e.g. prio < I915_PRIORITY_NORMAL maps to GUC_CLIENT_PRIORITY_NORMAL)2722 */2723 switch (prio) {2724 default:2725 MISSING_CASE(prio);2726 fallthrough;2727 case GUC_CLIENT_PRIORITY_KMD_NORMAL:2728 return GEN12_CTX_PRIORITY_NORMAL;2729 case GUC_CLIENT_PRIORITY_NORMAL:2730 return GEN12_CTX_PRIORITY_LOW;2731 case GUC_CLIENT_PRIORITY_HIGH:2732 case GUC_CLIENT_PRIORITY_KMD_HIGH:2733 return GEN12_CTX_PRIORITY_HIGH;2734 }2735}2736 2737static void prepare_context_registration_info_v69(struct intel_context *ce)2738{2739 struct intel_engine_cs *engine = ce->engine;2740 struct intel_guc *guc = gt_to_guc(engine->gt);2741 u32 ctx_id = ce->guc_id.id;2742 struct guc_lrc_desc_v69 *desc;2743 struct intel_context *child;2744 2745 GEM_BUG_ON(!engine->mask);2746 2747 /*2748 * Ensure LRC + CT vmas are is same region as write barrier is done2749 * based on CT vma region.2750 */2751 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=2752 i915_gem_object_is_lmem(ce->ring->vma->obj));2753 2754 desc = __get_lrc_desc_v69(guc, ctx_id);2755 GEM_BUG_ON(!desc);2756 desc->engine_class = engine_class_to_guc_class(engine->class);2757 desc->engine_submit_mask = engine->logical_mask;2758 desc->hw_context_desc = ce->lrc.lrca;2759 desc->priority = ce->guc_state.prio;2760 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;2761 guc_context_policy_init_v69(engine, desc);2762 2763 /*2764 * If context is a parent, we need to register a process descriptor2765 * describing a work queue and register all child contexts.2766 */2767 if (intel_context_is_parent(ce)) {2768 struct guc_process_desc_v69 *pdesc;2769 2770 ce->parallel.guc.wqi_tail = 0;2771 ce->parallel.guc.wqi_head = 0;2772 2773 desc->process_desc = i915_ggtt_offset(ce->state) +2774 __get_parent_scratch_offset(ce);2775 desc->wq_addr = i915_ggtt_offset(ce->state) +2776 __get_wq_offset(ce);2777 desc->wq_size = WQ_SIZE;2778 2779 pdesc = __get_process_desc_v69(ce);2780 memset(pdesc, 0, sizeof(*(pdesc)));2781 pdesc->stage_id = ce->guc_id.id;2782 pdesc->wq_base_addr = desc->wq_addr;2783 pdesc->wq_size_bytes = desc->wq_size;2784 pdesc->wq_status = WQ_STATUS_ACTIVE;2785 2786 ce->parallel.guc.wq_head = &pdesc->head;2787 ce->parallel.guc.wq_tail = &pdesc->tail;2788 ce->parallel.guc.wq_status = &pdesc->wq_status;2789 2790 for_each_child(ce, child) {2791 desc = __get_lrc_desc_v69(guc, child->guc_id.id);2792 2793 desc->engine_class =2794 engine_class_to_guc_class(engine->class);2795 desc->hw_context_desc = child->lrc.lrca;2796 desc->priority = ce->guc_state.prio;2797 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;2798 guc_context_policy_init_v69(engine, desc);2799 }2800 2801 clear_children_join_go_memory(ce);2802 }2803}2804 2805static void prepare_context_registration_info_v70(struct intel_context *ce,2806 struct guc_ctxt_registration_info *info)2807{2808 struct intel_engine_cs *engine = ce->engine;2809 struct intel_guc *guc = gt_to_guc(engine->gt);2810 u32 ctx_id = ce->guc_id.id;2811 2812 GEM_BUG_ON(!engine->mask);2813 2814 /*2815 * Ensure LRC + CT vmas are is same region as write barrier is done2816 * based on CT vma region.2817 */2818 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=2819 i915_gem_object_is_lmem(ce->ring->vma->obj));2820 2821 memset(info, 0, sizeof(*info));2822 info->context_idx = ctx_id;2823 info->engine_class = engine_class_to_guc_class(engine->class);2824 info->engine_submit_mask = engine->logical_mask;2825 /*2826 * NB: GuC interface supports 64 bit LRCA even though i915/HW2827 * only supports 32 bit currently.2828 */2829 info->hwlrca_lo = lower_32_bits(ce->lrc.lrca);2830 info->hwlrca_hi = upper_32_bits(ce->lrc.lrca);2831 if (engine->flags & I915_ENGINE_HAS_EU_PRIORITY)2832 info->hwlrca_lo |= map_guc_prio_to_lrc_desc_prio(ce->guc_state.prio);2833 info->flags = CONTEXT_REGISTRATION_FLAG_KMD;2834 2835 /*2836 * If context is a parent, we need to register a process descriptor2837 * describing a work queue and register all child contexts.2838 */2839 if (intel_context_is_parent(ce)) {2840 struct guc_sched_wq_desc *wq_desc;2841 u64 wq_desc_offset, wq_base_offset;2842 2843 ce->parallel.guc.wqi_tail = 0;2844 ce->parallel.guc.wqi_head = 0;2845 2846 wq_desc_offset = (u64)i915_ggtt_offset(ce->state) +2847 __get_parent_scratch_offset(ce);2848 wq_base_offset = (u64)i915_ggtt_offset(ce->state) +2849 __get_wq_offset(ce);2850 info->wq_desc_lo = lower_32_bits(wq_desc_offset);2851 info->wq_desc_hi = upper_32_bits(wq_desc_offset);2852 info->wq_base_lo = lower_32_bits(wq_base_offset);2853 info->wq_base_hi = upper_32_bits(wq_base_offset);2854 info->wq_size = WQ_SIZE;2855 2856 wq_desc = __get_wq_desc_v70(ce);2857 memset(wq_desc, 0, sizeof(*wq_desc));2858 wq_desc->wq_status = WQ_STATUS_ACTIVE;2859 2860 ce->parallel.guc.wq_head = &wq_desc->head;2861 ce->parallel.guc.wq_tail = &wq_desc->tail;2862 ce->parallel.guc.wq_status = &wq_desc->wq_status;2863 2864 clear_children_join_go_memory(ce);2865 }2866}2867 2868static int try_context_registration(struct intel_context *ce, bool loop)2869{2870 struct intel_engine_cs *engine = ce->engine;2871 struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;2872 struct intel_guc *guc = gt_to_guc(engine->gt);2873 intel_wakeref_t wakeref;2874 u32 ctx_id = ce->guc_id.id;2875 bool context_registered;2876 int ret = 0;2877 2878 GEM_BUG_ON(!sched_state_is_init(ce));2879 2880 context_registered = ctx_id_mapped(guc, ctx_id);2881 2882 clr_ctx_id_mapping(guc, ctx_id);2883 set_ctx_id_mapping(guc, ctx_id, ce);2884 2885 /*2886 * The context_lookup xarray is used to determine if the hardware2887 * context is currently registered. There are two cases in which it2888 * could be registered either the guc_id has been stolen from another2889 * context or the lrc descriptor address of this context has changed. In2890 * either case the context needs to be deregistered with the GuC before2891 * registering this context.2892 */2893 if (context_registered) {2894 bool disabled;2895 unsigned long flags;2896 2897 trace_intel_context_steal_guc_id(ce);2898 GEM_BUG_ON(!loop);2899 2900 /* Seal race with Reset */2901 spin_lock_irqsave(&ce->guc_state.lock, flags);2902 disabled = submission_disabled(guc);2903 if (likely(!disabled)) {2904 set_context_wait_for_deregister_to_register(ce);2905 intel_context_get(ce);2906 }2907 spin_unlock_irqrestore(&ce->guc_state.lock, flags);2908 if (unlikely(disabled)) {2909 clr_ctx_id_mapping(guc, ctx_id);2910 return 0; /* Will get registered later */2911 }2912 2913 /*2914 * If stealing the guc_id, this ce has the same guc_id as the2915 * context whose guc_id was stolen.2916 */2917 with_intel_runtime_pm(runtime_pm, wakeref)2918 ret = deregister_context(ce, ce->guc_id.id);2919 if (unlikely(ret == -ENODEV))2920 ret = 0; /* Will get registered later */2921 } else {2922 with_intel_runtime_pm(runtime_pm, wakeref)2923 ret = register_context(ce, loop);2924 if (unlikely(ret == -EBUSY)) {2925 clr_ctx_id_mapping(guc, ctx_id);2926 } else if (unlikely(ret == -ENODEV)) {2927 clr_ctx_id_mapping(guc, ctx_id);2928 ret = 0; /* Will get registered later */2929 }2930 }2931 2932 return ret;2933}2934 2935static int __guc_context_pre_pin(struct intel_context *ce,2936 struct intel_engine_cs *engine,2937 struct i915_gem_ww_ctx *ww,2938 void **vaddr)2939{2940 return lrc_pre_pin(ce, engine, ww, vaddr);2941}2942 2943static int __guc_context_pin(struct intel_context *ce,2944 struct intel_engine_cs *engine,2945 void *vaddr)2946{2947 if (i915_ggtt_offset(ce->state) !=2948 (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK))2949 set_bit(CONTEXT_LRCA_DIRTY, &ce->flags);2950 2951 /*2952 * GuC context gets pinned in guc_request_alloc. See that function for2953 * explaination of why.2954 */2955 2956 return lrc_pin(ce, engine, vaddr);2957}2958 2959static int guc_context_pre_pin(struct intel_context *ce,2960 struct i915_gem_ww_ctx *ww,2961 void **vaddr)2962{2963 return __guc_context_pre_pin(ce, ce->engine, ww, vaddr);2964}2965 2966static int guc_context_pin(struct intel_context *ce, void *vaddr)2967{2968 int ret = __guc_context_pin(ce, ce->engine, vaddr);2969 2970 if (likely(!ret && !intel_context_is_barrier(ce)))2971 intel_engine_pm_get(ce->engine);2972 2973 return ret;2974}2975 2976static void guc_context_unpin(struct intel_context *ce)2977{2978 struct intel_guc *guc = ce_to_guc(ce);2979 2980 __guc_context_update_stats(ce);2981 unpin_guc_id(guc, ce);2982 lrc_unpin(ce);2983 2984 if (likely(!intel_context_is_barrier(ce)))2985 intel_engine_pm_put_async(ce->engine);2986}2987 2988static void guc_context_post_unpin(struct intel_context *ce)2989{2990 lrc_post_unpin(ce);2991}2992 2993static void __guc_context_sched_enable(struct intel_guc *guc,2994 struct intel_context *ce)2995{2996 u32 action[] = {2997 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,2998 ce->guc_id.id,2999 GUC_CONTEXT_ENABLE3000 };3001 3002 trace_intel_context_sched_enable(ce);3003 3004 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),3005 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);3006}3007 3008static void __guc_context_sched_disable(struct intel_guc *guc,3009 struct intel_context *ce,3010 u16 guc_id)3011{3012 u32 action[] = {3013 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,3014 guc_id, /* ce->guc_id.id not stable */3015 GUC_CONTEXT_DISABLE3016 };3017 3018 GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID);3019 3020 GEM_BUG_ON(intel_context_is_child(ce));3021 trace_intel_context_sched_disable(ce);3022 3023 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),3024 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);3025}3026 3027static void guc_blocked_fence_complete(struct intel_context *ce)3028{3029 lockdep_assert_held(&ce->guc_state.lock);3030 3031 if (!i915_sw_fence_done(&ce->guc_state.blocked))3032 i915_sw_fence_complete(&ce->guc_state.blocked);3033}3034 3035static void guc_blocked_fence_reinit(struct intel_context *ce)3036{3037 lockdep_assert_held(&ce->guc_state.lock);3038 GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked));3039 3040 /*3041 * This fence is always complete unless a pending schedule disable is3042 * outstanding. We arm the fence here and complete it when we receive3043 * the pending schedule disable complete message.3044 */3045 i915_sw_fence_fini(&ce->guc_state.blocked);3046 i915_sw_fence_reinit(&ce->guc_state.blocked);3047 i915_sw_fence_await(&ce->guc_state.blocked);3048 i915_sw_fence_commit(&ce->guc_state.blocked);3049}3050 3051static u16 prep_context_pending_disable(struct intel_context *ce)3052{3053 lockdep_assert_held(&ce->guc_state.lock);3054 3055 set_context_pending_disable(ce);3056 clr_context_enabled(ce);3057 guc_blocked_fence_reinit(ce);3058 intel_context_get(ce);3059 3060 return ce->guc_id.id;3061}3062 3063static struct i915_sw_fence *guc_context_block(struct intel_context *ce)3064{3065 struct intel_guc *guc = ce_to_guc(ce);3066 unsigned long flags;3067 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;3068 intel_wakeref_t wakeref;3069 u16 guc_id;3070 bool enabled;3071 3072 GEM_BUG_ON(intel_context_is_child(ce));3073 3074 spin_lock_irqsave(&ce->guc_state.lock, flags);3075 3076 incr_context_blocked(ce);3077 3078 enabled = context_enabled(ce);3079 if (unlikely(!enabled || submission_disabled(guc))) {3080 if (enabled)3081 clr_context_enabled(ce);3082 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3083 return &ce->guc_state.blocked;3084 }3085 3086 /*3087 * We add +2 here as the schedule disable complete CTB handler calls3088 * intel_context_sched_disable_unpin (-2 to pin_count).3089 */3090 atomic_add(2, &ce->pin_count);3091 3092 guc_id = prep_context_pending_disable(ce);3093 3094 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3095 3096 with_intel_runtime_pm(runtime_pm, wakeref)3097 __guc_context_sched_disable(guc, ce, guc_id);3098 3099 return &ce->guc_state.blocked;3100}3101 3102#define SCHED_STATE_MULTI_BLOCKED_MASK \3103 (SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED)3104#define SCHED_STATE_NO_UNBLOCK \3105 (SCHED_STATE_MULTI_BLOCKED_MASK | \3106 SCHED_STATE_PENDING_DISABLE | \3107 SCHED_STATE_BANNED)3108 3109static bool context_cant_unblock(struct intel_context *ce)3110{3111 lockdep_assert_held(&ce->guc_state.lock);3112 3113 return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) ||3114 context_guc_id_invalid(ce) ||3115 !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id) ||3116 !intel_context_is_pinned(ce);3117}3118 3119static void guc_context_unblock(struct intel_context *ce)3120{3121 struct intel_guc *guc = ce_to_guc(ce);3122 unsigned long flags;3123 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;3124 intel_wakeref_t wakeref;3125 bool enable;3126 3127 GEM_BUG_ON(context_enabled(ce));3128 GEM_BUG_ON(intel_context_is_child(ce));3129 3130 spin_lock_irqsave(&ce->guc_state.lock, flags);3131 3132 if (unlikely(submission_disabled(guc) ||3133 context_cant_unblock(ce))) {3134 enable = false;3135 } else {3136 enable = true;3137 set_context_pending_enable(ce);3138 set_context_enabled(ce);3139 intel_context_get(ce);3140 }3141 3142 decr_context_blocked(ce);3143 3144 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3145 3146 if (enable) {3147 with_intel_runtime_pm(runtime_pm, wakeref)3148 __guc_context_sched_enable(guc, ce);3149 }3150}3151 3152static void guc_context_cancel_request(struct intel_context *ce,3153 struct i915_request *rq)3154{3155 struct intel_context *block_context =3156 request_to_scheduling_context(rq);3157 3158 if (i915_sw_fence_signaled(&rq->submit)) {3159 struct i915_sw_fence *fence;3160 3161 intel_context_get(ce);3162 fence = guc_context_block(block_context);3163 i915_sw_fence_wait(fence);3164 if (!i915_request_completed(rq)) {3165 __i915_request_skip(rq);3166 guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),3167 true);3168 }3169 3170 guc_context_unblock(block_context);3171 intel_context_put(ce);3172 }3173}3174 3175static void __guc_context_set_preemption_timeout(struct intel_guc *guc,3176 u16 guc_id,3177 u32 preemption_timeout)3178{3179 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {3180 struct context_policy policy;3181 3182 __guc_context_policy_start_klv(&policy, guc_id);3183 __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);3184 __guc_context_set_context_policies(guc, &policy, true);3185 } else {3186 u32 action[] = {3187 INTEL_GUC_ACTION_V69_SET_CONTEXT_PREEMPTION_TIMEOUT,3188 guc_id,3189 preemption_timeout3190 };3191 3192 intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);3193 }3194}3195 3196static void3197guc_context_revoke(struct intel_context *ce, struct i915_request *rq,3198 unsigned int preempt_timeout_ms)3199{3200 struct intel_guc *guc = ce_to_guc(ce);3201 struct intel_runtime_pm *runtime_pm =3202 &ce->engine->gt->i915->runtime_pm;3203 intel_wakeref_t wakeref;3204 unsigned long flags;3205 3206 GEM_BUG_ON(intel_context_is_child(ce));3207 3208 guc_flush_submissions(guc);3209 3210 spin_lock_irqsave(&ce->guc_state.lock, flags);3211 set_context_banned(ce);3212 3213 if (submission_disabled(guc) ||3214 (!context_enabled(ce) && !context_pending_disable(ce))) {3215 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3216 3217 guc_cancel_context_requests(ce);3218 intel_engine_signal_breadcrumbs(ce->engine);3219 } else if (!context_pending_disable(ce)) {3220 u16 guc_id;3221 3222 /*3223 * We add +2 here as the schedule disable complete CTB handler3224 * calls intel_context_sched_disable_unpin (-2 to pin_count).3225 */3226 atomic_add(2, &ce->pin_count);3227 3228 guc_id = prep_context_pending_disable(ce);3229 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3230 3231 /*3232 * In addition to disabling scheduling, set the preemption3233 * timeout to the minimum value (1 us) so the banned context3234 * gets kicked off the HW ASAP.3235 */3236 with_intel_runtime_pm(runtime_pm, wakeref) {3237 __guc_context_set_preemption_timeout(guc, guc_id,3238 preempt_timeout_ms);3239 __guc_context_sched_disable(guc, ce, guc_id);3240 }3241 } else {3242 if (!context_guc_id_invalid(ce))3243 with_intel_runtime_pm(runtime_pm, wakeref)3244 __guc_context_set_preemption_timeout(guc,3245 ce->guc_id.id,3246 preempt_timeout_ms);3247 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3248 }3249}3250 3251static void do_sched_disable(struct intel_guc *guc, struct intel_context *ce,3252 unsigned long flags)3253 __releases(ce->guc_state.lock)3254{3255 struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;3256 intel_wakeref_t wakeref;3257 u16 guc_id;3258 3259 lockdep_assert_held(&ce->guc_state.lock);3260 guc_id = prep_context_pending_disable(ce);3261 3262 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3263 3264 with_intel_runtime_pm(runtime_pm, wakeref)3265 __guc_context_sched_disable(guc, ce, guc_id);3266}3267 3268static bool bypass_sched_disable(struct intel_guc *guc,3269 struct intel_context *ce)3270{3271 lockdep_assert_held(&ce->guc_state.lock);3272 GEM_BUG_ON(intel_context_is_child(ce));3273 3274 if (submission_disabled(guc) || context_guc_id_invalid(ce) ||3275 !ctx_id_mapped(guc, ce->guc_id.id)) {3276 clr_context_enabled(ce);3277 return true;3278 }3279 3280 return !context_enabled(ce);3281}3282 3283static void __delay_sched_disable(struct work_struct *wrk)3284{3285 struct intel_context *ce =3286 container_of(wrk, typeof(*ce), guc_state.sched_disable_delay_work.work);3287 struct intel_guc *guc = ce_to_guc(ce);3288 unsigned long flags;3289 3290 spin_lock_irqsave(&ce->guc_state.lock, flags);3291 3292 if (bypass_sched_disable(guc, ce)) {3293 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3294 intel_context_sched_disable_unpin(ce);3295 } else {3296 do_sched_disable(guc, ce, flags);3297 }3298}3299 3300static bool guc_id_pressure(struct intel_guc *guc, struct intel_context *ce)3301{3302 /*3303 * parent contexts are perma-pinned, if we are unpinning do schedule3304 * disable immediately.3305 */3306 if (intel_context_is_parent(ce))3307 return true;3308 3309 /*3310 * If we are beyond the threshold for avail guc_ids, do schedule disable immediately.3311 */3312 return guc->submission_state.guc_ids_in_use >3313 guc->submission_state.sched_disable_gucid_threshold;3314}3315 3316static void guc_context_sched_disable(struct intel_context *ce)3317{3318 struct intel_guc *guc = ce_to_guc(ce);3319 u64 delay = guc->submission_state.sched_disable_delay_ms;3320 unsigned long flags;3321 3322 spin_lock_irqsave(&ce->guc_state.lock, flags);3323 3324 if (bypass_sched_disable(guc, ce)) {3325 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3326 intel_context_sched_disable_unpin(ce);3327 } else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&3328 delay) {3329 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3330 mod_delayed_work(system_unbound_wq,3331 &ce->guc_state.sched_disable_delay_work,3332 msecs_to_jiffies(delay));3333 } else {3334 do_sched_disable(guc, ce, flags);3335 }3336}3337 3338static void guc_context_close(struct intel_context *ce)3339{3340 unsigned long flags;3341 3342 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&3343 cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))3344 __delay_sched_disable(&ce->guc_state.sched_disable_delay_work.work);3345 3346 spin_lock_irqsave(&ce->guc_state.lock, flags);3347 set_context_close_done(ce);3348 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3349}3350 3351static inline int guc_lrc_desc_unpin(struct intel_context *ce)3352{3353 struct intel_guc *guc = ce_to_guc(ce);3354 struct intel_gt *gt = guc_to_gt(guc);3355 unsigned long flags;3356 bool disabled;3357 int ret;3358 3359 GEM_BUG_ON(!intel_gt_pm_is_awake(gt));3360 GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id));3361 GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id));3362 GEM_BUG_ON(context_enabled(ce));3363 3364 /* Seal race with Reset */3365 spin_lock_irqsave(&ce->guc_state.lock, flags);3366 disabled = submission_disabled(guc);3367 if (likely(!disabled)) {3368 /*3369 * Take a gt-pm ref and change context state to be destroyed.3370 * NOTE: a G2H IRQ that comes after will put this gt-pm ref back3371 */3372 __intel_gt_pm_get(gt);3373 set_context_destroyed(ce);3374 clr_context_registered(ce);3375 }3376 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3377 3378 if (unlikely(disabled)) {3379 release_guc_id(guc, ce);3380 __guc_context_destroy(ce);3381 return 0;3382 }3383 3384 /*3385 * GuC is active, lets destroy this context, but at this point we can still be racing3386 * with suspend, so we undo everything if the H2G fails in deregister_context so3387 * that GuC reset will find this context during clean up.3388 */3389 ret = deregister_context(ce, ce->guc_id.id);3390 if (ret) {3391 spin_lock(&ce->guc_state.lock);3392 set_context_registered(ce);3393 clr_context_destroyed(ce);3394 spin_unlock(&ce->guc_state.lock);3395 /*3396 * As gt-pm is awake at function entry, intel_wakeref_put_async merely decrements3397 * the wakeref immediately but per function spec usage call this after unlock.3398 */3399 intel_wakeref_put_async(>->wakeref);3400 }3401 3402 return ret;3403}3404 3405static void __guc_context_destroy(struct intel_context *ce)3406{3407 GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] ||3408 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] ||3409 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] ||3410 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL]);3411 3412 lrc_fini(ce);3413 intel_context_fini(ce);3414 3415 if (intel_engine_is_virtual(ce->engine)) {3416 struct guc_virtual_engine *ve =3417 container_of(ce, typeof(*ve), context);3418 3419 if (ve->base.breadcrumbs)3420 intel_breadcrumbs_put(ve->base.breadcrumbs);3421 3422 kfree(ve);3423 } else {3424 intel_context_free(ce);3425 }3426}3427 3428static void guc_flush_destroyed_contexts(struct intel_guc *guc)3429{3430 struct intel_context *ce;3431 unsigned long flags;3432 3433 GEM_BUG_ON(!submission_disabled(guc) &&3434 guc_submission_initialized(guc));3435 3436 while (!list_empty(&guc->submission_state.destroyed_contexts)) {3437 spin_lock_irqsave(&guc->submission_state.lock, flags);3438 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,3439 struct intel_context,3440 destroyed_link);3441 if (ce)3442 list_del_init(&ce->destroyed_link);3443 spin_unlock_irqrestore(&guc->submission_state.lock, flags);3444 3445 if (!ce)3446 break;3447 3448 release_guc_id(guc, ce);3449 __guc_context_destroy(ce);3450 }3451}3452 3453static void deregister_destroyed_contexts(struct intel_guc *guc)3454{3455 struct intel_context *ce;3456 unsigned long flags;3457 3458 while (!list_empty(&guc->submission_state.destroyed_contexts)) {3459 spin_lock_irqsave(&guc->submission_state.lock, flags);3460 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,3461 struct intel_context,3462 destroyed_link);3463 if (ce)3464 list_del_init(&ce->destroyed_link);3465 spin_unlock_irqrestore(&guc->submission_state.lock, flags);3466 3467 if (!ce)3468 break;3469 3470 if (guc_lrc_desc_unpin(ce)) {3471 /*3472 * This means GuC's CT link severed mid-way which could happen3473 * in suspend-resume corner cases. In this case, put the3474 * context back into the destroyed_contexts list which will3475 * get picked up on the next context deregistration event or3476 * purged in a GuC sanitization event (reset/unload/wedged/...).3477 */3478 spin_lock_irqsave(&guc->submission_state.lock, flags);3479 list_add_tail(&ce->destroyed_link,3480 &guc->submission_state.destroyed_contexts);3481 spin_unlock_irqrestore(&guc->submission_state.lock, flags);3482 /* Bail now since the list might never be emptied if h2gs fail */3483 break;3484 }3485 3486 }3487}3488 3489static void destroyed_worker_func(struct work_struct *w)3490{3491 struct intel_guc *guc = container_of(w, struct intel_guc,3492 submission_state.destroyed_worker);3493 struct intel_gt *gt = guc_to_gt(guc);3494 intel_wakeref_t wakeref;3495 3496 /*3497 * In rare cases we can get here via async context-free fence-signals that3498 * come very late in suspend flow or very early in resume flows. In these3499 * cases, GuC won't be ready but just skipping it here is fine as these3500 * pending-destroy-contexts get destroyed totally at GuC reset time at the3501 * end of suspend.. OR.. this worker can be picked up later on the next3502 * context destruction trigger after resume-completes3503 */3504 if (!intel_guc_is_ready(guc))3505 return;3506 3507 with_intel_gt_pm(gt, wakeref)3508 deregister_destroyed_contexts(guc);3509}3510 3511static void guc_context_destroy(struct kref *kref)3512{3513 struct intel_context *ce = container_of(kref, typeof(*ce), ref);3514 struct intel_guc *guc = ce_to_guc(ce);3515 unsigned long flags;3516 bool destroy;3517 3518 /*3519 * If the guc_id is invalid this context has been stolen and we can free3520 * it immediately. Also can be freed immediately if the context is not3521 * registered with the GuC or the GuC is in the middle of a reset.3522 */3523 spin_lock_irqsave(&guc->submission_state.lock, flags);3524 destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||3525 !ctx_id_mapped(guc, ce->guc_id.id);3526 if (likely(!destroy)) {3527 if (!list_empty(&ce->guc_id.link))3528 list_del_init(&ce->guc_id.link);3529 list_add_tail(&ce->destroyed_link,3530 &guc->submission_state.destroyed_contexts);3531 } else {3532 __release_guc_id(guc, ce);3533 }3534 spin_unlock_irqrestore(&guc->submission_state.lock, flags);3535 if (unlikely(destroy)) {3536 __guc_context_destroy(ce);3537 return;3538 }3539 3540 /*3541 * We use a worker to issue the H2G to deregister the context as we can3542 * take the GT PM for the first time which isn't allowed from an atomic3543 * context.3544 */3545 queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);3546}3547 3548static int guc_context_alloc(struct intel_context *ce)3549{3550 return lrc_alloc(ce, ce->engine);3551}3552 3553static void __guc_context_set_prio(struct intel_guc *guc,3554 struct intel_context *ce)3555{3556 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {3557 struct context_policy policy;3558 3559 __guc_context_policy_start_klv(&policy, ce->guc_id.id);3560 __guc_context_policy_add_priority(&policy, ce->guc_state.prio);3561 __guc_context_set_context_policies(guc, &policy, true);3562 } else {3563 u32 action[] = {3564 INTEL_GUC_ACTION_V69_SET_CONTEXT_PRIORITY,3565 ce->guc_id.id,3566 ce->guc_state.prio,3567 };3568 3569 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);3570 }3571}3572 3573static void guc_context_set_prio(struct intel_guc *guc,3574 struct intel_context *ce,3575 u8 prio)3576{3577 GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH ||3578 prio > GUC_CLIENT_PRIORITY_NORMAL);3579 lockdep_assert_held(&ce->guc_state.lock);3580 3581 if (ce->guc_state.prio == prio || submission_disabled(guc) ||3582 !context_registered(ce)) {3583 ce->guc_state.prio = prio;3584 return;3585 }3586 3587 ce->guc_state.prio = prio;3588 __guc_context_set_prio(guc, ce);3589 3590 trace_intel_context_set_prio(ce);3591}3592 3593static inline u8 map_i915_prio_to_guc_prio(int prio)3594{3595 if (prio == I915_PRIORITY_NORMAL)3596 return GUC_CLIENT_PRIORITY_KMD_NORMAL;3597 else if (prio < I915_PRIORITY_NORMAL)3598 return GUC_CLIENT_PRIORITY_NORMAL;3599 else if (prio < I915_PRIORITY_DISPLAY)3600 return GUC_CLIENT_PRIORITY_HIGH;3601 else3602 return GUC_CLIENT_PRIORITY_KMD_HIGH;3603}3604 3605static inline void add_context_inflight_prio(struct intel_context *ce,3606 u8 guc_prio)3607{3608 lockdep_assert_held(&ce->guc_state.lock);3609 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));3610 3611 ++ce->guc_state.prio_count[guc_prio];3612 3613 /* Overflow protection */3614 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);3615}3616 3617static inline void sub_context_inflight_prio(struct intel_context *ce,3618 u8 guc_prio)3619{3620 lockdep_assert_held(&ce->guc_state.lock);3621 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));3622 3623 /* Underflow protection */3624 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);3625 3626 --ce->guc_state.prio_count[guc_prio];3627}3628 3629static inline void update_context_prio(struct intel_context *ce)3630{3631 struct intel_guc *guc = &ce->engine->gt->uc.guc;3632 int i;3633 3634 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0);3635 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL);3636 3637 lockdep_assert_held(&ce->guc_state.lock);3638 3639 for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count); ++i) {3640 if (ce->guc_state.prio_count[i]) {3641 guc_context_set_prio(guc, ce, i);3642 break;3643 }3644 }3645}3646 3647static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio)3648{3649 /* Lower value is higher priority */3650 return new_guc_prio < old_guc_prio;3651}3652 3653static void add_to_context(struct i915_request *rq)3654{3655 struct intel_context *ce = request_to_scheduling_context(rq);3656 u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq));3657 3658 GEM_BUG_ON(intel_context_is_child(ce));3659 GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI);3660 3661 spin_lock(&ce->guc_state.lock);3662 list_move_tail(&rq->sched.link, &ce->guc_state.requests);3663 3664 if (rq->guc_prio == GUC_PRIO_INIT) {3665 rq->guc_prio = new_guc_prio;3666 add_context_inflight_prio(ce, rq->guc_prio);3667 } else if (new_guc_prio_higher(rq->guc_prio, new_guc_prio)) {3668 sub_context_inflight_prio(ce, rq->guc_prio);3669 rq->guc_prio = new_guc_prio;3670 add_context_inflight_prio(ce, rq->guc_prio);3671 }3672 update_context_prio(ce);3673 3674 spin_unlock(&ce->guc_state.lock);3675}3676 3677static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce)3678{3679 lockdep_assert_held(&ce->guc_state.lock);3680 3681 if (rq->guc_prio != GUC_PRIO_INIT &&3682 rq->guc_prio != GUC_PRIO_FINI) {3683 sub_context_inflight_prio(ce, rq->guc_prio);3684 update_context_prio(ce);3685 }3686 rq->guc_prio = GUC_PRIO_FINI;3687}3688 3689static void remove_from_context(struct i915_request *rq)3690{3691 struct intel_context *ce = request_to_scheduling_context(rq);3692 3693 GEM_BUG_ON(intel_context_is_child(ce));3694 3695 spin_lock_irq(&ce->guc_state.lock);3696 3697 list_del_init(&rq->sched.link);3698 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);3699 3700 /* Prevent further __await_execution() registering a cb, then flush */3701 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);3702 3703 guc_prio_fini(rq, ce);3704 3705 spin_unlock_irq(&ce->guc_state.lock);3706 3707 atomic_dec(&ce->guc_id.ref);3708 i915_request_notify_execute_cb_imm(rq);3709}3710 3711static const struct intel_context_ops guc_context_ops = {3712 .flags = COPS_RUNTIME_CYCLES,3713 .alloc = guc_context_alloc,3714 3715 .close = guc_context_close,3716 3717 .pre_pin = guc_context_pre_pin,3718 .pin = guc_context_pin,3719 .unpin = guc_context_unpin,3720 .post_unpin = guc_context_post_unpin,3721 3722 .revoke = guc_context_revoke,3723 3724 .cancel_request = guc_context_cancel_request,3725 3726 .enter = intel_context_enter_engine,3727 .exit = intel_context_exit_engine,3728 3729 .sched_disable = guc_context_sched_disable,3730 3731 .update_stats = guc_context_update_stats,3732 3733 .reset = lrc_reset,3734 .destroy = guc_context_destroy,3735 3736 .create_virtual = guc_create_virtual,3737 .create_parallel = guc_create_parallel,3738};3739 3740static void submit_work_cb(struct irq_work *wrk)3741{3742 struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work);3743 3744 might_lock(&rq->engine->sched_engine->lock);3745 i915_sw_fence_complete(&rq->submit);3746}3747 3748static void __guc_signal_context_fence(struct intel_context *ce)3749{3750 struct i915_request *rq, *rn;3751 3752 lockdep_assert_held(&ce->guc_state.lock);3753 3754 if (!list_empty(&ce->guc_state.fences))3755 trace_intel_context_fence_release(ce);3756 3757 /*3758 * Use an IRQ to ensure locking order of sched_engine->lock ->3759 * ce->guc_state.lock is preserved.3760 */3761 list_for_each_entry_safe(rq, rn, &ce->guc_state.fences,3762 guc_fence_link) {3763 list_del(&rq->guc_fence_link);3764 irq_work_queue(&rq->submit_work);3765 }3766 3767 INIT_LIST_HEAD(&ce->guc_state.fences);3768}3769 3770static void guc_signal_context_fence(struct intel_context *ce)3771{3772 unsigned long flags;3773 3774 GEM_BUG_ON(intel_context_is_child(ce));3775 3776 spin_lock_irqsave(&ce->guc_state.lock, flags);3777 clr_context_wait_for_deregister_to_register(ce);3778 __guc_signal_context_fence(ce);3779 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3780}3781 3782static bool context_needs_register(struct intel_context *ce, bool new_guc_id)3783{3784 return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) ||3785 !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)) &&3786 !submission_disabled(ce_to_guc(ce));3787}3788 3789static void guc_context_init(struct intel_context *ce)3790{3791 const struct i915_gem_context *ctx;3792 int prio = I915_CONTEXT_DEFAULT_PRIORITY;3793 3794 rcu_read_lock();3795 ctx = rcu_dereference(ce->gem_context);3796 if (ctx)3797 prio = ctx->sched.priority;3798 rcu_read_unlock();3799 3800 ce->guc_state.prio = map_i915_prio_to_guc_prio(prio);3801 3802 INIT_DELAYED_WORK(&ce->guc_state.sched_disable_delay_work,3803 __delay_sched_disable);3804 3805 set_bit(CONTEXT_GUC_INIT, &ce->flags);3806}3807 3808static int guc_request_alloc(struct i915_request *rq)3809{3810 struct intel_context *ce = request_to_scheduling_context(rq);3811 struct intel_guc *guc = ce_to_guc(ce);3812 unsigned long flags;3813 int ret;3814 3815 GEM_BUG_ON(!intel_context_is_pinned(rq->context));3816 3817 /*3818 * Flush enough space to reduce the likelihood of waiting after3819 * we start building the request - in which case we will just3820 * have to repeat work.3821 */3822 rq->reserved_space += GUC_REQUEST_SIZE;3823 3824 /*3825 * Note that after this point, we have committed to using3826 * this request as it is being used to both track the3827 * state of engine initialisation and liveness of the3828 * golden renderstate above. Think twice before you try3829 * to cancel/unwind this request now.3830 */3831 3832 /* Unconditionally invalidate GPU caches and TLBs. */3833 ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);3834 if (ret)3835 return ret;3836 3837 rq->reserved_space -= GUC_REQUEST_SIZE;3838 3839 if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags)))3840 guc_context_init(ce);3841 3842 /*3843 * If the context gets closed while the execbuf is ongoing, the context3844 * close code will race with the below code to cancel the delayed work.3845 * If the context close wins the race and cancels the work, it will3846 * immediately call the sched disable (see guc_context_close), so there3847 * is a chance we can get past this check while the sched_disable code3848 * is being executed. To make sure that code completes before we check3849 * the status further down, we wait for the close process to complete.3850 * Else, this code path could send a request down thinking that the3851 * context is still in a schedule-enable mode while the GuC ends up3852 * dropping the request completely because the disable did go from the3853 * context_close path right to GuC just prior. In the event the CT is3854 * full, we could potentially need to wait up to 1.5 seconds.3855 */3856 if (cancel_delayed_work_sync(&ce->guc_state.sched_disable_delay_work))3857 intel_context_sched_disable_unpin(ce);3858 else if (intel_context_is_closed(ce))3859 if (wait_for(context_close_done(ce), 1500))3860 guc_warn(guc, "timed out waiting on context sched close before realloc\n");3861 /*3862 * Call pin_guc_id here rather than in the pinning step as with3863 * dma_resv, contexts can be repeatedly pinned / unpinned trashing the3864 * guc_id and creating horrible race conditions. This is especially bad3865 * when guc_id are being stolen due to over subscription. By the time3866 * this function is reached, it is guaranteed that the guc_id will be3867 * persistent until the generated request is retired. Thus, sealing these3868 * race conditions. It is still safe to fail here if guc_id are3869 * exhausted and return -EAGAIN to the user indicating that they can try3870 * again in the future.3871 *3872 * There is no need for a lock here as the timeline mutex ensures at3873 * most one context can be executing this code path at once. The3874 * guc_id_ref is incremented once for every request in flight and3875 * decremented on each retire. When it is zero, a lock around the3876 * increment (in pin_guc_id) is needed to seal a race with unpin_guc_id.3877 */3878 if (atomic_add_unless(&ce->guc_id.ref, 1, 0))3879 goto out;3880 3881 ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */3882 if (unlikely(ret < 0))3883 return ret;3884 if (context_needs_register(ce, !!ret)) {3885 ret = try_context_registration(ce, true);3886 if (unlikely(ret)) { /* unwind */3887 if (ret == -EPIPE) {3888 disable_submission(guc);3889 goto out; /* GPU will be reset */3890 }3891 atomic_dec(&ce->guc_id.ref);3892 unpin_guc_id(guc, ce);3893 return ret;3894 }3895 }3896 3897 clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags);3898 3899out:3900 /*3901 * We block all requests on this context if a G2H is pending for a3902 * schedule disable or context deregistration as the GuC will fail a3903 * schedule enable or context registration if either G2H is pending3904 * respectfully. Once a G2H returns, the fence is released that is3905 * blocking these requests (see guc_signal_context_fence).3906 */3907 spin_lock_irqsave(&ce->guc_state.lock, flags);3908 if (context_wait_for_deregister_to_register(ce) ||3909 context_pending_disable(ce)) {3910 init_irq_work(&rq->submit_work, submit_work_cb);3911 i915_sw_fence_await(&rq->submit);3912 3913 list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences);3914 }3915 spin_unlock_irqrestore(&ce->guc_state.lock, flags);3916 3917 return 0;3918}3919 3920static int guc_virtual_context_pre_pin(struct intel_context *ce,3921 struct i915_gem_ww_ctx *ww,3922 void **vaddr)3923{3924 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);3925 3926 return __guc_context_pre_pin(ce, engine, ww, vaddr);3927}3928 3929static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr)3930{3931 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);3932 int ret = __guc_context_pin(ce, engine, vaddr);3933 intel_engine_mask_t tmp, mask = ce->engine->mask;3934 3935 if (likely(!ret))3936 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)3937 intel_engine_pm_get(engine);3938 3939 return ret;3940}3941 3942static void guc_virtual_context_unpin(struct intel_context *ce)3943{3944 intel_engine_mask_t tmp, mask = ce->engine->mask;3945 struct intel_engine_cs *engine;3946 struct intel_guc *guc = ce_to_guc(ce);3947 3948 GEM_BUG_ON(context_enabled(ce));3949 GEM_BUG_ON(intel_context_is_barrier(ce));3950 3951 unpin_guc_id(guc, ce);3952 lrc_unpin(ce);3953 3954 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)3955 intel_engine_pm_put_async(engine);3956}3957 3958static void guc_virtual_context_enter(struct intel_context *ce)3959{3960 intel_engine_mask_t tmp, mask = ce->engine->mask;3961 struct intel_engine_cs *engine;3962 3963 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)3964 intel_engine_pm_get(engine);3965 3966 intel_timeline_enter(ce->timeline);3967}3968 3969static void guc_virtual_context_exit(struct intel_context *ce)3970{3971 intel_engine_mask_t tmp, mask = ce->engine->mask;3972 struct intel_engine_cs *engine;3973 3974 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)3975 intel_engine_pm_put(engine);3976 3977 intel_timeline_exit(ce->timeline);3978}3979 3980static int guc_virtual_context_alloc(struct intel_context *ce)3981{3982 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);3983 3984 return lrc_alloc(ce, engine);3985}3986 3987static const struct intel_context_ops virtual_guc_context_ops = {3988 .flags = COPS_RUNTIME_CYCLES,3989 .alloc = guc_virtual_context_alloc,3990 3991 .close = guc_context_close,3992 3993 .pre_pin = guc_virtual_context_pre_pin,3994 .pin = guc_virtual_context_pin,3995 .unpin = guc_virtual_context_unpin,3996 .post_unpin = guc_context_post_unpin,3997 3998 .revoke = guc_context_revoke,3999 4000 .cancel_request = guc_context_cancel_request,4001 4002 .enter = guc_virtual_context_enter,4003 .exit = guc_virtual_context_exit,4004 4005 .sched_disable = guc_context_sched_disable,4006 .update_stats = guc_context_update_stats,4007 4008 .destroy = guc_context_destroy,4009 4010 .get_sibling = guc_virtual_get_sibling,4011};4012 4013static int guc_parent_context_pin(struct intel_context *ce, void *vaddr)4014{4015 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);4016 struct intel_guc *guc = ce_to_guc(ce);4017 int ret;4018 4019 GEM_BUG_ON(!intel_context_is_parent(ce));4020 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));4021 4022 ret = pin_guc_id(guc, ce);4023 if (unlikely(ret < 0))4024 return ret;4025 4026 return __guc_context_pin(ce, engine, vaddr);4027}4028 4029static int guc_child_context_pin(struct intel_context *ce, void *vaddr)4030{4031 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);4032 4033 GEM_BUG_ON(!intel_context_is_child(ce));4034 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));4035 4036 __intel_context_pin(ce->parallel.parent);4037 return __guc_context_pin(ce, engine, vaddr);4038}4039 4040static void guc_parent_context_unpin(struct intel_context *ce)4041{4042 struct intel_guc *guc = ce_to_guc(ce);4043 4044 GEM_BUG_ON(context_enabled(ce));4045 GEM_BUG_ON(intel_context_is_barrier(ce));4046 GEM_BUG_ON(!intel_context_is_parent(ce));4047 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));4048 4049 unpin_guc_id(guc, ce);4050 lrc_unpin(ce);4051}4052 4053static void guc_child_context_unpin(struct intel_context *ce)4054{4055 GEM_BUG_ON(context_enabled(ce));4056 GEM_BUG_ON(intel_context_is_barrier(ce));4057 GEM_BUG_ON(!intel_context_is_child(ce));4058 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));4059 4060 lrc_unpin(ce);4061}4062 4063static void guc_child_context_post_unpin(struct intel_context *ce)4064{4065 GEM_BUG_ON(!intel_context_is_child(ce));4066 GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent));4067 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));4068 4069 lrc_post_unpin(ce);4070 intel_context_unpin(ce->parallel.parent);4071}4072 4073static void guc_child_context_destroy(struct kref *kref)4074{4075 struct intel_context *ce = container_of(kref, typeof(*ce), ref);4076 4077 __guc_context_destroy(ce);4078}4079 4080static const struct intel_context_ops virtual_parent_context_ops = {4081 .alloc = guc_virtual_context_alloc,4082 4083 .close = guc_context_close,4084 4085 .pre_pin = guc_context_pre_pin,4086 .pin = guc_parent_context_pin,4087 .unpin = guc_parent_context_unpin,4088 .post_unpin = guc_context_post_unpin,4089 4090 .revoke = guc_context_revoke,4091 4092 .cancel_request = guc_context_cancel_request,4093 4094 .enter = guc_virtual_context_enter,4095 .exit = guc_virtual_context_exit,4096 4097 .sched_disable = guc_context_sched_disable,4098 4099 .destroy = guc_context_destroy,4100 4101 .get_sibling = guc_virtual_get_sibling,4102};4103 4104static const struct intel_context_ops virtual_child_context_ops = {4105 .alloc = guc_virtual_context_alloc,4106 4107 .pre_pin = guc_context_pre_pin,4108 .pin = guc_child_context_pin,4109 .unpin = guc_child_context_unpin,4110 .post_unpin = guc_child_context_post_unpin,4111 4112 .cancel_request = guc_context_cancel_request,4113 4114 .enter = guc_virtual_context_enter,4115 .exit = guc_virtual_context_exit,4116 4117 .destroy = guc_child_context_destroy,4118 4119 .get_sibling = guc_virtual_get_sibling,4120};4121 4122/*4123 * The below override of the breadcrumbs is enabled when the user configures a4124 * context for parallel submission (multi-lrc, parent-child).4125 *4126 * The overridden breadcrumbs implements an algorithm which allows the GuC to4127 * safely preempt all the hw contexts configured for parallel submission4128 * between each BB. The contract between the i915 and GuC is if the parent4129 * context can be preempted, all the children can be preempted, and the GuC will4130 * always try to preempt the parent before the children. A handshake between the4131 * parent / children breadcrumbs ensures the i915 holds up its end of the deal4132 * creating a window to preempt between each set of BBs.4133 */4134static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,4135 u64 offset, u32 len,4136 const unsigned int flags);4137static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,4138 u64 offset, u32 len,4139 const unsigned int flags);4140static u32 *4141emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,4142 u32 *cs);4143static u32 *4144emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,4145 u32 *cs);4146 4147static struct intel_context *4148guc_create_parallel(struct intel_engine_cs **engines,4149 unsigned int num_siblings,4150 unsigned int width)4151{4152 struct intel_engine_cs **siblings = NULL;4153 struct intel_context *parent = NULL, *ce, *err;4154 int i, j;4155 4156 siblings = kmalloc_array(num_siblings,4157 sizeof(*siblings),4158 GFP_KERNEL);4159 if (!siblings)4160 return ERR_PTR(-ENOMEM);4161 4162 for (i = 0; i < width; ++i) {4163 for (j = 0; j < num_siblings; ++j)4164 siblings[j] = engines[i * num_siblings + j];4165 4166 ce = intel_engine_create_virtual(siblings, num_siblings,4167 FORCE_VIRTUAL);4168 if (IS_ERR(ce)) {4169 err = ERR_CAST(ce);4170 goto unwind;4171 }4172 4173 if (i == 0) {4174 parent = ce;4175 parent->ops = &virtual_parent_context_ops;4176 } else {4177 ce->ops = &virtual_child_context_ops;4178 intel_context_bind_parent_child(parent, ce);4179 }4180 }4181 4182 parent->parallel.fence_context = dma_fence_context_alloc(1);4183 4184 parent->engine->emit_bb_start =4185 emit_bb_start_parent_no_preempt_mid_batch;4186 parent->engine->emit_fini_breadcrumb =4187 emit_fini_breadcrumb_parent_no_preempt_mid_batch;4188 parent->engine->emit_fini_breadcrumb_dw =4189 12 + 4 * parent->parallel.number_children;4190 for_each_child(parent, ce) {4191 ce->engine->emit_bb_start =4192 emit_bb_start_child_no_preempt_mid_batch;4193 ce->engine->emit_fini_breadcrumb =4194 emit_fini_breadcrumb_child_no_preempt_mid_batch;4195 ce->engine->emit_fini_breadcrumb_dw = 16;4196 }4197 4198 kfree(siblings);4199 return parent;4200 4201unwind:4202 if (parent)4203 intel_context_put(parent);4204 kfree(siblings);4205 return err;4206}4207 4208static bool4209guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b)4210{4211 struct intel_engine_cs *sibling;4212 intel_engine_mask_t tmp, mask = b->engine_mask;4213 bool result = false;4214 4215 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)4216 result |= intel_engine_irq_enable(sibling);4217 4218 return result;4219}4220 4221static void4222guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b)4223{4224 struct intel_engine_cs *sibling;4225 intel_engine_mask_t tmp, mask = b->engine_mask;4226 4227 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)4228 intel_engine_irq_disable(sibling);4229}4230 4231static void guc_init_breadcrumbs(struct intel_engine_cs *engine)4232{4233 int i;4234 4235 /*4236 * In GuC submission mode we do not know which physical engine a request4237 * will be scheduled on, this creates a problem because the breadcrumb4238 * interrupt is per physical engine. To work around this we attach4239 * requests and direct all breadcrumb interrupts to the first instance4240 * of an engine per class. In addition all breadcrumb interrupts are4241 * enabled / disabled across an engine class in unison.4242 */4243 for (i = 0; i < MAX_ENGINE_INSTANCE; ++i) {4244 struct intel_engine_cs *sibling =4245 engine->gt->engine_class[engine->class][i];4246 4247 if (sibling) {4248 if (engine->breadcrumbs != sibling->breadcrumbs) {4249 intel_breadcrumbs_put(engine->breadcrumbs);4250 engine->breadcrumbs =4251 intel_breadcrumbs_get(sibling->breadcrumbs);4252 }4253 break;4254 }4255 }4256 4257 if (engine->breadcrumbs) {4258 engine->breadcrumbs->engine_mask |= engine->mask;4259 engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs;4260 engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs;4261 }4262}4263 4264static void guc_bump_inflight_request_prio(struct i915_request *rq,4265 int prio)4266{4267 struct intel_context *ce = request_to_scheduling_context(rq);4268 u8 new_guc_prio = map_i915_prio_to_guc_prio(prio);4269 4270 /* Short circuit function */4271 if (prio < I915_PRIORITY_NORMAL)4272 return;4273 4274 spin_lock(&ce->guc_state.lock);4275 4276 if (rq->guc_prio == GUC_PRIO_FINI)4277 goto exit;4278 4279 if (!new_guc_prio_higher(rq->guc_prio, new_guc_prio))4280 goto exit;4281 4282 if (rq->guc_prio != GUC_PRIO_INIT)4283 sub_context_inflight_prio(ce, rq->guc_prio);4284 4285 rq->guc_prio = new_guc_prio;4286 add_context_inflight_prio(ce, rq->guc_prio);4287 update_context_prio(ce);4288 4289exit:4290 spin_unlock(&ce->guc_state.lock);4291}4292 4293static void guc_retire_inflight_request_prio(struct i915_request *rq)4294{4295 struct intel_context *ce = request_to_scheduling_context(rq);4296 4297 spin_lock(&ce->guc_state.lock);4298 guc_prio_fini(rq, ce);4299 spin_unlock(&ce->guc_state.lock);4300}4301 4302static void sanitize_hwsp(struct intel_engine_cs *engine)4303{4304 struct intel_timeline *tl;4305 4306 list_for_each_entry(tl, &engine->status_page.timelines, engine_link)4307 intel_timeline_reset_seqno(tl);4308}4309 4310static void guc_sanitize(struct intel_engine_cs *engine)4311{4312 /*4313 * Poison residual state on resume, in case the suspend didn't!4314 *4315 * We have to assume that across suspend/resume (or other loss4316 * of control) that the contents of our pinned buffers has been4317 * lost, replaced by garbage. Since this doesn't always happen,4318 * let's poison such state so that we more quickly spot when4319 * we falsely assume it has been preserved.4320 */4321 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))4322 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);4323 4324 /*4325 * The kernel_context HWSP is stored in the status_page. As above,4326 * that may be lost on resume/initialisation, and so we need to4327 * reset the value in the HWSP.4328 */4329 sanitize_hwsp(engine);4330 4331 /* And scrub the dirty cachelines for the HWSP */4332 drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);4333 4334 intel_engine_reset_pinned_contexts(engine);4335}4336 4337static void setup_hwsp(struct intel_engine_cs *engine)4338{4339 intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */4340 4341 ENGINE_WRITE_FW(engine,4342 RING_HWS_PGA,4343 i915_ggtt_offset(engine->status_page.vma));4344}4345 4346static void start_engine(struct intel_engine_cs *engine)4347{4348 ENGINE_WRITE_FW(engine,4349 RING_MODE_GEN7,4350 _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));4351 4352 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));4353 ENGINE_POSTING_READ(engine, RING_MI_MODE);4354}4355 4356static int guc_resume(struct intel_engine_cs *engine)4357{4358 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);4359 4360 intel_mocs_init_engine(engine);4361 4362 intel_breadcrumbs_reset(engine->breadcrumbs);4363 4364 setup_hwsp(engine);4365 start_engine(engine);4366 4367 if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)4368 xehp_enable_ccs_engines(engine);4369 4370 return 0;4371}4372 4373static bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine)4374{4375 return !sched_engine->tasklet.callback;4376}4377 4378static void guc_set_default_submission(struct intel_engine_cs *engine)4379{4380 engine->submit_request = guc_submit_request;4381}4382 4383static inline int guc_kernel_context_pin(struct intel_guc *guc,4384 struct intel_context *ce)4385{4386 int ret;4387 4388 /*4389 * Note: we purposefully do not check the returns below because4390 * the registration can only fail if a reset is just starting.4391 * This is called at the end of reset so presumably another reset4392 * isn't happening and even it did this code would be run again.4393 */4394 4395 if (context_guc_id_invalid(ce)) {4396 ret = pin_guc_id(guc, ce);4397 4398 if (ret < 0)4399 return ret;4400 }4401 4402 if (!test_bit(CONTEXT_GUC_INIT, &ce->flags))4403 guc_context_init(ce);4404 4405 ret = try_context_registration(ce, true);4406 if (ret)4407 unpin_guc_id(guc, ce);4408 4409 return ret;4410}4411 4412static inline int guc_init_submission(struct intel_guc *guc)4413{4414 struct intel_gt *gt = guc_to_gt(guc);4415 struct intel_engine_cs *engine;4416 enum intel_engine_id id;4417 4418 /* make sure all descriptors are clean... */4419 xa_destroy(&guc->context_lookup);4420 4421 /*4422 * A reset might have occurred while we had a pending stalled request,4423 * so make sure we clean that up.4424 */4425 guc->stalled_request = NULL;4426 guc->submission_stall_reason = STALL_NONE;4427 4428 /*4429 * Some contexts might have been pinned before we enabled GuC4430 * submission, so we need to add them to the GuC bookeeping.4431 * Also, after a reset the of the GuC we want to make sure that the4432 * information shared with GuC is properly reset. The kernel LRCs are4433 * not attached to the gem_context, so they need to be added separately.4434 */4435 for_each_engine(engine, gt, id) {4436 struct intel_context *ce;4437 4438 list_for_each_entry(ce, &engine->pinned_contexts_list,4439 pinned_contexts_link) {4440 int ret = guc_kernel_context_pin(guc, ce);4441 4442 if (ret) {4443 /* No point in trying to clean up as i915 will wedge on failure */4444 return ret;4445 }4446 }4447 }4448 4449 return 0;4450}4451 4452static void guc_release(struct intel_engine_cs *engine)4453{4454 engine->sanitize = NULL; /* no longer in control, nothing to sanitize */4455 4456 intel_engine_cleanup_common(engine);4457 lrc_fini_wa_ctx(engine);4458}4459 4460static void virtual_guc_bump_serial(struct intel_engine_cs *engine)4461{4462 struct intel_engine_cs *e;4463 intel_engine_mask_t tmp, mask = engine->mask;4464 4465 for_each_engine_masked(e, engine->gt, mask, tmp)4466 e->serial++;4467}4468 4469static void guc_default_vfuncs(struct intel_engine_cs *engine)4470{4471 /* Default vfuncs which can be overridden by each engine. */4472 4473 engine->resume = guc_resume;4474 4475 engine->cops = &guc_context_ops;4476 engine->request_alloc = guc_request_alloc;4477 engine->add_active_request = add_to_context;4478 engine->remove_active_request = remove_from_context;4479 4480 engine->sched_engine->schedule = i915_schedule;4481 4482 engine->reset.prepare = guc_engine_reset_prepare;4483 engine->reset.rewind = guc_rewind_nop;4484 engine->reset.cancel = guc_reset_nop;4485 engine->reset.finish = guc_reset_nop;4486 4487 engine->emit_flush = gen8_emit_flush_xcs;4488 engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;4489 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;4490 if (GRAPHICS_VER(engine->i915) >= 12) {4491 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;4492 engine->emit_flush = gen12_emit_flush_xcs;4493 }4494 engine->set_default_submission = guc_set_default_submission;4495 engine->busyness = guc_engine_busyness;4496 4497 engine->flags |= I915_ENGINE_SUPPORTS_STATS;4498 engine->flags |= I915_ENGINE_HAS_PREEMPTION;4499 engine->flags |= I915_ENGINE_HAS_TIMESLICES;4500 4501 /* Wa_14014475959:dg2 */4502 if (engine->class == COMPUTE_CLASS)4503 if (IS_GFX_GT_IP_STEP(engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||4504 IS_DG2(engine->i915))4505 engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;4506 4507 /* Wa_16019325821 */4508 /* Wa_14019159160 */4509 if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) &&4510 IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))4511 engine->flags |= I915_ENGINE_USES_WA_HOLD_SWITCHOUT;4512 4513 /*4514 * TODO: GuC supports timeslicing and semaphores as well, but they're4515 * handled by the firmware so some minor tweaks are required before4516 * enabling.4517 *4518 * engine->flags |= I915_ENGINE_HAS_SEMAPHORES;4519 */4520 4521 engine->emit_bb_start = gen8_emit_bb_start;4522 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))4523 engine->emit_bb_start = xehp_emit_bb_start;4524}4525 4526static void rcs_submission_override(struct intel_engine_cs *engine)4527{4528 switch (GRAPHICS_VER(engine->i915)) {4529 case 12:4530 engine->emit_flush = gen12_emit_flush_rcs;4531 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;4532 break;4533 case 11:4534 engine->emit_flush = gen11_emit_flush_rcs;4535 engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;4536 break;4537 default:4538 engine->emit_flush = gen8_emit_flush_rcs;4539 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;4540 break;4541 }4542}4543 4544static inline void guc_default_irqs(struct intel_engine_cs *engine)4545{4546 engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT;4547 intel_engine_set_irq_handler(engine, cs_irq_handler);4548}4549 4550static void guc_sched_engine_destroy(struct kref *kref)4551{4552 struct i915_sched_engine *sched_engine =4553 container_of(kref, typeof(*sched_engine), ref);4554 struct intel_guc *guc = sched_engine->private_data;4555 4556 guc->sched_engine = NULL;4557 tasklet_kill(&sched_engine->tasklet); /* flush the callback */4558 kfree(sched_engine);4559}4560 4561int intel_guc_submission_setup(struct intel_engine_cs *engine)4562{4563 struct drm_i915_private *i915 = engine->i915;4564 struct intel_guc *guc = gt_to_guc(engine->gt);4565 4566 /*4567 * The setup relies on several assumptions (e.g. irqs always enabled)4568 * that are only valid on gen11+4569 */4570 GEM_BUG_ON(GRAPHICS_VER(i915) < 11);4571 4572 if (!guc->sched_engine) {4573 guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);4574 if (!guc->sched_engine)4575 return -ENOMEM;4576 4577 guc->sched_engine->schedule = i915_schedule;4578 guc->sched_engine->disabled = guc_sched_engine_disabled;4579 guc->sched_engine->private_data = guc;4580 guc->sched_engine->destroy = guc_sched_engine_destroy;4581 guc->sched_engine->bump_inflight_request_prio =4582 guc_bump_inflight_request_prio;4583 guc->sched_engine->retire_inflight_request_prio =4584 guc_retire_inflight_request_prio;4585 tasklet_setup(&guc->sched_engine->tasklet,4586 guc_submission_tasklet);4587 }4588 i915_sched_engine_put(engine->sched_engine);4589 engine->sched_engine = i915_sched_engine_get(guc->sched_engine);4590 4591 guc_default_vfuncs(engine);4592 guc_default_irqs(engine);4593 guc_init_breadcrumbs(engine);4594 4595 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)4596 rcs_submission_override(engine);4597 4598 lrc_init_wa_ctx(engine);4599 4600 /* Finally, take ownership and responsibility for cleanup! */4601 engine->sanitize = guc_sanitize;4602 engine->release = guc_release;4603 4604 return 0;4605}4606 4607struct scheduling_policy {4608 /* internal data */4609 u32 max_words, num_words;4610 u32 count;4611 /* API data */4612 struct guc_update_scheduling_policy h2g;4613};4614 4615static u32 __guc_scheduling_policy_action_size(struct scheduling_policy *policy)4616{4617 u32 *start = (void *)&policy->h2g;4618 u32 *end = policy->h2g.data + policy->num_words;4619 size_t delta = end - start;4620 4621 return delta;4622}4623 4624static struct scheduling_policy *__guc_scheduling_policy_start_klv(struct scheduling_policy *policy)4625{4626 policy->h2g.header.action = INTEL_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;4627 policy->max_words = ARRAY_SIZE(policy->h2g.data);4628 policy->num_words = 0;4629 policy->count = 0;4630 4631 return policy;4632}4633 4634static void __guc_scheduling_policy_add_klv(struct scheduling_policy *policy,4635 u32 action, u32 *data, u32 len)4636{4637 u32 *klv_ptr = policy->h2g.data + policy->num_words;4638 4639 GEM_BUG_ON((policy->num_words + 1 + len) > policy->max_words);4640 *(klv_ptr++) = FIELD_PREP(GUC_KLV_0_KEY, action) |4641 FIELD_PREP(GUC_KLV_0_LEN, len);4642 memcpy(klv_ptr, data, sizeof(u32) * len);4643 policy->num_words += 1 + len;4644 policy->count++;4645}4646 4647static int __guc_action_set_scheduling_policies(struct intel_guc *guc,4648 struct scheduling_policy *policy)4649{4650 int ret;4651 4652 ret = intel_guc_send(guc, (u32 *)&policy->h2g,4653 __guc_scheduling_policy_action_size(policy));4654 if (ret < 0) {4655 guc_probe_error(guc, "Failed to configure global scheduling policies: %pe!\n",4656 ERR_PTR(ret));4657 return ret;4658 }4659 4660 if (ret != policy->count) {4661 guc_warn(guc, "global scheduler policy processed %d of %d KLVs!",4662 ret, policy->count);4663 if (ret > policy->count)4664 return -EPROTO;4665 }4666 4667 return 0;4668}4669 4670static int guc_init_global_schedule_policy(struct intel_guc *guc)4671{4672 struct scheduling_policy policy;4673 struct intel_gt *gt = guc_to_gt(guc);4674 intel_wakeref_t wakeref;4675 int ret;4676 4677 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))4678 return 0;4679 4680 __guc_scheduling_policy_start_klv(&policy);4681 4682 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) {4683 u32 yield[] = {4684 GLOBAL_SCHEDULE_POLICY_RC_YIELD_DURATION,4685 GLOBAL_SCHEDULE_POLICY_RC_YIELD_RATIO,4686 };4687 4688 __guc_scheduling_policy_add_klv(&policy,4689 GUC_SCHEDULING_POLICIES_KLV_ID_RENDER_COMPUTE_YIELD,4690 yield, ARRAY_SIZE(yield));4691 4692 ret = __guc_action_set_scheduling_policies(guc, &policy);4693 }4694 4695 return ret;4696}4697 4698static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)4699{4700 struct intel_gt *gt = guc_to_gt(guc);4701 u32 val;4702 4703 if (GRAPHICS_VER(gt->i915) < 12)4704 return;4705 4706 if (to_guc)4707 val = GUC_SEM_INTR_ROUTE_TO_GUC | GUC_SEM_INTR_ENABLE_ALL;4708 else4709 val = 0;4710 4711 intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, val);4712}4713 4714int intel_guc_submission_enable(struct intel_guc *guc)4715{4716 int ret;4717 4718 /* Semaphore interrupt enable and route to GuC */4719 guc_route_semaphores(guc, true);4720 4721 ret = guc_init_submission(guc);4722 if (ret)4723 goto fail_sem;4724 4725 ret = guc_init_engine_stats(guc);4726 if (ret)4727 goto fail_sem;4728 4729 ret = guc_init_global_schedule_policy(guc);4730 if (ret)4731 goto fail_stats;4732 4733 return 0;4734 4735fail_stats:4736 guc_fini_engine_stats(guc);4737fail_sem:4738 guc_route_semaphores(guc, false);4739 return ret;4740}4741 4742/* Note: By the time we're here, GuC may have already been reset */4743void intel_guc_submission_disable(struct intel_guc *guc)4744{4745 guc_cancel_busyness_worker(guc);4746 4747 /* Semaphore interrupt disable and route to host */4748 guc_route_semaphores(guc, false);4749}4750 4751static bool __guc_submission_supported(struct intel_guc *guc)4752{4753 /* GuC submission is unavailable for pre-Gen11 */4754 return intel_guc_is_supported(guc) &&4755 GRAPHICS_VER(guc_to_i915(guc)) >= 11;4756}4757 4758static bool __guc_submission_selected(struct intel_guc *guc)4759{4760 struct drm_i915_private *i915 = guc_to_i915(guc);4761 4762 if (!intel_guc_submission_is_supported(guc))4763 return false;4764 4765 return i915->params.enable_guc & ENABLE_GUC_SUBMISSION;4766}4767 4768int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc)4769{4770 return guc->submission_state.num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);4771}4772 4773/*4774 * This default value of 33 milisecs (+1 milisec round up) ensures 30fps or higher4775 * workloads are able to enjoy the latency reduction when delaying the schedule-disable4776 * operation. This matches the 30fps game-render + encode (real world) workload this4777 * knob was tested against.4778 */4779#define SCHED_DISABLE_DELAY_MS 344780 4781/*4782 * A threshold of 75% is a reasonable starting point considering that real world apps4783 * generally don't get anywhere near this.4784 */4785#define NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(__guc) \4786 (((intel_guc_sched_disable_gucid_threshold_max(guc)) * 3) / 4)4787 4788void intel_guc_submission_init_early(struct intel_guc *guc)4789{4790 xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);4791 4792 spin_lock_init(&guc->submission_state.lock);4793 INIT_LIST_HEAD(&guc->submission_state.guc_id_list);4794 ida_init(&guc->submission_state.guc_ids);4795 INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts);4796 INIT_WORK(&guc->submission_state.destroyed_worker,4797 destroyed_worker_func);4798 INIT_WORK(&guc->submission_state.reset_fail_worker,4799 reset_fail_worker_func);4800 4801 spin_lock_init(&guc->timestamp.lock);4802 INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);4803 4804 guc->submission_state.sched_disable_delay_ms = SCHED_DISABLE_DELAY_MS;4805 guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID;4806 guc->submission_state.sched_disable_gucid_threshold =4807 NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(guc);4808 guc->submission_supported = __guc_submission_supported(guc);4809 guc->submission_selected = __guc_submission_selected(guc);4810}4811 4812static inline struct intel_context *4813g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)4814{4815 struct intel_context *ce;4816 4817 if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)) {4818 guc_err(guc, "Invalid ctx_id %u\n", ctx_id);4819 return NULL;4820 }4821 4822 ce = __get_context(guc, ctx_id);4823 if (unlikely(!ce)) {4824 guc_err(guc, "Context is NULL, ctx_id %u\n", ctx_id);4825 return NULL;4826 }4827 4828 if (unlikely(intel_context_is_child(ce))) {4829 guc_err(guc, "Context is child, ctx_id %u\n", ctx_id);4830 return NULL;4831 }4832 4833 return ce;4834}4835 4836static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)4837{4838 struct intel_guc_tlb_wait *wait;4839 unsigned long flags;4840 4841 xa_lock_irqsave(&guc->tlb_lookup, flags);4842 wait = xa_load(&guc->tlb_lookup, seqno);4843 4844 if (wait)4845 wake_up(&wait->wq);4846 else4847 guc_dbg(guc,4848 "Stale TLB invalidation response with seqno %d\n", seqno);4849 4850 xa_unlock_irqrestore(&guc->tlb_lookup, flags);4851}4852 4853int intel_guc_tlb_invalidation_done(struct intel_guc *guc,4854 const u32 *payload, u32 len)4855{4856 if (len < 1)4857 return -EPROTO;4858 4859 wait_wake_outstanding_tlb_g2h(guc, payload[0]);4860 return 0;4861}4862 4863static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)4864{4865 /*4866 * This is equivalent to wait_woken() with the exception that4867 * we do not wake up early if the kthread task has been completed.4868 * As we are called from page reclaim in any task context,4869 * we may be invoked from stopped kthreads, but we *must*4870 * complete the wait from the HW.4871 */4872 do {4873 set_current_state(TASK_UNINTERRUPTIBLE);4874 if (wq_entry->flags & WQ_FLAG_WOKEN)4875 break;4876 4877 timeout = schedule_timeout(timeout);4878 } while (timeout);4879 4880 /* See wait_woken() and woken_wake_function() */4881 __set_current_state(TASK_RUNNING);4882 smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN);4883 4884 return timeout;4885}4886 4887static bool intel_gt_is_enabled(const struct intel_gt *gt)4888{4889 /* Check if GT is wedged or suspended */4890 if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915))4891 return false;4892 return true;4893}4894 4895static int guc_send_invalidate_tlb(struct intel_guc *guc,4896 enum intel_guc_tlb_invalidation_type type)4897{4898 struct intel_guc_tlb_wait _wq, *wq = &_wq;4899 struct intel_gt *gt = guc_to_gt(guc);4900 DEFINE_WAIT_FUNC(wait, woken_wake_function);4901 int err;4902 u32 seqno;4903 u32 action[] = {4904 INTEL_GUC_ACTION_TLB_INVALIDATION,4905 0,4906 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_TYPE_MASK, type) |4907 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_MODE_MASK,4908 INTEL_GUC_TLB_INVAL_MODE_HEAVY) |4909 INTEL_GUC_TLB_INVAL_FLUSH_CACHE,4910 };4911 u32 size = ARRAY_SIZE(action);4912 4913 /*4914 * Early guard against GT enablement. TLB invalidation should not be4915 * attempted if the GT is disabled due to suspend/wedge.4916 */4917 if (!intel_gt_is_enabled(gt))4918 return -EINVAL;4919 4920 init_waitqueue_head(&_wq.wq);4921 4922 if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq,4923 xa_limit_32b, &guc->next_seqno,4924 GFP_ATOMIC | __GFP_NOWARN) < 0) {4925 /* Under severe memory pressure? Serialise TLB allocations */4926 xa_lock_irq(&guc->tlb_lookup);4927 wq = xa_load(&guc->tlb_lookup, guc->serial_slot);4928 wait_event_lock_irq(wq->wq,4929 !READ_ONCE(wq->busy),4930 guc->tlb_lookup.xa_lock);4931 /*4932 * Update wq->busy under lock to ensure only one waiter can4933 * issue the TLB invalidation command using the serial slot at a4934 * time. The condition is set to true before releasing the lock4935 * so that other caller continue to wait until woken up again.4936 */4937 wq->busy = true;4938 xa_unlock_irq(&guc->tlb_lookup);4939 4940 seqno = guc->serial_slot;4941 }4942 4943 action[1] = seqno;4944 4945 add_wait_queue(&wq->wq, &wait);4946 4947 /* This is a critical reclaim path and thus we must loop here. */4948 err = intel_guc_send_busy_loop(guc, action, size, G2H_LEN_DW_INVALIDATE_TLB, true);4949 if (err)4950 goto out;4951 4952 /*4953 * Late guard against GT enablement. It is not an error for the TLB4954 * invalidation to time out if the GT is disabled during the process4955 * due to suspend/wedge. In fact, the TLB invalidation is cancelled4956 * in this case.4957 */4958 if (!must_wait_woken(&wait, intel_guc_ct_max_queue_time_jiffies()) &&4959 intel_gt_is_enabled(gt)) {4960 guc_err(guc,4961 "TLB invalidation response timed out for seqno %u\n", seqno);4962 err = -ETIME;4963 }4964out:4965 remove_wait_queue(&wq->wq, &wait);4966 if (seqno != guc->serial_slot)4967 xa_erase_irq(&guc->tlb_lookup, seqno);4968 4969 return err;4970}4971 4972/* Send a H2G command to invalidate the TLBs at engine level and beyond. */4973int intel_guc_invalidate_tlb_engines(struct intel_guc *guc)4974{4975 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_ENGINES);4976}4977 4978/* Send a H2G command to invalidate the GuC's internal TLB. */4979int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)4980{4981 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC);4982}4983 4984int intel_guc_deregister_done_process_msg(struct intel_guc *guc,4985 const u32 *msg,4986 u32 len)4987{4988 struct intel_context *ce;4989 u32 ctx_id;4990 4991 if (unlikely(len < 1)) {4992 guc_err(guc, "Invalid length %u\n", len);4993 return -EPROTO;4994 }4995 ctx_id = msg[0];4996 4997 ce = g2h_context_lookup(guc, ctx_id);4998 if (unlikely(!ce))4999 return -EPROTO;5000 5001 trace_intel_context_deregister_done(ce);5002 5003#ifdef CONFIG_DRM_I915_SELFTEST5004 if (unlikely(ce->drop_deregister)) {5005 ce->drop_deregister = false;5006 return 0;5007 }5008#endif5009 5010 if (context_wait_for_deregister_to_register(ce)) {5011 struct intel_runtime_pm *runtime_pm =5012 &ce->engine->gt->i915->runtime_pm;5013 intel_wakeref_t wakeref;5014 5015 /*5016 * Previous owner of this guc_id has been deregistered, now safe5017 * register this context.5018 */5019 with_intel_runtime_pm(runtime_pm, wakeref)5020 register_context(ce, true);5021 guc_signal_context_fence(ce);5022 intel_context_put(ce);5023 } else if (context_destroyed(ce)) {5024 /* Context has been destroyed */5025 intel_gt_pm_put_async_untracked(guc_to_gt(guc));5026 release_guc_id(guc, ce);5027 __guc_context_destroy(ce);5028 }5029 5030 decr_outstanding_submission_g2h(guc);5031 5032 return 0;5033}5034 5035int intel_guc_sched_done_process_msg(struct intel_guc *guc,5036 const u32 *msg,5037 u32 len)5038{5039 struct intel_context *ce;5040 unsigned long flags;5041 u32 ctx_id;5042 5043 if (unlikely(len < 2)) {5044 guc_err(guc, "Invalid length %u\n", len);5045 return -EPROTO;5046 }5047 ctx_id = msg[0];5048 5049 ce = g2h_context_lookup(guc, ctx_id);5050 if (unlikely(!ce))5051 return -EPROTO;5052 5053 if (unlikely(context_destroyed(ce) ||5054 (!context_pending_enable(ce) &&5055 !context_pending_disable(ce)))) {5056 guc_err(guc, "Bad context sched_state 0x%x, ctx_id %u\n",5057 ce->guc_state.sched_state, ctx_id);5058 return -EPROTO;5059 }5060 5061 trace_intel_context_sched_done(ce);5062 5063 if (context_pending_enable(ce)) {5064#ifdef CONFIG_DRM_I915_SELFTEST5065 if (unlikely(ce->drop_schedule_enable)) {5066 ce->drop_schedule_enable = false;5067 return 0;5068 }5069#endif5070 5071 spin_lock_irqsave(&ce->guc_state.lock, flags);5072 clr_context_pending_enable(ce);5073 spin_unlock_irqrestore(&ce->guc_state.lock, flags);5074 } else if (context_pending_disable(ce)) {5075 bool banned;5076 5077#ifdef CONFIG_DRM_I915_SELFTEST5078 if (unlikely(ce->drop_schedule_disable)) {5079 ce->drop_schedule_disable = false;5080 return 0;5081 }5082#endif5083 5084 /*5085 * Unpin must be done before __guc_signal_context_fence,5086 * otherwise a race exists between the requests getting5087 * submitted + retired before this unpin completes resulting in5088 * the pin_count going to zero and the context still being5089 * enabled.5090 */5091 intel_context_sched_disable_unpin(ce);5092 5093 spin_lock_irqsave(&ce->guc_state.lock, flags);5094 banned = context_banned(ce);5095 clr_context_banned(ce);5096 clr_context_pending_disable(ce);5097 __guc_signal_context_fence(ce);5098 guc_blocked_fence_complete(ce);5099 spin_unlock_irqrestore(&ce->guc_state.lock, flags);5100 5101 if (banned) {5102 guc_cancel_context_requests(ce);5103 intel_engine_signal_breadcrumbs(ce->engine);5104 }5105 }5106 5107 decr_outstanding_submission_g2h(guc);5108 intel_context_put(ce);5109 5110 return 0;5111}5112 5113static void capture_error_state(struct intel_guc *guc,5114 struct intel_context *ce)5115{5116 struct intel_gt *gt = guc_to_gt(guc);5117 struct drm_i915_private *i915 = gt->i915;5118 intel_wakeref_t wakeref;5119 intel_engine_mask_t engine_mask;5120 5121 if (intel_engine_is_virtual(ce->engine)) {5122 struct intel_engine_cs *e;5123 intel_engine_mask_t tmp, virtual_mask = ce->engine->mask;5124 5125 engine_mask = 0;5126 for_each_engine_masked(e, ce->engine->gt, virtual_mask, tmp) {5127 bool match = intel_guc_capture_is_matching_engine(gt, ce, e);5128 5129 if (match) {5130 intel_engine_set_hung_context(e, ce);5131 engine_mask |= e->mask;5132 i915_increase_reset_engine_count(&i915->gpu_error,5133 e);5134 }5135 }5136 5137 if (!engine_mask) {5138 guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s",5139 ce->guc_id.id, ce->engine->name);5140 engine_mask = ~0U;5141 }5142 } else {5143 intel_engine_set_hung_context(ce->engine, ce);5144 engine_mask = ce->engine->mask;5145 i915_increase_reset_engine_count(&i915->gpu_error, ce->engine);5146 }5147 5148 with_intel_runtime_pm(&i915->runtime_pm, wakeref)5149 i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE);5150}5151 5152static void guc_context_replay(struct intel_context *ce)5153{5154 struct i915_sched_engine *sched_engine = ce->engine->sched_engine;5155 5156 __guc_reset_context(ce, ce->engine->mask);5157 tasklet_hi_schedule(&sched_engine->tasklet);5158}5159 5160static void guc_handle_context_reset(struct intel_guc *guc,5161 struct intel_context *ce)5162{5163 bool capture = intel_context_is_schedulable(ce);5164 5165 trace_intel_context_reset(ce);5166 5167 guc_dbg(guc, "%s context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",5168 capture ? "Got" : "Ignoring",5169 ce->guc_id.id, ce->engine->name,5170 str_yes_no(intel_context_is_exiting(ce)),5171 str_yes_no(intel_context_is_banned(ce)));5172 5173 if (capture) {5174 capture_error_state(guc, ce);5175 guc_context_replay(ce);5176 }5177}5178 5179int intel_guc_context_reset_process_msg(struct intel_guc *guc,5180 const u32 *msg, u32 len)5181{5182 struct intel_context *ce;5183 unsigned long flags;5184 int ctx_id;5185 5186 if (unlikely(len != 1)) {5187 guc_err(guc, "Invalid length %u", len);5188 return -EPROTO;5189 }5190 5191 ctx_id = msg[0];5192 5193 /*5194 * The context lookup uses the xarray but lookups only require an RCU lock5195 * not the full spinlock. So take the lock explicitly and keep it until the5196 * context has been reference count locked to ensure it can't be destroyed5197 * asynchronously until the reset is done.5198 */5199 xa_lock_irqsave(&guc->context_lookup, flags);5200 ce = g2h_context_lookup(guc, ctx_id);5201 if (ce)5202 intel_context_get(ce);5203 xa_unlock_irqrestore(&guc->context_lookup, flags);5204 5205 if (unlikely(!ce))5206 return -EPROTO;5207 5208 guc_handle_context_reset(guc, ce);5209 intel_context_put(ce);5210 5211 return 0;5212}5213 5214int intel_guc_error_capture_process_msg(struct intel_guc *guc,5215 const u32 *msg, u32 len)5216{5217 u32 status;5218 5219 if (unlikely(len != 1)) {5220 guc_dbg(guc, "Invalid length %u", len);5221 return -EPROTO;5222 }5223 5224 status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;5225 if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)5226 guc_warn(guc, "No space for error capture");5227 5228 intel_guc_capture_process(guc);5229 5230 return 0;5231}5232 5233struct intel_engine_cs *5234intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)5235{5236 struct intel_gt *gt = guc_to_gt(guc);5237 u8 engine_class = guc_class_to_engine_class(guc_class);5238 5239 /* Class index is checked in class converter */5240 GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE);5241 5242 return gt->engine_class[engine_class][instance];5243}5244 5245static void reset_fail_worker_func(struct work_struct *w)5246{5247 struct intel_guc *guc = container_of(w, struct intel_guc,5248 submission_state.reset_fail_worker);5249 struct intel_gt *gt = guc_to_gt(guc);5250 intel_engine_mask_t reset_fail_mask;5251 unsigned long flags;5252 5253 spin_lock_irqsave(&guc->submission_state.lock, flags);5254 reset_fail_mask = guc->submission_state.reset_fail_mask;5255 guc->submission_state.reset_fail_mask = 0;5256 spin_unlock_irqrestore(&guc->submission_state.lock, flags);5257 5258 if (likely(reset_fail_mask)) {5259 struct intel_engine_cs *engine;5260 enum intel_engine_id id;5261 5262 /*5263 * GuC is toast at this point - it dead loops after sending the failed5264 * reset notification. So need to manually determine the guilty context.5265 * Note that it should be reliable to do this here because the GuC is5266 * toast and will not be scheduling behind the KMD's back.5267 */5268 for_each_engine_masked(engine, gt, reset_fail_mask, id)5269 intel_guc_find_hung_context(engine);5270 5271 intel_gt_handle_error(gt, reset_fail_mask,5272 I915_ERROR_CAPTURE,5273 "GuC failed to reset engine mask=0x%x",5274 reset_fail_mask);5275 }5276}5277 5278int intel_guc_engine_failure_process_msg(struct intel_guc *guc,5279 const u32 *msg, u32 len)5280{5281 struct intel_engine_cs *engine;5282 u8 guc_class, instance;5283 u32 reason;5284 unsigned long flags;5285 5286 if (unlikely(len != 3)) {5287 guc_err(guc, "Invalid length %u", len);5288 return -EPROTO;5289 }5290 5291 guc_class = msg[0];5292 instance = msg[1];5293 reason = msg[2];5294 5295 engine = intel_guc_lookup_engine(guc, guc_class, instance);5296 if (unlikely(!engine)) {5297 guc_err(guc, "Invalid engine %d:%d", guc_class, instance);5298 return -EPROTO;5299 }5300 5301 /*5302 * This is an unexpected failure of a hardware feature. So, log a real5303 * error message not just the informational that comes with the reset.5304 */5305 guc_err(guc, "Engine reset failed on %d:%d (%s) because 0x%08X",5306 guc_class, instance, engine->name, reason);5307 5308 spin_lock_irqsave(&guc->submission_state.lock, flags);5309 guc->submission_state.reset_fail_mask |= engine->mask;5310 spin_unlock_irqrestore(&guc->submission_state.lock, flags);5311 5312 /*5313 * A GT reset flushes this worker queue (G2H handler) so we must use5314 * another worker to trigger a GT reset.5315 */5316 queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);5317 5318 return 0;5319}5320 5321void intel_guc_find_hung_context(struct intel_engine_cs *engine)5322{5323 struct intel_guc *guc = gt_to_guc(engine->gt);5324 struct intel_context *ce;5325 struct i915_request *rq;5326 unsigned long index;5327 unsigned long flags;5328 5329 /* Reset called during driver load? GuC not yet initialised! */5330 if (unlikely(!guc_submission_initialized(guc)))5331 return;5332 5333 xa_lock_irqsave(&guc->context_lookup, flags);5334 xa_for_each(&guc->context_lookup, index, ce) {5335 bool found;5336 5337 if (!kref_get_unless_zero(&ce->ref))5338 continue;5339 5340 xa_unlock(&guc->context_lookup);5341 5342 if (!intel_context_is_pinned(ce))5343 goto next;5344 5345 if (intel_engine_is_virtual(ce->engine)) {5346 if (!(ce->engine->mask & engine->mask))5347 goto next;5348 } else {5349 if (ce->engine != engine)5350 goto next;5351 }5352 5353 found = false;5354 spin_lock(&ce->guc_state.lock);5355 list_for_each_entry(rq, &ce->guc_state.requests, sched.link) {5356 if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE)5357 continue;5358 5359 found = true;5360 break;5361 }5362 spin_unlock(&ce->guc_state.lock);5363 5364 if (found) {5365 intel_engine_set_hung_context(engine, ce);5366 5367 /* Can only cope with one hang at a time... */5368 intel_context_put(ce);5369 xa_lock(&guc->context_lookup);5370 goto done;5371 }5372 5373next:5374 intel_context_put(ce);5375 xa_lock(&guc->context_lookup);5376 }5377done:5378 xa_unlock_irqrestore(&guc->context_lookup, flags);5379}5380 5381void intel_guc_dump_active_requests(struct intel_engine_cs *engine,5382 struct i915_request *hung_rq,5383 struct drm_printer *m)5384{5385 struct intel_guc *guc = gt_to_guc(engine->gt);5386 struct intel_context *ce;5387 unsigned long index;5388 unsigned long flags;5389 5390 /* Reset called during driver load? GuC not yet initialised! */5391 if (unlikely(!guc_submission_initialized(guc)))5392 return;5393 5394 xa_lock_irqsave(&guc->context_lookup, flags);5395 xa_for_each(&guc->context_lookup, index, ce) {5396 if (!kref_get_unless_zero(&ce->ref))5397 continue;5398 5399 xa_unlock(&guc->context_lookup);5400 5401 if (!intel_context_is_pinned(ce))5402 goto next;5403 5404 if (intel_engine_is_virtual(ce->engine)) {5405 if (!(ce->engine->mask & engine->mask))5406 goto next;5407 } else {5408 if (ce->engine != engine)5409 goto next;5410 }5411 5412 spin_lock(&ce->guc_state.lock);5413 intel_engine_dump_active_requests(&ce->guc_state.requests,5414 hung_rq, m);5415 spin_unlock(&ce->guc_state.lock);5416 5417next:5418 intel_context_put(ce);5419 xa_lock(&guc->context_lookup);5420 }5421 xa_unlock_irqrestore(&guc->context_lookup, flags);5422}5423 5424void intel_guc_submission_print_info(struct intel_guc *guc,5425 struct drm_printer *p)5426{5427 struct i915_sched_engine *sched_engine = guc->sched_engine;5428 struct rb_node *rb;5429 unsigned long flags;5430 5431 if (!sched_engine)5432 return;5433 5434 drm_printf(p, "GuC Submission API Version: %d.%d.%d\n",5435 guc->submission_version.major, guc->submission_version.minor,5436 guc->submission_version.patch);5437 drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n",5438 atomic_read(&guc->outstanding_submission_g2h));5439 drm_printf(p, "GuC tasklet count: %u\n",5440 atomic_read(&sched_engine->tasklet.count));5441 5442 spin_lock_irqsave(&sched_engine->lock, flags);5443 drm_printf(p, "Requests in GuC submit tasklet:\n");5444 for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {5445 struct i915_priolist *pl = to_priolist(rb);5446 struct i915_request *rq;5447 5448 priolist_for_each_request(rq, pl)5449 drm_printf(p, "guc_id=%u, seqno=%llu\n",5450 rq->context->guc_id.id,5451 rq->fence.seqno);5452 }5453 spin_unlock_irqrestore(&sched_engine->lock, flags);5454 drm_printf(p, "\n");5455}5456 5457static inline void guc_log_context_priority(struct drm_printer *p,5458 struct intel_context *ce)5459{5460 int i;5461 5462 drm_printf(p, "\t\tPriority: %d\n", ce->guc_state.prio);5463 drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n");5464 for (i = GUC_CLIENT_PRIORITY_KMD_HIGH;5465 i < GUC_CLIENT_PRIORITY_NUM; ++i) {5466 drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n",5467 i, ce->guc_state.prio_count[i]);5468 }5469 drm_printf(p, "\n");5470}5471 5472static inline void guc_log_context(struct drm_printer *p,5473 struct intel_context *ce)5474{5475 drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id.id);5476 drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);5477 drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n",5478 ce->ring->head,5479 ce->lrc_reg_state[CTX_RING_HEAD]);5480 drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n",5481 ce->ring->tail,5482 ce->lrc_reg_state[CTX_RING_TAIL]);5483 drm_printf(p, "\t\tContext Pin Count: %u\n",5484 atomic_read(&ce->pin_count));5485 drm_printf(p, "\t\tGuC ID Ref Count: %u\n",5486 atomic_read(&ce->guc_id.ref));5487 drm_printf(p, "\t\tSchedule State: 0x%x\n",5488 ce->guc_state.sched_state);5489}5490 5491void intel_guc_submission_print_context_info(struct intel_guc *guc,5492 struct drm_printer *p)5493{5494 struct intel_context *ce;5495 unsigned long index;5496 unsigned long flags;5497 5498 xa_lock_irqsave(&guc->context_lookup, flags);5499 xa_for_each(&guc->context_lookup, index, ce) {5500 GEM_BUG_ON(intel_context_is_child(ce));5501 5502 guc_log_context(p, ce);5503 guc_log_context_priority(p, ce);5504 5505 if (intel_context_is_parent(ce)) {5506 struct intel_context *child;5507 5508 drm_printf(p, "\t\tNumber children: %u\n",5509 ce->parallel.number_children);5510 5511 if (ce->parallel.guc.wq_status) {5512 drm_printf(p, "\t\tWQI Head: %u\n",5513 READ_ONCE(*ce->parallel.guc.wq_head));5514 drm_printf(p, "\t\tWQI Tail: %u\n",5515 READ_ONCE(*ce->parallel.guc.wq_tail));5516 drm_printf(p, "\t\tWQI Status: %u\n",5517 READ_ONCE(*ce->parallel.guc.wq_status));5518 }5519 5520 if (ce->engine->emit_bb_start ==5521 emit_bb_start_parent_no_preempt_mid_batch) {5522 u8 i;5523 5524 drm_printf(p, "\t\tChildren Go: %u\n",5525 get_children_go_value(ce));5526 for (i = 0; i < ce->parallel.number_children; ++i)5527 drm_printf(p, "\t\tChildren Join: %u\n",5528 get_children_join_value(ce, i));5529 }5530 5531 for_each_child(ce, child)5532 guc_log_context(p, child);5533 }5534 }5535 xa_unlock_irqrestore(&guc->context_lookup, flags);5536}5537 5538static inline u32 get_children_go_addr(struct intel_context *ce)5539{5540 GEM_BUG_ON(!intel_context_is_parent(ce));5541 5542 return i915_ggtt_offset(ce->state) +5543 __get_parent_scratch_offset(ce) +5544 offsetof(struct parent_scratch, go.semaphore);5545}5546 5547static inline u32 get_children_join_addr(struct intel_context *ce,5548 u8 child_index)5549{5550 GEM_BUG_ON(!intel_context_is_parent(ce));5551 5552 return i915_ggtt_offset(ce->state) +5553 __get_parent_scratch_offset(ce) +5554 offsetof(struct parent_scratch, join[child_index].semaphore);5555}5556 5557#define PARENT_GO_BB 15558#define PARENT_GO_FINI_BREADCRUMB 05559#define CHILD_GO_BB 15560#define CHILD_GO_FINI_BREADCRUMB 05561static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,5562 u64 offset, u32 len,5563 const unsigned int flags)5564{5565 struct intel_context *ce = rq->context;5566 u32 *cs;5567 u8 i;5568 5569 GEM_BUG_ON(!intel_context_is_parent(ce));5570 5571 cs = intel_ring_begin(rq, 10 + 4 * ce->parallel.number_children);5572 if (IS_ERR(cs))5573 return PTR_ERR(cs);5574 5575 /* Wait on children */5576 for (i = 0; i < ce->parallel.number_children; ++i) {5577 *cs++ = (MI_SEMAPHORE_WAIT |5578 MI_SEMAPHORE_GLOBAL_GTT |5579 MI_SEMAPHORE_POLL |5580 MI_SEMAPHORE_SAD_EQ_SDD);5581 *cs++ = PARENT_GO_BB;5582 *cs++ = get_children_join_addr(ce, i);5583 *cs++ = 0;5584 }5585 5586 /* Turn off preemption */5587 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;5588 *cs++ = MI_NOOP;5589 5590 /* Tell children go */5591 cs = gen8_emit_ggtt_write(cs,5592 CHILD_GO_BB,5593 get_children_go_addr(ce),5594 0);5595 5596 /* Jump to batch */5597 *cs++ = MI_BATCH_BUFFER_START_GEN8 |5598 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));5599 *cs++ = lower_32_bits(offset);5600 *cs++ = upper_32_bits(offset);5601 *cs++ = MI_NOOP;5602 5603 intel_ring_advance(rq, cs);5604 5605 return 0;5606}5607 5608static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,5609 u64 offset, u32 len,5610 const unsigned int flags)5611{5612 struct intel_context *ce = rq->context;5613 struct intel_context *parent = intel_context_to_parent(ce);5614 u32 *cs;5615 5616 GEM_BUG_ON(!intel_context_is_child(ce));5617 5618 cs = intel_ring_begin(rq, 12);5619 if (IS_ERR(cs))5620 return PTR_ERR(cs);5621 5622 /* Signal parent */5623 cs = gen8_emit_ggtt_write(cs,5624 PARENT_GO_BB,5625 get_children_join_addr(parent,5626 ce->parallel.child_index),5627 0);5628 5629 /* Wait on parent for go */5630 *cs++ = (MI_SEMAPHORE_WAIT |5631 MI_SEMAPHORE_GLOBAL_GTT |5632 MI_SEMAPHORE_POLL |5633 MI_SEMAPHORE_SAD_EQ_SDD);5634 *cs++ = CHILD_GO_BB;5635 *cs++ = get_children_go_addr(parent);5636 *cs++ = 0;5637 5638 /* Turn off preemption */5639 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;5640 5641 /* Jump to batch */5642 *cs++ = MI_BATCH_BUFFER_START_GEN8 |5643 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));5644 *cs++ = lower_32_bits(offset);5645 *cs++ = upper_32_bits(offset);5646 5647 intel_ring_advance(rq, cs);5648 5649 return 0;5650}5651 5652static u32 *5653__emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,5654 u32 *cs)5655{5656 struct intel_context *ce = rq->context;5657 u8 i;5658 5659 GEM_BUG_ON(!intel_context_is_parent(ce));5660 5661 /* Wait on children */5662 for (i = 0; i < ce->parallel.number_children; ++i) {5663 *cs++ = (MI_SEMAPHORE_WAIT |5664 MI_SEMAPHORE_GLOBAL_GTT |5665 MI_SEMAPHORE_POLL |5666 MI_SEMAPHORE_SAD_EQ_SDD);5667 *cs++ = PARENT_GO_FINI_BREADCRUMB;5668 *cs++ = get_children_join_addr(ce, i);5669 *cs++ = 0;5670 }5671 5672 /* Turn on preemption */5673 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;5674 *cs++ = MI_NOOP;5675 5676 /* Tell children go */5677 cs = gen8_emit_ggtt_write(cs,5678 CHILD_GO_FINI_BREADCRUMB,5679 get_children_go_addr(ce),5680 0);5681 5682 return cs;5683}5684 5685/*5686 * If this true, a submission of multi-lrc requests had an error and the5687 * requests need to be skipped. The front end (execuf IOCTL) should've called5688 * i915_request_skip which squashes the BB but we still need to emit the fini5689 * breadrcrumbs seqno write. At this point we don't know how many of the5690 * requests in the multi-lrc submission were generated so we can't do the5691 * handshake between the parent and children (e.g. if 4 requests should be5692 * generated but 2nd hit an error only 1 would be seen by the GuC backend).5693 * Simply skip the handshake, but still emit the breadcrumbd seqno, if an error5694 * has occurred on any of the requests in submission / relationship.5695 */5696static inline bool skip_handshake(struct i915_request *rq)5697{5698 return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags);5699}5700 5701#define NON_SKIP_LEN 65702static u32 *5703emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,5704 u32 *cs)5705{5706 struct intel_context *ce = rq->context;5707 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;5708 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;5709 5710 GEM_BUG_ON(!intel_context_is_parent(ce));5711 5712 if (unlikely(skip_handshake(rq))) {5713 /*5714 * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch,5715 * the NON_SKIP_LEN comes from the length of the emits below.5716 */5717 memset(cs, 0, sizeof(u32) *5718 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));5719 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;5720 } else {5721 cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs);5722 }5723 5724 /* Emit fini breadcrumb */5725 before_fini_breadcrumb_user_interrupt_cs = cs;5726 cs = gen8_emit_ggtt_write(cs,5727 rq->fence.seqno,5728 i915_request_active_timeline(rq)->hwsp_offset,5729 0);5730 5731 /* User interrupt */5732 *cs++ = MI_USER_INTERRUPT;5733 *cs++ = MI_NOOP;5734 5735 /* Ensure our math for skip + emit is correct */5736 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=5737 cs);5738 GEM_BUG_ON(start_fini_breadcrumb_cs +5739 ce->engine->emit_fini_breadcrumb_dw != cs);5740 5741 rq->tail = intel_ring_offset(rq, cs);5742 5743 return cs;5744}5745 5746static u32 *5747__emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,5748 u32 *cs)5749{5750 struct intel_context *ce = rq->context;5751 struct intel_context *parent = intel_context_to_parent(ce);5752 5753 GEM_BUG_ON(!intel_context_is_child(ce));5754 5755 /* Turn on preemption */5756 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;5757 *cs++ = MI_NOOP;5758 5759 /* Signal parent */5760 cs = gen8_emit_ggtt_write(cs,5761 PARENT_GO_FINI_BREADCRUMB,5762 get_children_join_addr(parent,5763 ce->parallel.child_index),5764 0);5765 5766 /* Wait parent on for go */5767 *cs++ = (MI_SEMAPHORE_WAIT |5768 MI_SEMAPHORE_GLOBAL_GTT |5769 MI_SEMAPHORE_POLL |5770 MI_SEMAPHORE_SAD_EQ_SDD);5771 *cs++ = CHILD_GO_FINI_BREADCRUMB;5772 *cs++ = get_children_go_addr(parent);5773 *cs++ = 0;5774 5775 return cs;5776}5777 5778static u32 *5779emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,5780 u32 *cs)5781{5782 struct intel_context *ce = rq->context;5783 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;5784 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;5785 5786 GEM_BUG_ON(!intel_context_is_child(ce));5787 5788 if (unlikely(skip_handshake(rq))) {5789 /*5790 * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch,5791 * the NON_SKIP_LEN comes from the length of the emits below.5792 */5793 memset(cs, 0, sizeof(u32) *5794 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));5795 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;5796 } else {5797 cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs);5798 }5799 5800 /* Emit fini breadcrumb */5801 before_fini_breadcrumb_user_interrupt_cs = cs;5802 cs = gen8_emit_ggtt_write(cs,5803 rq->fence.seqno,5804 i915_request_active_timeline(rq)->hwsp_offset,5805 0);5806 5807 /* User interrupt */5808 *cs++ = MI_USER_INTERRUPT;5809 *cs++ = MI_NOOP;5810 5811 /* Ensure our math for skip + emit is correct */5812 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=5813 cs);5814 GEM_BUG_ON(start_fini_breadcrumb_cs +5815 ce->engine->emit_fini_breadcrumb_dw != cs);5816 5817 rq->tail = intel_ring_offset(rq, cs);5818 5819 return cs;5820}5821 5822#undef NON_SKIP_LEN5823 5824static struct intel_context *5825guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,5826 unsigned long flags)5827{5828 struct guc_virtual_engine *ve;5829 struct intel_guc *guc;5830 unsigned int n;5831 int err;5832 5833 ve = kzalloc(sizeof(*ve), GFP_KERNEL);5834 if (!ve)5835 return ERR_PTR(-ENOMEM);5836 5837 guc = gt_to_guc(siblings[0]->gt);5838 5839 ve->base.i915 = siblings[0]->i915;5840 ve->base.gt = siblings[0]->gt;5841 ve->base.uncore = siblings[0]->uncore;5842 ve->base.id = -1;5843 5844 ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;5845 ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;5846 ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;5847 ve->base.saturated = ALL_ENGINES;5848 5849 snprintf(ve->base.name, sizeof(ve->base.name), "virtual");5850 5851 ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine);5852 5853 ve->base.cops = &virtual_guc_context_ops;5854 ve->base.request_alloc = guc_request_alloc;5855 ve->base.bump_serial = virtual_guc_bump_serial;5856 5857 ve->base.submit_request = guc_submit_request;5858 5859 ve->base.flags = I915_ENGINE_IS_VIRTUAL;5860 5861 BUILD_BUG_ON(ilog2(VIRTUAL_ENGINES) < I915_NUM_ENGINES);5862 ve->base.mask = VIRTUAL_ENGINES;5863 5864 intel_context_init(&ve->context, &ve->base);5865 5866 for (n = 0; n < count; n++) {5867 struct intel_engine_cs *sibling = siblings[n];5868 5869 GEM_BUG_ON(!is_power_of_2(sibling->mask));5870 if (sibling->mask & ve->base.mask) {5871 guc_dbg(guc, "duplicate %s entry in load balancer\n",5872 sibling->name);5873 err = -EINVAL;5874 goto err_put;5875 }5876 5877 ve->base.mask |= sibling->mask;5878 ve->base.logical_mask |= sibling->logical_mask;5879 5880 if (n != 0 && ve->base.class != sibling->class) {5881 guc_dbg(guc, "invalid mixing of engine class, sibling %d, already %d\n",5882 sibling->class, ve->base.class);5883 err = -EINVAL;5884 goto err_put;5885 } else if (n == 0) {5886 ve->base.class = sibling->class;5887 ve->base.uabi_class = sibling->uabi_class;5888 snprintf(ve->base.name, sizeof(ve->base.name),5889 "v%dx%d", ve->base.class, count);5890 ve->base.context_size = sibling->context_size;5891 5892 ve->base.add_active_request =5893 sibling->add_active_request;5894 ve->base.remove_active_request =5895 sibling->remove_active_request;5896 ve->base.emit_bb_start = sibling->emit_bb_start;5897 ve->base.emit_flush = sibling->emit_flush;5898 ve->base.emit_init_breadcrumb =5899 sibling->emit_init_breadcrumb;5900 ve->base.emit_fini_breadcrumb =5901 sibling->emit_fini_breadcrumb;5902 ve->base.emit_fini_breadcrumb_dw =5903 sibling->emit_fini_breadcrumb_dw;5904 ve->base.breadcrumbs =5905 intel_breadcrumbs_get(sibling->breadcrumbs);5906 5907 ve->base.flags |= sibling->flags;5908 5909 ve->base.props.timeslice_duration_ms =5910 sibling->props.timeslice_duration_ms;5911 ve->base.props.preempt_timeout_ms =5912 sibling->props.preempt_timeout_ms;5913 }5914 }5915 5916 return &ve->context;5917 5918err_put:5919 intel_context_put(&ve->context);5920 return ERR_PTR(err);5921}5922 5923bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)5924{5925 struct intel_engine_cs *engine;5926 intel_engine_mask_t tmp, mask = ve->mask;5927 5928 for_each_engine_masked(engine, ve->gt, mask, tmp)5929 if (READ_ONCE(engine->props.heartbeat_interval_ms))5930 return true;5931 5932 return false;5933}5934 5935#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)5936#include "selftest_guc.c"5937#include "selftest_guc_multi_lrc.c"5938#include "selftest_guc_hangcheck.c"5939#endif5940