555 lines · c
1/* SPDX-License-Identifier: MIT */2/*3 * Copyright © 2014-2019 Intel Corporation4 */5 6#ifndef _INTEL_GUC_H_7#define _INTEL_GUC_H_8 9#include <linux/delay.h>10#include <linux/iosys-map.h>11#include <linux/xarray.h>12 13#include "intel_guc_ct.h"14#include "intel_guc_fw.h"15#include "intel_guc_fwif.h"16#include "intel_guc_log.h"17#include "intel_guc_reg.h"18#include "intel_guc_slpc_types.h"19#include "intel_uc_fw.h"20#include "intel_uncore.h"21#include "i915_utils.h"22#include "i915_vma.h"23 24struct __guc_ads_blob;25struct intel_guc_state_capture;26 27/**28 * struct intel_guc - Top level structure of GuC.29 *30 * It handles firmware loading and manages client pool. intel_guc owns an31 * i915_sched_engine for submission.32 */33struct intel_guc {34 /** @fw: the GuC firmware */35 struct intel_uc_fw fw;36 /** @log: sub-structure containing GuC log related data and objects */37 struct intel_guc_log log;38 /** @ct: the command transport communication channel */39 struct intel_guc_ct ct;40 /** @slpc: sub-structure containing SLPC related data and objects */41 struct intel_guc_slpc slpc;42 /** @capture: the error-state-capture module's data and objects */43 struct intel_guc_state_capture *capture;44 45 /** @dbgfs_node: debugfs node */46 struct dentry *dbgfs_node;47 48 /** @sched_engine: Global engine used to submit requests to GuC */49 struct i915_sched_engine *sched_engine;50 /**51 * @stalled_request: if GuC can't process a request for any reason, we52 * save it until GuC restarts processing. No other request can be53 * submitted until the stalled request is processed.54 */55 struct i915_request *stalled_request;56 /**57 * @submission_stall_reason: reason why submission is stalled58 */59 enum {60 STALL_NONE,61 STALL_REGISTER_CONTEXT,62 STALL_MOVE_LRC_TAIL,63 STALL_ADD_REQUEST,64 } submission_stall_reason;65 66 /* intel_guc_recv interrupt related state */67 /** @irq_lock: protects GuC irq state */68 spinlock_t irq_lock;69 /**70 * @msg_enabled_mask: mask of events that are processed when receiving71 * an INTEL_GUC_ACTION_DEFAULT G2H message.72 */73 unsigned int msg_enabled_mask;74 75 /**76 * @outstanding_submission_g2h: number of outstanding GuC to Host77 * responses related to GuC submission, used to determine if the GT is78 * idle79 */80 atomic_t outstanding_submission_g2h;81 82 /** @tlb_lookup: xarray to store all pending TLB invalidation requests */83 struct xarray tlb_lookup;84 85 /**86 * @serial_slot: id to the initial waiter created in tlb_lookup,87 * which is used only when failed to allocate new waiter.88 */89 u32 serial_slot;90 91 /** @next_seqno: the next id (sequence number) to allocate. */92 u32 next_seqno;93 94 /** @interrupts: pointers to GuC interrupt-managing functions. */95 struct {96 bool enabled;97 void (*reset)(struct intel_guc *guc);98 void (*enable)(struct intel_guc *guc);99 void (*disable)(struct intel_guc *guc);100 } interrupts;101 102 /**103 * @submission_state: sub-structure for submission state protected by104 * single lock105 */106 struct {107 /**108 * @submission_state.lock: protects everything in109 * submission_state, ce->guc_id.id, and ce->guc_id.ref110 * when transitioning in and out of zero111 */112 spinlock_t lock;113 /**114 * @submission_state.guc_ids: used to allocate new115 * guc_ids, single-lrc116 */117 struct ida guc_ids;118 /**119 * @submission_state.num_guc_ids: Number of guc_ids, selftest120 * feature to be able to reduce this number while testing.121 */122 int num_guc_ids;123 /**124 * @submission_state.guc_ids_bitmap: used to allocate125 * new guc_ids, multi-lrc126 */127 unsigned long *guc_ids_bitmap;128 /**129 * @submission_state.guc_id_list: list of intel_context130 * with valid guc_ids but no refs131 */132 struct list_head guc_id_list;133 /**134 * @submission_state.guc_ids_in_use: Number single-lrc135 * guc_ids in use136 */137 unsigned int guc_ids_in_use;138 /**139 * @submission_state.destroyed_contexts: list of contexts140 * waiting to be destroyed (deregistered with the GuC)141 */142 struct list_head destroyed_contexts;143 /**144 * @submission_state.destroyed_worker: worker to deregister145 * contexts, need as we need to take a GT PM reference and146 * can't from destroy function as it might be in an atomic147 * context (no sleeping)148 */149 struct work_struct destroyed_worker;150 /**151 * @submission_state.reset_fail_worker: worker to trigger152 * a GT reset after an engine reset fails153 */154 struct work_struct reset_fail_worker;155 /**156 * @submission_state.reset_fail_mask: mask of engines that157 * failed to reset158 */159 intel_engine_mask_t reset_fail_mask;160 /**161 * @submission_state.sched_disable_delay_ms: schedule162 * disable delay, in ms, for contexts163 */164 unsigned int sched_disable_delay_ms;165 /**166 * @submission_state.sched_disable_gucid_threshold:167 * threshold of min remaining available guc_ids before168 * we start bypassing the schedule disable delay169 */170 unsigned int sched_disable_gucid_threshold;171 } submission_state;172 173 /**174 * @submission_supported: tracks whether we support GuC submission on175 * the current platform176 */177 bool submission_supported;178 /** @submission_selected: tracks whether the user enabled GuC submission */179 bool submission_selected;180 /** @submission_initialized: tracks whether GuC submission has been initialised */181 bool submission_initialized;182 /** @submission_version: Submission API version of the currently loaded firmware */183 struct intel_uc_fw_ver submission_version;184 185 /**186 * @rc_supported: tracks whether we support GuC rc on the current platform187 */188 bool rc_supported;189 /** @rc_selected: tracks whether the user enabled GuC rc */190 bool rc_selected;191 192 /** @ads_vma: object allocated to hold the GuC ADS */193 struct i915_vma *ads_vma;194 /** @ads_map: contents of the GuC ADS */195 struct iosys_map ads_map;196 /** @ads_regset_size: size of the save/restore regsets in the ADS */197 u32 ads_regset_size;198 /**199 * @ads_regset_count: number of save/restore registers in the ADS for200 * each engine201 */202 u32 ads_regset_count[I915_NUM_ENGINES];203 /** @ads_regset: save/restore regsets in the ADS */204 struct guc_mmio_reg *ads_regset;205 /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */206 u32 ads_golden_ctxt_size;207 /** @ads_waklv_size: size of workaround KLVs */208 u32 ads_waklv_size;209 /** @ads_capture_size: size of register lists in the ADS used for error capture */210 u32 ads_capture_size;211 212 /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */213 struct i915_vma *lrc_desc_pool_v69;214 /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */215 void *lrc_desc_pool_vaddr_v69;216 217 /**218 * @context_lookup: used to resolve intel_context from guc_id, if a219 * context is present in this structure it is registered with the GuC220 */221 struct xarray context_lookup;222 223 /** @params: Control params for fw initialization */224 u32 params[GUC_CTL_MAX_DWORDS];225 226 /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */227 struct {228 u32 base;229 unsigned int count;230 enum forcewake_domains fw_domains;231 } send_regs;232 233 /** @notify_reg: register used to send interrupts to the GuC FW */234 i915_reg_t notify_reg;235 236 /**237 * @mmio_msg: notification bitmask that the GuC writes in one of its238 * registers when the CT channel is disabled, to be processed when the239 * channel is back up.240 */241 u32 mmio_msg;242 243 /** @send_mutex: used to serialize the intel_guc_send actions */244 struct mutex send_mutex;245 246 /**247 * @timestamp: GT timestamp object that stores a copy of the timestamp248 * and adjusts it for overflow using a worker.249 */250 struct {251 /**252 * @timestamp.lock: Lock protecting the below fields and253 * the engine stats.254 */255 spinlock_t lock;256 257 /**258 * @timestamp.gt_stamp: 64-bit extended value of the GT259 * timestamp.260 */261 u64 gt_stamp;262 263 /**264 * @timestamp.ping_delay: Period for polling the GT265 * timestamp for overflow.266 */267 unsigned long ping_delay;268 269 /**270 * @timestamp.work: Periodic work to adjust GT timestamp,271 * engine and context usage for overflows.272 */273 struct delayed_work work;274 275 /**276 * @timestamp.shift: Right shift value for the gpm timestamp277 */278 u32 shift;279 280 /**281 * @timestamp.last_stat_jiffies: jiffies at last actual282 * stats collection time. We use this timestamp to ensure283 * we don't oversample the stats because runtime power284 * management events can trigger stats collection at much285 * higher rates than required.286 */287 unsigned long last_stat_jiffies;288 } timestamp;289 290 /**291 * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset.292 * Specifically used when the G2H handler wants to issue a reset. Resets293 * require flushing the G2H queue. So, the G2H processing itself must not294 * trigger a reset directly. Instead, go via this worker.295 */296 struct work_struct dead_guc_worker;297 /**298 * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance299 * used to prevent a fundamentally broken system from continuously300 * reloading the GuC.301 */302 unsigned long last_dead_guc_jiffies;303 304#ifdef CONFIG_DRM_I915_SELFTEST305 /**306 * @number_guc_id_stolen: The number of guc_ids that have been stolen307 */308 int number_guc_id_stolen;309 /**310 * @fast_response_selftest: Backdoor to CT handler for fast response selftest311 */312 u32 fast_response_selftest;313#endif314};315 316struct intel_guc_tlb_wait {317 struct wait_queue_head wq;318 bool busy;319};320 321/*322 * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8323 * integer works.324 */325#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat))326#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)327#define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version)328#define GUC_FIRMWARE_VER(guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)329 330static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)331{332 return container_of(log, struct intel_guc, log);333}334 335static336inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)337{338 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0);339}340 341static342inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len,343 u32 g2h_len_dw)344{345 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0,346 MAKE_SEND_FLAGS(g2h_len_dw));347}348 349static inline int350intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len,351 u32 *response_buf, u32 response_buf_size)352{353 return intel_guc_ct_send(&guc->ct, action, len,354 response_buf, response_buf_size, 0);355}356 357static inline int intel_guc_send_busy_loop(struct intel_guc *guc,358 const u32 *action,359 u32 len,360 u32 g2h_len_dw,361 bool loop)362{363 int err;364 unsigned int sleep_period_ms = 1;365 bool not_atomic = !in_atomic() && !irqs_disabled();366 367 /*368 * FIXME: Have caller pass in if we are in an atomic context to avoid369 * using in_atomic(). It is likely safe here as we check for irqs370 * disabled which basically all the spin locks in the i915 do but371 * regardless this should be cleaned up.372 */373 374 /* No sleeping with spin locks, just busy loop */375 might_sleep_if(loop && not_atomic);376 377retry:378 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);379 if (unlikely(err == -EBUSY && loop)) {380 if (likely(not_atomic)) {381 if (msleep_interruptible(sleep_period_ms))382 return -EINTR;383 sleep_period_ms = sleep_period_ms << 1;384 } else {385 cpu_relax();386 }387 goto retry;388 }389 390 return err;391}392 393/* Only call this from the interrupt handler code */394static inline void intel_guc_to_host_event_handler(struct intel_guc *guc)395{396 if (guc->interrupts.enabled)397 intel_guc_ct_event_handler(&guc->ct);398}399 400/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */401#define GUC_GGTT_TOP 0xFEE00000402 403/**404 * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma405 * @guc: intel_guc structure.406 * @vma: i915 graphics virtual memory area.407 *408 * GuC does not allow any gfx GGTT address that falls into range409 * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.410 * Currently, in order to exclude [0, ggtt.pin_bias) address space from411 * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()412 * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias.413 *414 * Return: GGTT offset of the @vma.415 */416static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,417 struct i915_vma *vma)418{419 u32 offset = i915_ggtt_offset(vma);420 421 GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma));422 GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP));423 424 return offset;425}426 427void intel_guc_init_early(struct intel_guc *guc);428void intel_guc_init_late(struct intel_guc *guc);429void intel_guc_init_send_regs(struct intel_guc *guc);430void intel_guc_write_params(struct intel_guc *guc);431int intel_guc_init(struct intel_guc *guc);432void intel_guc_fini(struct intel_guc *guc);433void intel_guc_notify(struct intel_guc *guc);434int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,435 u32 *response_buf, u32 response_buf_size);436int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,437 const u32 *payload, u32 len);438int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);439int intel_guc_suspend(struct intel_guc *guc);440int intel_guc_resume(struct intel_guc *guc);441struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);442int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,443 struct i915_vma **out_vma, void **out_vaddr);444int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);445int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);446 447static inline bool intel_guc_is_supported(struct intel_guc *guc)448{449 return intel_uc_fw_is_supported(&guc->fw);450}451 452static inline bool intel_guc_is_wanted(struct intel_guc *guc)453{454 return intel_uc_fw_is_enabled(&guc->fw);455}456 457static inline bool intel_guc_is_used(struct intel_guc *guc)458{459 GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED);460 return intel_uc_fw_is_available(&guc->fw);461}462 463static inline bool intel_guc_is_fw_running(struct intel_guc *guc)464{465 return intel_uc_fw_is_running(&guc->fw);466}467 468static inline bool intel_guc_is_ready(struct intel_guc *guc)469{470 return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct);471}472 473static inline void intel_guc_reset_interrupts(struct intel_guc *guc)474{475 guc->interrupts.reset(guc);476}477 478static inline void intel_guc_enable_interrupts(struct intel_guc *guc)479{480 guc->interrupts.enable(guc);481}482 483static inline void intel_guc_disable_interrupts(struct intel_guc *guc)484{485 guc->interrupts.disable(guc);486}487 488static inline int intel_guc_sanitize(struct intel_guc *guc)489{490 intel_uc_fw_sanitize(&guc->fw);491 intel_guc_disable_interrupts(guc);492 intel_guc_ct_sanitize(&guc->ct);493 guc->mmio_msg = 0;494 495 return 0;496}497 498static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask)499{500 spin_lock_irq(&guc->irq_lock);501 guc->msg_enabled_mask |= mask;502 spin_unlock_irq(&guc->irq_lock);503}504 505static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask)506{507 spin_lock_irq(&guc->irq_lock);508 guc->msg_enabled_mask &= ~mask;509 spin_unlock_irq(&guc->irq_lock);510}511 512int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout);513 514int intel_guc_deregister_done_process_msg(struct intel_guc *guc,515 const u32 *msg, u32 len);516int intel_guc_sched_done_process_msg(struct intel_guc *guc,517 const u32 *msg, u32 len);518int intel_guc_context_reset_process_msg(struct intel_guc *guc,519 const u32 *msg, u32 len);520int intel_guc_engine_failure_process_msg(struct intel_guc *guc,521 const u32 *msg, u32 len);522int intel_guc_error_capture_process_msg(struct intel_guc *guc,523 const u32 *msg, u32 len);524int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action);525 526struct intel_engine_cs *527intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance);528 529void intel_guc_find_hung_context(struct intel_engine_cs *engine);530 531int intel_guc_global_policies_update(struct intel_guc *guc);532 533void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq);534 535void intel_guc_submission_reset_prepare(struct intel_guc *guc);536void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled);537void intel_guc_submission_reset_finish(struct intel_guc *guc);538void intel_guc_submission_cancel_requests(struct intel_guc *guc);539 540void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p);541 542void intel_guc_write_barrier(struct intel_guc *guc);543 544void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);545 546int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);547 548bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc);549int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);550int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);551int intel_guc_tlb_invalidation_done(struct intel_guc *guc,552 const u32 *payload, u32 len);553void wake_up_all_tlb_invalidate(struct intel_guc *guc);554#endif555