1587 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_guc_ct.h"7 8#include <linux/bitfield.h>9#include <linux/circ_buf.h>10#include <linux/delay.h>11 12#include <kunit/static_stub.h>13 14#include <drm/drm_managed.h>15 16#include "abi/guc_actions_abi.h"17#include "abi/guc_actions_sriov_abi.h"18#include "abi/guc_klvs_abi.h"19#include "xe_bo.h"20#include "xe_device.h"21#include "xe_gt.h"22#include "xe_gt_pagefault.h"23#include "xe_gt_printk.h"24#include "xe_gt_sriov_pf_control.h"25#include "xe_gt_sriov_pf_monitor.h"26#include "xe_gt_tlb_invalidation.h"27#include "xe_guc.h"28#include "xe_guc_relay.h"29#include "xe_guc_submit.h"30#include "xe_map.h"31#include "xe_pm.h"32#include "xe_trace_guc.h"33 34/* Used when a CT send wants to block and / or receive data */35struct g2h_fence {36 u32 *response_buffer;37 u32 seqno;38 u32 response_data;39 u16 response_len;40 u16 error;41 u16 hint;42 u16 reason;43 bool retry;44 bool fail;45 bool done;46};47 48static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)49{50 g2h_fence->response_buffer = response_buffer;51 g2h_fence->response_data = 0;52 g2h_fence->response_len = 0;53 g2h_fence->fail = false;54 g2h_fence->retry = false;55 g2h_fence->done = false;56 g2h_fence->seqno = ~0x0;57}58 59static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)60{61 return g2h_fence->seqno == ~0x0;62}63 64static struct xe_guc *65ct_to_guc(struct xe_guc_ct *ct)66{67 return container_of(ct, struct xe_guc, ct);68}69 70static struct xe_gt *71ct_to_gt(struct xe_guc_ct *ct)72{73 return container_of(ct, struct xe_gt, uc.guc.ct);74}75 76static struct xe_device *77ct_to_xe(struct xe_guc_ct *ct)78{79 return gt_to_xe(ct_to_gt(ct));80}81 82/**83 * DOC: GuC CTB Blob84 *85 * We allocate single blob to hold both CTB descriptors and buffers:86 *87 * +--------+-----------------------------------------------+------+88 * | offset | contents | size |89 * +========+===============================================+======+90 * | 0x0000 | H2G CTB Descriptor (send) | |91 * +--------+-----------------------------------------------+ 4K |92 * | 0x0800 | G2H CTB Descriptor (g2h) | |93 * +--------+-----------------------------------------------+------+94 * | 0x1000 | H2G CT Buffer (send) | n*4K |95 * | | | |96 * +--------+-----------------------------------------------+------+97 * | 0x1000 | G2H CT Buffer (g2h) | m*4K |98 * | + n*4K | | |99 * +--------+-----------------------------------------------+------+100 *101 * Size of each ``CT Buffer`` must be multiple of 4K.102 * We don't expect too many messages in flight at any time, unless we are103 * using the GuC submission. In that case each request requires a minimum104 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this105 * enough space to avoid backpressure on the driver. We increase the size106 * of the receive buffer (relative to the send) to ensure a G2H response107 * CTB has a landing spot.108 *109 * In addition to submissions, the G2H buffer needs to be able to hold110 * enough space for recoverable page fault notifications. The number of111 * page faults is interrupt driven and can be as much as the number of112 * compute resources available. However, most of the actual work for these113 * is in a separate page fault worker thread. Therefore we only need to114 * make sure the queue has enough space to handle all of the submissions115 * and responses and an extra buffer for incoming page faults.116 */117 118#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)119#define CTB_H2G_BUFFER_SIZE (SZ_4K)120#define CTB_G2H_BUFFER_SIZE (SZ_128K)121#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)122 123/**124 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full125 * CT command queue126 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future.127 *128 * Observation is that a 4KiB buffer full of commands takes a little over a129 * second to process. Use that to calculate maximum time to process a full CT130 * command queue.131 *132 * Return: Maximum time to process a full CT queue in jiffies.133 */134long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct)135{136 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4));137 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ;138}139 140static size_t guc_ct_size(void)141{142 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +143 CTB_G2H_BUFFER_SIZE;144}145 146static void guc_ct_fini(struct drm_device *drm, void *arg)147{148 struct xe_guc_ct *ct = arg;149 150 destroy_workqueue(ct->g2h_wq);151 xa_destroy(&ct->fence_lookup);152}153 154static void receive_g2h(struct xe_guc_ct *ct);155static void g2h_worker_func(struct work_struct *w);156static void safe_mode_worker_func(struct work_struct *w);157 158static void primelockdep(struct xe_guc_ct *ct)159{160 if (!IS_ENABLED(CONFIG_LOCKDEP))161 return;162 163 fs_reclaim_acquire(GFP_KERNEL);164 might_lock(&ct->lock);165 fs_reclaim_release(GFP_KERNEL);166}167 168int xe_guc_ct_init(struct xe_guc_ct *ct)169{170 struct xe_device *xe = ct_to_xe(ct);171 struct xe_gt *gt = ct_to_gt(ct);172 struct xe_tile *tile = gt_to_tile(gt);173 struct xe_bo *bo;174 int err;175 176 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE));177 178 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0);179 if (!ct->g2h_wq)180 return -ENOMEM;181 182 spin_lock_init(&ct->fast_lock);183 xa_init(&ct->fence_lookup);184 INIT_WORK(&ct->g2h_worker, g2h_worker_func);185 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);186 init_waitqueue_head(&ct->wq);187 init_waitqueue_head(&ct->g2h_fence_wq);188 189 err = drmm_mutex_init(&xe->drm, &ct->lock);190 if (err)191 return err;192 193 primelockdep(ct);194 195 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),196 XE_BO_FLAG_SYSTEM |197 XE_BO_FLAG_GGTT |198 XE_BO_FLAG_GGTT_INVALIDATE);199 if (IS_ERR(bo))200 return PTR_ERR(bo);201 202 ct->bo = bo;203 204 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);205 if (err)206 return err;207 208 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED);209 ct->state = XE_GUC_CT_STATE_DISABLED;210 return 0;211}212 213#define desc_read(xe_, guc_ctb__, field_) \214 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \215 struct guc_ct_buffer_desc, field_)216 217#define desc_write(xe_, guc_ctb__, field_, val_) \218 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \219 struct guc_ct_buffer_desc, field_, val_)220 221static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,222 struct iosys_map *map)223{224 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);225 h2g->info.resv_space = 0;226 h2g->info.tail = 0;227 h2g->info.head = 0;228 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,229 h2g->info.size) -230 h2g->info.resv_space;231 h2g->info.broken = false;232 233 h2g->desc = *map;234 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));235 236 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);237}238 239static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,240 struct iosys_map *map)241{242 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);243 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);244 g2h->info.head = 0;245 g2h->info.tail = 0;246 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,247 g2h->info.size) -248 g2h->info.resv_space;249 g2h->info.broken = false;250 251 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);252 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));253 254 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +255 CTB_H2G_BUFFER_SIZE);256}257 258static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)259{260 struct xe_guc *guc = ct_to_guc(ct);261 u32 desc_addr, ctb_addr, size;262 int err;263 264 desc_addr = xe_bo_ggtt_addr(ct->bo);265 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;266 size = ct->ctbs.h2g.info.size * sizeof(u32);267 268 err = xe_guc_self_cfg64(guc,269 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,270 desc_addr);271 if (err)272 return err;273 274 err = xe_guc_self_cfg64(guc,275 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,276 ctb_addr);277 if (err)278 return err;279 280 return xe_guc_self_cfg32(guc,281 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,282 size);283}284 285static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)286{287 struct xe_guc *guc = ct_to_guc(ct);288 u32 desc_addr, ctb_addr, size;289 int err;290 291 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;292 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +293 CTB_H2G_BUFFER_SIZE;294 size = ct->ctbs.g2h.info.size * sizeof(u32);295 296 err = xe_guc_self_cfg64(guc,297 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,298 desc_addr);299 if (err)300 return err;301 302 err = xe_guc_self_cfg64(guc,303 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,304 ctb_addr);305 if (err)306 return err;307 308 return xe_guc_self_cfg32(guc,309 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,310 size);311}312 313static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)314{315 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {316 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |317 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |318 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,319 GUC_ACTION_HOST2GUC_CONTROL_CTB),320 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,321 enable ? GUC_CTB_CONTROL_ENABLE :322 GUC_CTB_CONTROL_DISABLE),323 };324 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));325 326 return ret > 0 ? -EPROTO : ret;327}328 329static void xe_guc_ct_set_state(struct xe_guc_ct *ct,330 enum xe_guc_ct_state state)331{332 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */333 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */334 335 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||336 state == XE_GUC_CT_STATE_STOPPED);337 338 if (ct->g2h_outstanding)339 xe_pm_runtime_put(ct_to_xe(ct));340 ct->g2h_outstanding = 0;341 ct->state = state;342 343 spin_unlock_irq(&ct->fast_lock);344 345 /*346 * Lockdep doesn't like this under the fast lock and he destroy only347 * needs to be serialized with the send path which ct lock provides.348 */349 xa_destroy(&ct->fence_lookup);350 351 mutex_unlock(&ct->lock);352}353 354static bool ct_needs_safe_mode(struct xe_guc_ct *ct)355{356 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev));357}358 359static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct)360{361 if (!ct_needs_safe_mode(ct))362 return false;363 364 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10);365 return true;366}367 368static void safe_mode_worker_func(struct work_struct *w)369{370 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work);371 372 receive_g2h(ct);373 374 if (!ct_restart_safe_mode_worker(ct))375 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n");376}377 378static void ct_enter_safe_mode(struct xe_guc_ct *ct)379{380 if (ct_restart_safe_mode_worker(ct))381 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n");382}383 384static void ct_exit_safe_mode(struct xe_guc_ct *ct)385{386 if (cancel_delayed_work_sync(&ct->safe_mode_worker))387 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n");388}389 390int xe_guc_ct_enable(struct xe_guc_ct *ct)391{392 struct xe_device *xe = ct_to_xe(ct);393 struct xe_gt *gt = ct_to_gt(ct);394 int err;395 396 xe_gt_assert(gt, !xe_guc_ct_enabled(ct));397 398 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);399 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);400 401 err = guc_ct_ctb_h2g_register(ct);402 if (err)403 goto err_out;404 405 err = guc_ct_ctb_g2h_register(ct);406 if (err)407 goto err_out;408 409 err = guc_ct_control_toggle(ct, true);410 if (err)411 goto err_out;412 413 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED);414 415 smp_mb();416 wake_up_all(&ct->wq);417 xe_gt_dbg(gt, "GuC CT communication channel enabled\n");418 419 if (ct_needs_safe_mode(ct))420 ct_enter_safe_mode(ct);421 422 return 0;423 424err_out:425 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err));426 427 return err;428}429 430static void stop_g2h_handler(struct xe_guc_ct *ct)431{432 cancel_work_sync(&ct->g2h_worker);433}434 435/**436 * xe_guc_ct_disable - Set GuC to disabled state437 * @ct: the &xe_guc_ct438 *439 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected440 * in this transition.441 */442void xe_guc_ct_disable(struct xe_guc_ct *ct)443{444 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED);445 ct_exit_safe_mode(ct);446 stop_g2h_handler(ct);447}448 449/**450 * xe_guc_ct_stop - Set GuC to stopped state451 * @ct: the &xe_guc_ct452 *453 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h454 */455void xe_guc_ct_stop(struct xe_guc_ct *ct)456{457 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED);458 stop_g2h_handler(ct);459}460 461static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)462{463 struct guc_ctb *h2g = &ct->ctbs.h2g;464 465 lockdep_assert_held(&ct->lock);466 467 if (cmd_len > h2g->info.space) {468 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);469 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,470 h2g->info.size) -471 h2g->info.resv_space;472 if (cmd_len > h2g->info.space)473 return false;474 }475 476 return true;477}478 479static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)480{481 if (!g2h_len)482 return true;483 484 lockdep_assert_held(&ct->fast_lock);485 486 return ct->ctbs.g2h.info.space > g2h_len;487}488 489static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)490{491 lockdep_assert_held(&ct->lock);492 493 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))494 return -EBUSY;495 496 return 0;497}498 499static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)500{501 lockdep_assert_held(&ct->lock);502 ct->ctbs.h2g.info.space -= cmd_len;503}504 505static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)506{507 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);508 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||509 (g2h_len && num_g2h));510 511 if (g2h_len) {512 lockdep_assert_held(&ct->fast_lock);513 514 if (!ct->g2h_outstanding)515 xe_pm_runtime_get_noresume(ct_to_xe(ct));516 517 ct->ctbs.g2h.info.space -= g2h_len;518 ct->g2h_outstanding += num_g2h;519 }520}521 522static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)523{524 lockdep_assert_held(&ct->fast_lock);525 xe_gt_assert(ct_to_gt(ct), ct->ctbs.g2h.info.space + g2h_len <=526 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);527 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding);528 529 ct->ctbs.g2h.info.space += g2h_len;530 if (!--ct->g2h_outstanding)531 xe_pm_runtime_put(ct_to_xe(ct));532}533 534static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)535{536 spin_lock_irq(&ct->fast_lock);537 __g2h_release_space(ct, g2h_len);538 spin_unlock_irq(&ct->fast_lock);539}540 541#define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */542 543static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,544 u32 ct_fence_value, bool want_response)545{546 struct xe_device *xe = ct_to_xe(ct);547 struct xe_gt *gt = ct_to_gt(ct);548 struct guc_ctb *h2g = &ct->ctbs.h2g;549 u32 cmd[H2G_CT_HEADERS];550 u32 tail = h2g->info.tail;551 u32 full_len;552 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,553 tail * sizeof(u32));554 555 full_len = len + GUC_CTB_HDR_LEN;556 557 lockdep_assert_held(&ct->lock);558 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);559 xe_gt_assert(gt, tail <= h2g->info.size);560 561 /* Command will wrap, zero fill (NOPs), return and check credits again */562 if (tail + full_len > h2g->info.size) {563 xe_map_memset(xe, &map, 0, 0,564 (h2g->info.size - tail) * sizeof(u32));565 h2g_reserve_space(ct, (h2g->info.size - tail));566 h2g->info.tail = 0;567 desc_write(xe, h2g, tail, h2g->info.tail);568 569 return -EAGAIN;570 }571 572 /*573 * dw0: CT header (including fence)574 * dw1: HXG header (including action code)575 * dw2+: action data576 */577 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |578 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |579 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);580 if (want_response) {581 cmd[1] =582 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |583 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |584 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);585 } else {586 cmd[1] =587 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) |588 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |589 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);590 }591 592 /* H2G header in cmd[1] replaces action[0] so: */593 --len;594 ++action;595 596 /* Write H2G ensuring visable before descriptor update */597 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));598 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));599 xe_device_wmb(xe);600 601 /* Update local copies */602 h2g->info.tail = (tail + full_len) % h2g->info.size;603 h2g_reserve_space(ct, full_len);604 605 /* Update descriptor */606 desc_write(xe, h2g, tail, h2g->info.tail);607 608 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len,609 desc_read(xe, h2g, head), h2g->info.tail);610 611 return 0;612}613 614/*615 * The CT protocol accepts a 16 bits fence. This field is fully owned by the616 * driver, the GuC will just copy it to the reply message. Since we need to617 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages,618 * we use one bit of the seqno as an indicator for that and a rolling counter619 * for the remaining 15 bits.620 */621#define CT_SEQNO_MASK GENMASK(14, 0)622#define CT_SEQNO_UNTRACKED BIT(15)623static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence)624{625 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK;626 627 if (!is_g2h_fence)628 seqno |= CT_SEQNO_UNTRACKED;629 630 return seqno;631}632 633static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,634 u32 len, u32 g2h_len, u32 num_g2h,635 struct g2h_fence *g2h_fence)636{637 struct xe_gt *gt __maybe_unused = ct_to_gt(ct);638 u16 seqno;639 int ret;640 641 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);642 xe_gt_assert(gt, !g2h_len || !g2h_fence);643 xe_gt_assert(gt, !num_g2h || !g2h_fence);644 xe_gt_assert(gt, !g2h_len || num_g2h);645 xe_gt_assert(gt, g2h_len || !num_g2h);646 lockdep_assert_held(&ct->lock);647 648 if (unlikely(ct->ctbs.h2g.info.broken)) {649 ret = -EPIPE;650 goto out;651 }652 653 if (ct->state == XE_GUC_CT_STATE_DISABLED) {654 ret = -ENODEV;655 goto out;656 }657 658 if (ct->state == XE_GUC_CT_STATE_STOPPED) {659 ret = -ECANCELED;660 goto out;661 }662 663 xe_gt_assert(gt, xe_guc_ct_enabled(ct));664 665 if (g2h_fence) {666 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;667 num_g2h = 1;668 669 if (g2h_fence_needs_alloc(g2h_fence)) {670 g2h_fence->seqno = next_ct_seqno(ct, true);671 ret = xa_err(xa_store(&ct->fence_lookup,672 g2h_fence->seqno, g2h_fence,673 GFP_ATOMIC));674 if (ret)675 goto out;676 }677 678 seqno = g2h_fence->seqno;679 } else {680 seqno = next_ct_seqno(ct, false);681 }682 683 if (g2h_len)684 spin_lock_irq(&ct->fast_lock);685retry:686 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);687 if (unlikely(ret))688 goto out_unlock;689 690 ret = h2g_write(ct, action, len, seqno, !!g2h_fence);691 if (unlikely(ret)) {692 if (ret == -EAGAIN)693 goto retry;694 goto out_unlock;695 }696 697 __g2h_reserve_space(ct, g2h_len, num_g2h);698 xe_guc_notify(ct_to_guc(ct));699out_unlock:700 if (g2h_len)701 spin_unlock_irq(&ct->fast_lock);702out:703 return ret;704}705 706static void kick_reset(struct xe_guc_ct *ct)707{708 xe_gt_reset_async(ct_to_gt(ct));709}710 711static int dequeue_one_g2h(struct xe_guc_ct *ct);712 713static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,714 u32 g2h_len, u32 num_g2h,715 struct g2h_fence *g2h_fence)716{717 struct xe_device *xe = ct_to_xe(ct);718 struct xe_gt *gt = ct_to_gt(ct);719 struct drm_printer p = xe_gt_info_printer(gt);720 unsigned int sleep_period_ms = 1;721 int ret;722 723 xe_gt_assert(gt, !g2h_len || !g2h_fence);724 lockdep_assert_held(&ct->lock);725 xe_device_assert_mem_access(ct_to_xe(ct));726 727try_again:728 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,729 g2h_fence);730 731 /*732 * We wait to try to restore credits for about 1 second before bailing.733 * In the case of H2G credits we have no choice but just to wait for the734 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In735 * the case of G2H we process any G2H in the channel, hopefully freeing736 * credits as we consume the G2H messages.737 */738 if (unlikely(ret == -EBUSY &&739 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {740 struct guc_ctb *h2g = &ct->ctbs.h2g;741 742 if (sleep_period_ms == 1024)743 goto broken;744 745 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail,746 h2g->info.size,747 h2g->info.space,748 len + GUC_CTB_HDR_LEN);749 msleep(sleep_period_ms);750 sleep_period_ms <<= 1;751 752 goto try_again;753 } else if (unlikely(ret == -EBUSY)) {754 struct xe_device *xe = ct_to_xe(ct);755 struct guc_ctb *g2h = &ct->ctbs.g2h;756 757 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head,758 desc_read(xe, g2h, tail),759 g2h->info.size,760 g2h->info.space,761 g2h_fence ?762 GUC_CTB_HXG_MSG_MAX_LEN :763 g2h_len);764 765#define g2h_avail(ct) \766 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)767 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||768 g2h_avail(ct), HZ))769 goto broken;770#undef g2h_avail771 772 if (dequeue_one_g2h(ct) < 0)773 goto broken;774 775 goto try_again;776 }777 778 return ret;779 780broken:781 xe_gt_err(gt, "No forward process on H2G, reset required\n");782 xe_guc_ct_print(ct, &p, true);783 ct->ctbs.h2g.info.broken = true;784 785 return -EDEADLK;786}787 788static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,789 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)790{791 int ret;792 793 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);794 795 mutex_lock(&ct->lock);796 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);797 mutex_unlock(&ct->lock);798 799 return ret;800}801 802int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,803 u32 g2h_len, u32 num_g2h)804{805 int ret;806 807 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);808 if (ret == -EDEADLK)809 kick_reset(ct);810 811 return ret;812}813 814int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,815 u32 g2h_len, u32 num_g2h)816{817 int ret;818 819 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);820 if (ret == -EDEADLK)821 kick_reset(ct);822 823 return ret;824}825 826int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)827{828 int ret;829 830 lockdep_assert_held(&ct->lock);831 832 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);833 if (ret == -EDEADLK)834 kick_reset(ct);835 836 return ret;837}838 839/*840 * Check if a GT reset is in progress or will occur and if GT reset brought the841 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.842 */843static bool retry_failure(struct xe_guc_ct *ct, int ret)844{845 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))846 return false;847 848#define ct_alive(ct) \849 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \850 !ct->ctbs.g2h.info.broken)851 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))852 return false;853#undef ct_alive854 855 return true;856}857 858static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,859 u32 *response_buffer, bool no_fail)860{861 struct xe_gt *gt = ct_to_gt(ct);862 struct g2h_fence g2h_fence;863 int ret = 0;864 865 /*866 * We use a fence to implement blocking sends / receiving response data.867 * The seqno of the fence is sent in the H2G, returned in the G2H, and868 * an xarray is used as storage media with the seqno being to key.869 * Fields in the fence hold success, failure, retry status and the870 * response data. Safe to allocate on the stack as the xarray is the871 * only reference and it cannot be present after this function exits.872 */873retry:874 g2h_fence_init(&g2h_fence, response_buffer);875retry_same_fence:876 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);877 if (unlikely(ret == -ENOMEM)) {878 /* Retry allocation /w GFP_KERNEL */879 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno,880 &g2h_fence, GFP_KERNEL));881 if (ret)882 return ret;883 884 goto retry_same_fence;885 } else if (unlikely(ret)) {886 if (ret == -EDEADLK)887 kick_reset(ct);888 889 if (no_fail && retry_failure(ct, ret))890 goto retry_same_fence;891 892 if (!g2h_fence_needs_alloc(&g2h_fence))893 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);894 895 return ret;896 }897 898 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);899 900 if (!ret) {901 LNL_FLUSH_WORK(&ct->g2h_worker);902 if (g2h_fence.done) {903 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n",904 g2h_fence.seqno, action[0]);905 ret = 1;906 }907 }908 909 /*910 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on911 * the stack, since we have no clue if it will fire after the timeout before we can erase912 * from the xa. Also we have some dependent loads and stores below for which we need the913 * correct ordering, and we lack the needed barriers.914 */915 mutex_lock(&ct->lock);916 if (!ret) {917 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s",918 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done));919 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);920 mutex_unlock(&ct->lock);921 return -ETIME;922 }923 924 if (g2h_fence.retry) {925 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",926 action[0], g2h_fence.reason);927 mutex_unlock(&ct->lock);928 goto retry;929 }930 if (g2h_fence.fail) {931 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",932 action[0], g2h_fence.error, g2h_fence.hint);933 ret = -EIO;934 }935 936 if (ret > 0)937 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data;938 939 mutex_unlock(&ct->lock);940 941 return ret;942}943 944/**945 * xe_guc_ct_send_recv - Send and receive HXG to the GuC946 * @ct: the &xe_guc_ct947 * @action: the dword array with `HXG Request`_ message (can't be NULL)948 * @len: length of the `HXG Request`_ message (in dwords, can't be 0)949 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL)950 *951 * Send a `HXG Request`_ message to the GuC over CT communication channel and952 * blocks until GuC replies with a `HXG Response`_ message.953 *954 * For non-blocking communication with GuC use xe_guc_ct_send().955 *956 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_.957 *958 * Return: response length (in dwords) if &response_buffer was not NULL, or959 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or960 * a negative error code on failure.961 */962int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,963 u32 *response_buffer)964{965 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer);966 return guc_ct_send_recv(ct, action, len, response_buffer, false);967}968 969int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,970 u32 len, u32 *response_buffer)971{972 return guc_ct_send_recv(ct, action, len, response_buffer, true);973}974 975static u32 *msg_to_hxg(u32 *msg)976{977 return msg + GUC_CTB_MSG_MIN_LEN;978}979 980static u32 msg_len_to_hxg_len(u32 len)981{982 return len - GUC_CTB_MSG_MIN_LEN;983}984 985static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)986{987 u32 *hxg = msg_to_hxg(msg);988 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);989 990 lockdep_assert_held(&ct->lock);991 992 switch (action) {993 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:994 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:995 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:996 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:997 g2h_release_space(ct, len);998 }999 1000 return 0;1001}1002 1003static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)1004{1005 struct xe_gt *gt = ct_to_gt(ct);1006 u32 *hxg = msg_to_hxg(msg);1007 u32 hxg_len = msg_len_to_hxg_len(len);1008 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);1009 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);1010 struct g2h_fence *g2h_fence;1011 1012 lockdep_assert_held(&ct->lock);1013 1014 /*1015 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup.1016 * Those messages should never fail, so if we do get an error back it1017 * means we're likely doing an illegal operation and the GuC is1018 * rejecting it. We have no way to inform the code that submitted the1019 * H2G that the message was rejected, so we need to escalate the1020 * failure to trigger a reset.1021 */1022 if (fence & CT_SEQNO_UNTRACKED) {1023 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE)1024 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n",1025 fence,1026 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]),1027 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]));1028 else1029 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n",1030 type, fence);1031 1032 return -EPROTO;1033 }1034 1035 g2h_fence = xa_erase(&ct->fence_lookup, fence);1036 if (unlikely(!g2h_fence)) {1037 /* Don't tear down channel, as send could've timed out */1038 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence);1039 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);1040 return 0;1041 }1042 1043 xe_gt_assert(gt, fence == g2h_fence->seqno);1044 1045 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {1046 g2h_fence->fail = true;1047 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);1048 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]);1049 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {1050 g2h_fence->retry = true;1051 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);1052 } else if (g2h_fence->response_buffer) {1053 g2h_fence->response_len = hxg_len;1054 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));1055 } else {1056 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);1057 }1058 1059 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);1060 1061 g2h_fence->done = true;1062 smp_mb();1063 1064 wake_up_all(&ct->g2h_fence_wq);1065 1066 return 0;1067}1068 1069static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)1070{1071 struct xe_gt *gt = ct_to_gt(ct);1072 u32 *hxg = msg_to_hxg(msg);1073 u32 origin, type;1074 int ret;1075 1076 lockdep_assert_held(&ct->lock);1077 1078 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);1079 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {1080 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n",1081 origin);1082 ct->ctbs.g2h.info.broken = true;1083 1084 return -EPROTO;1085 }1086 1087 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);1088 switch (type) {1089 case GUC_HXG_TYPE_EVENT:1090 ret = parse_g2h_event(ct, msg, len);1091 break;1092 case GUC_HXG_TYPE_RESPONSE_SUCCESS:1093 case GUC_HXG_TYPE_RESPONSE_FAILURE:1094 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:1095 ret = parse_g2h_response(ct, msg, len);1096 break;1097 default:1098 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n",1099 type);1100 ct->ctbs.g2h.info.broken = true;1101 1102 ret = -EOPNOTSUPP;1103 }1104 1105 return ret;1106}1107 1108static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)1109{1110 struct xe_guc *guc = ct_to_guc(ct);1111 struct xe_gt *gt = ct_to_gt(ct);1112 u32 hxg_len = msg_len_to_hxg_len(len);1113 u32 *hxg = msg_to_hxg(msg);1114 u32 action, adj_len;1115 u32 *payload;1116 int ret = 0;1117 1118 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)1119 return 0;1120 1121 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);1122 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN;1123 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN;1124 1125 switch (action) {1126 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:1127 ret = xe_guc_sched_done_handler(guc, payload, adj_len);1128 break;1129 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:1130 ret = xe_guc_deregister_done_handler(guc, payload, adj_len);1131 break;1132 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:1133 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);1134 break;1135 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:1136 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,1137 adj_len);1138 break;1139 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:1140 /* Selftest only at the moment */1141 break;1142 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:1143 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:1144 /* FIXME: Handle this */1145 break;1146 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:1147 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,1148 adj_len);1149 break;1150 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:1151 ret = xe_guc_pagefault_handler(guc, payload, adj_len);1152 break;1153 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:1154 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,1155 adj_len);1156 break;1157 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:1158 ret = xe_guc_access_counter_notify_handler(guc, payload,1159 adj_len);1160 break;1161 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:1162 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);1163 break;1164 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF:1165 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len);1166 break;1167 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY:1168 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len);1169 break;1170 case GUC_ACTION_GUC2PF_ADVERSE_EVENT:1171 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len);1172 break;1173 default:1174 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);1175 }1176 1177 if (ret)1178 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",1179 action, ERR_PTR(ret));1180 1181 return 0;1182}1183 1184static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)1185{1186 struct xe_device *xe = ct_to_xe(ct);1187 struct xe_gt *gt = ct_to_gt(ct);1188 struct guc_ctb *g2h = &ct->ctbs.g2h;1189 u32 tail, head, len;1190 s32 avail;1191 u32 action;1192 u32 *hxg;1193 1194 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED);1195 lockdep_assert_held(&ct->fast_lock);1196 1197 if (ct->state == XE_GUC_CT_STATE_DISABLED)1198 return -ENODEV;1199 1200 if (ct->state == XE_GUC_CT_STATE_STOPPED)1201 return -ECANCELED;1202 1203 if (g2h->info.broken)1204 return -EPIPE;1205 1206 xe_gt_assert(gt, xe_guc_ct_enabled(ct));1207 1208 /* Calculate DW available to read */1209 tail = desc_read(xe, g2h, tail);1210 avail = tail - g2h->info.head;1211 if (unlikely(avail == 0))1212 return 0;1213 1214 if (avail < 0)1215 avail += g2h->info.size;1216 1217 /* Read header */1218 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,1219 sizeof(u32));1220 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;1221 if (len > avail) {1222 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n",1223 avail, len);1224 g2h->info.broken = true;1225 1226 return -EPROTO;1227 }1228 1229 head = (g2h->info.head + 1) % g2h->info.size;1230 avail = len - 1;1231 1232 /* Read G2H message */1233 if (avail + head > g2h->info.size) {1234 u32 avail_til_wrap = g2h->info.size - head;1235 1236 xe_map_memcpy_from(xe, msg + 1,1237 &g2h->cmds, sizeof(u32) * head,1238 avail_til_wrap * sizeof(u32));1239 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,1240 &g2h->cmds, 0,1241 (avail - avail_til_wrap) * sizeof(u32));1242 } else {1243 xe_map_memcpy_from(xe, msg + 1,1244 &g2h->cmds, sizeof(u32) * head,1245 avail * sizeof(u32));1246 }1247 1248 hxg = msg_to_hxg(msg);1249 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);1250 1251 if (fast_path) {1252 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT)1253 return 0;1254 1255 switch (action) {1256 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:1257 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:1258 break; /* Process these in fast-path */1259 default:1260 return 0;1261 }1262 }1263 1264 /* Update local / descriptor header */1265 g2h->info.head = (head + avail) % g2h->info.size;1266 desc_write(xe, g2h, head, g2h->info.head);1267 1268 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id,1269 action, len, g2h->info.head, tail);1270 1271 return len;1272}1273 1274static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)1275{1276 struct xe_gt *gt = ct_to_gt(ct);1277 struct xe_guc *guc = ct_to_guc(ct);1278 u32 hxg_len = msg_len_to_hxg_len(len);1279 u32 *hxg = msg_to_hxg(msg);1280 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);1281 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN;1282 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN;1283 int ret = 0;1284 1285 switch (action) {1286 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:1287 ret = xe_guc_pagefault_handler(guc, payload, adj_len);1288 break;1289 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:1290 __g2h_release_space(ct, len);1291 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,1292 adj_len);1293 break;1294 default:1295 xe_gt_warn(gt, "NOT_POSSIBLE");1296 }1297 1298 if (ret)1299 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n",1300 action, ERR_PTR(ret));1301}1302 1303/**1304 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler1305 * @ct: GuC CT object1306 *1307 * Anything related to page faults is critical for performance, process these1308 * critical G2H in the IRQ. This is safe as these handlers either just wake up1309 * waiters or queue another worker.1310 */1311void xe_guc_ct_fast_path(struct xe_guc_ct *ct)1312{1313 struct xe_device *xe = ct_to_xe(ct);1314 bool ongoing;1315 int len;1316 1317 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));1318 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)1319 return;1320 1321 spin_lock(&ct->fast_lock);1322 do {1323 len = g2h_read(ct, ct->fast_msg, true);1324 if (len > 0)1325 g2h_fast_path(ct, ct->fast_msg, len);1326 } while (len > 0);1327 spin_unlock(&ct->fast_lock);1328 1329 if (ongoing)1330 xe_pm_runtime_put(xe);1331}1332 1333/* Returns less than zero on error, 0 on done, 1 on more available */1334static int dequeue_one_g2h(struct xe_guc_ct *ct)1335{1336 int len;1337 int ret;1338 1339 lockdep_assert_held(&ct->lock);1340 1341 spin_lock_irq(&ct->fast_lock);1342 len = g2h_read(ct, ct->msg, false);1343 spin_unlock_irq(&ct->fast_lock);1344 if (len <= 0)1345 return len;1346 1347 ret = parse_g2h_msg(ct, ct->msg, len);1348 if (unlikely(ret < 0))1349 return ret;1350 1351 ret = process_g2h_msg(ct, ct->msg, len);1352 if (unlikely(ret < 0))1353 return ret;1354 1355 return 1;1356}1357 1358static void receive_g2h(struct xe_guc_ct *ct)1359{1360 struct xe_gt *gt = ct_to_gt(ct);1361 bool ongoing;1362 int ret;1363 1364 /*1365 * Normal users must always hold mem_access.ref around CT calls. However1366 * during the runtime pm callbacks we rely on CT to talk to the GuC, but1367 * at this stage we can't rely on mem_access.ref and even the1368 * callback_task will be different than current. For such cases we just1369 * need to ensure we always process the responses from any blocking1370 * ct_send requests or where we otherwise expect some response when1371 * initiated from those callbacks (which will need to wait for the below1372 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if1373 * the device has suspended to the point that the CT communication has1374 * been disabled.1375 *1376 * If we are inside the runtime pm callback, we can be the only task1377 * still issuing CT requests (since that requires having the1378 * mem_access.ref). It seems like it might in theory be possible to1379 * receive unsolicited events from the GuC just as we are1380 * suspending-resuming, but those will currently anyway be lost when1381 * eventually exiting from suspend, hence no need to wake up the device1382 * here. If we ever need something stronger than get_if_ongoing() then1383 * we need to be careful with blocking the pm callbacks from getting CT1384 * responses, if the worker here is blocked on those callbacks1385 * completing, creating a deadlock.1386 */1387 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct));1388 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)1389 return;1390 1391 do {1392 mutex_lock(&ct->lock);1393 ret = dequeue_one_g2h(ct);1394 mutex_unlock(&ct->lock);1395 1396 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {1397 struct drm_printer p = xe_gt_info_printer(gt);1398 1399 xe_guc_ct_print(ct, &p, false);1400 kick_reset(ct);1401 }1402 } while (ret == 1);1403 1404 if (ongoing)1405 xe_pm_runtime_put(ct_to_xe(ct));1406}1407 1408static void g2h_worker_func(struct work_struct *w)1409{1410 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);1411 1412 receive_g2h(ct);1413}1414 1415static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,1416 struct guc_ctb_snapshot *snapshot,1417 bool atomic)1418{1419 u32 head, tail;1420 1421 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,1422 sizeof(struct guc_ct_buffer_desc));1423 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));1424 1425 snapshot->cmds = kmalloc_array(ctb->info.size, sizeof(u32),1426 atomic ? GFP_ATOMIC : GFP_KERNEL);1427 1428 if (!snapshot->cmds) {1429 drm_err(&xe->drm, "Skipping CTB commands snapshot. Only CTB info will be available.\n");1430 return;1431 }1432 1433 head = snapshot->desc.head;1434 tail = snapshot->desc.tail;1435 1436 if (head != tail) {1437 struct iosys_map map =1438 IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32));1439 1440 while (head != tail) {1441 snapshot->cmds[head] = xe_map_rd(xe, &map, 0, u32);1442 ++head;1443 if (head == ctb->info.size) {1444 head = 0;1445 map = ctb->cmds;1446 } else {1447 iosys_map_incr(&map, sizeof(u32));1448 }1449 }1450 }1451}1452 1453static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,1454 struct drm_printer *p)1455{1456 u32 head, tail;1457 1458 drm_printf(p, "\tsize: %d\n", snapshot->info.size);1459 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);1460 drm_printf(p, "\thead: %d\n", snapshot->info.head);1461 drm_printf(p, "\ttail: %d\n", snapshot->info.tail);1462 drm_printf(p, "\tspace: %d\n", snapshot->info.space);1463 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);1464 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);1465 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);1466 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);1467 1468 if (!snapshot->cmds)1469 return;1470 1471 head = snapshot->desc.head;1472 tail = snapshot->desc.tail;1473 1474 while (head != tail) {1475 drm_printf(p, "\tcmd[%d]: 0x%08x\n", head,1476 snapshot->cmds[head]);1477 ++head;1478 if (head == snapshot->info.size)1479 head = 0;1480 }1481}1482 1483static void guc_ctb_snapshot_free(struct guc_ctb_snapshot *snapshot)1484{1485 kfree(snapshot->cmds);1486}1487 1488/**1489 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.1490 * @ct: GuC CT object.1491 * @atomic: Boolean to indicate if this is called from atomic context like1492 * reset or CTB handler or from some regular path like debugfs.1493 *1494 * This can be printed out in a later stage like during dev_coredump1495 * analysis.1496 *1497 * Returns: a GuC CT snapshot object that must be freed by the caller1498 * by using `xe_guc_ct_snapshot_free`.1499 */1500struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,1501 bool atomic)1502{1503 struct xe_device *xe = ct_to_xe(ct);1504 struct xe_guc_ct_snapshot *snapshot;1505 1506 snapshot = kzalloc(sizeof(*snapshot),1507 atomic ? GFP_ATOMIC : GFP_KERNEL);1508 1509 if (!snapshot) {1510 drm_err(&xe->drm, "Skipping CTB snapshot entirely.\n");1511 return NULL;1512 }1513 1514 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) {1515 snapshot->ct_enabled = true;1516 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);1517 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,1518 &snapshot->h2g, atomic);1519 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h,1520 &snapshot->g2h, atomic);1521 }1522 1523 return snapshot;1524}1525 1526/**1527 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.1528 * @snapshot: GuC CT snapshot object.1529 * @p: drm_printer where it will be printed out.1530 *1531 * This function prints out a given GuC CT snapshot object.1532 */1533void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,1534 struct drm_printer *p)1535{1536 if (!snapshot)1537 return;1538 1539 if (snapshot->ct_enabled) {1540 drm_puts(p, "H2G CTB (all sizes in DW):\n");1541 guc_ctb_snapshot_print(&snapshot->h2g, p);1542 1543 drm_puts(p, "\nG2H CTB (all sizes in DW):\n");1544 guc_ctb_snapshot_print(&snapshot->g2h, p);1545 1546 drm_printf(p, "\tg2h outstanding: %d\n",1547 snapshot->g2h_outstanding);1548 } else {1549 drm_puts(p, "CT disabled\n");1550 }1551}1552 1553/**1554 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.1555 * @snapshot: GuC CT snapshot object.1556 *1557 * This function free all the memory that needed to be allocated at capture1558 * time.1559 */1560void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)1561{1562 if (!snapshot)1563 return;1564 1565 guc_ctb_snapshot_free(&snapshot->h2g);1566 guc_ctb_snapshot_free(&snapshot->g2h);1567 kfree(snapshot);1568}1569 1570/**1571 * xe_guc_ct_print - GuC CT Print.1572 * @ct: GuC CT.1573 * @p: drm_printer where it will be printed out.1574 * @atomic: Boolean to indicate if this is called from atomic context like1575 * reset or CTB handler or from some regular path like debugfs.1576 *1577 * This function quickly capture a snapshot and immediately print it out.1578 */1579void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool atomic)1580{1581 struct xe_guc_ct_snapshot *snapshot;1582 1583 snapshot = xe_guc_ct_snapshot_capture(ct, atomic);1584 xe_guc_ct_snapshot_print(snapshot, p);1585 xe_guc_ct_snapshot_free(snapshot);1586}1587