1395 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2016-2019 Intel Corporation4 */5 6#include <linux/circ_buf.h>7#include <linux/ktime.h>8#include <linux/time64.h>9#include <linux/string_helpers.h>10#include <linux/timekeeping.h>11 12#include "i915_drv.h"13#include "intel_guc_ct.h"14#include "intel_guc_print.h"15 16#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)17enum {18 CT_DEAD_ALIVE = 0,19 CT_DEAD_SETUP,20 CT_DEAD_WRITE,21 CT_DEAD_DEADLOCK,22 CT_DEAD_H2G_HAS_ROOM,23 CT_DEAD_READ,24 CT_DEAD_PROCESS_FAILED,25};26 27static void ct_dead_ct_worker_func(struct work_struct *w);28 29#define CT_DEAD(ct, reason) \30 do { \31 if (!(ct)->dead_ct_reported) { \32 (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \33 queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \34 } \35 } while (0)36#else37#define CT_DEAD(ct, reason) do { } while (0)38#endif39 40static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)41{42 return container_of(ct, struct intel_guc, ct);43}44 45#define CT_ERROR(_ct, _fmt, ...) \46 guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)47#ifdef CONFIG_DRM_I915_DEBUG_GUC48#define CT_DEBUG(_ct, _fmt, ...) \49 guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)50#else51#define CT_DEBUG(...) do { } while (0)52#endif53#define CT_PROBE_ERROR(_ct, _fmt, ...) \54 guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)55 56/**57 * DOC: CTB Blob58 *59 * We allocate single blob to hold both CTB descriptors and buffers:60 *61 * +--------+-----------------------------------------------+------+62 * | offset | contents | size |63 * +========+===============================================+======+64 * | 0x0000 | H2G `CTB Descriptor`_ (send) | |65 * +--------+-----------------------------------------------+ 4K |66 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | |67 * +--------+-----------------------------------------------+------+68 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K |69 * | | | |70 * +--------+-----------------------------------------------+------+71 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K |72 * | + n*4K | | |73 * +--------+-----------------------------------------------+------+74 *75 * Size of each `CT Buffer`_ must be multiple of 4K.76 * We don't expect too many messages in flight at any time, unless we are77 * using the GuC submission. In that case each request requires a minimum78 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this79 * enough space to avoid backpressure on the driver. We increase the size80 * of the receive buffer (relative to the send) to ensure a G2H response81 * CTB has a landing spot.82 */83#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)84#define CTB_H2G_BUFFER_SIZE (SZ_4K)85#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE)86#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4)87 88struct ct_request {89 struct list_head link;90 u32 fence;91 u32 status;92 u32 response_len;93 u32 *response_buf;94};95 96struct ct_incoming_msg {97 struct list_head link;98 u32 size;99 u32 msg[] __counted_by(size);100};101 102enum { CTB_SEND = 0, CTB_RECV = 1 };103 104enum { CTB_OWNER_HOST = 0 };105 106/*107 * Some H2G commands involve a synchronous response that the driver needs108 * to wait for. In such cases, a timeout is required to prevent the driver109 * from waiting forever in the case of an error (either no error response110 * is defined in the protocol or something has died and requires a reset).111 * The specific command may be defined as having a time bound response but112 * the CT is a queue and that time guarantee only starts from the point113 * when the command reaches the head of the queue and is processed by GuC.114 *115 * Ideally there would be a helper to report the progress of a given116 * command through the CT. However, that would require a significant117 * amount of work in the CT layer. In the meantime, provide a reasonable118 * estimation of the worst case latency it should take for the entire119 * queue to drain. And therefore, how long a caller should wait before120 * giving up on their request. The current estimate is based on empirical121 * measurement of a test that fills the buffer with context creation and122 * destruction requests as they seem to be the slowest operation.123 */124long intel_guc_ct_max_queue_time_jiffies(void)125{126 /*127 * A 4KB buffer full of context destroy commands takes a little128 * over a second to process so bump that to 2s to be super safe.129 */130 return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;131}132 133static void ct_receive_tasklet_func(struct tasklet_struct *t);134static void ct_incoming_request_worker_func(struct work_struct *w);135 136/**137 * intel_guc_ct_init_early - Initialize CT state without requiring device access138 * @ct: pointer to CT struct139 */140void intel_guc_ct_init_early(struct intel_guc_ct *ct)141{142 spin_lock_init(&ct->ctbs.send.lock);143 spin_lock_init(&ct->ctbs.recv.lock);144 spin_lock_init(&ct->requests.lock);145 INIT_LIST_HEAD(&ct->requests.pending);146 INIT_LIST_HEAD(&ct->requests.incoming);147#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)148 INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);149#endif150 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);151 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);152 init_waitqueue_head(&ct->wq);153}154 155static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)156{157 memset(desc, 0, sizeof(*desc));158}159 160static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)161{162 u32 space;163 164 ctb->broken = false;165 ctb->tail = 0;166 ctb->head = 0;167 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;168 atomic_set(&ctb->space, space);169 170 guc_ct_buffer_desc_init(ctb->desc);171}172 173static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,174 struct guc_ct_buffer_desc *desc,175 u32 *cmds, u32 size_in_bytes, u32 resv_space)176{177 GEM_BUG_ON(size_in_bytes % 4);178 179 ctb->desc = desc;180 ctb->cmds = cmds;181 ctb->size = size_in_bytes / 4;182 ctb->resv_space = resv_space / 4;183 184 guc_ct_buffer_reset(ctb);185}186 187static int guc_action_control_ctb(struct intel_guc *guc, u32 control)188{189 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {190 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |191 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |192 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),193 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),194 };195 int ret;196 197 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);198 199 /* CT control must go over MMIO */200 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);201 202 return ret > 0 ? -EPROTO : ret;203}204 205static int ct_control_enable(struct intel_guc_ct *ct, bool enable)206{207 int err;208 209 err = guc_action_control_ctb(ct_to_guc(ct), enable ?210 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);211 if (unlikely(err))212 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",213 str_enable_disable(enable), ERR_PTR(err));214 215 return err;216}217 218static int ct_register_buffer(struct intel_guc_ct *ct, bool send,219 u32 desc_addr, u32 buff_addr, u32 size)220{221 int err;222 223 err = intel_guc_self_cfg64(ct_to_guc(ct), send ?224 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :225 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,226 desc_addr);227 if (unlikely(err))228 goto failed;229 230 err = intel_guc_self_cfg64(ct_to_guc(ct), send ?231 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :232 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,233 buff_addr);234 if (unlikely(err))235 goto failed;236 237 err = intel_guc_self_cfg32(ct_to_guc(ct), send ?238 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :239 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,240 size);241 if (unlikely(err))242failed:243 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",244 send ? "SEND" : "RECV", ERR_PTR(err));245 246 return err;247}248 249/**250 * intel_guc_ct_init - Init buffer-based communication251 * @ct: pointer to CT struct252 *253 * Allocate memory required for buffer-based communication.254 *255 * Return: 0 on success, a negative errno code on failure.256 */257int intel_guc_ct_init(struct intel_guc_ct *ct)258{259 struct intel_guc *guc = ct_to_guc(ct);260 struct guc_ct_buffer_desc *desc;261 u32 blob_size;262 u32 cmds_size;263 u32 resv_space;264 void *blob;265 u32 *cmds;266 int err;267 268 err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);269 if (err)270 return err;271 272 GEM_BUG_ON(ct->vma);273 274 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;275 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);276 if (unlikely(err)) {277 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",278 blob_size, ERR_PTR(err));279 return err;280 }281 282 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);283 284 /* store pointers to desc and cmds for send ctb */285 desc = blob;286 cmds = blob + 2 * CTB_DESC_SIZE;287 cmds_size = CTB_H2G_BUFFER_SIZE;288 resv_space = 0;289 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",290 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,291 resv_space);292 293 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);294 295 /* store pointers to desc and cmds for recv ctb */296 desc = blob + CTB_DESC_SIZE;297 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;298 cmds_size = CTB_G2H_BUFFER_SIZE;299 resv_space = G2H_ROOM_BUFFER_SIZE;300 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",301 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,302 resv_space);303 304 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);305 306 return 0;307}308 309/**310 * intel_guc_ct_fini - Fini buffer-based communication311 * @ct: pointer to CT struct312 *313 * Deallocate memory required for buffer-based communication.314 */315void intel_guc_ct_fini(struct intel_guc_ct *ct)316{317 GEM_BUG_ON(ct->enabled);318 319 tasklet_kill(&ct->receive_tasklet);320 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);321 memset(ct, 0, sizeof(*ct));322}323 324/**325 * intel_guc_ct_enable - Enable buffer based command transport.326 * @ct: pointer to CT struct327 *328 * Return: 0 on success, a negative errno code on failure.329 */330int intel_guc_ct_enable(struct intel_guc_ct *ct)331{332 struct intel_guc *guc = ct_to_guc(ct);333 u32 base, desc, cmds, size;334 void *blob;335 int err;336 337 GEM_BUG_ON(ct->enabled);338 339 /* vma should be already allocated and map'ed */340 GEM_BUG_ON(!ct->vma);341 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));342 base = intel_guc_ggtt_offset(guc, ct->vma);343 344 /* blob should start with send descriptor */345 blob = __px_vaddr(ct->vma->obj);346 GEM_BUG_ON(blob != ct->ctbs.send.desc);347 348 /* (re)initialize descriptors */349 guc_ct_buffer_reset(&ct->ctbs.send);350 guc_ct_buffer_reset(&ct->ctbs.recv);351 352 /*353 * Register both CT buffers starting with RECV buffer.354 * Descriptors are in first half of the blob.355 */356 desc = base + ptrdiff(ct->ctbs.recv.desc, blob);357 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);358 size = ct->ctbs.recv.size * 4;359 err = ct_register_buffer(ct, false, desc, cmds, size);360 if (unlikely(err))361 goto err_out;362 363 desc = base + ptrdiff(ct->ctbs.send.desc, blob);364 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);365 size = ct->ctbs.send.size * 4;366 err = ct_register_buffer(ct, true, desc, cmds, size);367 if (unlikely(err))368 goto err_out;369 370 err = ct_control_enable(ct, true);371 if (unlikely(err))372 goto err_out;373 374 ct->enabled = true;375 ct->stall_time = KTIME_MAX;376#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)377 ct->dead_ct_reported = false;378 ct->dead_ct_reason = CT_DEAD_ALIVE;379#endif380 381 return 0;382 383err_out:384 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));385 CT_DEAD(ct, SETUP);386 return err;387}388 389/**390 * intel_guc_ct_disable - Disable buffer based command transport.391 * @ct: pointer to CT struct392 */393void intel_guc_ct_disable(struct intel_guc_ct *ct)394{395 struct intel_guc *guc = ct_to_guc(ct);396 397 GEM_BUG_ON(!ct->enabled);398 399 ct->enabled = false;400 401 if (intel_guc_is_fw_running(guc)) {402 ct_control_enable(ct, false);403 }404}405 406#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)407static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)408{409 unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);410#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)411 unsigned long entries[SZ_32];412 unsigned int n;413 414 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);415 416 /* May be called under spinlock, so avoid sleeping */417 ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);418#endif419 ct->requests.lost_and_found[lost].fence = fence;420 ct->requests.lost_and_found[lost].action = action;421}422#endif423 424static u32 ct_get_next_fence(struct intel_guc_ct *ct)425{426 /* For now it's trivial */427 return ++ct->requests.last_fence;428}429 430static int ct_write(struct intel_guc_ct *ct,431 const u32 *action,432 u32 len /* in dwords */,433 u32 fence, u32 flags)434{435 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;436 struct guc_ct_buffer_desc *desc = ctb->desc;437 u32 tail = ctb->tail;438 u32 size = ctb->size;439 u32 header;440 u32 hxg;441 u32 type;442 u32 *cmds = ctb->cmds;443 unsigned int i;444 445 if (unlikely(desc->status))446 goto corrupted;447 448 GEM_BUG_ON(tail > size);449 450#ifdef CONFIG_DRM_I915_DEBUG_GUC451 if (unlikely(tail != READ_ONCE(desc->tail))) {452 CT_ERROR(ct, "Tail was modified %u != %u\n",453 desc->tail, tail);454 desc->status |= GUC_CTB_STATUS_MISMATCH;455 goto corrupted;456 }457 if (unlikely(READ_ONCE(desc->head) >= size)) {458 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",459 desc->head, size);460 desc->status |= GUC_CTB_STATUS_OVERFLOW;461 goto corrupted;462 }463#endif464 465 /*466 * dw0: CT header (including fence)467 * dw1: HXG header (including action code)468 * dw2+: action data469 */470 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |471 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |472 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);473 474 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :475 GUC_HXG_TYPE_REQUEST;476 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |477 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |478 GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);479 480 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",481 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);482 483 cmds[tail] = header;484 tail = (tail + 1) % size;485 486 cmds[tail] = hxg;487 tail = (tail + 1) % size;488 489 for (i = 1; i < len; i++) {490 cmds[tail] = action[i];491 tail = (tail + 1) % size;492 }493 GEM_BUG_ON(tail > size);494 495#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)496 ct_track_lost_and_found(ct, fence,497 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));498#endif499 500 /*501 * make sure H2G buffer update and LRC tail update (if this triggering a502 * submission) are visible before updating the descriptor tail503 */504 intel_guc_write_barrier(ct_to_guc(ct));505 506 /* update local copies */507 ctb->tail = tail;508 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);509 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);510 511 /* now update descriptor */512 WRITE_ONCE(desc->tail, tail);513 514 return 0;515 516corrupted:517 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",518 desc->head, desc->tail, desc->status);519 CT_DEAD(ct, WRITE);520 ctb->broken = true;521 return -EPIPE;522}523 524/**525 * wait_for_ct_request_update - Wait for CT request state update.526 * @ct: pointer to CT527 * @req: pointer to pending request528 * @status: placeholder for status529 *530 * For each sent request, GuC shall send back CT response message.531 * Our message handler will update status of tracked request once532 * response message with given fence is received. Wait here and533 * check for valid response status value.534 *535 * Return:536 * * 0 response received (status is valid)537 * * -ETIMEDOUT no response within hardcoded timeout538 */539static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)540{541 int err;542 bool ct_enabled;543 544 /*545 * Fast commands should complete in less than 10us, so sample quickly546 * up to that length of time, then switch to a slower sleep-wait loop.547 * No GuC command should ever take longer than 10ms but many GuC548 * commands can be inflight at time, so use a 1s timeout on the slower549 * sleep-wait loop.550 */551#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10552#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000553#define done \554 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \555 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \556 GUC_HXG_ORIGIN_GUC)557 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);558 if (err)559 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);560#undef done561 if (!ct_enabled)562 err = -ENODEV;563 564 *status = req->status;565 return err;566}567 568#define GUC_CTB_TIMEOUT_MS 1500569static inline bool ct_deadlocked(struct intel_guc_ct *ct)570{571 long timeout = GUC_CTB_TIMEOUT_MS;572 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;573 574 if (unlikely(ret)) {575 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;576 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;577 578 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",579 ktime_ms_delta(ktime_get(), ct->stall_time),580 send->status, recv->status);581 CT_ERROR(ct, "H2G Space: %u (Bytes)\n",582 atomic_read(&ct->ctbs.send.space) * 4);583 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);584 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);585 CT_ERROR(ct, "G2H Space: %u (Bytes)\n",586 atomic_read(&ct->ctbs.recv.space) * 4);587 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);588 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);589 590 CT_DEAD(ct, DEADLOCK);591 ct->ctbs.send.broken = true;592 }593 594 return ret;595}596 597static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)598{599 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;600 601 /*602 * We leave a certain amount of space in the G2H CTB buffer for603 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)604 */605 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;606}607 608static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)609{610 lockdep_assert_held(&ct->ctbs.send.lock);611 612 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));613 614 if (g2h_len_dw)615 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);616}617 618static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)619{620 atomic_add(g2h_len_dw, &ct->ctbs.recv.space);621}622 623static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)624{625 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;626 struct guc_ct_buffer_desc *desc = ctb->desc;627 u32 head;628 u32 space;629 630 if (atomic_read(&ctb->space) >= len_dw)631 return true;632 633 head = READ_ONCE(desc->head);634 if (unlikely(head > ctb->size)) {635 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",636 head, ctb->size);637 desc->status |= GUC_CTB_STATUS_OVERFLOW;638 ctb->broken = true;639 CT_DEAD(ct, H2G_HAS_ROOM);640 return false;641 }642 643 space = CIRC_SPACE(ctb->tail, head, ctb->size);644 atomic_set(&ctb->space, space);645 646 return space >= len_dw;647}648 649static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)650{651 bool h2g = h2g_has_room(ct, h2g_dw);652 bool g2h = g2h_has_room(ct, g2h_dw);653 654 lockdep_assert_held(&ct->ctbs.send.lock);655 656 if (unlikely(!h2g || !g2h)) {657 if (ct->stall_time == KTIME_MAX)658 ct->stall_time = ktime_get();659 660 /* Be paranoid and kick G2H tasklet to free credits */661 if (!g2h)662 tasklet_hi_schedule(&ct->receive_tasklet);663 664 if (unlikely(ct_deadlocked(ct)))665 return -EPIPE;666 else667 return -EBUSY;668 }669 670 ct->stall_time = KTIME_MAX;671 return 0;672}673 674#define G2H_LEN_DW(f) ({ \675 typeof(f) f_ = (f); \676 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \677 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \678 GUC_CTB_HXG_MSG_MIN_LEN : 0; \679})680static int ct_send_nb(struct intel_guc_ct *ct,681 const u32 *action,682 u32 len,683 u32 flags)684{685 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;686 unsigned long spin_flags;687 u32 g2h_len_dw = G2H_LEN_DW(flags);688 u32 fence;689 int ret;690 691 spin_lock_irqsave(&ctb->lock, spin_flags);692 693 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);694 if (unlikely(ret))695 goto out;696 697 fence = ct_get_next_fence(ct);698 ret = ct_write(ct, action, len, fence, flags);699 if (unlikely(ret))700 goto out;701 702 g2h_reserve_space(ct, g2h_len_dw);703 intel_guc_notify(ct_to_guc(ct));704 705out:706 spin_unlock_irqrestore(&ctb->lock, spin_flags);707 708 return ret;709}710 711static int ct_send(struct intel_guc_ct *ct,712 const u32 *action,713 u32 len,714 u32 *response_buf,715 u32 response_buf_size,716 u32 *status)717{718 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;719 struct ct_request request;720 unsigned long flags;721 unsigned int sleep_period_ms = 1;722 bool send_again;723 u32 fence;724 int err;725 726 GEM_BUG_ON(!ct->enabled);727 GEM_BUG_ON(!len);728 GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);729 GEM_BUG_ON(!response_buf && response_buf_size);730 might_sleep();731 732resend:733 send_again = false;734 735 /*736 * We use a lazy spin wait loop here as we believe that if the CT737 * buffers are sized correctly the flow control condition should be738 * rare. Reserving the maximum size in the G2H credits as we don't know739 * how big the response is going to be.740 */741retry:742 spin_lock_irqsave(&ctb->lock, flags);743 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||744 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {745 if (ct->stall_time == KTIME_MAX)746 ct->stall_time = ktime_get();747 spin_unlock_irqrestore(&ctb->lock, flags);748 749 if (unlikely(ct_deadlocked(ct)))750 return -EPIPE;751 752 if (msleep_interruptible(sleep_period_ms))753 return -EINTR;754 sleep_period_ms = sleep_period_ms << 1;755 756 goto retry;757 }758 759 ct->stall_time = KTIME_MAX;760 761 fence = ct_get_next_fence(ct);762 request.fence = fence;763 request.status = 0;764 request.response_len = response_buf_size;765 request.response_buf = response_buf;766 767 spin_lock(&ct->requests.lock);768 list_add_tail(&request.link, &ct->requests.pending);769 spin_unlock(&ct->requests.lock);770 771 err = ct_write(ct, action, len, fence, 0);772 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);773 774 spin_unlock_irqrestore(&ctb->lock, flags);775 776 if (unlikely(err))777 goto unlink;778 779 intel_guc_notify(ct_to_guc(ct));780 781 err = wait_for_ct_request_update(ct, &request, status);782 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);783 if (unlikely(err)) {784 if (err == -ENODEV)785 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.786 * In this case, output is debug rather than error info787 */788 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",789 action[0], request.fence);790 else791 CT_ERROR(ct, "No response for request %#x (fence %u)\n",792 action[0], request.fence);793 goto unlink;794 }795 796 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {797 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,798 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));799 send_again = true;800 goto unlink;801 }802 803 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {804 err = -EIO;805 goto unlink;806 }807 808 if (response_buf) {809 /* There shall be no data in the status */810 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));811 /* Return actual response len */812 err = request.response_len;813 } else {814 /* There shall be no response payload */815 WARN_ON(request.response_len);816 /* Return data decoded from the status dword */817 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);818 }819 820unlink:821 spin_lock_irqsave(&ct->requests.lock, flags);822 list_del(&request.link);823 spin_unlock_irqrestore(&ct->requests.lock, flags);824 825 if (unlikely(send_again))826 goto resend;827 828 return err;829}830 831/*832 * Command Transport (CT) buffer based GuC send function.833 */834int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,835 u32 *response_buf, u32 response_buf_size, u32 flags)836{837 u32 status = ~0; /* undefined */838 int ret;839 840 if (unlikely(!ct->enabled)) {841 struct intel_guc *guc = ct_to_guc(ct);842 struct intel_uc *uc = container_of(guc, struct intel_uc, guc);843 844 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);845 return -ENODEV;846 }847 848 if (unlikely(ct->ctbs.send.broken))849 return -EPIPE;850 851 if (flags & INTEL_GUC_CT_SEND_NB)852 return ct_send_nb(ct, action, len, flags);853 854 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);855 if (unlikely(ret < 0)) {856 if (ret != -ENODEV)857 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",858 action[0], ERR_PTR(ret), status);859 } else if (unlikely(ret)) {860 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",861 action[0], ret, ret);862 }863 864 return ret;865}866 867static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)868{869 struct ct_incoming_msg *msg;870 871 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);872 if (msg)873 msg->size = num_dwords;874 return msg;875}876 877static void ct_free_msg(struct ct_incoming_msg *msg)878{879 kfree(msg);880}881 882/*883 * Return: number available remaining dwords to read (0 if empty)884 * or a negative error code on failure885 */886static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)887{888 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;889 struct guc_ct_buffer_desc *desc = ctb->desc;890 u32 head = ctb->head;891 u32 tail = READ_ONCE(desc->tail);892 u32 size = ctb->size;893 u32 *cmds = ctb->cmds;894 s32 available;895 unsigned int len;896 unsigned int i;897 u32 header;898 899 if (unlikely(ctb->broken))900 return -EPIPE;901 902 if (unlikely(desc->status)) {903 u32 status = desc->status;904 905 if (status & GUC_CTB_STATUS_UNUSED) {906 /*907 * Potentially valid if a CLIENT_RESET request resulted in908 * contexts/engines being reset. But should never happen as909 * no contexts should be active when CLIENT_RESET is sent.910 */911 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");912 status &= ~GUC_CTB_STATUS_UNUSED;913 }914 915 if (status)916 goto corrupted;917 }918 919 GEM_BUG_ON(head > size);920 921#ifdef CONFIG_DRM_I915_DEBUG_GUC922 if (unlikely(head != READ_ONCE(desc->head))) {923 CT_ERROR(ct, "Head was modified %u != %u\n",924 desc->head, head);925 desc->status |= GUC_CTB_STATUS_MISMATCH;926 goto corrupted;927 }928#endif929 if (unlikely(tail >= size)) {930 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",931 tail, size);932 desc->status |= GUC_CTB_STATUS_OVERFLOW;933 goto corrupted;934 }935 936 /* tail == head condition indicates empty */937 available = tail - head;938 if (unlikely(available == 0)) {939 *msg = NULL;940 return 0;941 }942 943 /* beware of buffer wrap case */944 if (unlikely(available < 0))945 available += size;946 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);947 GEM_BUG_ON(available < 0);948 949 header = cmds[head];950 head = (head + 1) % size;951 952 /* message len with header */953 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;954 if (unlikely(len > (u32)available)) {955 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",956 4, &header,957 4 * (head + available - 1 > size ?958 size - head : available - 1), &cmds[head],959 4 * (head + available - 1 > size ?960 available - 1 - size + head : 0), &cmds[0]);961 desc->status |= GUC_CTB_STATUS_UNDERFLOW;962 goto corrupted;963 }964 965 *msg = ct_alloc_msg(len);966 if (!*msg) {967 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",968 4, &header,969 4 * (head + available - 1 > size ?970 size - head : available - 1), &cmds[head],971 4 * (head + available - 1 > size ?972 available - 1 - size + head : 0), &cmds[0]);973 return available;974 }975 976 (*msg)->msg[0] = header;977 978 for (i = 1; i < len; i++) {979 (*msg)->msg[i] = cmds[head];980 head = (head + 1) % size;981 }982 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);983 984 /* update local copies */985 ctb->head = head;986 987 /* now update descriptor */988 WRITE_ONCE(desc->head, head);989 990 intel_guc_write_barrier(ct_to_guc(ct));991 992 return available - len;993 994corrupted:995 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",996 desc->head, desc->tail, desc->status);997 ctb->broken = true;998 CT_DEAD(ct, READ);999 return -EPIPE;1000}1001 1002#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)1003static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)1004{1005 unsigned int n;1006 char *buf = NULL;1007 bool found = false;1008 1009 lockdep_assert_held(&ct->requests.lock);1010 1011 for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {1012 if (ct->requests.lost_and_found[n].fence != fence)1013 continue;1014 found = true;1015 1016#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)1017 buf = kmalloc(SZ_4K, GFP_NOWAIT);1018 if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,1019 buf, SZ_4K, 0)) {1020 CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",1021 fence, ct->requests.lost_and_found[n].action, buf);1022 break;1023 }1024#endif1025 CT_ERROR(ct, "Fence %u was used by action %#04x\n",1026 fence, ct->requests.lost_and_found[n].action);1027 break;1028 }1029 kfree(buf);1030 return found;1031}1032#else1033static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)1034{1035 return false;1036}1037#endif1038 1039static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)1040{1041 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);1042 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);1043 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];1044 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];1045 u32 datalen = len - GUC_HXG_MSG_MIN_LEN;1046 struct ct_request *req;1047 unsigned long flags;1048 bool found = false;1049 int err = 0;1050 1051 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);1052 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);1053 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&1054 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&1055 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);1056 1057 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);1058 1059 spin_lock_irqsave(&ct->requests.lock, flags);1060 list_for_each_entry(req, &ct->requests.pending, link) {1061 if (unlikely(fence != req->fence)) {1062 CT_DEBUG(ct, "request %u awaits response\n",1063 req->fence);1064 continue;1065 }1066 if (unlikely(datalen > req->response_len)) {1067 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",1068 req->fence, datalen, req->response_len);1069 datalen = min(datalen, req->response_len);1070 err = -EMSGSIZE;1071 }1072 if (datalen)1073 memcpy(req->response_buf, data, 4 * datalen);1074 req->response_len = datalen;1075 WRITE_ONCE(req->status, hxg[0]);1076 found = true;1077 break;1078 }1079 1080#ifdef CONFIG_DRM_I915_SELFTEST1081 if (!found && ct_to_guc(ct)->fast_response_selftest) {1082 CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n");1083 ct_to_guc(ct)->fast_response_selftest++;1084 found = true;1085 }1086#endif1087 1088 if (!found) {1089 CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",1090 len, hxg[0], fence, ct->requests.last_fence);1091 if (!ct_check_lost_and_found(ct, fence)) {1092 list_for_each_entry(req, &ct->requests.pending, link)1093 CT_ERROR(ct, "request %u awaits response\n",1094 req->fence);1095 }1096 err = -ENOKEY;1097 }1098 spin_unlock_irqrestore(&ct->requests.lock, flags);1099 1100 if (unlikely(err))1101 return err;1102 1103 ct_free_msg(response);1104 return 0;1105}1106 1107static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)1108{1109 struct intel_guc *guc = ct_to_guc(ct);1110 const u32 *hxg;1111 const u32 *payload;1112 u32 hxg_len, action, len;1113 int ret;1114 1115 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];1116 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;1117 payload = &hxg[GUC_HXG_MSG_MIN_LEN];1118 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);1119 len = hxg_len - GUC_HXG_MSG_MIN_LEN;1120 1121 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);1122 1123 switch (action) {1124 case INTEL_GUC_ACTION_DEFAULT:1125 ret = intel_guc_to_host_process_recv_msg(guc, payload, len);1126 break;1127 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:1128 ret = intel_guc_deregister_done_process_msg(guc, payload,1129 len);1130 break;1131 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:1132 ret = intel_guc_sched_done_process_msg(guc, payload, len);1133 break;1134 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:1135 ret = intel_guc_context_reset_process_msg(guc, payload, len);1136 break;1137 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:1138 ret = intel_guc_error_capture_process_msg(guc, payload, len);1139 if (unlikely(ret))1140 CT_ERROR(ct, "error capture notification failed %x %*ph\n",1141 action, 4 * len, payload);1142 break;1143 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:1144 ret = intel_guc_engine_failure_process_msg(guc, payload, len);1145 break;1146 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:1147 intel_guc_log_handle_flush_event(&guc->log);1148 ret = 0;1149 break;1150 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:1151 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:1152 ret = intel_guc_crash_process_msg(guc, action);1153 break;1154 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:1155 ret = intel_guc_tlb_invalidation_done(guc, payload, len);1156 break;1157 default:1158 ret = -EOPNOTSUPP;1159 break;1160 }1161 1162 if (unlikely(ret)) {1163 CT_ERROR(ct, "Failed to process request %04x (%pe)\n",1164 action, ERR_PTR(ret));1165 return ret;1166 }1167 1168 ct_free_msg(request);1169 return 0;1170}1171 1172static bool ct_process_incoming_requests(struct intel_guc_ct *ct)1173{1174 unsigned long flags;1175 struct ct_incoming_msg *request;1176 bool done;1177 int err;1178 1179 spin_lock_irqsave(&ct->requests.lock, flags);1180 request = list_first_entry_or_null(&ct->requests.incoming,1181 struct ct_incoming_msg, link);1182 if (request)1183 list_del(&request->link);1184 done = !!list_empty(&ct->requests.incoming);1185 spin_unlock_irqrestore(&ct->requests.lock, flags);1186 1187 if (!request)1188 return true;1189 1190 err = ct_process_request(ct, request);1191 if (unlikely(err)) {1192 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",1193 ERR_PTR(err), 4 * request->size, request->msg);1194 CT_DEAD(ct, PROCESS_FAILED);1195 ct_free_msg(request);1196 }1197 1198 return done;1199}1200 1201static void ct_incoming_request_worker_func(struct work_struct *w)1202{1203 struct intel_guc_ct *ct =1204 container_of(w, struct intel_guc_ct, requests.worker);1205 bool done;1206 1207 do {1208 done = ct_process_incoming_requests(ct);1209 } while (!done);1210}1211 1212static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)1213{1214 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];1215 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);1216 unsigned long flags;1217 1218 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);1219 1220 /*1221 * Adjusting the space must be done in IRQ or deadlock can occur as the1222 * CTB processing in the below workqueue can send CTBs which creates a1223 * circular dependency if the space was returned there.1224 */1225 switch (action) {1226 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:1227 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:1228 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:1229 g2h_release_space(ct, request->size);1230 }1231 1232 /*1233 * TLB invalidation responses must be handled immediately as processing1234 * of other G2H notifications may be blocked by an invalidation request.1235 */1236 if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)1237 return ct_process_request(ct, request);1238 1239 spin_lock_irqsave(&ct->requests.lock, flags);1240 list_add_tail(&request->link, &ct->requests.incoming);1241 spin_unlock_irqrestore(&ct->requests.lock, flags);1242 1243 queue_work(system_unbound_wq, &ct->requests.worker);1244 return 0;1245}1246 1247static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)1248{1249 u32 origin, type;1250 u32 *hxg;1251 int err;1252 1253 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))1254 return -EBADMSG;1255 1256 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];1257 1258 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);1259 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {1260 err = -EPROTO;1261 goto failed;1262 }1263 1264 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);1265 switch (type) {1266 case GUC_HXG_TYPE_EVENT:1267 err = ct_handle_event(ct, msg);1268 break;1269 case GUC_HXG_TYPE_RESPONSE_SUCCESS:1270 case GUC_HXG_TYPE_RESPONSE_FAILURE:1271 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:1272 err = ct_handle_response(ct, msg);1273 break;1274 default:1275 err = -EOPNOTSUPP;1276 }1277 1278 if (unlikely(err)) {1279failed:1280 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",1281 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);1282 }1283 return err;1284}1285 1286static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)1287{1288 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);1289 int err;1290 1291 if (format == GUC_CTB_FORMAT_HXG)1292 err = ct_handle_hxg(ct, msg);1293 else1294 err = -EOPNOTSUPP;1295 1296 if (unlikely(err)) {1297 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",1298 ERR_PTR(err), 4 * msg->size, msg->msg);1299 ct_free_msg(msg);1300 }1301}1302 1303/*1304 * Return: number available remaining dwords to read (0 if empty)1305 * or a negative error code on failure1306 */1307static int ct_receive(struct intel_guc_ct *ct)1308{1309 struct ct_incoming_msg *msg = NULL;1310 unsigned long flags;1311 int ret;1312 1313 spin_lock_irqsave(&ct->ctbs.recv.lock, flags);1314 ret = ct_read(ct, &msg);1315 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);1316 if (ret < 0)1317 return ret;1318 1319 if (msg)1320 ct_handle_msg(ct, msg);1321 1322 return ret;1323}1324 1325static void ct_try_receive_message(struct intel_guc_ct *ct)1326{1327 int ret;1328 1329 if (GEM_WARN_ON(!ct->enabled))1330 return;1331 1332 ret = ct_receive(ct);1333 if (ret > 0)1334 tasklet_hi_schedule(&ct->receive_tasklet);1335}1336 1337static void ct_receive_tasklet_func(struct tasklet_struct *t)1338{1339 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);1340 1341 ct_try_receive_message(ct);1342}1343 1344/*1345 * When we're communicating with the GuC over CT, GuC uses events1346 * to notify us about new messages being posted on the RECV buffer.1347 */1348void intel_guc_ct_event_handler(struct intel_guc_ct *ct)1349{1350 if (unlikely(!ct->enabled)) {1351 WARN(1, "Unexpected GuC event received while CT disabled!\n");1352 return;1353 }1354 1355 ct_try_receive_message(ct);1356}1357 1358void intel_guc_ct_print_info(struct intel_guc_ct *ct,1359 struct drm_printer *p)1360{1361 drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));1362 1363 if (!ct->enabled)1364 return;1365 1366 drm_printf(p, "H2G Space: %u\n",1367 atomic_read(&ct->ctbs.send.space) * 4);1368 drm_printf(p, "Head: %u\n",1369 ct->ctbs.send.desc->head);1370 drm_printf(p, "Tail: %u\n",1371 ct->ctbs.send.desc->tail);1372 drm_printf(p, "G2H Space: %u\n",1373 atomic_read(&ct->ctbs.recv.space) * 4);1374 drm_printf(p, "Head: %u\n",1375 ct->ctbs.recv.desc->head);1376 drm_printf(p, "Tail: %u\n",1377 ct->ctbs.recv.desc->tail);1378}1379 1380#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)1381static void ct_dead_ct_worker_func(struct work_struct *w)1382{1383 struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);1384 struct intel_guc *guc = ct_to_guc(ct);1385 1386 if (ct->dead_ct_reported)1387 return;1388 1389 ct->dead_ct_reported = true;1390 1391 guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);1392 intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);1393}1394#endif1395