432 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include <drm/drm_managed.h>7 8#include "regs/xe_gt_regs.h"9#include "regs/xe_guc_regs.h"10#include "regs/xe_regs.h"11 12#include "xe_assert.h"13#include "xe_bo.h"14#include "xe_device.h"15#include "xe_device_types.h"16#include "xe_gt.h"17#include "xe_gt_printk.h"18#include "xe_guc.h"19#include "xe_hw_engine.h"20#include "xe_map.h"21#include "xe_memirq.h"22#include "xe_sriov.h"23#include "xe_sriov_printk.h"24 25#define memirq_assert(m, condition) xe_tile_assert(memirq_to_tile(m), condition)26#define memirq_debug(m, msg...) xe_sriov_dbg_verbose(memirq_to_xe(m), "MEMIRQ: " msg)27 28static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq)29{30 return container_of(memirq, struct xe_tile, sriov.vf.memirq);31}32 33static struct xe_device *memirq_to_xe(struct xe_memirq *memirq)34{35 return tile_to_xe(memirq_to_tile(memirq));36}37 38static const char *guc_name(struct xe_guc *guc)39{40 return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC";41}42 43/**44 * DOC: Memory Based Interrupts45 *46 * MMIO register based interrupts infrastructure used for non-virtualized mode47 * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently48 * to allow delivering interrupts to a large number of Virtual machines or49 * containers. Memory based interrupt status reporting provides an efficient50 * and scalable infrastructure.51 *52 * For memory based interrupt status reporting hardware sequence is:53 * * Engine writes the interrupt event to memory54 * (Pointer to memory location is provided by SW. This memory surface must55 * be mapped to system memory and must be marked as un-cacheable (UC) on56 * Graphics IP Caches)57 * * Engine triggers an interrupt to host.58 */59 60/**61 * DOC: Memory Based Interrupts Page Layout62 *63 * `Memory Based Interrupts`_ requires three different objects, which are64 * called "page" in the specs, even if they aren't page-sized or aligned.65 *66 * To simplify the code we allocate a single page size object and then use67 * offsets to embedded "pages". The address of those "pages" are then68 * programmed in the HW via LRI and LRM in the context image.69 *70 * - _`Interrupt Status Report Page`: this page contains the interrupt71 * status vectors for each unit. Each bit in the interrupt vectors is72 * converted to a byte, with the byte being set to 0xFF when an73 * interrupt is triggered; interrupt vectors are 16b big so each unit74 * gets 16B. One space is reserved for each bit in one of the75 * GT_INTR_DWx registers, so this object needs a total of 1024B.76 * This object needs to be 4KiB aligned.77 *78 * - _`Interrupt Source Report Page`: this is the equivalent of the79 * GEN11_GT_INTR_DWx registers, with each bit in those registers being80 * mapped to a byte here. The offsets are the same, just bytes instead81 * of bits. This object needs to be cacheline aligned.82 *83 * - Interrupt Mask: the HW needs a location to fetch the interrupt84 * mask vector to be used by the LRM in the context, so we just use85 * the next available space in the interrupt page.86 *87 * ::88 *89 * 0x0000 +===========+ <== Interrupt Status Report Page90 * | |91 * | | ____ +----+----------------+92 * | | / | 0 | USER INTERRUPT |93 * +-----------+ __/ | 1 | |94 * | HWE(n) | __ | | CTX SWITCH |95 * +-----------+ \ | | WAIT SEMAPHORE |96 * | | \____ | 15 | |97 * | | +----+----------------+98 * | |99 * 0x0400 +===========+ <== Interrupt Source Report Page100 * | HWE(0) |101 * | HWE(1) |102 * | |103 * | HWE(x) |104 * 0x0440 +===========+ <== Interrupt Enable Mask105 * | |106 * | |107 * +-----------+108 */109 110static void __release_xe_bo(struct drm_device *drm, void *arg)111{112 struct xe_bo *bo = arg;113 114 xe_bo_unpin_map_no_vm(bo);115}116 117static int memirq_alloc_pages(struct xe_memirq *memirq)118{119 struct xe_device *xe = memirq_to_xe(memirq);120 struct xe_tile *tile = memirq_to_tile(memirq);121 struct xe_bo *bo;122 int err;123 124 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET, SZ_64));125 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET, SZ_4K));126 127 /* XXX: convert to managed bo */128 bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K,129 ttm_bo_type_kernel,130 XE_BO_FLAG_SYSTEM |131 XE_BO_FLAG_GGTT |132 XE_BO_FLAG_GGTT_INVALIDATE |133 XE_BO_FLAG_NEEDS_UC |134 XE_BO_FLAG_NEEDS_CPU_ACCESS);135 if (IS_ERR(bo)) {136 err = PTR_ERR(bo);137 goto out;138 }139 140 memirq_assert(memirq, !xe_bo_is_vram(bo));141 memirq_assert(memirq, !memirq->bo);142 143 iosys_map_memset(&bo->vmap, 0, 0, SZ_4K);144 145 memirq->bo = bo;146 memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET);147 memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET);148 memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET);149 150 memirq_assert(memirq, !memirq->source.is_iomem);151 memirq_assert(memirq, !memirq->status.is_iomem);152 memirq_assert(memirq, !memirq->mask.is_iomem);153 154 memirq_debug(memirq, "page offsets: source %#x status %#x\n",155 xe_memirq_source_ptr(memirq), xe_memirq_status_ptr(memirq));156 157 return drmm_add_action_or_reset(&xe->drm, __release_xe_bo, memirq->bo);158 159out:160 xe_sriov_err(memirq_to_xe(memirq),161 "Failed to allocate memirq page (%pe)\n", ERR_PTR(err));162 return err;163}164 165static void memirq_set_enable(struct xe_memirq *memirq, bool enable)166{167 iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0);168 169 memirq->enabled = enable;170}171 172/**173 * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_.174 * @memirq: the &xe_memirq to initialize175 *176 * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_177 * used by `Memory Based Interrupts`_.178 *179 * These allocations are managed and will be implicitly released on unload.180 *181 * Note: This function shall be called only by the VF driver.182 *183 * If this function fails then VF driver won't be able to operate correctly.184 * If `Memory Based Interrupts`_ are not used this function will return 0.185 *186 * Return: 0 on success or a negative error code on failure.187 */188int xe_memirq_init(struct xe_memirq *memirq)189{190 struct xe_device *xe = memirq_to_xe(memirq);191 int err;192 193 memirq_assert(memirq, IS_SRIOV_VF(xe));194 195 if (!xe_device_has_memirq(xe))196 return 0;197 198 err = memirq_alloc_pages(memirq);199 if (unlikely(err))200 return err;201 202 /* we need to start with all irqs enabled */203 memirq_set_enable(memirq, true);204 205 return 0;206}207 208/**209 * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_.210 * @memirq: the &xe_memirq to query211 *212 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used213 * and xe_memirq_init() didn't fail.214 *215 * Return: GGTT's offset of the `Interrupt Source Report Page`_.216 */217u32 xe_memirq_source_ptr(struct xe_memirq *memirq)218{219 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));220 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));221 memirq_assert(memirq, memirq->bo);222 223 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET;224}225 226/**227 * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_.228 * @memirq: the &xe_memirq to query229 *230 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used231 * and xe_memirq_init() didn't fail.232 *233 * Return: GGTT's offset of the `Interrupt Status Report Page`_.234 */235u32 xe_memirq_status_ptr(struct xe_memirq *memirq)236{237 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));238 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));239 memirq_assert(memirq, memirq->bo);240 241 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET;242}243 244/**245 * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask.246 * @memirq: the &xe_memirq to query247 *248 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used249 * and xe_memirq_init() didn't fail.250 *251 * Return: GGTT's offset of the Interrupt Enable Mask.252 */253u32 xe_memirq_enable_ptr(struct xe_memirq *memirq)254{255 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));256 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));257 memirq_assert(memirq, memirq->bo);258 259 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET;260}261 262/**263 * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_.264 * @memirq: the &xe_memirq265 * @guc: the &xe_guc to setup266 *267 * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_268 * to be used by the GuC when `Memory Based Interrupts`_ are required.269 *270 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used271 * and xe_memirq_init() didn't fail.272 *273 * Return: 0 on success or a negative error code on failure.274 */275int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc)276{277 bool is_media = xe_gt_is_media_type(guc_to_gt(guc));278 u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);279 u32 source, status;280 int err;281 282 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));283 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));284 memirq_assert(memirq, memirq->bo);285 286 source = xe_memirq_source_ptr(memirq) + offset;287 status = xe_memirq_status_ptr(memirq) + offset * SZ_16;288 289 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY,290 source);291 if (unlikely(err))292 goto failed;293 294 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY,295 status);296 if (unlikely(err))297 goto failed;298 299 return 0;300 301failed:302 xe_sriov_err(memirq_to_xe(memirq),303 "Failed to setup report pages in %s (%pe)\n",304 guc_name(guc), ERR_PTR(err));305 return err;306}307 308/**309 * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_.310 * @memirq: struct xe_memirq311 *312 * This is part of the driver IRQ setup flow.313 *314 * This function shall only be used by the VF driver on platforms that use315 * `Memory Based Interrupts`_.316 */317void xe_memirq_reset(struct xe_memirq *memirq)318{319 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));320 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));321 322 if (memirq->bo)323 memirq_set_enable(memirq, false);324}325 326/**327 * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_.328 * @memirq: the &xe_memirq329 *330 * This is part of the driver IRQ setup flow.331 *332 * This function shall only be used by the VF driver on platforms that use333 * `Memory Based Interrupts`_.334 */335void xe_memirq_postinstall(struct xe_memirq *memirq)336{337 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq)));338 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq)));339 340 if (memirq->bo)341 memirq_set_enable(memirq, true);342}343 344static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,345 u16 offset, const char *name)346{347 u8 value;348 349 value = iosys_map_rd(vector, offset, u8);350 if (value) {351 if (value != 0xff)352 xe_sriov_err_ratelimited(memirq_to_xe(memirq),353 "Unexpected memirq value %#x from %s at %u\n",354 value, name, offset);355 iosys_map_wr(vector, offset, u8, 0x00);356 }357 358 return value;359}360 361static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status,362 struct xe_hw_engine *hwe)363{364 memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);365 366 if (memirq_received(memirq, status, ilog2(GT_RENDER_USER_INTERRUPT), hwe->name))367 xe_hw_engine_handle_irq(hwe, GT_RENDER_USER_INTERRUPT);368}369 370static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status,371 struct xe_guc *guc)372{373 const char *name = guc_name(guc);374 375 memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);376 377 if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))378 xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);379}380 381/**382 * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.383 * @memirq: the &xe_memirq384 *385 * This function reads and dispatches `Memory Based Interrupts`.386 */387void xe_memirq_handler(struct xe_memirq *memirq)388{389 struct xe_device *xe = memirq_to_xe(memirq);390 struct xe_tile *tile = memirq_to_tile(memirq);391 struct xe_hw_engine *hwe;392 enum xe_hw_engine_id id;393 struct iosys_map map;394 unsigned int gtid;395 struct xe_gt *gt;396 397 if (!memirq->bo)398 return;399 400 memirq_assert(memirq, !memirq->source.is_iomem);401 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr);402 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32);403 404 for_each_gt(gt, xe, gtid) {405 if (gt->tile != tile)406 continue;407 408 for_each_hw_engine(hwe, gt, id) {409 if (memirq_received(memirq, &memirq->source, hwe->irq_offset, "SRC")) {410 map = IOSYS_MAP_INIT_OFFSET(&memirq->status,411 hwe->irq_offset * SZ_16);412 memirq_dispatch_engine(memirq, &map, hwe);413 }414 }415 }416 417 /* GuC and media GuC (if present) must be checked separately */418 419 if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) {420 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);421 memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc);422 }423 424 if (!tile->media_gt)425 return;426 427 if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) {428 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);429 memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc);430 }431}432