266 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include "xe_hw_fence.h"7 8#include <linux/device.h>9#include <linux/slab.h>10 11#include "xe_bo.h"12#include "xe_device.h"13#include "xe_gt.h"14#include "xe_hw_engine.h"15#include "xe_macros.h"16#include "xe_map.h"17#include "xe_trace.h"18 19static struct kmem_cache *xe_hw_fence_slab;20 21int __init xe_hw_fence_module_init(void)22{23 xe_hw_fence_slab = kmem_cache_create("xe_hw_fence",24 sizeof(struct xe_hw_fence), 0,25 SLAB_HWCACHE_ALIGN, NULL);26 if (!xe_hw_fence_slab)27 return -ENOMEM;28 29 return 0;30}31 32void xe_hw_fence_module_exit(void)33{34 rcu_barrier();35 kmem_cache_destroy(xe_hw_fence_slab);36}37 38static struct xe_hw_fence *fence_alloc(void)39{40 return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL);41}42 43static void fence_free(struct rcu_head *rcu)44{45 struct xe_hw_fence *fence =46 container_of(rcu, struct xe_hw_fence, dma.rcu);47 48 if (!WARN_ON_ONCE(!fence))49 kmem_cache_free(xe_hw_fence_slab, fence);50}51 52static void hw_fence_irq_run_cb(struct irq_work *work)53{54 struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work);55 struct xe_hw_fence *fence, *next;56 bool tmp;57 58 tmp = dma_fence_begin_signalling();59 spin_lock(&irq->lock);60 if (irq->enabled) {61 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {62 struct dma_fence *dma_fence = &fence->dma;63 64 trace_xe_hw_fence_try_signal(fence);65 if (dma_fence_is_signaled_locked(dma_fence)) {66 trace_xe_hw_fence_signal(fence);67 list_del_init(&fence->irq_link);68 dma_fence_put(dma_fence);69 }70 }71 }72 spin_unlock(&irq->lock);73 dma_fence_end_signalling(tmp);74}75 76void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq)77{78 spin_lock_init(&irq->lock);79 init_irq_work(&irq->work, hw_fence_irq_run_cb);80 INIT_LIST_HEAD(&irq->pending);81 irq->enabled = true;82}83 84void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)85{86 struct xe_hw_fence *fence, *next;87 unsigned long flags;88 int err;89 bool tmp;90 91 if (XE_WARN_ON(!list_empty(&irq->pending))) {92 tmp = dma_fence_begin_signalling();93 spin_lock_irqsave(&irq->lock, flags);94 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {95 list_del_init(&fence->irq_link);96 err = dma_fence_signal_locked(&fence->dma);97 dma_fence_put(&fence->dma);98 XE_WARN_ON(err);99 }100 spin_unlock_irqrestore(&irq->lock, flags);101 dma_fence_end_signalling(tmp);102 }103}104 105void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq)106{107 irq_work_queue(&irq->work);108}109 110void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq)111{112 spin_lock_irq(&irq->lock);113 irq->enabled = false;114 spin_unlock_irq(&irq->lock);115}116 117void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq)118{119 spin_lock_irq(&irq->lock);120 irq->enabled = true;121 spin_unlock_irq(&irq->lock);122 123 irq_work_queue(&irq->work);124}125 126void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,127 struct xe_hw_fence_irq *irq, const char *name)128{129 ctx->gt = gt;130 ctx->irq = irq;131 ctx->dma_fence_ctx = dma_fence_context_alloc(1);132 ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;133 snprintf(ctx->name, sizeof(ctx->name), "%s", name);134}135 136void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)137{138}139 140static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);141 142static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)143{144 return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);145}146 147static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)148{149 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);150 151 return dev_name(fence->xe->drm.dev);152}153 154static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence)155{156 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);157 158 return fence->name;159}160 161static bool xe_hw_fence_signaled(struct dma_fence *dma_fence)162{163 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);164 struct xe_device *xe = fence->xe;165 u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32);166 167 return dma_fence->error ||168 !__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops);169}170 171static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence)172{173 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);174 struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence);175 176 dma_fence_get(dma_fence);177 list_add_tail(&fence->irq_link, &irq->pending);178 179 /* SW completed (no HW IRQ) so kick handler to signal fence */180 if (xe_hw_fence_signaled(dma_fence))181 xe_hw_fence_irq_run(irq);182 183 return true;184}185 186static void xe_hw_fence_release(struct dma_fence *dma_fence)187{188 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);189 190 XE_WARN_ON(!list_empty(&fence->irq_link));191 call_rcu(&dma_fence->rcu, fence_free);192}193 194static const struct dma_fence_ops xe_hw_fence_ops = {195 .get_driver_name = xe_hw_fence_get_driver_name,196 .get_timeline_name = xe_hw_fence_get_timeline_name,197 .enable_signaling = xe_hw_fence_enable_signaling,198 .signaled = xe_hw_fence_signaled,199 .release = xe_hw_fence_release,200};201 202static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)203{204 if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops))205 return NULL;206 207 return container_of(fence, struct xe_hw_fence, dma);208}209 210/**211 * xe_hw_fence_alloc() - Allocate an hw fence.212 *213 * Allocate but don't initialize an hw fence.214 *215 * Return: Pointer to the allocated fence or216 * negative error pointer on error.217 */218struct dma_fence *xe_hw_fence_alloc(void)219{220 struct xe_hw_fence *hw_fence = fence_alloc();221 222 if (!hw_fence)223 return ERR_PTR(-ENOMEM);224 225 return &hw_fence->dma;226}227 228/**229 * xe_hw_fence_free() - Free an hw fence.230 * @fence: Pointer to the fence to free.231 *232 * Frees an hw fence that hasn't yet been233 * initialized.234 */235void xe_hw_fence_free(struct dma_fence *fence)236{237 fence_free(&fence->rcu);238}239 240/**241 * xe_hw_fence_init() - Initialize an hw fence.242 * @fence: Pointer to the fence to initialize.243 * @ctx: Pointer to the struct xe_hw_fence_ctx fence context.244 * @seqno_map: Pointer to the map into where the seqno is blitted.245 *246 * Initializes a pre-allocated hw fence.247 * After initialization, the fence is subject to normal248 * dma-fence refcounting.249 */250void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx,251 struct iosys_map seqno_map)252{253 struct xe_hw_fence *hw_fence =254 container_of(fence, typeof(*hw_fence), dma);255 256 hw_fence->xe = gt_to_xe(ctx->gt);257 snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name);258 hw_fence->seqno_map = seqno_map;259 INIT_LIST_HEAD(&hw_fence->irq_link);260 261 dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock,262 ctx->dma_fence_ctx, ctx->next_seqno++);263 264 trace_xe_hw_fence_create(hw_fence);265}266