174 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_preempt_fence.h"7 8#include <linux/slab.h>9 10#include "xe_exec_queue.h"11#include "xe_vm.h"12 13static void preempt_fence_work_func(struct work_struct *w)14{15 bool cookie = dma_fence_begin_signalling();16 struct xe_preempt_fence *pfence =17 container_of(w, typeof(*pfence), preempt_work);18 struct xe_exec_queue *q = pfence->q;19 20 if (pfence->error) {21 dma_fence_set_error(&pfence->base, pfence->error);22 } else if (!q->ops->reset_status(q)) {23 int err = q->ops->suspend_wait(q);24 25 if (err)26 dma_fence_set_error(&pfence->base, err);27 } else {28 dma_fence_set_error(&pfence->base, -ENOENT);29 }30 31 dma_fence_signal(&pfence->base);32 /*33 * Opt for keep everything in the fence critical section. This looks really strange since we34 * have just signalled the fence, however the preempt fences are all signalled via single35 * global ordered-wq, therefore anything that happens in this callback can easily block36 * progress on the entire wq, which itself may prevent other published preempt fences from37 * ever signalling. Therefore try to keep everything here in the callback in the fence38 * critical section. For example if something below grabs a scary lock like vm->lock,39 * lockdep should complain since we also hold that lock whilst waiting on preempt fences to40 * complete.41 */42 xe_vm_queue_rebind_worker(q->vm);43 xe_exec_queue_put(q);44 dma_fence_end_signalling(cookie);45}46 47static const char *48preempt_fence_get_driver_name(struct dma_fence *fence)49{50 return "xe";51}52 53static const char *54preempt_fence_get_timeline_name(struct dma_fence *fence)55{56 return "preempt";57}58 59static bool preempt_fence_enable_signaling(struct dma_fence *fence)60{61 struct xe_preempt_fence *pfence =62 container_of(fence, typeof(*pfence), base);63 struct xe_exec_queue *q = pfence->q;64 65 pfence->error = q->ops->suspend(q);66 queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work);67 return true;68}69 70static const struct dma_fence_ops preempt_fence_ops = {71 .get_driver_name = preempt_fence_get_driver_name,72 .get_timeline_name = preempt_fence_get_timeline_name,73 .enable_signaling = preempt_fence_enable_signaling,74};75 76/**77 * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal78 * initialization79 *80 * Allocate a preempt fence, and initialize its list head.81 * If the preempt_fence allocated has been armed with82 * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not,83 * it must be freed using xe_preempt_fence_free().84 *85 * Return: A struct xe_preempt_fence pointer used for calling into86 * xe_preempt_fence_arm() or xe_preempt_fence_free().87 * An error pointer on error.88 */89struct xe_preempt_fence *xe_preempt_fence_alloc(void)90{91 struct xe_preempt_fence *pfence;92 93 pfence = kmalloc(sizeof(*pfence), GFP_KERNEL);94 if (!pfence)95 return ERR_PTR(-ENOMEM);96 97 INIT_LIST_HEAD(&pfence->link);98 INIT_WORK(&pfence->preempt_work, preempt_fence_work_func);99 100 return pfence;101}102 103/**104 * xe_preempt_fence_free() - Free a preempt fence allocated using105 * xe_preempt_fence_alloc().106 * @pfence: pointer obtained from xe_preempt_fence_alloc();107 *108 * Free a preempt fence that has not yet been armed.109 */110void xe_preempt_fence_free(struct xe_preempt_fence *pfence)111{112 list_del(&pfence->link);113 kfree(pfence);114}115 116/**117 * xe_preempt_fence_arm() - Arm a preempt fence allocated using118 * xe_preempt_fence_alloc().119 * @pfence: The struct xe_preempt_fence pointer returned from120 * xe_preempt_fence_alloc().121 * @q: The struct xe_exec_queue used for arming.122 * @context: The dma-fence context used for arming.123 * @seqno: The dma-fence seqno used for arming.124 *125 * Inserts the preempt fence into @context's timeline, takes @link off any126 * list, and registers the struct xe_exec_queue as the xe_engine to be preempted.127 *128 * Return: A pointer to a struct dma_fence embedded into the preempt fence.129 * This function doesn't error.130 */131struct dma_fence *132xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q,133 u64 context, u32 seqno)134{135 list_del_init(&pfence->link);136 pfence->q = xe_exec_queue_get(q);137 spin_lock_init(&pfence->lock);138 dma_fence_init(&pfence->base, &preempt_fence_ops,139 &pfence->lock, context, seqno);140 141 return &pfence->base;142}143 144/**145 * xe_preempt_fence_create() - Helper to create and arm a preempt fence.146 * @q: The struct xe_exec_queue used for arming.147 * @context: The dma-fence context used for arming.148 * @seqno: The dma-fence seqno used for arming.149 *150 * Allocates and inserts the preempt fence into @context's timeline,151 * and registers @e as the struct xe_exec_queue to be preempted.152 *153 * Return: A pointer to the resulting struct dma_fence on success. An error154 * pointer on error. In particular if allocation fails it returns155 * ERR_PTR(-ENOMEM);156 */157struct dma_fence *158xe_preempt_fence_create(struct xe_exec_queue *q,159 u64 context, u32 seqno)160{161 struct xe_preempt_fence *pfence;162 163 pfence = xe_preempt_fence_alloc();164 if (IS_ERR(pfence))165 return ERR_CAST(pfence);166 167 return xe_preempt_fence_arm(pfence, q, context, seqno);168}169 170bool xe_fence_is_xe_preempt(const struct dma_fence *fence)171{172 return fence->ops == &preempt_fence_ops;173}174