299 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * dma-fence-array: aggregate fences to be waited together4 *5 * Copyright (C) 2016 Collabora Ltd6 * Copyright (C) 2016 Advanced Micro Devices, Inc.7 * Authors:8 * Gustavo Padovan <gustavo@padovan.org>9 * Christian König <christian.koenig@amd.com>10 */11 12#include <linux/export.h>13#include <linux/slab.h>14#include <linux/dma-fence-array.h>15 16#define PENDING_ERROR 117 18static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)19{20 return "dma_fence_array";21}22 23static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)24{25 return "unbound";26}27 28static void dma_fence_array_set_pending_error(struct dma_fence_array *array,29 int error)30{31 /*32 * Propagate the first error reported by any of our fences, but only33 * before we ourselves are signaled.34 */35 if (error)36 cmpxchg(&array->base.error, PENDING_ERROR, error);37}38 39static void dma_fence_array_clear_pending_error(struct dma_fence_array *array)40{41 /* Clear the error flag if not actually set. */42 cmpxchg(&array->base.error, PENDING_ERROR, 0);43}44 45static void irq_dma_fence_array_work(struct irq_work *wrk)46{47 struct dma_fence_array *array = container_of(wrk, typeof(*array), work);48 49 dma_fence_array_clear_pending_error(array);50 51 dma_fence_signal(&array->base);52 dma_fence_put(&array->base);53}54 55static void dma_fence_array_cb_func(struct dma_fence *f,56 struct dma_fence_cb *cb)57{58 struct dma_fence_array_cb *array_cb =59 container_of(cb, struct dma_fence_array_cb, cb);60 struct dma_fence_array *array = array_cb->array;61 62 dma_fence_array_set_pending_error(array, f->error);63 64 if (atomic_dec_and_test(&array->num_pending))65 irq_work_queue(&array->work);66 else67 dma_fence_put(&array->base);68}69 70static bool dma_fence_array_enable_signaling(struct dma_fence *fence)71{72 struct dma_fence_array *array = to_dma_fence_array(fence);73 struct dma_fence_array_cb *cb = array->callbacks;74 unsigned i;75 76 for (i = 0; i < array->num_fences; ++i) {77 cb[i].array = array;78 /*79 * As we may report that the fence is signaled before all80 * callbacks are complete, we need to take an additional81 * reference count on the array so that we do not free it too82 * early. The core fence handling will only hold the reference83 * until we signal the array as complete (but that is now84 * insufficient).85 */86 dma_fence_get(&array->base);87 if (dma_fence_add_callback(array->fences[i], &cb[i].cb,88 dma_fence_array_cb_func)) {89 int error = array->fences[i]->error;90 91 dma_fence_array_set_pending_error(array, error);92 dma_fence_put(&array->base);93 if (atomic_dec_and_test(&array->num_pending)) {94 dma_fence_array_clear_pending_error(array);95 return false;96 }97 }98 }99 100 return true;101}102 103static bool dma_fence_array_signaled(struct dma_fence *fence)104{105 struct dma_fence_array *array = to_dma_fence_array(fence);106 107 if (atomic_read(&array->num_pending) > 0)108 return false;109 110 dma_fence_array_clear_pending_error(array);111 return true;112}113 114static void dma_fence_array_release(struct dma_fence *fence)115{116 struct dma_fence_array *array = to_dma_fence_array(fence);117 unsigned i;118 119 for (i = 0; i < array->num_fences; ++i)120 dma_fence_put(array->fences[i]);121 122 kfree(array->fences);123 dma_fence_free(fence);124}125 126static void dma_fence_array_set_deadline(struct dma_fence *fence,127 ktime_t deadline)128{129 struct dma_fence_array *array = to_dma_fence_array(fence);130 unsigned i;131 132 for (i = 0; i < array->num_fences; ++i)133 dma_fence_set_deadline(array->fences[i], deadline);134}135 136const struct dma_fence_ops dma_fence_array_ops = {137 .get_driver_name = dma_fence_array_get_driver_name,138 .get_timeline_name = dma_fence_array_get_timeline_name,139 .enable_signaling = dma_fence_array_enable_signaling,140 .signaled = dma_fence_array_signaled,141 .release = dma_fence_array_release,142 .set_deadline = dma_fence_array_set_deadline,143};144EXPORT_SYMBOL(dma_fence_array_ops);145 146/**147 * dma_fence_array_alloc - Allocate a custom fence array148 * @num_fences: [in] number of fences to add in the array149 *150 * Return dma fence array on success, NULL on failure151 */152struct dma_fence_array *dma_fence_array_alloc(int num_fences)153{154 struct dma_fence_array *array;155 156 return kzalloc(struct_size(array, callbacks, num_fences), GFP_KERNEL);157}158EXPORT_SYMBOL(dma_fence_array_alloc);159 160/**161 * dma_fence_array_init - Init a custom fence array162 * @array: [in] dma fence array to arm163 * @num_fences: [in] number of fences to add in the array164 * @fences: [in] array containing the fences165 * @context: [in] fence context to use166 * @seqno: [in] sequence number to use167 * @signal_on_any: [in] signal on any fence in the array168 *169 * Implementation of @dma_fence_array_create without allocation. Useful to init170 * a preallocated dma fence array in the path of reclaim or dma fence signaling.171 */172void dma_fence_array_init(struct dma_fence_array *array,173 int num_fences, struct dma_fence **fences,174 u64 context, unsigned seqno,175 bool signal_on_any)176{177 WARN_ON(!num_fences || !fences);178 179 array->num_fences = num_fences;180 181 spin_lock_init(&array->lock);182 dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,183 context, seqno);184 init_irq_work(&array->work, irq_dma_fence_array_work);185 186 atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);187 array->fences = fences;188 189 array->base.error = PENDING_ERROR;190 191 /*192 * dma_fence_array objects should never contain any other fence193 * containers or otherwise we run into recursion and potential kernel194 * stack overflow on operations on the dma_fence_array.195 *196 * The correct way of handling this is to flatten out the array by the197 * caller instead.198 *199 * Enforce this here by checking that we don't create a dma_fence_array200 * with any container inside.201 */202 while (num_fences--)203 WARN_ON(dma_fence_is_container(fences[num_fences]));204}205EXPORT_SYMBOL(dma_fence_array_init);206 207/**208 * dma_fence_array_create - Create a custom fence array209 * @num_fences: [in] number of fences to add in the array210 * @fences: [in] array containing the fences211 * @context: [in] fence context to use212 * @seqno: [in] sequence number to use213 * @signal_on_any: [in] signal on any fence in the array214 *215 * Allocate a dma_fence_array object and initialize the base fence with216 * dma_fence_init().217 * In case of error it returns NULL.218 *219 * The caller should allocate the fences array with num_fences size220 * and fill it with the fences it wants to add to the object. Ownership of this221 * array is taken and dma_fence_put() is used on each fence on release.222 *223 * If @signal_on_any is true the fence array signals if any fence in the array224 * signals, otherwise it signals when all fences in the array signal.225 */226struct dma_fence_array *dma_fence_array_create(int num_fences,227 struct dma_fence **fences,228 u64 context, unsigned seqno,229 bool signal_on_any)230{231 struct dma_fence_array *array;232 233 array = dma_fence_array_alloc(num_fences);234 if (!array)235 return NULL;236 237 dma_fence_array_init(array, num_fences, fences,238 context, seqno, signal_on_any);239 240 return array;241}242EXPORT_SYMBOL(dma_fence_array_create);243 244/**245 * dma_fence_match_context - Check if all fences are from the given context246 * @fence: [in] fence or fence array247 * @context: [in] fence context to check all fences against248 *249 * Checks the provided fence or, for a fence array, all fences in the array250 * against the given context. Returns false if any fence is from a different251 * context.252 */253bool dma_fence_match_context(struct dma_fence *fence, u64 context)254{255 struct dma_fence_array *array = to_dma_fence_array(fence);256 unsigned i;257 258 if (!dma_fence_is_array(fence))259 return fence->context == context;260 261 for (i = 0; i < array->num_fences; i++) {262 if (array->fences[i]->context != context)263 return false;264 }265 266 return true;267}268EXPORT_SYMBOL(dma_fence_match_context);269 270struct dma_fence *dma_fence_array_first(struct dma_fence *head)271{272 struct dma_fence_array *array;273 274 if (!head)275 return NULL;276 277 array = to_dma_fence_array(head);278 if (!array)279 return head;280 281 if (!array->num_fences)282 return NULL;283 284 return array->fences[0];285}286EXPORT_SYMBOL(dma_fence_array_first);287 288struct dma_fence *dma_fence_array_next(struct dma_fence *head,289 unsigned int index)290{291 struct dma_fence_array *array = to_dma_fence_array(head);292 293 if (!array || index >= array->num_fences)294 return NULL;295 296 return array->fences[index];297}298EXPORT_SYMBOL(dma_fence_array_next);299