278 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * fence-chain: chain fences together in a timeline4 *5 * Copyright (C) 2018 Advanced Micro Devices, Inc.6 * Authors:7 * Christian König <christian.koenig@amd.com>8 */9 10#include <linux/dma-fence-chain.h>11 12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);13 14/**15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence16 * @chain: chain node to get the previous node from17 *18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the19 * chain node.20 */21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)22{23 struct dma_fence *prev;24 25 rcu_read_lock();26 prev = dma_fence_get_rcu_safe(&chain->prev);27 rcu_read_unlock();28 return prev;29}30 31/**32 * dma_fence_chain_walk - chain walking function33 * @fence: current chain node34 *35 * Walk the chain to the next node. Returns the next fence or NULL if we are at36 * the end of the chain. Garbage collects chain nodes which are already37 * signaled.38 */39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)40{41 struct dma_fence_chain *chain, *prev_chain;42 struct dma_fence *prev, *replacement, *tmp;43 44 chain = to_dma_fence_chain(fence);45 if (!chain) {46 dma_fence_put(fence);47 return NULL;48 }49 50 while ((prev = dma_fence_chain_get_prev(chain))) {51 52 prev_chain = to_dma_fence_chain(prev);53 if (prev_chain) {54 if (!dma_fence_is_signaled(prev_chain->fence))55 break;56 57 replacement = dma_fence_chain_get_prev(prev_chain);58 } else {59 if (!dma_fence_is_signaled(prev))60 break;61 62 replacement = NULL;63 }64 65 tmp = unrcu_pointer(cmpxchg(&chain->prev, RCU_INITIALIZER(prev),66 RCU_INITIALIZER(replacement)));67 if (tmp == prev)68 dma_fence_put(tmp);69 else70 dma_fence_put(replacement);71 dma_fence_put(prev);72 }73 74 dma_fence_put(fence);75 return prev;76}77EXPORT_SYMBOL(dma_fence_chain_walk);78 79/**80 * dma_fence_chain_find_seqno - find fence chain node by seqno81 * @pfence: pointer to the chain node where to start82 * @seqno: the sequence number to search for83 *84 * Advance the fence pointer to the chain node which will signal this sequence85 * number. If no sequence number is provided then this is a no-op.86 *87 * Returns EINVAL if the fence is not a chain node or the sequence number has88 * not yet advanced far enough.89 */90int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)91{92 struct dma_fence_chain *chain;93 94 if (!seqno)95 return 0;96 97 chain = to_dma_fence_chain(*pfence);98 if (!chain || chain->base.seqno < seqno)99 return -EINVAL;100 101 dma_fence_chain_for_each(*pfence, &chain->base) {102 if ((*pfence)->context != chain->base.context ||103 to_dma_fence_chain(*pfence)->prev_seqno < seqno)104 break;105 }106 dma_fence_put(&chain->base);107 108 return 0;109}110EXPORT_SYMBOL(dma_fence_chain_find_seqno);111 112static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)113{114 return "dma_fence_chain";115}116 117static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)118{119 return "unbound";120}121 122static void dma_fence_chain_irq_work(struct irq_work *work)123{124 struct dma_fence_chain *chain;125 126 chain = container_of(work, typeof(*chain), work);127 128 /* Try to rearm the callback */129 if (!dma_fence_chain_enable_signaling(&chain->base))130 /* Ok, we are done. No more unsignaled fences left */131 dma_fence_signal(&chain->base);132 dma_fence_put(&chain->base);133}134 135static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)136{137 struct dma_fence_chain *chain;138 139 chain = container_of(cb, typeof(*chain), cb);140 init_irq_work(&chain->work, dma_fence_chain_irq_work);141 irq_work_queue(&chain->work);142 dma_fence_put(f);143}144 145static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)146{147 struct dma_fence_chain *head = to_dma_fence_chain(fence);148 149 dma_fence_get(&head->base);150 dma_fence_chain_for_each(fence, &head->base) {151 struct dma_fence *f = dma_fence_chain_contained(fence);152 153 dma_fence_get(f);154 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {155 dma_fence_put(fence);156 return true;157 }158 dma_fence_put(f);159 }160 dma_fence_put(&head->base);161 return false;162}163 164static bool dma_fence_chain_signaled(struct dma_fence *fence)165{166 dma_fence_chain_for_each(fence, fence) {167 struct dma_fence *f = dma_fence_chain_contained(fence);168 169 if (!dma_fence_is_signaled(f)) {170 dma_fence_put(fence);171 return false;172 }173 }174 175 return true;176}177 178static void dma_fence_chain_release(struct dma_fence *fence)179{180 struct dma_fence_chain *chain = to_dma_fence_chain(fence);181 struct dma_fence *prev;182 183 /* Manually unlink the chain as much as possible to avoid recursion184 * and potential stack overflow.185 */186 while ((prev = rcu_dereference_protected(chain->prev, true))) {187 struct dma_fence_chain *prev_chain;188 189 if (kref_read(&prev->refcount) > 1)190 break;191 192 prev_chain = to_dma_fence_chain(prev);193 if (!prev_chain)194 break;195 196 /* No need for atomic operations since we hold the last197 * reference to prev_chain.198 */199 chain->prev = prev_chain->prev;200 RCU_INIT_POINTER(prev_chain->prev, NULL);201 dma_fence_put(prev);202 }203 dma_fence_put(prev);204 205 dma_fence_put(chain->fence);206 dma_fence_free(fence);207}208 209 210static void dma_fence_chain_set_deadline(struct dma_fence *fence,211 ktime_t deadline)212{213 dma_fence_chain_for_each(fence, fence) {214 struct dma_fence *f = dma_fence_chain_contained(fence);215 216 dma_fence_set_deadline(f, deadline);217 }218}219 220const struct dma_fence_ops dma_fence_chain_ops = {221 .use_64bit_seqno = true,222 .get_driver_name = dma_fence_chain_get_driver_name,223 .get_timeline_name = dma_fence_chain_get_timeline_name,224 .enable_signaling = dma_fence_chain_enable_signaling,225 .signaled = dma_fence_chain_signaled,226 .release = dma_fence_chain_release,227 .set_deadline = dma_fence_chain_set_deadline,228};229EXPORT_SYMBOL(dma_fence_chain_ops);230 231/**232 * dma_fence_chain_init - initialize a fence chain233 * @chain: the chain node to initialize234 * @prev: the previous fence235 * @fence: the current fence236 * @seqno: the sequence number to use for the fence chain237 *238 * Initialize a new chain node and either start a new chain or add the node to239 * the existing chain of the previous fence.240 */241void dma_fence_chain_init(struct dma_fence_chain *chain,242 struct dma_fence *prev,243 struct dma_fence *fence,244 uint64_t seqno)245{246 struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);247 uint64_t context;248 249 spin_lock_init(&chain->lock);250 rcu_assign_pointer(chain->prev, prev);251 chain->fence = fence;252 chain->prev_seqno = 0;253 254 /* Try to reuse the context of the previous chain node. */255 if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {256 context = prev->context;257 chain->prev_seqno = prev->seqno;258 } else {259 context = dma_fence_context_alloc(1);260 /* Make sure that we always have a valid sequence number. */261 if (prev_chain)262 seqno = max(prev->seqno, seqno);263 }264 265 dma_fence_init(&chain->base, &dma_fence_chain_ops,266 &chain->lock, context, seqno);267 268 /*269 * Chaining dma_fence_chain container together is only allowed through270 * the prev fence and not through the contained fence.271 *272 * The correct way of handling this is to flatten out the fence273 * structure into a dma_fence_array by the caller instead.274 */275 WARN_ON(dma_fence_is_chain(fence));276}277EXPORT_SYMBOL(dma_fence_chain_init);278