239 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2019 Intel Corporation5 */6 7#ifndef _I915_ACTIVE_H_8#define _I915_ACTIVE_H_9 10#include <linux/lockdep.h>11 12#include "i915_active_types.h"13#include "i915_request.h"14 15struct i915_request;16struct intel_engine_cs;17struct intel_timeline;18 19/*20 * We treat requests as fences. This is not be to confused with our21 * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.22 * We use the fences to synchronize access from the CPU with activity on the23 * GPU, for example, we should not rewrite an object's PTE whilst the GPU24 * is reading them. We also track fences at a higher level to provide25 * implicit synchronisation around GEM objects, e.g. set-domain will wait26 * for outstanding GPU rendering before marking the object ready for CPU27 * access, or a pageflip will wait until the GPU is complete before showing28 * the frame on the scanout.29 *30 * In order to use a fence, the object must track the fence it needs to31 * serialise with. For example, GEM objects want to track both read and32 * write access so that we can perform concurrent read operations between33 * the CPU and GPU engines, as well as waiting for all rendering to34 * complete, or waiting for the last GPU user of a "fence register". The35 * object then embeds a #i915_active_fence to track the most recent (in36 * retirement order) request relevant for the desired mode of access.37 * The #i915_active_fence is updated with i915_active_fence_set() to38 * track the most recent fence request, typically this is done as part of39 * i915_vma_move_to_active().40 *41 * When the #i915_active_fence completes (is retired), it will42 * signal its completion to the owner through a callback as well as mark43 * itself as idle (i915_active_fence.request == NULL). The owner44 * can then perform any action, such as delayed freeing of an active45 * resource including itself.46 */47 48void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb);49 50/**51 * __i915_active_fence_init - prepares the activity tracker for use52 * @active: the active tracker53 * @fence: initial fence to track, can be NULL54 * @fn: a callback when then the tracker is retired (becomes idle),55 * can be NULL56 *57 * i915_active_fence_init() prepares the embedded @active struct for use as58 * an activity tracker, that is for tracking the last known active fence59 * associated with it. When the last fence becomes idle, when it is retired60 * after completion, the optional callback @func is invoked.61 */62static inline void63__i915_active_fence_init(struct i915_active_fence *active,64 void *fence,65 dma_fence_func_t fn)66{67 RCU_INIT_POINTER(active->fence, fence);68 active->cb.func = fn ?: i915_active_noop;69}70 71#define INIT_ACTIVE_FENCE(A) \72 __i915_active_fence_init((A), NULL, NULL)73 74struct dma_fence *75__i915_active_fence_set(struct i915_active_fence *active,76 struct dma_fence *fence);77 78/**79 * i915_active_fence_set - updates the tracker to watch the current fence80 * @active: the active tracker81 * @rq: the request to watch82 *83 * i915_active_fence_set() watches the given @rq for completion. While84 * that @rq is busy, the @active reports busy. When that @rq is signaled85 * (or else retired) the @active tracker is updated to report idle.86 */87int __must_check88i915_active_fence_set(struct i915_active_fence *active,89 struct i915_request *rq);90/**91 * i915_active_fence_get - return a reference to the active fence92 * @active: the active tracker93 *94 * i915_active_fence_get() returns a reference to the active fence,95 * or NULL if the active tracker is idle. The reference is obtained under RCU,96 * so no locking is required by the caller.97 *98 * The reference should be freed with dma_fence_put().99 */100static inline struct dma_fence *101i915_active_fence_get(struct i915_active_fence *active)102{103 struct dma_fence *fence;104 105 rcu_read_lock();106 fence = dma_fence_get_rcu_safe(&active->fence);107 rcu_read_unlock();108 109 return fence;110}111 112/**113 * i915_active_fence_isset - report whether the active tracker is assigned114 * @active: the active tracker115 *116 * i915_active_fence_isset() returns true if the active tracker is currently117 * assigned to a fence. Due to the lazy retiring, that fence may be idle118 * and this may report stale information.119 */120static inline bool121i915_active_fence_isset(const struct i915_active_fence *active)122{123 return rcu_access_pointer(active->fence);124}125 126/*127 * GPU activity tracking128 *129 * Each set of commands submitted to the GPU compromises a single request that130 * signals a fence upon completion. struct i915_request combines the131 * command submission, scheduling and fence signaling roles. If we want to see132 * if a particular task is complete, we need to grab the fence (struct133 * i915_request) for that task and check or wait for it to be signaled. More134 * often though we want to track the status of a bunch of tasks, for example135 * to wait for the GPU to finish accessing some memory across a variety of136 * different command pipelines from different clients. We could choose to137 * track every single request associated with the task, but knowing that138 * each request belongs to an ordered timeline (later requests within a139 * timeline must wait for earlier requests), we need only track the140 * latest request in each timeline to determine the overall status of the141 * task.142 *143 * struct i915_active provides this tracking across timelines. It builds a144 * composite shared-fence, and is updated as new work is submitted to the task,145 * forming a snapshot of the current status. It should be embedded into the146 * different resources that need to track their associated GPU activity to147 * provide a callback when that GPU activity has ceased, or otherwise to148 * provide a serialisation point either for request submission or for CPU149 * synchronisation.150 */151 152void __i915_active_init(struct i915_active *ref,153 int (*active)(struct i915_active *ref),154 void (*retire)(struct i915_active *ref),155 unsigned long flags,156 struct lock_class_key *mkey,157 struct lock_class_key *wkey);158 159/* Specialise each class of i915_active to avoid impossible lockdep cycles. */160#define i915_active_init(ref, active, retire, flags) do { \161 static struct lock_class_key __mkey; \162 static struct lock_class_key __wkey; \163 \164 __i915_active_init(ref, active, retire, flags, &__mkey, &__wkey); \165} while (0)166 167int i915_active_add_request(struct i915_active *ref, struct i915_request *rq);168 169struct dma_fence *170i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f);171 172int __i915_active_wait(struct i915_active *ref, int state);173static inline int i915_active_wait(struct i915_active *ref)174{175 return __i915_active_wait(ref, TASK_INTERRUPTIBLE);176}177 178int i915_sw_fence_await_active(struct i915_sw_fence *fence,179 struct i915_active *ref,180 unsigned int flags);181int i915_request_await_active(struct i915_request *rq,182 struct i915_active *ref,183 unsigned int flags);184#define I915_ACTIVE_AWAIT_EXCL BIT(0)185#define I915_ACTIVE_AWAIT_ACTIVE BIT(1)186#define I915_ACTIVE_AWAIT_BARRIER BIT(2)187 188int i915_active_acquire(struct i915_active *ref);189int i915_active_acquire_for_context(struct i915_active *ref, u64 idx);190bool i915_active_acquire_if_busy(struct i915_active *ref);191 192void i915_active_release(struct i915_active *ref);193 194static inline void __i915_active_acquire(struct i915_active *ref)195{196 GEM_BUG_ON(!atomic_read(&ref->count));197 atomic_inc(&ref->count);198}199 200static inline bool201i915_active_is_idle(const struct i915_active *ref)202{203 return !atomic_read(&ref->count);204}205 206void i915_active_fini(struct i915_active *ref);207 208int i915_active_acquire_preallocate_barrier(struct i915_active *ref,209 struct intel_engine_cs *engine);210void i915_active_acquire_barrier(struct i915_active *ref);211void i915_request_add_active_barriers(struct i915_request *rq);212 213void i915_active_print(struct i915_active *ref, struct drm_printer *m);214void i915_active_unlock_wait(struct i915_active *ref);215 216struct i915_active *i915_active_create(void);217struct i915_active *i915_active_get(struct i915_active *ref);218void i915_active_put(struct i915_active *ref);219 220static inline int __i915_request_await_exclusive(struct i915_request *rq,221 struct i915_active *active)222{223 struct dma_fence *fence;224 int err = 0;225 226 fence = i915_active_fence_get(&active->excl);227 if (fence) {228 err = i915_request_await_dma_fence(rq, fence);229 dma_fence_put(fence);230 }231 232 return err;233}234 235void i915_active_module_exit(void);236int i915_active_module_init(void);237 238#endif /* _I915_ACTIVE_H_ */239