238 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include <linux/dma-fence.h>7#include <linux/slab.h>8 9#include <drm/ttm/ttm_bo.h>10 11#include "i915_deps.h"12 13/**14 * DOC: Set of utilities to dynamically collect dependencies into a15 * structure which is fed into the GT migration code.16 *17 * Once we can do async unbinding, this is also needed to coalesce18 * the migration fence with the unbind fences if these are coalesced19 * post-migration.20 *21 * While collecting the individual dependencies, we store the refcounted22 * struct dma_fence pointers in a realloc-managed pointer array, since23 * that can be easily fed into a dma_fence_array. Other options are24 * available, like for example an xarray for similarity with drm/sched.25 * Can be changed easily if needed.26 *27 * A struct i915_deps need to be initialized using i915_deps_init().28 * If i915_deps_add_dependency() or i915_deps_add_resv() return an29 * error code they will internally call i915_deps_fini(), which frees30 * all internal references and allocations.31 */32 33/* Min number of fence pointers in the array when an allocation occurs. */34#define I915_DEPS_MIN_ALLOC_CHUNK 8U35 36static void i915_deps_reset_fences(struct i915_deps *deps)37{38 if (deps->fences != &deps->single)39 kfree(deps->fences);40 deps->num_deps = 0;41 deps->fences_size = 1;42 deps->fences = &deps->single;43}44 45/**46 * i915_deps_init - Initialize an i915_deps structure47 * @deps: Pointer to the i915_deps structure to initialize.48 * @gfp: The allocation mode for subsequenst allocations.49 */50void i915_deps_init(struct i915_deps *deps, gfp_t gfp)51{52 deps->fences = NULL;53 deps->gfp = gfp;54 i915_deps_reset_fences(deps);55}56 57/**58 * i915_deps_fini - Finalize an i915_deps structure59 * @deps: Pointer to the i915_deps structure to finalize.60 *61 * This function drops all fence references taken, conditionally frees and62 * then resets the fences array.63 */64void i915_deps_fini(struct i915_deps *deps)65{66 unsigned int i;67 68 for (i = 0; i < deps->num_deps; ++i)69 dma_fence_put(deps->fences[i]);70 71 if (deps->fences != &deps->single)72 kfree(deps->fences);73}74 75static int i915_deps_grow(struct i915_deps *deps, struct dma_fence *fence,76 const struct ttm_operation_ctx *ctx)77{78 int ret;79 80 if (deps->num_deps >= deps->fences_size) {81 unsigned int new_size = 2 * deps->fences_size;82 struct dma_fence **new_fences;83 84 new_size = max(new_size, I915_DEPS_MIN_ALLOC_CHUNK);85 new_fences = kmalloc_array(new_size, sizeof(*new_fences), deps->gfp);86 if (!new_fences)87 goto sync;88 89 memcpy(new_fences, deps->fences,90 deps->fences_size * sizeof(*new_fences));91 swap(new_fences, deps->fences);92 if (new_fences != &deps->single)93 kfree(new_fences);94 deps->fences_size = new_size;95 }96 deps->fences[deps->num_deps++] = dma_fence_get(fence);97 return 0;98 99sync:100 if (ctx->no_wait_gpu && !dma_fence_is_signaled(fence)) {101 ret = -EBUSY;102 goto unref;103 }104 105 ret = dma_fence_wait(fence, ctx->interruptible);106 if (ret)107 goto unref;108 109 ret = fence->error;110 if (ret)111 goto unref;112 113 return 0;114 115unref:116 i915_deps_fini(deps);117 return ret;118}119 120/**121 * i915_deps_sync - Wait for all the fences in the dependency collection122 * @deps: Pointer to the i915_deps structure the fences of which to wait for.123 * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits124 * should be performed.125 *126 * This function waits for fences in the dependency collection. If it127 * encounters an error during the wait or a fence error, the wait for128 * further fences is aborted and the error returned.129 *130 * Return: Zero if successful, Negative error code on error.131 */132int i915_deps_sync(const struct i915_deps *deps, const struct ttm_operation_ctx *ctx)133{134 struct dma_fence **fences = deps->fences;135 unsigned int i;136 int ret = 0;137 138 for (i = 0; i < deps->num_deps; ++i, ++fences) {139 if (ctx->no_wait_gpu && !dma_fence_is_signaled(*fences)) {140 ret = -EBUSY;141 break;142 }143 144 ret = dma_fence_wait(*fences, ctx->interruptible);145 if (!ret)146 ret = (*fences)->error;147 if (ret)148 break;149 }150 151 return ret;152}153 154/**155 * i915_deps_add_dependency - Add a fence to the dependency collection156 * @deps: Pointer to the i915_deps structure a fence is to be added to.157 * @fence: The fence to add.158 * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to159 * be performed if waiting.160 *161 * Adds a fence to the dependency collection, and takes a reference on it.162 * If the fence context is not zero and there was a later fence from the163 * same fence context already added, then the fence is not added to the164 * dependency collection. If the fence context is not zero and there was165 * an earlier fence already added, then the fence will replace the older166 * fence from the same context and the reference on the earlier fence will167 * be dropped.168 * If there is a failure to allocate memory to accommodate the new fence to169 * be added, the new fence will instead be waited for and an error may170 * be returned; depending on the value of @ctx, or if there was a fence171 * error. If an error was returned, the dependency collection will be172 * finalized and all fence reference dropped.173 *174 * Return: 0 if success. Negative error code on error.175 */176int i915_deps_add_dependency(struct i915_deps *deps,177 struct dma_fence *fence,178 const struct ttm_operation_ctx *ctx)179{180 unsigned int i;181 int ret;182 183 if (!fence)184 return 0;185 186 if (dma_fence_is_signaled(fence)) {187 ret = fence->error;188 if (ret)189 i915_deps_fini(deps);190 return ret;191 }192 193 for (i = 0; i < deps->num_deps; ++i) {194 struct dma_fence *entry = deps->fences[i];195 196 if (!entry->context || entry->context != fence->context)197 continue;198 199 if (dma_fence_is_later(fence, entry)) {200 dma_fence_put(entry);201 deps->fences[i] = dma_fence_get(fence);202 }203 204 return 0;205 }206 207 return i915_deps_grow(deps, fence, ctx);208}209 210/**211 * i915_deps_add_resv - Add the fences of a reservation object to a dependency212 * collection.213 * @deps: Pointer to the i915_deps structure a fence is to be added to.214 * @resv: The reservation object, then fences of which to add.215 * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to216 * be performed if waiting.217 *218 * Calls i915_deps_add_depencency() on the indicated fences of @resv.219 *220 * Return: Zero on success. Negative error code on error.221 */222int i915_deps_add_resv(struct i915_deps *deps, struct dma_resv *resv,223 const struct ttm_operation_ctx *ctx)224{225 struct dma_resv_iter iter;226 struct dma_fence *fence;227 228 dma_resv_assert_held(resv);229 dma_resv_for_each_fence(&iter, resv, dma_resv_usage_rw(true), fence) {230 int ret = i915_deps_add_dependency(deps, fence, ctx);231 232 if (ret)233 return ret;234 }235 236 return 0;237}238