290 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include <uapi/drm/pvr_drm.h>5 6#include <drm/drm_syncobj.h>7#include <drm/gpu_scheduler.h>8#include <linux/xarray.h>9#include <linux/dma-fence-unwrap.h>10 11#include "pvr_device.h"12#include "pvr_queue.h"13#include "pvr_sync.h"14 15static int16pvr_check_sync_op(const struct drm_pvr_sync_op *sync_op)17{18 u8 handle_type;19 20 if (sync_op->flags & ~DRM_PVR_SYNC_OP_FLAGS_MASK)21 return -EINVAL;22 23 handle_type = sync_op->flags & DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK;24 if (handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&25 handle_type != DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ)26 return -EINVAL;27 28 if (handle_type == DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ &&29 sync_op->value != 0)30 return -EINVAL;31 32 return 0;33}34 35static void36pvr_sync_signal_free(struct pvr_sync_signal *sig_sync)37{38 if (!sig_sync)39 return;40 41 drm_syncobj_put(sig_sync->syncobj);42 dma_fence_chain_free(sig_sync->chain);43 dma_fence_put(sig_sync->fence);44 kfree(sig_sync);45}46 47void48pvr_sync_signal_array_cleanup(struct xarray *array)49{50 struct pvr_sync_signal *sig_sync;51 unsigned long i;52 53 xa_for_each(array, i, sig_sync)54 pvr_sync_signal_free(sig_sync);55 56 xa_destroy(array);57}58 59static struct pvr_sync_signal *60pvr_sync_signal_array_add(struct xarray *array, struct drm_file *file, u32 handle, u64 point)61{62 struct pvr_sync_signal *sig_sync;63 struct dma_fence *cur_fence;64 int err;65 u32 id;66 67 sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL);68 if (!sig_sync)69 return ERR_PTR(-ENOMEM);70 71 sig_sync->handle = handle;72 sig_sync->point = point;73 74 if (point > 0) {75 sig_sync->chain = dma_fence_chain_alloc();76 if (!sig_sync->chain) {77 err = -ENOMEM;78 goto err_free_sig_sync;79 }80 }81 82 sig_sync->syncobj = drm_syncobj_find(file, handle);83 if (!sig_sync->syncobj) {84 err = -EINVAL;85 goto err_free_sig_sync;86 }87 88 /* Retrieve the current fence attached to that point. It's89 * perfectly fine to get a NULL fence here, it just means there's90 * no fence attached to that point yet.91 */92 if (!drm_syncobj_find_fence(file, handle, point, 0, &cur_fence))93 sig_sync->fence = cur_fence;94 95 err = xa_alloc(array, &id, sig_sync, xa_limit_32b, GFP_KERNEL);96 if (err)97 goto err_free_sig_sync;98 99 return sig_sync;100 101err_free_sig_sync:102 pvr_sync_signal_free(sig_sync);103 return ERR_PTR(err);104}105 106static struct pvr_sync_signal *107pvr_sync_signal_array_search(struct xarray *array, u32 handle, u64 point)108{109 struct pvr_sync_signal *sig_sync;110 unsigned long i;111 112 xa_for_each(array, i, sig_sync) {113 if (handle == sig_sync->handle && point == sig_sync->point)114 return sig_sync;115 }116 117 return NULL;118}119 120static struct pvr_sync_signal *121pvr_sync_signal_array_get(struct xarray *array, struct drm_file *file, u32 handle, u64 point)122{123 struct pvr_sync_signal *sig_sync;124 125 sig_sync = pvr_sync_signal_array_search(array, handle, point);126 if (sig_sync)127 return sig_sync;128 129 return pvr_sync_signal_array_add(array, file, handle, point);130}131 132int133pvr_sync_signal_array_collect_ops(struct xarray *array,134 struct drm_file *file,135 u32 sync_op_count,136 const struct drm_pvr_sync_op *sync_ops)137{138 for (u32 i = 0; i < sync_op_count; i++) {139 struct pvr_sync_signal *sig_sync;140 int ret;141 142 if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))143 continue;144 145 ret = pvr_check_sync_op(&sync_ops[i]);146 if (ret)147 return ret;148 149 sig_sync = pvr_sync_signal_array_get(array, file,150 sync_ops[i].handle,151 sync_ops[i].value);152 if (IS_ERR(sig_sync))153 return PTR_ERR(sig_sync);154 }155 156 return 0;157}158 159int160pvr_sync_signal_array_update_fences(struct xarray *array,161 u32 sync_op_count,162 const struct drm_pvr_sync_op *sync_ops,163 struct dma_fence *done_fence)164{165 for (u32 i = 0; i < sync_op_count; i++) {166 struct dma_fence *old_fence;167 struct pvr_sync_signal *sig_sync;168 169 if (!(sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL))170 continue;171 172 sig_sync = pvr_sync_signal_array_search(array, sync_ops[i].handle,173 sync_ops[i].value);174 if (WARN_ON(!sig_sync))175 return -EINVAL;176 177 old_fence = sig_sync->fence;178 sig_sync->fence = dma_fence_get(done_fence);179 dma_fence_put(old_fence);180 181 if (WARN_ON(!sig_sync->fence))182 return -EINVAL;183 }184 185 return 0;186}187 188void189pvr_sync_signal_array_push_fences(struct xarray *array)190{191 struct pvr_sync_signal *sig_sync;192 unsigned long i;193 194 xa_for_each(array, i, sig_sync) {195 if (sig_sync->chain) {196 drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain,197 sig_sync->fence, sig_sync->point);198 sig_sync->chain = NULL;199 } else {200 drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence);201 }202 }203}204 205static int206pvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f)207{208 struct dma_fence_unwrap iter;209 u32 native_fence_count = 0;210 struct dma_fence *uf;211 int err = 0;212 213 dma_fence_unwrap_for_each(uf, &iter, f) {214 if (pvr_queue_fence_is_ufo_backed(uf))215 native_fence_count++;216 }217 218 /* No need to unwrap the fence if it's fully non-native. */219 if (!native_fence_count)220 return drm_sched_job_add_dependency(job, f);221 222 dma_fence_unwrap_for_each(uf, &iter, f) {223 /* There's no dma_fence_unwrap_stop() helper cleaning up the refs224 * owned by dma_fence_unwrap(), so let's just iterate over all225 * entries without doing anything when something failed.226 */227 if (err)228 continue;229 230 if (pvr_queue_fence_is_ufo_backed(uf)) {231 struct drm_sched_fence *s_fence = to_drm_sched_fence(uf);232 233 /* If this is a native dependency, we wait for the scheduled fence,234 * and we will let pvr_queue_run_job() issue FW waits.235 */236 err = drm_sched_job_add_dependency(job,237 dma_fence_get(&s_fence->scheduled));238 } else {239 err = drm_sched_job_add_dependency(job, dma_fence_get(uf));240 }241 }242 243 dma_fence_put(f);244 return err;245}246 247int248pvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job,249 u32 sync_op_count,250 const struct drm_pvr_sync_op *sync_ops,251 struct xarray *signal_array)252{253 int err = 0;254 255 if (!sync_op_count)256 return 0;257 258 for (u32 i = 0; i < sync_op_count; i++) {259 struct pvr_sync_signal *sig_sync;260 struct dma_fence *fence;261 262 if (sync_ops[i].flags & DRM_PVR_SYNC_OP_FLAG_SIGNAL)263 continue;264 265 err = pvr_check_sync_op(&sync_ops[i]);266 if (err)267 return err;268 269 sig_sync = pvr_sync_signal_array_search(signal_array, sync_ops[i].handle,270 sync_ops[i].value);271 if (sig_sync) {272 if (WARN_ON(!sig_sync->fence))273 return -EINVAL;274 275 fence = dma_fence_get(sig_sync->fence);276 } else {277 err = drm_syncobj_find_fence(from_pvr_file(pvr_file), sync_ops[i].handle,278 sync_ops[i].value, 0, &fence);279 if (err)280 return err;281 }282 283 err = pvr_sync_add_dep_to_job(job, fence);284 if (err)285 return err;286 }287 288 return 0;289}290