379 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include "xe_sync.h"7 8#include <linux/dma-fence-array.h>9#include <linux/kthread.h>10#include <linux/sched/mm.h>11#include <linux/uaccess.h>12 13#include <drm/drm_print.h>14#include <drm/drm_syncobj.h>15#include <uapi/drm/xe_drm.h>16 17#include "xe_device_types.h"18#include "xe_exec_queue.h"19#include "xe_macros.h"20#include "xe_sched_job_types.h"21 22struct xe_user_fence {23 struct xe_device *xe;24 struct kref refcount;25 struct dma_fence_cb cb;26 struct work_struct worker;27 struct mm_struct *mm;28 u64 __user *addr;29 u64 value;30 int signalled;31};32 33static void user_fence_destroy(struct kref *kref)34{35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,36 refcount);37 38 mmdrop(ufence->mm);39 kfree(ufence);40}41 42static void user_fence_get(struct xe_user_fence *ufence)43{44 kref_get(&ufence->refcount);45}46 47static void user_fence_put(struct xe_user_fence *ufence)48{49 kref_put(&ufence->refcount, user_fence_destroy);50}51 52static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,53 u64 value)54{55 struct xe_user_fence *ufence;56 u64 __user *ptr = u64_to_user_ptr(addr);57 u64 __maybe_unused prefetch_val;58 59 if (get_user(prefetch_val, ptr))60 return ERR_PTR(-EFAULT);61 62 ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);63 if (!ufence)64 return ERR_PTR(-ENOMEM);65 66 ufence->xe = xe;67 kref_init(&ufence->refcount);68 ufence->addr = ptr;69 ufence->value = value;70 ufence->mm = current->mm;71 mmgrab(ufence->mm);72 73 return ufence;74}75 76static void user_fence_worker(struct work_struct *w)77{78 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);79 80 if (mmget_not_zero(ufence->mm)) {81 kthread_use_mm(ufence->mm);82 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))83 XE_WARN_ON("Copy to user failed");84 kthread_unuse_mm(ufence->mm);85 mmput(ufence->mm);86 }87 88 wake_up_all(&ufence->xe->ufence_wq);89 WRITE_ONCE(ufence->signalled, 1);90 user_fence_put(ufence);91}92 93static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)94{95 INIT_WORK(&ufence->worker, user_fence_worker);96 queue_work(ufence->xe->ordered_wq, &ufence->worker);97 dma_fence_put(fence);98}99 100static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)101{102 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);103 104 kick_ufence(ufence, fence);105}106 107int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,108 struct xe_sync_entry *sync,109 struct drm_xe_sync __user *sync_user,110 unsigned int flags)111{112 struct drm_xe_sync sync_in;113 int err;114 bool exec = flags & SYNC_PARSE_FLAG_EXEC;115 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;116 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;117 bool signal;118 119 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))120 return -EFAULT;121 122 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||123 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))124 return -EINVAL;125 126 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;127 switch (sync_in.type) {128 case DRM_XE_SYNC_TYPE_SYNCOBJ:129 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))130 return -EOPNOTSUPP;131 132 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))133 return -EINVAL;134 135 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);136 if (XE_IOCTL_DBG(xe, !sync->syncobj))137 return -ENOENT;138 139 if (!signal) {140 sync->fence = drm_syncobj_fence_get(sync->syncobj);141 if (XE_IOCTL_DBG(xe, !sync->fence))142 return -EINVAL;143 }144 break;145 146 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:147 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))148 return -EOPNOTSUPP;149 150 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))151 return -EINVAL;152 153 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))154 return -EINVAL;155 156 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);157 if (XE_IOCTL_DBG(xe, !sync->syncobj))158 return -ENOENT;159 160 if (signal) {161 sync->chain_fence = dma_fence_chain_alloc();162 if (!sync->chain_fence)163 return -ENOMEM;164 } else {165 sync->fence = drm_syncobj_fence_get(sync->syncobj);166 if (XE_IOCTL_DBG(xe, !sync->fence))167 return -EINVAL;168 169 err = dma_fence_chain_find_seqno(&sync->fence,170 sync_in.timeline_value);171 if (err)172 return err;173 }174 break;175 176 case DRM_XE_SYNC_TYPE_USER_FENCE:177 if (XE_IOCTL_DBG(xe, disallow_user_fence))178 return -EOPNOTSUPP;179 180 if (XE_IOCTL_DBG(xe, !signal))181 return -EOPNOTSUPP;182 183 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))184 return -EINVAL;185 186 if (exec) {187 sync->addr = sync_in.addr;188 } else {189 sync->ufence = user_fence_create(xe, sync_in.addr,190 sync_in.timeline_value);191 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))192 return PTR_ERR(sync->ufence);193 }194 195 break;196 197 default:198 return -EINVAL;199 }200 201 sync->type = sync_in.type;202 sync->flags = sync_in.flags;203 sync->timeline_value = sync_in.timeline_value;204 205 return 0;206}207 208int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)209{210 if (sync->fence)211 return drm_sched_job_add_dependency(&job->drm,212 dma_fence_get(sync->fence));213 214 return 0;215}216 217void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)218{219 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))220 return;221 222 if (sync->chain_fence) {223 drm_syncobj_add_point(sync->syncobj, sync->chain_fence,224 fence, sync->timeline_value);225 /*226 * The chain's ownership is transferred to the227 * timeline.228 */229 sync->chain_fence = NULL;230 } else if (sync->syncobj) {231 drm_syncobj_replace_fence(sync->syncobj, fence);232 } else if (sync->ufence) {233 int err;234 235 dma_fence_get(fence);236 user_fence_get(sync->ufence);237 err = dma_fence_add_callback(fence, &sync->ufence->cb,238 user_fence_cb);239 if (err == -ENOENT) {240 kick_ufence(sync->ufence, fence);241 } else if (err) {242 XE_WARN_ON("failed to add user fence");243 user_fence_put(sync->ufence);244 dma_fence_put(fence);245 }246 }247}248 249void xe_sync_entry_cleanup(struct xe_sync_entry *sync)250{251 if (sync->syncobj)252 drm_syncobj_put(sync->syncobj);253 dma_fence_put(sync->fence);254 dma_fence_chain_free(sync->chain_fence);255 if (sync->ufence)256 user_fence_put(sync->ufence);257}258 259/**260 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM261 * @sync: input syncs262 * @num_sync: number of syncs263 * @q: exec queue264 * @vm: VM265 *266 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create267 * and return a composite fence of all in-fences + last fence. If no in-fences268 * return last fence on input exec queue. Caller must drop reference to269 * returned fence.270 *271 * Return: fence on success, ERR_PTR(-ENOMEM) on failure272 */273struct dma_fence *274xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,275 struct xe_exec_queue *q, struct xe_vm *vm)276{277 struct dma_fence **fences = NULL;278 struct dma_fence_array *cf = NULL;279 struct dma_fence *fence;280 int i, num_in_fence = 0, current_fence = 0;281 282 lockdep_assert_held(&vm->lock);283 284 /* Count in-fences */285 for (i = 0; i < num_sync; ++i) {286 if (sync[i].fence) {287 ++num_in_fence;288 fence = sync[i].fence;289 }290 }291 292 /* Easy case... */293 if (!num_in_fence) {294 fence = xe_exec_queue_last_fence_get(q, vm);295 return fence;296 }297 298 /* Create composite fence */299 fences = kmalloc_array(num_in_fence + 1, sizeof(*fences), GFP_KERNEL);300 if (!fences)301 return ERR_PTR(-ENOMEM);302 for (i = 0; i < num_sync; ++i) {303 if (sync[i].fence) {304 dma_fence_get(sync[i].fence);305 fences[current_fence++] = sync[i].fence;306 }307 }308 fences[current_fence++] = xe_exec_queue_last_fence_get(q, vm);309 cf = dma_fence_array_create(num_in_fence, fences,310 vm->composite_fence_ctx,311 vm->composite_fence_seqno++,312 false);313 if (!cf) {314 --vm->composite_fence_seqno;315 goto err_out;316 }317 318 return &cf->base;319 320err_out:321 while (current_fence)322 dma_fence_put(fences[--current_fence]);323 kfree(fences);324 kfree(cf);325 326 return ERR_PTR(-ENOMEM);327}328 329/**330 * __xe_sync_ufence_get() - Get user fence from user fence331 * @ufence: input user fence332 *333 * Get a user fence reference from user fence334 *335 * Return: xe_user_fence pointer with reference336 */337struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence)338{339 user_fence_get(ufence);340 341 return ufence;342}343 344/**345 * xe_sync_ufence_get() - Get user fence from sync346 * @sync: input sync347 *348 * Get a user fence reference from sync.349 *350 * Return: xe_user_fence pointer with reference351 */352struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)353{354 user_fence_get(sync->ufence);355 356 return sync->ufence;357}358 359/**360 * xe_sync_ufence_put() - Put user fence reference361 * @ufence: user fence reference362 *363 */364void xe_sync_ufence_put(struct xe_user_fence *ufence)365{366 user_fence_put(ufence);367}368 369/**370 * xe_sync_ufence_get_status() - Get user fence status371 * @ufence: user fence372 *373 * Return: 1 if signalled, 0 not signalled, <0 on error374 */375int xe_sync_ufence_get_status(struct xe_user_fence *ufence)376{377 return READ_ONCE(ufence->signalled);378}379