1720 lines · c
1/*2 * Copyright 2017 Red Hat3 * Parts ported from amdgpu (fence wait code).4 * Copyright 2016 Advanced Micro Devices, Inc.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice (including the next14 * paragraph) shall be included in all copies or substantial portions of the15 * Software.16 *17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS23 * IN THE SOFTWARE.24 *25 * Authors:26 *27 */28 29/**30 * DOC: Overview31 *32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a33 * container for a synchronization primitive which can be used by userspace34 * to explicitly synchronize GPU commands, can be shared between userspace35 * processes, and can be shared between different DRM drivers.36 * Their primary use-case is to implement Vulkan fences and semaphores.37 * The syncobj userspace API provides ioctls for several operations:38 *39 * - Creation and destruction of syncobjs40 * - Import and export of syncobjs to/from a syncobj file descriptor41 * - Import and export a syncobj's underlying fence to/from a sync file42 * - Reset a syncobj (set its fence to NULL)43 * - Signal a syncobj (set a trivially signaled fence)44 * - Wait for a syncobj's fence to appear and be signaled45 *46 * The syncobj userspace API also provides operations to manipulate a syncobj47 * in terms of a timeline of struct &dma_fence_chain rather than a single48 * struct &dma_fence, through the following operations:49 *50 * - Signal a given point on the timeline51 * - Wait for a given point to appear and/or be signaled52 * - Import and export from/to a given point of a timeline53 *54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct55 * &dma_fence which may be NULL.56 * When a syncobj is first created, its pointer is either NULL or a pointer57 * to an already signaled fence depending on whether the58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to59 * &DRM_IOCTL_SYNCOBJ_CREATE.60 *61 * If the syncobj is considered as a binary (its state is either signaled or62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal63 * the syncobj, the syncobj's fence is replaced with a fence which will be64 * signaled by the completion of that work.65 * If the syncobj is considered as a timeline primitive, when GPU work is66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new67 * struct &dma_fence_chain pointing to the DRM driver's fence and also68 * pointing to the previous fence that was in the syncobj. The new struct69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by70 * completion of the DRM driver's work and also any work associated with the71 * fence previously in the syncobj.72 *73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the74 * time the work is enqueued, it waits on the syncobj's fence before75 * submitting the work to hardware. That fence is either :76 *77 * - The syncobj's current fence if the syncobj is considered as a binary78 * primitive.79 * - The struct &dma_fence associated with a given point if the syncobj is80 * considered as a timeline primitive.81 *82 * If the syncobj's fence is NULL or not present in the syncobj's timeline,83 * the enqueue operation is expected to fail.84 *85 * With binary syncobj, all manipulation of the syncobjs's fence happens in86 * terms of the current fence at the time the ioctl is called by userspace87 * regardless of whether that operation is an immediate host-side operation88 * (signal or reset) or or an operation which is enqueued in some driver89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used90 * to manipulate a syncobj from the host by resetting its pointer to NULL or91 * setting its pointer to a fence which is already signaled.92 *93 * With a timeline syncobj, all manipulation of the synobj's fence happens in94 * terms of a u64 value referring to point in the timeline. See95 * dma_fence_chain_find_seqno() to see how a given point is found in the96 * timeline.97 *98 * Note that applications should be careful to always use timeline set of99 * ioctl() when dealing with syncobj considered as timeline. Using a binary100 * set of ioctl() with a syncobj considered as timeline could result incorrect101 * synchronization. The use of binary syncobj is supported through the102 * timeline set of ioctl() by using a point value of 0, this will reproduce103 * the behavior of the binary set of ioctl() (for example replace the104 * syncobj's fence when signaling).105 *106 *107 * Host-side wait on syncobjs108 * --------------------------109 *110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a111 * host-side wait on all of the syncobj fences simultaneously.112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on113 * all of the syncobj fences to be signaled before it returns.114 * Otherwise, it returns once at least one syncobj fence has been signaled115 * and the index of a signaled fence is written back to the client.116 *117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,119 * the host-side wait will first wait for the syncobj to receive a non-NULL120 * fence and then wait on that fence.121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the122 * syncobjs in the array has a NULL fence, -EINVAL will be returned.123 * Assuming the syncobj starts off with a NULL fence, this allows a client124 * to do a host wait in one thread (or process) which waits on GPU work125 * submitted in another thread (or process) without having to manually126 * synchronize between the two.127 * This requirement is inherited from the Vulkan fence API.128 *129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set130 * a fence deadline hint on the backing fences before waiting, to provide the131 * fence signaler with an appropriate sense of urgency. The deadline is132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns.133 *134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj135 * handles as well as an array of u64 points and does a host-side wait on all136 * of syncobj fences at the given points simultaneously.137 *138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given139 * fence to materialize on the timeline without waiting for the fence to be140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This141 * requirement is inherited from the wait-before-signal behavior required by142 * the Vulkan timeline semaphore API.143 *144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to146 * integrate the wait in an event loop.147 *148 *149 * Import/export of syncobjs150 * -------------------------151 *152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD153 * provide two mechanisms for import/export of syncobjs.154 *155 * The first lets the client import or export an entire syncobj to a file156 * descriptor.157 * These fd's are opaque and have no other use case, except passing the158 * syncobj between processes.159 * All exported file descriptors and any syncobj handles created as a160 * result of importing those file descriptors own a reference to the161 * same underlying struct &drm_syncobj and the syncobj can be used162 * persistently across all the processes with which it is shared.163 * The syncobj is freed only once the last reference is dropped.164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own165 * reference) for every import instead of de-duplicating.166 * The primary use-case of this persistent import/export is for shared167 * Vulkan fences and semaphores.168 *169 * The second import/export mechanism, which is indicated by170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client172 * import/export the syncobj's current fence from/to a &sync_file.173 * When a syncobj is exported to a sync file, that sync file wraps the174 * sycnobj's fence at the time of export and any later signal or reset175 * operations on the syncobj will not affect the exported sync file.176 * When a sync file is imported into a syncobj, the syncobj's fence is set177 * to the fence wrapped by that sync file.178 * Because sync files are immutable, resetting or signaling the syncobj179 * will not affect any sync files whose fences have been imported into the180 * syncobj.181 *182 *183 * Import/export of timeline points in timeline syncobjs184 * -----------------------------------------------------185 *186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point188 * into another syncobj.189 *190 * Note that if you want to transfer a struct &dma_fence_chain from a given191 * point on a timeline syncobj from/into a binary syncobj, you can use the192 * point 0 to mean take/replace the fence in the syncobj.193 */194 195#include <linux/anon_inodes.h>196#include <linux/dma-fence-unwrap.h>197#include <linux/eventfd.h>198#include <linux/file.h>199#include <linux/fs.h>200#include <linux/sched/signal.h>201#include <linux/sync_file.h>202#include <linux/uaccess.h>203 204#include <drm/drm.h>205#include <drm/drm_drv.h>206#include <drm/drm_file.h>207#include <drm/drm_gem.h>208#include <drm/drm_print.h>209#include <drm/drm_syncobj.h>210#include <drm/drm_utils.h>211 212#include "drm_internal.h"213 214struct syncobj_wait_entry {215 struct list_head node;216 struct task_struct *task;217 struct dma_fence *fence;218 struct dma_fence_cb fence_cb;219 u64 point;220};221 222static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,223 struct syncobj_wait_entry *wait);224 225struct syncobj_eventfd_entry {226 struct list_head node;227 struct dma_fence *fence;228 struct dma_fence_cb fence_cb;229 struct drm_syncobj *syncobj;230 struct eventfd_ctx *ev_fd_ctx;231 u64 point;232 u32 flags;233};234 235static void236syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,237 struct syncobj_eventfd_entry *entry);238 239/**240 * drm_syncobj_find - lookup and reference a sync object.241 * @file_private: drm file private pointer242 * @handle: sync object handle to lookup.243 *244 * Returns a reference to the syncobj pointed to by handle or NULL. The245 * reference must be released by calling drm_syncobj_put().246 */247struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,248 u32 handle)249{250 struct drm_syncobj *syncobj;251 252 spin_lock(&file_private->syncobj_table_lock);253 254 /* Check if we currently have a reference on the object */255 syncobj = idr_find(&file_private->syncobj_idr, handle);256 if (syncobj)257 drm_syncobj_get(syncobj);258 259 spin_unlock(&file_private->syncobj_table_lock);260 261 return syncobj;262}263EXPORT_SYMBOL(drm_syncobj_find);264 265static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,266 struct syncobj_wait_entry *wait)267{268 struct dma_fence *fence;269 270 if (wait->fence)271 return;272 273 spin_lock(&syncobj->lock);274 /* We've already tried once to get a fence and failed. Now that we275 * have the lock, try one more time just to be sure we don't add a276 * callback when a fence has already been set.277 */278 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));279 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {280 dma_fence_put(fence);281 list_add_tail(&wait->node, &syncobj->cb_list);282 } else if (!fence) {283 wait->fence = dma_fence_get_stub();284 } else {285 wait->fence = fence;286 }287 spin_unlock(&syncobj->lock);288}289 290static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,291 struct syncobj_wait_entry *wait)292{293 if (!wait->node.next)294 return;295 296 spin_lock(&syncobj->lock);297 list_del_init(&wait->node);298 spin_unlock(&syncobj->lock);299}300 301static void302syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)303{304 eventfd_ctx_put(entry->ev_fd_ctx);305 dma_fence_put(entry->fence);306 /* This happens either inside the syncobj lock, or after the node has307 * already been removed from the list.308 */309 list_del(&entry->node);310 kfree(entry);311}312 313static void314drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,315 struct syncobj_eventfd_entry *entry)316{317 spin_lock(&syncobj->lock);318 list_add_tail(&entry->node, &syncobj->ev_fd_list);319 syncobj_eventfd_entry_func(syncobj, entry);320 spin_unlock(&syncobj->lock);321}322 323/**324 * drm_syncobj_add_point - add new timeline point to the syncobj325 * @syncobj: sync object to add timeline point do326 * @chain: chain node to use to add the point327 * @fence: fence to encapsulate in the chain node328 * @point: sequence number to use for the point329 *330 * Add the chain node as new timeline point to the syncobj.331 */332void drm_syncobj_add_point(struct drm_syncobj *syncobj,333 struct dma_fence_chain *chain,334 struct dma_fence *fence,335 uint64_t point)336{337 struct syncobj_wait_entry *wait_cur, *wait_tmp;338 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;339 struct dma_fence *prev;340 341 dma_fence_get(fence);342 343 spin_lock(&syncobj->lock);344 345 prev = drm_syncobj_fence_get(syncobj);346 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */347 if (prev && prev->seqno >= point)348 DRM_DEBUG("You are adding an unorder point to timeline!\n");349 dma_fence_chain_init(chain, prev, fence, point);350 rcu_assign_pointer(syncobj->fence, &chain->base);351 352 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)353 syncobj_wait_syncobj_func(syncobj, wait_cur);354 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)355 syncobj_eventfd_entry_func(syncobj, ev_fd_cur);356 spin_unlock(&syncobj->lock);357 358 /* Walk the chain once to trigger garbage collection */359 dma_fence_chain_for_each(fence, prev);360 dma_fence_put(prev);361}362EXPORT_SYMBOL(drm_syncobj_add_point);363 364/**365 * drm_syncobj_replace_fence - replace fence in a sync object.366 * @syncobj: Sync object to replace fence in367 * @fence: fence to install in sync file.368 *369 * This replaces the fence on a sync object.370 */371void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,372 struct dma_fence *fence)373{374 struct dma_fence *old_fence;375 struct syncobj_wait_entry *wait_cur, *wait_tmp;376 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;377 378 if (fence)379 dma_fence_get(fence);380 381 spin_lock(&syncobj->lock);382 383 old_fence = rcu_dereference_protected(syncobj->fence,384 lockdep_is_held(&syncobj->lock));385 rcu_assign_pointer(syncobj->fence, fence);386 387 if (fence != old_fence) {388 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)389 syncobj_wait_syncobj_func(syncobj, wait_cur);390 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)391 syncobj_eventfd_entry_func(syncobj, ev_fd_cur);392 }393 394 spin_unlock(&syncobj->lock);395 396 dma_fence_put(old_fence);397}398EXPORT_SYMBOL(drm_syncobj_replace_fence);399 400/**401 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object402 * @syncobj: sync object to assign the fence on403 *404 * Assign a already signaled stub fence to the sync object.405 */406static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)407{408 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());409 410 if (!fence)411 return -ENOMEM;412 413 drm_syncobj_replace_fence(syncobj, fence);414 dma_fence_put(fence);415 return 0;416}417 418/* 5s default for wait submission */419#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL420/**421 * drm_syncobj_find_fence - lookup and reference the fence in a sync object422 * @file_private: drm file private pointer423 * @handle: sync object handle to lookup.424 * @point: timeline point425 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not426 * @fence: out parameter for the fence427 *428 * This is just a convenience function that combines drm_syncobj_find() and429 * drm_syncobj_fence_get().430 *431 * Returns 0 on success or a negative error value on failure. On success @fence432 * contains a reference to the fence, which must be released by calling433 * dma_fence_put().434 */435int drm_syncobj_find_fence(struct drm_file *file_private,436 u32 handle, u64 point, u64 flags,437 struct dma_fence **fence)438{439 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);440 struct syncobj_wait_entry wait;441 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);442 int ret;443 444 if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)445 return -EINVAL;446 447 if (!syncobj)448 return -ENOENT;449 450 /* Waiting for userspace with locks help is illegal cause that can451 * trivial deadlock with page faults for example. Make lockdep complain452 * about it early on.453 */454 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {455 might_sleep();456 lockdep_assert_none_held_once();457 }458 459 *fence = drm_syncobj_fence_get(syncobj);460 461 if (*fence) {462 ret = dma_fence_chain_find_seqno(fence, point);463 if (!ret) {464 /* If the requested seqno is already signaled465 * drm_syncobj_find_fence may return a NULL466 * fence. To make sure the recipient gets467 * signalled, use a new fence instead.468 */469 if (!*fence)470 *fence = dma_fence_get_stub();471 472 goto out;473 }474 dma_fence_put(*fence);475 } else {476 ret = -EINVAL;477 }478 479 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))480 goto out;481 482 memset(&wait, 0, sizeof(wait));483 wait.task = current;484 wait.point = point;485 drm_syncobj_fence_add_wait(syncobj, &wait);486 487 do {488 set_current_state(TASK_INTERRUPTIBLE);489 if (wait.fence) {490 ret = 0;491 break;492 }493 if (timeout == 0) {494 ret = -ETIME;495 break;496 }497 498 if (signal_pending(current)) {499 ret = -ERESTARTSYS;500 break;501 }502 503 timeout = schedule_timeout(timeout);504 } while (1);505 506 __set_current_state(TASK_RUNNING);507 *fence = wait.fence;508 509 if (wait.node.next)510 drm_syncobj_remove_wait(syncobj, &wait);511 512out:513 drm_syncobj_put(syncobj);514 515 return ret;516}517EXPORT_SYMBOL(drm_syncobj_find_fence);518 519/**520 * drm_syncobj_free - free a sync object.521 * @kref: kref to free.522 *523 * Only to be called from kref_put in drm_syncobj_put.524 */525void drm_syncobj_free(struct kref *kref)526{527 struct drm_syncobj *syncobj = container_of(kref,528 struct drm_syncobj,529 refcount);530 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;531 532 drm_syncobj_replace_fence(syncobj, NULL);533 534 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)535 syncobj_eventfd_entry_free(ev_fd_cur);536 537 kfree(syncobj);538}539EXPORT_SYMBOL(drm_syncobj_free);540 541/**542 * drm_syncobj_create - create a new syncobj543 * @out_syncobj: returned syncobj544 * @flags: DRM_SYNCOBJ_* flags545 * @fence: if non-NULL, the syncobj will represent this fence546 *547 * This is the first function to create a sync object. After creating, drivers548 * probably want to make it available to userspace, either through549 * drm_syncobj_get_handle() or drm_syncobj_get_fd().550 *551 * Returns 0 on success or a negative error value on failure.552 */553int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,554 struct dma_fence *fence)555{556 int ret;557 struct drm_syncobj *syncobj;558 559 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);560 if (!syncobj)561 return -ENOMEM;562 563 kref_init(&syncobj->refcount);564 INIT_LIST_HEAD(&syncobj->cb_list);565 INIT_LIST_HEAD(&syncobj->ev_fd_list);566 spin_lock_init(&syncobj->lock);567 568 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {569 ret = drm_syncobj_assign_null_handle(syncobj);570 if (ret < 0) {571 drm_syncobj_put(syncobj);572 return ret;573 }574 }575 576 if (fence)577 drm_syncobj_replace_fence(syncobj, fence);578 579 *out_syncobj = syncobj;580 return 0;581}582EXPORT_SYMBOL(drm_syncobj_create);583 584/**585 * drm_syncobj_get_handle - get a handle from a syncobj586 * @file_private: drm file private pointer587 * @syncobj: Sync object to export588 * @handle: out parameter with the new handle589 *590 * Exports a sync object created with drm_syncobj_create() as a handle on591 * @file_private to userspace.592 *593 * Returns 0 on success or a negative error value on failure.594 */595int drm_syncobj_get_handle(struct drm_file *file_private,596 struct drm_syncobj *syncobj, u32 *handle)597{598 int ret;599 600 /* take a reference to put in the idr */601 drm_syncobj_get(syncobj);602 603 idr_preload(GFP_KERNEL);604 spin_lock(&file_private->syncobj_table_lock);605 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);606 spin_unlock(&file_private->syncobj_table_lock);607 608 idr_preload_end();609 610 if (ret < 0) {611 drm_syncobj_put(syncobj);612 return ret;613 }614 615 *handle = ret;616 return 0;617}618EXPORT_SYMBOL(drm_syncobj_get_handle);619 620static int drm_syncobj_create_as_handle(struct drm_file *file_private,621 u32 *handle, uint32_t flags)622{623 int ret;624 struct drm_syncobj *syncobj;625 626 ret = drm_syncobj_create(&syncobj, flags, NULL);627 if (ret)628 return ret;629 630 ret = drm_syncobj_get_handle(file_private, syncobj, handle);631 drm_syncobj_put(syncobj);632 return ret;633}634 635static int drm_syncobj_destroy(struct drm_file *file_private,636 u32 handle)637{638 struct drm_syncobj *syncobj;639 640 spin_lock(&file_private->syncobj_table_lock);641 syncobj = idr_remove(&file_private->syncobj_idr, handle);642 spin_unlock(&file_private->syncobj_table_lock);643 644 if (!syncobj)645 return -EINVAL;646 647 drm_syncobj_put(syncobj);648 return 0;649}650 651static int drm_syncobj_file_release(struct inode *inode, struct file *file)652{653 struct drm_syncobj *syncobj = file->private_data;654 655 drm_syncobj_put(syncobj);656 return 0;657}658 659static const struct file_operations drm_syncobj_file_fops = {660 .release = drm_syncobj_file_release,661};662 663/**664 * drm_syncobj_get_fd - get a file descriptor from a syncobj665 * @syncobj: Sync object to export666 * @p_fd: out parameter with the new file descriptor667 *668 * Exports a sync object created with drm_syncobj_create() as a file descriptor.669 *670 * Returns 0 on success or a negative error value on failure.671 */672int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)673{674 struct file *file;675 int fd;676 677 fd = get_unused_fd_flags(O_CLOEXEC);678 if (fd < 0)679 return fd;680 681 file = anon_inode_getfile("syncobj_file",682 &drm_syncobj_file_fops,683 syncobj, 0);684 if (IS_ERR(file)) {685 put_unused_fd(fd);686 return PTR_ERR(file);687 }688 689 drm_syncobj_get(syncobj);690 fd_install(fd, file);691 692 *p_fd = fd;693 return 0;694}695EXPORT_SYMBOL(drm_syncobj_get_fd);696 697static int drm_syncobj_handle_to_fd(struct drm_file *file_private,698 u32 handle, int *p_fd)699{700 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);701 int ret;702 703 if (!syncobj)704 return -EINVAL;705 706 ret = drm_syncobj_get_fd(syncobj, p_fd);707 drm_syncobj_put(syncobj);708 return ret;709}710 711static int drm_syncobj_fd_to_handle(struct drm_file *file_private,712 int fd, u32 *handle)713{714 struct drm_syncobj *syncobj;715 struct fd f = fdget(fd);716 int ret;717 718 if (!fd_file(f))719 return -EINVAL;720 721 if (fd_file(f)->f_op != &drm_syncobj_file_fops) {722 fdput(f);723 return -EINVAL;724 }725 726 /* take a reference to put in the idr */727 syncobj = fd_file(f)->private_data;728 drm_syncobj_get(syncobj);729 730 idr_preload(GFP_KERNEL);731 spin_lock(&file_private->syncobj_table_lock);732 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);733 spin_unlock(&file_private->syncobj_table_lock);734 idr_preload_end();735 736 if (ret > 0) {737 *handle = ret;738 ret = 0;739 } else740 drm_syncobj_put(syncobj);741 742 fdput(f);743 return ret;744}745 746static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,747 int fd, int handle)748{749 struct dma_fence *fence = sync_file_get_fence(fd);750 struct drm_syncobj *syncobj;751 752 if (!fence)753 return -EINVAL;754 755 syncobj = drm_syncobj_find(file_private, handle);756 if (!syncobj) {757 dma_fence_put(fence);758 return -ENOENT;759 }760 761 drm_syncobj_replace_fence(syncobj, fence);762 dma_fence_put(fence);763 drm_syncobj_put(syncobj);764 return 0;765}766 767static int drm_syncobj_export_sync_file(struct drm_file *file_private,768 int handle, int *p_fd)769{770 int ret;771 struct dma_fence *fence;772 struct sync_file *sync_file;773 int fd = get_unused_fd_flags(O_CLOEXEC);774 775 if (fd < 0)776 return fd;777 778 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);779 if (ret)780 goto err_put_fd;781 782 sync_file = sync_file_create(fence);783 784 dma_fence_put(fence);785 786 if (!sync_file) {787 ret = -EINVAL;788 goto err_put_fd;789 }790 791 fd_install(fd, sync_file->file);792 793 *p_fd = fd;794 return 0;795err_put_fd:796 put_unused_fd(fd);797 return ret;798}799/**800 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time801 * @file_private: drm file-private structure to set up802 *803 * Called at device open time, sets up the structure for handling refcounting804 * of sync objects.805 */806void807drm_syncobj_open(struct drm_file *file_private)808{809 idr_init_base(&file_private->syncobj_idr, 1);810 spin_lock_init(&file_private->syncobj_table_lock);811}812 813static int814drm_syncobj_release_handle(int id, void *ptr, void *data)815{816 struct drm_syncobj *syncobj = ptr;817 818 drm_syncobj_put(syncobj);819 return 0;820}821 822/**823 * drm_syncobj_release - release file-private sync object resources824 * @file_private: drm file-private structure to clean up825 *826 * Called at close time when the filp is going away.827 *828 * Releases any remaining references on objects by this filp.829 */830void831drm_syncobj_release(struct drm_file *file_private)832{833 idr_for_each(&file_private->syncobj_idr,834 &drm_syncobj_release_handle, file_private);835 idr_destroy(&file_private->syncobj_idr);836}837 838int839drm_syncobj_create_ioctl(struct drm_device *dev, void *data,840 struct drm_file *file_private)841{842 struct drm_syncobj_create *args = data;843 844 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))845 return -EOPNOTSUPP;846 847 /* no valid flags yet */848 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)849 return -EINVAL;850 851 return drm_syncobj_create_as_handle(file_private,852 &args->handle, args->flags);853}854 855int856drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,857 struct drm_file *file_private)858{859 struct drm_syncobj_destroy *args = data;860 861 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))862 return -EOPNOTSUPP;863 864 /* make sure padding is empty */865 if (args->pad)866 return -EINVAL;867 return drm_syncobj_destroy(file_private, args->handle);868}869 870int871drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,872 struct drm_file *file_private)873{874 struct drm_syncobj_handle *args = data;875 876 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))877 return -EOPNOTSUPP;878 879 if (args->pad)880 return -EINVAL;881 882 if (args->flags != 0 &&883 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)884 return -EINVAL;885 886 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)887 return drm_syncobj_export_sync_file(file_private, args->handle,888 &args->fd);889 890 return drm_syncobj_handle_to_fd(file_private, args->handle,891 &args->fd);892}893 894int895drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,896 struct drm_file *file_private)897{898 struct drm_syncobj_handle *args = data;899 900 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))901 return -EOPNOTSUPP;902 903 if (args->pad)904 return -EINVAL;905 906 if (args->flags != 0 &&907 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)908 return -EINVAL;909 910 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)911 return drm_syncobj_import_sync_file_fence(file_private,912 args->fd,913 args->handle);914 915 return drm_syncobj_fd_to_handle(file_private, args->fd,916 &args->handle);917}918 919static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,920 struct drm_syncobj_transfer *args)921{922 struct drm_syncobj *timeline_syncobj = NULL;923 struct dma_fence *fence, *tmp;924 struct dma_fence_chain *chain;925 int ret;926 927 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);928 if (!timeline_syncobj) {929 return -ENOENT;930 }931 ret = drm_syncobj_find_fence(file_private, args->src_handle,932 args->src_point, args->flags,933 &tmp);934 if (ret)935 goto err_put_timeline;936 937 fence = dma_fence_unwrap_merge(tmp);938 dma_fence_put(tmp);939 if (!fence) {940 ret = -ENOMEM;941 goto err_put_timeline;942 }943 944 chain = dma_fence_chain_alloc();945 if (!chain) {946 ret = -ENOMEM;947 goto err_free_fence;948 }949 950 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);951err_free_fence:952 dma_fence_put(fence);953err_put_timeline:954 drm_syncobj_put(timeline_syncobj);955 956 return ret;957}958 959static int960drm_syncobj_transfer_to_binary(struct drm_file *file_private,961 struct drm_syncobj_transfer *args)962{963 struct drm_syncobj *binary_syncobj = NULL;964 struct dma_fence *fence;965 int ret;966 967 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);968 if (!binary_syncobj)969 return -ENOENT;970 ret = drm_syncobj_find_fence(file_private, args->src_handle,971 args->src_point, args->flags, &fence);972 if (ret)973 goto err;974 drm_syncobj_replace_fence(binary_syncobj, fence);975 dma_fence_put(fence);976err:977 drm_syncobj_put(binary_syncobj);978 979 return ret;980}981int982drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,983 struct drm_file *file_private)984{985 struct drm_syncobj_transfer *args = data;986 int ret;987 988 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))989 return -EOPNOTSUPP;990 991 if (args->pad)992 return -EINVAL;993 994 if (args->dst_point)995 ret = drm_syncobj_transfer_to_timeline(file_private, args);996 else997 ret = drm_syncobj_transfer_to_binary(file_private, args);998 999 return ret;1000}1001 1002static void syncobj_wait_fence_func(struct dma_fence *fence,1003 struct dma_fence_cb *cb)1004{1005 struct syncobj_wait_entry *wait =1006 container_of(cb, struct syncobj_wait_entry, fence_cb);1007 1008 wake_up_process(wait->task);1009}1010 1011static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,1012 struct syncobj_wait_entry *wait)1013{1014 struct dma_fence *fence;1015 1016 /* This happens inside the syncobj lock */1017 fence = rcu_dereference_protected(syncobj->fence,1018 lockdep_is_held(&syncobj->lock));1019 dma_fence_get(fence);1020 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {1021 dma_fence_put(fence);1022 return;1023 } else if (!fence) {1024 wait->fence = dma_fence_get_stub();1025 } else {1026 wait->fence = fence;1027 }1028 1029 wake_up_process(wait->task);1030 list_del_init(&wait->node);1031}1032 1033static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,1034 void __user *user_points,1035 uint32_t count,1036 uint32_t flags,1037 signed long timeout,1038 uint32_t *idx,1039 ktime_t *deadline)1040{1041 struct syncobj_wait_entry *entries;1042 struct dma_fence *fence;1043 uint64_t *points;1044 uint32_t signaled_count, i;1045 1046 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |1047 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {1048 might_sleep();1049 lockdep_assert_none_held_once();1050 }1051 1052 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);1053 if (points == NULL)1054 return -ENOMEM;1055 1056 if (!user_points) {1057 memset(points, 0, count * sizeof(uint64_t));1058 1059 } else if (copy_from_user(points, user_points,1060 sizeof(uint64_t) * count)) {1061 timeout = -EFAULT;1062 goto err_free_points;1063 }1064 1065 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);1066 if (!entries) {1067 timeout = -ENOMEM;1068 goto err_free_points;1069 }1070 /* Walk the list of sync objects and initialize entries. We do1071 * this up-front so that we can properly return -EINVAL if there is1072 * a syncobj with a missing fence and then never have the chance of1073 * returning -EINVAL again.1074 */1075 signaled_count = 0;1076 for (i = 0; i < count; ++i) {1077 struct dma_fence *fence;1078 1079 entries[i].task = current;1080 entries[i].point = points[i];1081 fence = drm_syncobj_fence_get(syncobjs[i]);1082 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {1083 dma_fence_put(fence);1084 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |1085 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {1086 continue;1087 } else {1088 timeout = -EINVAL;1089 goto cleanup_entries;1090 }1091 }1092 1093 if (fence)1094 entries[i].fence = fence;1095 else1096 entries[i].fence = dma_fence_get_stub();1097 1098 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||1099 dma_fence_is_signaled(entries[i].fence)) {1100 if (signaled_count == 0 && idx)1101 *idx = i;1102 signaled_count++;1103 }1104 }1105 1106 if (signaled_count == count ||1107 (signaled_count > 0 &&1108 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))1109 goto cleanup_entries;1110 1111 /* There's a very annoying laxness in the dma_fence API here, in1112 * that backends are not required to automatically report when a1113 * fence is signaled prior to fence->ops->enable_signaling() being1114 * called. So here if we fail to match signaled_count, we need to1115 * fallthough and try a 0 timeout wait!1116 */1117 1118 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |1119 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {1120 for (i = 0; i < count; ++i)1121 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);1122 }1123 1124 if (deadline) {1125 for (i = 0; i < count; ++i) {1126 fence = entries[i].fence;1127 if (!fence)1128 continue;1129 dma_fence_set_deadline(fence, *deadline);1130 }1131 }1132 1133 do {1134 set_current_state(TASK_INTERRUPTIBLE);1135 1136 signaled_count = 0;1137 for (i = 0; i < count; ++i) {1138 fence = entries[i].fence;1139 if (!fence)1140 continue;1141 1142 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||1143 dma_fence_is_signaled(fence) ||1144 (!entries[i].fence_cb.func &&1145 dma_fence_add_callback(fence,1146 &entries[i].fence_cb,1147 syncobj_wait_fence_func))) {1148 /* The fence has been signaled */1149 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {1150 signaled_count++;1151 } else {1152 if (idx)1153 *idx = i;1154 goto done_waiting;1155 }1156 }1157 }1158 1159 if (signaled_count == count)1160 goto done_waiting;1161 1162 if (timeout == 0) {1163 timeout = -ETIME;1164 goto done_waiting;1165 }1166 1167 if (signal_pending(current)) {1168 timeout = -ERESTARTSYS;1169 goto done_waiting;1170 }1171 1172 timeout = schedule_timeout(timeout);1173 } while (1);1174 1175done_waiting:1176 __set_current_state(TASK_RUNNING);1177 1178cleanup_entries:1179 for (i = 0; i < count; ++i) {1180 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);1181 if (entries[i].fence_cb.func)1182 dma_fence_remove_callback(entries[i].fence,1183 &entries[i].fence_cb);1184 dma_fence_put(entries[i].fence);1185 }1186 kfree(entries);1187 1188err_free_points:1189 kfree(points);1190 1191 return timeout;1192}1193 1194/**1195 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value1196 *1197 * @timeout_nsec: timeout nsec component in ns, 0 for poll1198 *1199 * Calculate the timeout in jiffies from an absolute time in sec/nsec.1200 */1201signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)1202{1203 ktime_t abs_timeout, now;1204 u64 timeout_ns, timeout_jiffies64;1205 1206 /* make 0 timeout means poll - absolute 0 doesn't seem valid */1207 if (timeout_nsec == 0)1208 return 0;1209 1210 abs_timeout = ns_to_ktime(timeout_nsec);1211 now = ktime_get();1212 1213 if (!ktime_after(abs_timeout, now))1214 return 0;1215 1216 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));1217 1218 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);1219 /* clamp timeout to avoid infinite timeout */1220 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)1221 return MAX_SCHEDULE_TIMEOUT - 1;1222 1223 return timeout_jiffies64 + 1;1224}1225EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);1226 1227static int drm_syncobj_array_wait(struct drm_device *dev,1228 struct drm_file *file_private,1229 struct drm_syncobj_wait *wait,1230 struct drm_syncobj_timeline_wait *timeline_wait,1231 struct drm_syncobj **syncobjs, bool timeline,1232 ktime_t *deadline)1233{1234 signed long timeout = 0;1235 uint32_t first = ~0;1236 1237 if (!timeline) {1238 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);1239 timeout = drm_syncobj_array_wait_timeout(syncobjs,1240 NULL,1241 wait->count_handles,1242 wait->flags,1243 timeout, &first,1244 deadline);1245 if (timeout < 0)1246 return timeout;1247 wait->first_signaled = first;1248 } else {1249 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);1250 timeout = drm_syncobj_array_wait_timeout(syncobjs,1251 u64_to_user_ptr(timeline_wait->points),1252 timeline_wait->count_handles,1253 timeline_wait->flags,1254 timeout, &first,1255 deadline);1256 if (timeout < 0)1257 return timeout;1258 timeline_wait->first_signaled = first;1259 }1260 return 0;1261}1262 1263static int drm_syncobj_array_find(struct drm_file *file_private,1264 void __user *user_handles,1265 uint32_t count_handles,1266 struct drm_syncobj ***syncobjs_out)1267{1268 uint32_t i, *handles;1269 struct drm_syncobj **syncobjs;1270 int ret;1271 1272 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);1273 if (handles == NULL)1274 return -ENOMEM;1275 1276 if (copy_from_user(handles, user_handles,1277 sizeof(uint32_t) * count_handles)) {1278 ret = -EFAULT;1279 goto err_free_handles;1280 }1281 1282 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);1283 if (syncobjs == NULL) {1284 ret = -ENOMEM;1285 goto err_free_handles;1286 }1287 1288 for (i = 0; i < count_handles; i++) {1289 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);1290 if (!syncobjs[i]) {1291 ret = -ENOENT;1292 goto err_put_syncobjs;1293 }1294 }1295 1296 kfree(handles);1297 *syncobjs_out = syncobjs;1298 return 0;1299 1300err_put_syncobjs:1301 while (i-- > 0)1302 drm_syncobj_put(syncobjs[i]);1303 kfree(syncobjs);1304err_free_handles:1305 kfree(handles);1306 1307 return ret;1308}1309 1310static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,1311 uint32_t count)1312{1313 uint32_t i;1314 1315 for (i = 0; i < count; i++)1316 drm_syncobj_put(syncobjs[i]);1317 kfree(syncobjs);1318}1319 1320int1321drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,1322 struct drm_file *file_private)1323{1324 struct drm_syncobj_wait *args = data;1325 struct drm_syncobj **syncobjs;1326 unsigned int possible_flags;1327 ktime_t t, *tp = NULL;1328 int ret = 0;1329 1330 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))1331 return -EOPNOTSUPP;1332 1333 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |1334 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |1335 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;1336 1337 if (args->flags & ~possible_flags)1338 return -EINVAL;1339 1340 if (args->count_handles == 0)1341 return 0;1342 1343 ret = drm_syncobj_array_find(file_private,1344 u64_to_user_ptr(args->handles),1345 args->count_handles,1346 &syncobjs);1347 if (ret < 0)1348 return ret;1349 1350 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {1351 t = ns_to_ktime(args->deadline_nsec);1352 tp = &t;1353 }1354 1355 ret = drm_syncobj_array_wait(dev, file_private,1356 args, NULL, syncobjs, false, tp);1357 1358 drm_syncobj_array_free(syncobjs, args->count_handles);1359 1360 return ret;1361}1362 1363int1364drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,1365 struct drm_file *file_private)1366{1367 struct drm_syncobj_timeline_wait *args = data;1368 struct drm_syncobj **syncobjs;1369 unsigned int possible_flags;1370 ktime_t t, *tp = NULL;1371 int ret = 0;1372 1373 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))1374 return -EOPNOTSUPP;1375 1376 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |1377 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |1378 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE |1379 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE;1380 1381 if (args->flags & ~possible_flags)1382 return -EINVAL;1383 1384 if (args->count_handles == 0)1385 return 0;1386 1387 ret = drm_syncobj_array_find(file_private,1388 u64_to_user_ptr(args->handles),1389 args->count_handles,1390 &syncobjs);1391 if (ret < 0)1392 return ret;1393 1394 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) {1395 t = ns_to_ktime(args->deadline_nsec);1396 tp = &t;1397 }1398 1399 ret = drm_syncobj_array_wait(dev, file_private,1400 NULL, args, syncobjs, true, tp);1401 1402 drm_syncobj_array_free(syncobjs, args->count_handles);1403 1404 return ret;1405}1406 1407static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,1408 struct dma_fence_cb *cb)1409{1410 struct syncobj_eventfd_entry *entry =1411 container_of(cb, struct syncobj_eventfd_entry, fence_cb);1412 1413 eventfd_signal(entry->ev_fd_ctx);1414 syncobj_eventfd_entry_free(entry);1415}1416 1417static void1418syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,1419 struct syncobj_eventfd_entry *entry)1420{1421 int ret;1422 struct dma_fence *fence;1423 1424 /* This happens inside the syncobj lock */1425 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));1426 if (!fence)1427 return;1428 1429 ret = dma_fence_chain_find_seqno(&fence, entry->point);1430 if (ret != 0) {1431 /* The given seqno has not been submitted yet. */1432 dma_fence_put(fence);1433 return;1434 } else if (!fence) {1435 /* If dma_fence_chain_find_seqno returns 0 but sets the fence1436 * to NULL, it implies that the given seqno is signaled and a1437 * later seqno has already been submitted. Assign a stub fence1438 * so that the eventfd still gets signaled below.1439 */1440 fence = dma_fence_get_stub();1441 }1442 1443 list_del_init(&entry->node);1444 entry->fence = fence;1445 1446 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {1447 eventfd_signal(entry->ev_fd_ctx);1448 syncobj_eventfd_entry_free(entry);1449 } else {1450 ret = dma_fence_add_callback(fence, &entry->fence_cb,1451 syncobj_eventfd_entry_fence_func);1452 if (ret == -ENOENT) {1453 eventfd_signal(entry->ev_fd_ctx);1454 syncobj_eventfd_entry_free(entry);1455 }1456 }1457}1458 1459int1460drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,1461 struct drm_file *file_private)1462{1463 struct drm_syncobj_eventfd *args = data;1464 struct drm_syncobj *syncobj;1465 struct eventfd_ctx *ev_fd_ctx;1466 struct syncobj_eventfd_entry *entry;1467 int ret;1468 1469 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))1470 return -EOPNOTSUPP;1471 1472 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)1473 return -EINVAL;1474 1475 if (args->pad)1476 return -EINVAL;1477 1478 syncobj = drm_syncobj_find(file_private, args->handle);1479 if (!syncobj)1480 return -ENOENT;1481 1482 ev_fd_ctx = eventfd_ctx_fdget(args->fd);1483 if (IS_ERR(ev_fd_ctx)) {1484 ret = PTR_ERR(ev_fd_ctx);1485 goto err_fdget;1486 }1487 1488 entry = kzalloc(sizeof(*entry), GFP_KERNEL);1489 if (!entry) {1490 ret = -ENOMEM;1491 goto err_kzalloc;1492 }1493 entry->syncobj = syncobj;1494 entry->ev_fd_ctx = ev_fd_ctx;1495 entry->point = args->point;1496 entry->flags = args->flags;1497 1498 drm_syncobj_add_eventfd(syncobj, entry);1499 drm_syncobj_put(syncobj);1500 1501 return 0;1502 1503err_kzalloc:1504 eventfd_ctx_put(ev_fd_ctx);1505err_fdget:1506 drm_syncobj_put(syncobj);1507 return ret;1508}1509 1510int1511drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,1512 struct drm_file *file_private)1513{1514 struct drm_syncobj_array *args = data;1515 struct drm_syncobj **syncobjs;1516 uint32_t i;1517 int ret;1518 1519 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))1520 return -EOPNOTSUPP;1521 1522 if (args->pad != 0)1523 return -EINVAL;1524 1525 if (args->count_handles == 0)1526 return -EINVAL;1527 1528 ret = drm_syncobj_array_find(file_private,1529 u64_to_user_ptr(args->handles),1530 args->count_handles,1531 &syncobjs);1532 if (ret < 0)1533 return ret;1534 1535 for (i = 0; i < args->count_handles; i++)1536 drm_syncobj_replace_fence(syncobjs[i], NULL);1537 1538 drm_syncobj_array_free(syncobjs, args->count_handles);1539 1540 return 0;1541}1542 1543int1544drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,1545 struct drm_file *file_private)1546{1547 struct drm_syncobj_array *args = data;1548 struct drm_syncobj **syncobjs;1549 uint32_t i;1550 int ret;1551 1552 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))1553 return -EOPNOTSUPP;1554 1555 if (args->pad != 0)1556 return -EINVAL;1557 1558 if (args->count_handles == 0)1559 return -EINVAL;1560 1561 ret = drm_syncobj_array_find(file_private,1562 u64_to_user_ptr(args->handles),1563 args->count_handles,1564 &syncobjs);1565 if (ret < 0)1566 return ret;1567 1568 for (i = 0; i < args->count_handles; i++) {1569 ret = drm_syncobj_assign_null_handle(syncobjs[i]);1570 if (ret < 0)1571 break;1572 }1573 1574 drm_syncobj_array_free(syncobjs, args->count_handles);1575 1576 return ret;1577}1578 1579int1580drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,1581 struct drm_file *file_private)1582{1583 struct drm_syncobj_timeline_array *args = data;1584 struct drm_syncobj **syncobjs;1585 struct dma_fence_chain **chains;1586 uint64_t *points;1587 uint32_t i, j;1588 int ret;1589 1590 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))1591 return -EOPNOTSUPP;1592 1593 if (args->flags != 0)1594 return -EINVAL;1595 1596 if (args->count_handles == 0)1597 return -EINVAL;1598 1599 ret = drm_syncobj_array_find(file_private,1600 u64_to_user_ptr(args->handles),1601 args->count_handles,1602 &syncobjs);1603 if (ret < 0)1604 return ret;1605 1606 points = kmalloc_array(args->count_handles, sizeof(*points),1607 GFP_KERNEL);1608 if (!points) {1609 ret = -ENOMEM;1610 goto out;1611 }1612 if (!u64_to_user_ptr(args->points)) {1613 memset(points, 0, args->count_handles * sizeof(uint64_t));1614 } else if (copy_from_user(points, u64_to_user_ptr(args->points),1615 sizeof(uint64_t) * args->count_handles)) {1616 ret = -EFAULT;1617 goto err_points;1618 }1619 1620 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);1621 if (!chains) {1622 ret = -ENOMEM;1623 goto err_points;1624 }1625 for (i = 0; i < args->count_handles; i++) {1626 chains[i] = dma_fence_chain_alloc();1627 if (!chains[i]) {1628 for (j = 0; j < i; j++)1629 dma_fence_chain_free(chains[j]);1630 ret = -ENOMEM;1631 goto err_chains;1632 }1633 }1634 1635 for (i = 0; i < args->count_handles; i++) {1636 struct dma_fence *fence = dma_fence_get_stub();1637 1638 drm_syncobj_add_point(syncobjs[i], chains[i],1639 fence, points[i]);1640 dma_fence_put(fence);1641 }1642err_chains:1643 kfree(chains);1644err_points:1645 kfree(points);1646out:1647 drm_syncobj_array_free(syncobjs, args->count_handles);1648 1649 return ret;1650}1651 1652int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,1653 struct drm_file *file_private)1654{1655 struct drm_syncobj_timeline_array *args = data;1656 struct drm_syncobj **syncobjs;1657 uint64_t __user *points = u64_to_user_ptr(args->points);1658 uint32_t i;1659 int ret;1660 1661 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))1662 return -EOPNOTSUPP;1663 1664 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)1665 return -EINVAL;1666 1667 if (args->count_handles == 0)1668 return -EINVAL;1669 1670 ret = drm_syncobj_array_find(file_private,1671 u64_to_user_ptr(args->handles),1672 args->count_handles,1673 &syncobjs);1674 if (ret < 0)1675 return ret;1676 1677 for (i = 0; i < args->count_handles; i++) {1678 struct dma_fence_chain *chain;1679 struct dma_fence *fence;1680 uint64_t point;1681 1682 fence = drm_syncobj_fence_get(syncobjs[i]);1683 chain = to_dma_fence_chain(fence);1684 if (chain) {1685 struct dma_fence *iter, *last_signaled =1686 dma_fence_get(fence);1687 1688 if (args->flags &1689 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {1690 point = fence->seqno;1691 } else {1692 dma_fence_chain_for_each(iter, fence) {1693 if (iter->context != fence->context) {1694 dma_fence_put(iter);1695 /* It is most likely that timeline has1696 * unorder points. */1697 break;1698 }1699 dma_fence_put(last_signaled);1700 last_signaled = dma_fence_get(iter);1701 }1702 point = dma_fence_is_signaled(last_signaled) ?1703 last_signaled->seqno :1704 to_dma_fence_chain(last_signaled)->prev_seqno;1705 }1706 dma_fence_put(last_signaled);1707 } else {1708 point = 0;1709 }1710 dma_fence_put(fence);1711 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));1712 ret = ret ? -EFAULT : 0;1713 if (ret)1714 break;1715 }1716 drm_syncobj_array_free(syncobjs, args->count_handles);1717 1718 return ret;1719}1720