495 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Sync File validation framework4 *5 * Copyright (C) 2012 Google, Inc.6 */7 8#include <linux/file.h>9#include <linux/fs.h>10#include <linux/uaccess.h>11#include <linux/slab.h>12#include <linux/sync_file.h>13 14#include "sync_debug.h"15 16#define CREATE_TRACE_POINTS17#include "sync_trace.h"18 19/*20 * SW SYNC validation framework21 *22 * A sync object driver that uses a 32bit counter to coordinate23 * synchronization. Useful when there is no hardware primitive backing24 * the synchronization.25 *26 * To start the framework just open:27 *28 * <debugfs>/sync/sw_sync29 *30 * That will create a sync timeline, all fences created under this timeline31 * file descriptor will belong to the this timeline.32 *33 * The 'sw_sync' file can be opened many times as to create different34 * timelines.35 *36 * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct37 * sw_sync_create_fence_data as parameter.38 *39 * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used40 * with the increment as u32. This will update the last signaled value41 * from the timeline and signal any fence that has a seqno smaller or equal42 * to it.43 *44 * struct sw_sync_create_fence_data45 * @value: the seqno to initialise the fence with46 * @name: the name of the new sync point47 * @fence: return the fd of the new sync_file with the created fence48 */49struct sw_sync_create_fence_data {50 __u32 value;51 char name[32];52 __s32 fence; /* fd of new fence */53};54 55/**56 * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence57 * @deadline_ns: absolute time of the deadline58 * @pad: must be zero59 * @fence_fd: the sw_sync fence fd (in)60 *61 * Return the earliest deadline set on the fence. The timebase for the62 * deadline is CLOCK_MONOTONIC (same as vblank). If there is no deadline63 * set on the fence, this ioctl will return -ENOENT.64 */65struct sw_sync_get_deadline {66 __u64 deadline_ns;67 __u32 pad;68 __s32 fence_fd;69};70 71#define SW_SYNC_IOC_MAGIC 'W'72 73#define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\74 struct sw_sync_create_fence_data)75 76#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)77#define SW_SYNC_GET_DEADLINE _IOWR(SW_SYNC_IOC_MAGIC, 2, \78 struct sw_sync_get_deadline)79 80 81#define SW_SYNC_HAS_DEADLINE_BIT DMA_FENCE_FLAG_USER_BITS82 83static const struct dma_fence_ops timeline_fence_ops;84 85static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence)86{87 if (fence->ops != &timeline_fence_ops)88 return NULL;89 return container_of(fence, struct sync_pt, base);90}91 92/**93 * sync_timeline_create() - creates a sync object94 * @name: sync_timeline name95 *96 * Creates a new sync_timeline. Returns the sync_timeline object or NULL in97 * case of error.98 */99static struct sync_timeline *sync_timeline_create(const char *name)100{101 struct sync_timeline *obj;102 103 obj = kzalloc(sizeof(*obj), GFP_KERNEL);104 if (!obj)105 return NULL;106 107 kref_init(&obj->kref);108 obj->context = dma_fence_context_alloc(1);109 strscpy(obj->name, name, sizeof(obj->name));110 111 obj->pt_tree = RB_ROOT;112 INIT_LIST_HEAD(&obj->pt_list);113 spin_lock_init(&obj->lock);114 115 sync_timeline_debug_add(obj);116 117 return obj;118}119 120static void sync_timeline_free(struct kref *kref)121{122 struct sync_timeline *obj =123 container_of(kref, struct sync_timeline, kref);124 125 sync_timeline_debug_remove(obj);126 127 kfree(obj);128}129 130static void sync_timeline_get(struct sync_timeline *obj)131{132 kref_get(&obj->kref);133}134 135static void sync_timeline_put(struct sync_timeline *obj)136{137 kref_put(&obj->kref, sync_timeline_free);138}139 140static const char *timeline_fence_get_driver_name(struct dma_fence *fence)141{142 return "sw_sync";143}144 145static const char *timeline_fence_get_timeline_name(struct dma_fence *fence)146{147 struct sync_timeline *parent = dma_fence_parent(fence);148 149 return parent->name;150}151 152static void timeline_fence_release(struct dma_fence *fence)153{154 struct sync_pt *pt = dma_fence_to_sync_pt(fence);155 struct sync_timeline *parent = dma_fence_parent(fence);156 unsigned long flags;157 158 spin_lock_irqsave(fence->lock, flags);159 if (!list_empty(&pt->link)) {160 list_del(&pt->link);161 rb_erase(&pt->node, &parent->pt_tree);162 }163 spin_unlock_irqrestore(fence->lock, flags);164 165 sync_timeline_put(parent);166 dma_fence_free(fence);167}168 169static bool timeline_fence_signaled(struct dma_fence *fence)170{171 struct sync_timeline *parent = dma_fence_parent(fence);172 173 return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops);174}175 176static bool timeline_fence_enable_signaling(struct dma_fence *fence)177{178 return true;179}180 181static void timeline_fence_value_str(struct dma_fence *fence,182 char *str, int size)183{184 snprintf(str, size, "%lld", fence->seqno);185}186 187static void timeline_fence_timeline_value_str(struct dma_fence *fence,188 char *str, int size)189{190 struct sync_timeline *parent = dma_fence_parent(fence);191 192 snprintf(str, size, "%d", parent->value);193}194 195static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)196{197 struct sync_pt *pt = dma_fence_to_sync_pt(fence);198 unsigned long flags;199 200 spin_lock_irqsave(fence->lock, flags);201 if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {202 if (ktime_before(deadline, pt->deadline))203 pt->deadline = deadline;204 } else {205 pt->deadline = deadline;206 __set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags);207 }208 spin_unlock_irqrestore(fence->lock, flags);209}210 211static const struct dma_fence_ops timeline_fence_ops = {212 .get_driver_name = timeline_fence_get_driver_name,213 .get_timeline_name = timeline_fence_get_timeline_name,214 .enable_signaling = timeline_fence_enable_signaling,215 .signaled = timeline_fence_signaled,216 .release = timeline_fence_release,217 .fence_value_str = timeline_fence_value_str,218 .timeline_value_str = timeline_fence_timeline_value_str,219 .set_deadline = timeline_fence_set_deadline,220};221 222/**223 * sync_timeline_signal() - signal a status change on a sync_timeline224 * @obj: sync_timeline to signal225 * @inc: num to increment on timeline->value226 *227 * A sync implementation should call this any time one of it's fences228 * has signaled or has an error condition.229 */230static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)231{232 LIST_HEAD(signalled);233 struct sync_pt *pt, *next;234 235 trace_sync_timeline(obj);236 237 spin_lock_irq(&obj->lock);238 239 obj->value += inc;240 241 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {242 if (!timeline_fence_signaled(&pt->base))243 break;244 245 dma_fence_get(&pt->base);246 247 list_move_tail(&pt->link, &signalled);248 rb_erase(&pt->node, &obj->pt_tree);249 250 dma_fence_signal_locked(&pt->base);251 }252 253 spin_unlock_irq(&obj->lock);254 255 list_for_each_entry_safe(pt, next, &signalled, link) {256 list_del_init(&pt->link);257 dma_fence_put(&pt->base);258 }259}260 261/**262 * sync_pt_create() - creates a sync pt263 * @obj: parent sync_timeline264 * @value: value of the fence265 *266 * Creates a new sync_pt (fence) as a child of @parent. @size bytes will be267 * allocated allowing for implementation specific data to be kept after268 * the generic sync_timeline struct. Returns the sync_pt object or269 * NULL in case of error.270 */271static struct sync_pt *sync_pt_create(struct sync_timeline *obj,272 unsigned int value)273{274 struct sync_pt *pt;275 276 pt = kzalloc(sizeof(*pt), GFP_KERNEL);277 if (!pt)278 return NULL;279 280 sync_timeline_get(obj);281 dma_fence_init(&pt->base, &timeline_fence_ops, &obj->lock,282 obj->context, value);283 INIT_LIST_HEAD(&pt->link);284 285 spin_lock_irq(&obj->lock);286 if (!dma_fence_is_signaled_locked(&pt->base)) {287 struct rb_node **p = &obj->pt_tree.rb_node;288 struct rb_node *parent = NULL;289 290 while (*p) {291 struct sync_pt *other;292 int cmp;293 294 parent = *p;295 other = rb_entry(parent, typeof(*pt), node);296 cmp = value - other->base.seqno;297 if (cmp > 0) {298 p = &parent->rb_right;299 } else if (cmp < 0) {300 p = &parent->rb_left;301 } else {302 if (dma_fence_get_rcu(&other->base)) {303 sync_timeline_put(obj);304 kfree(pt);305 pt = other;306 goto unlock;307 }308 p = &parent->rb_left;309 }310 }311 rb_link_node(&pt->node, parent, p);312 rb_insert_color(&pt->node, &obj->pt_tree);313 314 parent = rb_next(&pt->node);315 list_add_tail(&pt->link,316 parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list);317 }318unlock:319 spin_unlock_irq(&obj->lock);320 321 return pt;322}323 324/*325 * *WARNING*326 *327 * improper use of this can result in deadlocking kernel drivers from userspace.328 */329 330/* opening sw_sync create a new sync obj */331static int sw_sync_debugfs_open(struct inode *inode, struct file *file)332{333 struct sync_timeline *obj;334 char task_comm[TASK_COMM_LEN];335 336 get_task_comm(task_comm, current);337 338 obj = sync_timeline_create(task_comm);339 if (!obj)340 return -ENOMEM;341 342 file->private_data = obj;343 344 return 0;345}346 347static int sw_sync_debugfs_release(struct inode *inode, struct file *file)348{349 struct sync_timeline *obj = file->private_data;350 struct sync_pt *pt, *next;351 352 spin_lock_irq(&obj->lock);353 354 list_for_each_entry_safe(pt, next, &obj->pt_list, link) {355 dma_fence_set_error(&pt->base, -ENOENT);356 dma_fence_signal_locked(&pt->base);357 }358 359 spin_unlock_irq(&obj->lock);360 361 sync_timeline_put(obj);362 return 0;363}364 365static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,366 unsigned long arg)367{368 int fd = get_unused_fd_flags(O_CLOEXEC);369 int err;370 struct sync_pt *pt;371 struct sync_file *sync_file;372 struct sw_sync_create_fence_data data;373 374 if (fd < 0)375 return fd;376 377 if (copy_from_user(&data, (void __user *)arg, sizeof(data))) {378 err = -EFAULT;379 goto err;380 }381 382 pt = sync_pt_create(obj, data.value);383 if (!pt) {384 err = -ENOMEM;385 goto err;386 }387 388 sync_file = sync_file_create(&pt->base);389 dma_fence_put(&pt->base);390 if (!sync_file) {391 err = -ENOMEM;392 goto err;393 }394 395 data.fence = fd;396 if (copy_to_user((void __user *)arg, &data, sizeof(data))) {397 fput(sync_file->file);398 err = -EFAULT;399 goto err;400 }401 402 fd_install(fd, sync_file->file);403 404 return 0;405 406err:407 put_unused_fd(fd);408 return err;409}410 411static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg)412{413 u32 value;414 415 if (copy_from_user(&value, (void __user *)arg, sizeof(value)))416 return -EFAULT;417 418 while (value > INT_MAX) {419 sync_timeline_signal(obj, INT_MAX);420 value -= INT_MAX;421 }422 423 sync_timeline_signal(obj, value);424 425 return 0;426}427 428static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg)429{430 struct sw_sync_get_deadline data;431 struct dma_fence *fence;432 unsigned long flags;433 struct sync_pt *pt;434 int ret = 0;435 436 if (copy_from_user(&data, (void __user *)arg, sizeof(data)))437 return -EFAULT;438 439 if (data.deadline_ns || data.pad)440 return -EINVAL;441 442 fence = sync_file_get_fence(data.fence_fd);443 if (!fence)444 return -EINVAL;445 446 pt = dma_fence_to_sync_pt(fence);447 if (!pt)448 return -EINVAL;449 450 spin_lock_irqsave(fence->lock, flags);451 if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) {452 data.deadline_ns = ktime_to_ns(pt->deadline);453 } else {454 ret = -ENOENT;455 }456 spin_unlock_irqrestore(fence->lock, flags);457 458 dma_fence_put(fence);459 460 if (ret)461 return ret;462 463 if (copy_to_user((void __user *)arg, &data, sizeof(data)))464 return -EFAULT;465 466 return 0;467}468 469static long sw_sync_ioctl(struct file *file, unsigned int cmd,470 unsigned long arg)471{472 struct sync_timeline *obj = file->private_data;473 474 switch (cmd) {475 case SW_SYNC_IOC_CREATE_FENCE:476 return sw_sync_ioctl_create_fence(obj, arg);477 478 case SW_SYNC_IOC_INC:479 return sw_sync_ioctl_inc(obj, arg);480 481 case SW_SYNC_GET_DEADLINE:482 return sw_sync_ioctl_get_deadline(obj, arg);483 484 default:485 return -ENOTTY;486 }487}488 489const struct file_operations sw_sync_debugfs_fops = {490 .open = sw_sync_debugfs_open,491 .release = sw_sync_debugfs_release,492 .unlocked_ioctl = sw_sync_ioctl,493 .compat_ioctl = compat_ptr_ioctl,494};495