598 lines · c
1/* SPDX-License-Identifier: MIT */2 3/*4 * Copyright © 2019 Intel Corporation5 */6 7#include <linux/delay.h>8#include <linux/dma-fence.h>9#include <linux/kernel.h>10#include <linux/kthread.h>11#include <linux/sched/signal.h>12#include <linux/slab.h>13#include <linux/spinlock.h>14 15#include "selftest.h"16 17static struct kmem_cache *slab_fences;18 19static struct mock_fence {20 struct dma_fence base;21 struct spinlock lock;22} *to_mock_fence(struct dma_fence *f) {23 return container_of(f, struct mock_fence, base);24}25 26static const char *mock_name(struct dma_fence *f)27{28 return "mock";29}30 31static void mock_fence_release(struct dma_fence *f)32{33 kmem_cache_free(slab_fences, to_mock_fence(f));34}35 36struct wait_cb {37 struct dma_fence_cb cb;38 struct task_struct *task;39};40 41static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)42{43 wake_up_process(container_of(cb, struct wait_cb, cb)->task);44}45 46static long mock_wait(struct dma_fence *f, bool intr, long timeout)47{48 const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;49 struct wait_cb cb = { .task = current };50 51 if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))52 return timeout;53 54 while (timeout) {55 set_current_state(state);56 57 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))58 break;59 60 if (signal_pending_state(state, current))61 break;62 63 timeout = schedule_timeout(timeout);64 }65 __set_current_state(TASK_RUNNING);66 67 if (!dma_fence_remove_callback(f, &cb.cb))68 return timeout;69 70 if (signal_pending_state(state, current))71 return -ERESTARTSYS;72 73 return -ETIME;74}75 76static const struct dma_fence_ops mock_ops = {77 .get_driver_name = mock_name,78 .get_timeline_name = mock_name,79 .wait = mock_wait,80 .release = mock_fence_release,81};82 83static struct dma_fence *mock_fence(void)84{85 struct mock_fence *f;86 87 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);88 if (!f)89 return NULL;90 91 spin_lock_init(&f->lock);92 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);93 94 return &f->base;95}96 97static int sanitycheck(void *arg)98{99 struct dma_fence *f;100 101 f = mock_fence();102 if (!f)103 return -ENOMEM;104 105 dma_fence_enable_sw_signaling(f);106 107 dma_fence_signal(f);108 dma_fence_put(f);109 110 return 0;111}112 113static int test_signaling(void *arg)114{115 struct dma_fence *f;116 int err = -EINVAL;117 118 f = mock_fence();119 if (!f)120 return -ENOMEM;121 122 dma_fence_enable_sw_signaling(f);123 124 if (dma_fence_is_signaled(f)) {125 pr_err("Fence unexpectedly signaled on creation\n");126 goto err_free;127 }128 129 if (dma_fence_signal(f)) {130 pr_err("Fence reported being already signaled\n");131 goto err_free;132 }133 134 if (!dma_fence_is_signaled(f)) {135 pr_err("Fence not reporting signaled\n");136 goto err_free;137 }138 139 if (!dma_fence_signal(f)) {140 pr_err("Fence reported not being already signaled\n");141 goto err_free;142 }143 144 err = 0;145err_free:146 dma_fence_put(f);147 return err;148}149 150struct simple_cb {151 struct dma_fence_cb cb;152 bool seen;153};154 155static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)156{157 smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);158}159 160static int test_add_callback(void *arg)161{162 struct simple_cb cb = {};163 struct dma_fence *f;164 int err = -EINVAL;165 166 f = mock_fence();167 if (!f)168 return -ENOMEM;169 170 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {171 pr_err("Failed to add callback, fence already signaled!\n");172 goto err_free;173 }174 175 dma_fence_signal(f);176 if (!cb.seen) {177 pr_err("Callback failed!\n");178 goto err_free;179 }180 181 err = 0;182err_free:183 dma_fence_put(f);184 return err;185}186 187static int test_late_add_callback(void *arg)188{189 struct simple_cb cb = {};190 struct dma_fence *f;191 int err = -EINVAL;192 193 f = mock_fence();194 if (!f)195 return -ENOMEM;196 197 dma_fence_enable_sw_signaling(f);198 199 dma_fence_signal(f);200 201 if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {202 pr_err("Added callback, but fence was already signaled!\n");203 goto err_free;204 }205 206 dma_fence_signal(f);207 if (cb.seen) {208 pr_err("Callback called after failed attachment !\n");209 goto err_free;210 }211 212 err = 0;213err_free:214 dma_fence_put(f);215 return err;216}217 218static int test_rm_callback(void *arg)219{220 struct simple_cb cb = {};221 struct dma_fence *f;222 int err = -EINVAL;223 224 f = mock_fence();225 if (!f)226 return -ENOMEM;227 228 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {229 pr_err("Failed to add callback, fence already signaled!\n");230 goto err_free;231 }232 233 if (!dma_fence_remove_callback(f, &cb.cb)) {234 pr_err("Failed to remove callback!\n");235 goto err_free;236 }237 238 dma_fence_signal(f);239 if (cb.seen) {240 pr_err("Callback still signaled after removal!\n");241 goto err_free;242 }243 244 err = 0;245err_free:246 dma_fence_put(f);247 return err;248}249 250static int test_late_rm_callback(void *arg)251{252 struct simple_cb cb = {};253 struct dma_fence *f;254 int err = -EINVAL;255 256 f = mock_fence();257 if (!f)258 return -ENOMEM;259 260 if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {261 pr_err("Failed to add callback, fence already signaled!\n");262 goto err_free;263 }264 265 dma_fence_signal(f);266 if (!cb.seen) {267 pr_err("Callback failed!\n");268 goto err_free;269 }270 271 if (dma_fence_remove_callback(f, &cb.cb)) {272 pr_err("Callback removal succeed after being executed!\n");273 goto err_free;274 }275 276 err = 0;277err_free:278 dma_fence_put(f);279 return err;280}281 282static int test_status(void *arg)283{284 struct dma_fence *f;285 int err = -EINVAL;286 287 f = mock_fence();288 if (!f)289 return -ENOMEM;290 291 dma_fence_enable_sw_signaling(f);292 293 if (dma_fence_get_status(f)) {294 pr_err("Fence unexpectedly has signaled status on creation\n");295 goto err_free;296 }297 298 dma_fence_signal(f);299 if (!dma_fence_get_status(f)) {300 pr_err("Fence not reporting signaled status\n");301 goto err_free;302 }303 304 err = 0;305err_free:306 dma_fence_put(f);307 return err;308}309 310static int test_error(void *arg)311{312 struct dma_fence *f;313 int err = -EINVAL;314 315 f = mock_fence();316 if (!f)317 return -ENOMEM;318 319 dma_fence_enable_sw_signaling(f);320 321 dma_fence_set_error(f, -EIO);322 323 if (dma_fence_get_status(f)) {324 pr_err("Fence unexpectedly has error status before signal\n");325 goto err_free;326 }327 328 dma_fence_signal(f);329 if (dma_fence_get_status(f) != -EIO) {330 pr_err("Fence not reporting error status, got %d\n",331 dma_fence_get_status(f));332 goto err_free;333 }334 335 err = 0;336err_free:337 dma_fence_put(f);338 return err;339}340 341static int test_wait(void *arg)342{343 struct dma_fence *f;344 int err = -EINVAL;345 346 f = mock_fence();347 if (!f)348 return -ENOMEM;349 350 dma_fence_enable_sw_signaling(f);351 352 if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {353 pr_err("Wait reported complete before being signaled\n");354 goto err_free;355 }356 357 dma_fence_signal(f);358 359 if (dma_fence_wait_timeout(f, false, 0) != 0) {360 pr_err("Wait reported incomplete after being signaled\n");361 goto err_free;362 }363 364 err = 0;365err_free:366 dma_fence_signal(f);367 dma_fence_put(f);368 return err;369}370 371struct wait_timer {372 struct timer_list timer;373 struct dma_fence *f;374};375 376static void wait_timer(struct timer_list *timer)377{378 struct wait_timer *wt = from_timer(wt, timer, timer);379 380 dma_fence_signal(wt->f);381}382 383static int test_wait_timeout(void *arg)384{385 struct wait_timer wt;386 int err = -EINVAL;387 388 timer_setup_on_stack(&wt.timer, wait_timer, 0);389 390 wt.f = mock_fence();391 if (!wt.f)392 return -ENOMEM;393 394 dma_fence_enable_sw_signaling(wt.f);395 396 if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {397 pr_err("Wait reported complete before being signaled\n");398 goto err_free;399 }400 401 mod_timer(&wt.timer, jiffies + 1);402 403 if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {404 if (timer_pending(&wt.timer)) {405 pr_notice("Timer did not fire within the jiffy!\n");406 err = 0; /* not our fault! */407 } else {408 pr_err("Wait reported incomplete after timeout\n");409 }410 goto err_free;411 }412 413 err = 0;414err_free:415 del_timer_sync(&wt.timer);416 destroy_timer_on_stack(&wt.timer);417 dma_fence_signal(wt.f);418 dma_fence_put(wt.f);419 return err;420}421 422static int test_stub(void *arg)423{424 struct dma_fence *f[64];425 int err = -EINVAL;426 int i;427 428 for (i = 0; i < ARRAY_SIZE(f); i++) {429 f[i] = dma_fence_get_stub();430 if (!dma_fence_is_signaled(f[i])) {431 pr_err("Obtained unsignaled stub fence!\n");432 goto err;433 }434 }435 436 err = 0;437err:438 while (i--)439 dma_fence_put(f[i]);440 return err;441}442 443/* Now off to the races! */444 445struct race_thread {446 struct dma_fence __rcu **fences;447 struct task_struct *task;448 bool before;449 int id;450};451 452static void __wait_for_callbacks(struct dma_fence *f)453{454 spin_lock_irq(f->lock);455 spin_unlock_irq(f->lock);456}457 458static int thread_signal_callback(void *arg)459{460 const struct race_thread *t = arg;461 unsigned long pass = 0;462 unsigned long miss = 0;463 int err = 0;464 465 while (!err && !kthread_should_stop()) {466 struct dma_fence *f1, *f2;467 struct simple_cb cb;468 469 f1 = mock_fence();470 if (!f1) {471 err = -ENOMEM;472 break;473 }474 475 dma_fence_enable_sw_signaling(f1);476 477 rcu_assign_pointer(t->fences[t->id], f1);478 smp_wmb();479 480 rcu_read_lock();481 do {482 f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);483 } while (!f2 && !kthread_should_stop());484 rcu_read_unlock();485 486 if (t->before)487 dma_fence_signal(f1);488 489 smp_store_mb(cb.seen, false);490 if (!f2 ||491 dma_fence_add_callback(f2, &cb.cb, simple_callback)) {492 miss++;493 cb.seen = true;494 }495 496 if (!t->before)497 dma_fence_signal(f1);498 499 if (!cb.seen) {500 dma_fence_wait(f2, false);501 __wait_for_callbacks(f2);502 }503 504 if (!READ_ONCE(cb.seen)) {505 pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",506 t->id, pass, miss,507 t->before ? "before" : "after",508 dma_fence_is_signaled(f2) ? "yes" : "no");509 err = -EINVAL;510 }511 512 dma_fence_put(f2);513 514 rcu_assign_pointer(t->fences[t->id], NULL);515 smp_wmb();516 517 dma_fence_put(f1);518 519 pass++;520 }521 522 pr_info("%s[%d] completed %lu passes, %lu misses\n",523 __func__, t->id, pass, miss);524 return err;525}526 527static int race_signal_callback(void *arg)528{529 struct dma_fence __rcu *f[2] = {};530 int ret = 0;531 int pass;532 533 for (pass = 0; !ret && pass <= 1; pass++) {534 struct race_thread t[2];535 int i;536 537 for (i = 0; i < ARRAY_SIZE(t); i++) {538 t[i].fences = f;539 t[i].id = i;540 t[i].before = pass;541 t[i].task = kthread_run(thread_signal_callback, &t[i],542 "dma-fence:%d", i);543 if (IS_ERR(t[i].task)) {544 ret = PTR_ERR(t[i].task);545 while (--i >= 0)546 kthread_stop_put(t[i].task);547 return ret;548 }549 get_task_struct(t[i].task);550 }551 552 msleep(50);553 554 for (i = 0; i < ARRAY_SIZE(t); i++) {555 int err;556 557 err = kthread_stop_put(t[i].task);558 if (err && !ret)559 ret = err;560 }561 }562 563 return ret;564}565 566int dma_fence(void)567{568 static const struct subtest tests[] = {569 SUBTEST(sanitycheck),570 SUBTEST(test_signaling),571 SUBTEST(test_add_callback),572 SUBTEST(test_late_add_callback),573 SUBTEST(test_rm_callback),574 SUBTEST(test_late_rm_callback),575 SUBTEST(test_status),576 SUBTEST(test_error),577 SUBTEST(test_wait),578 SUBTEST(test_wait_timeout),579 SUBTEST(test_stub),580 SUBTEST(race_signal_callback),581 };582 int ret;583 584 pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));585 586 slab_fences = KMEM_CACHE(mock_fence,587 SLAB_TYPESAFE_BY_RCU |588 SLAB_HWCACHE_ALIGN);589 if (!slab_fences)590 return -ENOMEM;591 592 ret = subtests(tests, NULL);593 594 kmem_cache_destroy(slab_fences);595 596 return ret;597}598