707 lines · c
1// SPDX-License-Identifier: MIT2 3/*4 * Copyright © 2019 Intel Corporation5 */6 7#include <linux/delay.h>8#include <linux/dma-fence.h>9#include <linux/dma-fence-chain.h>10#include <linux/kernel.h>11#include <linux/kthread.h>12#include <linux/mm.h>13#include <linux/sched/signal.h>14#include <linux/slab.h>15#include <linux/spinlock.h>16#include <linux/random.h>17 18#include "selftest.h"19 20#define CHAIN_SZ (4 << 10)21 22static struct kmem_cache *slab_fences;23 24static inline struct mock_fence {25 struct dma_fence base;26 spinlock_t lock;27} *to_mock_fence(struct dma_fence *f) {28 return container_of(f, struct mock_fence, base);29}30 31static const char *mock_name(struct dma_fence *f)32{33 return "mock";34}35 36static void mock_fence_release(struct dma_fence *f)37{38 kmem_cache_free(slab_fences, to_mock_fence(f));39}40 41static const struct dma_fence_ops mock_ops = {42 .get_driver_name = mock_name,43 .get_timeline_name = mock_name,44 .release = mock_fence_release,45};46 47static struct dma_fence *mock_fence(void)48{49 struct mock_fence *f;50 51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);52 if (!f)53 return NULL;54 55 spin_lock_init(&f->lock);56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);57 58 return &f->base;59}60 61static struct dma_fence *mock_chain(struct dma_fence *prev,62 struct dma_fence *fence,63 u64 seqno)64{65 struct dma_fence_chain *f;66 67 f = dma_fence_chain_alloc();68 if (!f)69 return NULL;70 71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence),72 seqno);73 74 return &f->base;75}76 77static int sanitycheck(void *arg)78{79 struct dma_fence *f, *chain;80 int err = 0;81 82 f = mock_fence();83 if (!f)84 return -ENOMEM;85 86 chain = mock_chain(NULL, f, 1);87 if (chain)88 dma_fence_enable_sw_signaling(chain);89 else90 err = -ENOMEM;91 92 dma_fence_signal(f);93 dma_fence_put(f);94 95 dma_fence_put(chain);96 97 return err;98}99 100struct fence_chains {101 unsigned int chain_length;102 struct dma_fence **fences;103 struct dma_fence **chains;104 105 struct dma_fence *tail;106};107 108static uint64_t seqno_inc(unsigned int i)109{110 return i + 1;111}112 113static int fence_chains_init(struct fence_chains *fc, unsigned int count,114 uint64_t (*seqno_fn)(unsigned int))115{116 unsigned int i;117 int err = 0;118 119 fc->chains = kvmalloc_array(count, sizeof(*fc->chains),120 GFP_KERNEL | __GFP_ZERO);121 if (!fc->chains)122 return -ENOMEM;123 124 fc->fences = kvmalloc_array(count, sizeof(*fc->fences),125 GFP_KERNEL | __GFP_ZERO);126 if (!fc->fences) {127 err = -ENOMEM;128 goto err_chains;129 }130 131 fc->tail = NULL;132 for (i = 0; i < count; i++) {133 fc->fences[i] = mock_fence();134 if (!fc->fences[i]) {135 err = -ENOMEM;136 goto unwind;137 }138 139 fc->chains[i] = mock_chain(fc->tail,140 fc->fences[i],141 seqno_fn(i));142 if (!fc->chains[i]) {143 err = -ENOMEM;144 goto unwind;145 }146 147 fc->tail = fc->chains[i];148 149 dma_fence_enable_sw_signaling(fc->chains[i]);150 }151 152 fc->chain_length = i;153 return 0;154 155unwind:156 for (i = 0; i < count; i++) {157 dma_fence_put(fc->fences[i]);158 dma_fence_put(fc->chains[i]);159 }160 kvfree(fc->fences);161err_chains:162 kvfree(fc->chains);163 return err;164}165 166static void fence_chains_fini(struct fence_chains *fc)167{168 unsigned int i;169 170 for (i = 0; i < fc->chain_length; i++) {171 dma_fence_signal(fc->fences[i]);172 dma_fence_put(fc->fences[i]);173 }174 kvfree(fc->fences);175 176 for (i = 0; i < fc->chain_length; i++)177 dma_fence_put(fc->chains[i]);178 kvfree(fc->chains);179}180 181static int find_seqno(void *arg)182{183 struct fence_chains fc;184 struct dma_fence *fence;185 int err;186 int i;187 188 err = fence_chains_init(&fc, 64, seqno_inc);189 if (err)190 return err;191 192 fence = dma_fence_get(fc.tail);193 err = dma_fence_chain_find_seqno(&fence, 0);194 dma_fence_put(fence);195 if (err) {196 pr_err("Reported %d for find_seqno(0)!\n", err);197 goto err;198 }199 200 for (i = 0; i < fc.chain_length; i++) {201 fence = dma_fence_get(fc.tail);202 err = dma_fence_chain_find_seqno(&fence, i + 1);203 dma_fence_put(fence);204 if (err) {205 pr_err("Reported %d for find_seqno(%d:%d)!\n",206 err, fc.chain_length + 1, i + 1);207 goto err;208 }209 if (fence != fc.chains[i]) {210 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",211 fc.chain_length + 1, i + 1);212 err = -EINVAL;213 goto err;214 }215 216 dma_fence_get(fence);217 err = dma_fence_chain_find_seqno(&fence, i + 1);218 dma_fence_put(fence);219 if (err) {220 pr_err("Error reported for finding self\n");221 goto err;222 }223 if (fence != fc.chains[i]) {224 pr_err("Incorrect fence reported by find self\n");225 err = -EINVAL;226 goto err;227 }228 229 dma_fence_get(fence);230 err = dma_fence_chain_find_seqno(&fence, i + 2);231 dma_fence_put(fence);232 if (!err) {233 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",234 i + 1, i + 2);235 err = -EINVAL;236 goto err;237 }238 239 dma_fence_get(fence);240 err = dma_fence_chain_find_seqno(&fence, i);241 dma_fence_put(fence);242 if (err) {243 pr_err("Error reported for previous fence!\n");244 goto err;245 }246 if (i > 0 && fence != fc.chains[i - 1]) {247 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",248 i + 1, i);249 err = -EINVAL;250 goto err;251 }252 }253 254err:255 fence_chains_fini(&fc);256 return err;257}258 259static int find_signaled(void *arg)260{261 struct fence_chains fc;262 struct dma_fence *fence;263 int err;264 265 err = fence_chains_init(&fc, 2, seqno_inc);266 if (err)267 return err;268 269 dma_fence_signal(fc.fences[0]);270 271 fence = dma_fence_get(fc.tail);272 err = dma_fence_chain_find_seqno(&fence, 1);273 dma_fence_put(fence);274 if (err) {275 pr_err("Reported %d for find_seqno()!\n", err);276 goto err;277 }278 279 if (fence && fence != fc.chains[0]) {280 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",281 fence->seqno);282 283 dma_fence_get(fence);284 err = dma_fence_chain_find_seqno(&fence, 1);285 dma_fence_put(fence);286 if (err)287 pr_err("Reported %d for finding self!\n", err);288 289 err = -EINVAL;290 }291 292err:293 fence_chains_fini(&fc);294 return err;295}296 297static int find_out_of_order(void *arg)298{299 struct fence_chains fc;300 struct dma_fence *fence;301 int err;302 303 err = fence_chains_init(&fc, 3, seqno_inc);304 if (err)305 return err;306 307 dma_fence_signal(fc.fences[1]);308 309 fence = dma_fence_get(fc.tail);310 err = dma_fence_chain_find_seqno(&fence, 2);311 dma_fence_put(fence);312 if (err) {313 pr_err("Reported %d for find_seqno()!\n", err);314 goto err;315 }316 317 /*318 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior319 * of the dma-fence-chain is to make us wait for all the fences up to320 * the point we want. Since fence 1 is still not signaled, this what321 * we should get as fence to wait upon (fence 2 being garbage322 * collected during the traversal of the chain).323 */324 if (fence != fc.chains[0]) {325 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",326 fence ? fence->seqno : 0);327 328 err = -EINVAL;329 }330 331err:332 fence_chains_fini(&fc);333 return err;334}335 336static uint64_t seqno_inc2(unsigned int i)337{338 return 2 * i + 2;339}340 341static int find_gap(void *arg)342{343 struct fence_chains fc;344 struct dma_fence *fence;345 int err;346 int i;347 348 err = fence_chains_init(&fc, 64, seqno_inc2);349 if (err)350 return err;351 352 for (i = 0; i < fc.chain_length; i++) {353 fence = dma_fence_get(fc.tail);354 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);355 dma_fence_put(fence);356 if (err) {357 pr_err("Reported %d for find_seqno(%d:%d)!\n",358 err, fc.chain_length + 1, 2 * i + 1);359 goto err;360 }361 if (fence != fc.chains[i]) {362 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",363 fence->seqno,364 fc.chain_length + 1,365 2 * i + 1);366 err = -EINVAL;367 goto err;368 }369 370 dma_fence_get(fence);371 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);372 dma_fence_put(fence);373 if (err) {374 pr_err("Error reported for finding self\n");375 goto err;376 }377 if (fence != fc.chains[i]) {378 pr_err("Incorrect fence reported by find self\n");379 err = -EINVAL;380 goto err;381 }382 }383 384err:385 fence_chains_fini(&fc);386 return err;387}388 389struct find_race {390 struct fence_chains fc;391 atomic_t children;392};393 394static int __find_race(void *arg)395{396 struct find_race *data = arg;397 int err = 0;398 399 while (!kthread_should_stop()) {400 struct dma_fence *fence = dma_fence_get(data->fc.tail);401 int seqno;402 403 seqno = get_random_u32_inclusive(1, data->fc.chain_length);404 405 err = dma_fence_chain_find_seqno(&fence, seqno);406 if (err) {407 pr_err("Failed to find fence seqno:%d\n",408 seqno);409 dma_fence_put(fence);410 break;411 }412 if (!fence)413 goto signal;414 415 /*416 * We can only find ourselves if we are on fence we were417 * looking for.418 */419 if (fence->seqno == seqno) {420 err = dma_fence_chain_find_seqno(&fence, seqno);421 if (err) {422 pr_err("Reported an invalid fence for find-self:%d\n",423 seqno);424 dma_fence_put(fence);425 break;426 }427 }428 429 dma_fence_put(fence);430 431signal:432 seqno = get_random_u32_below(data->fc.chain_length - 1);433 dma_fence_signal(data->fc.fences[seqno]);434 cond_resched();435 }436 437 if (atomic_dec_and_test(&data->children))438 wake_up_var(&data->children);439 return err;440}441 442static int find_race(void *arg)443{444 struct find_race data;445 int ncpus = num_online_cpus();446 struct task_struct **threads;447 unsigned long count;448 int err;449 int i;450 451 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);452 if (err)453 return err;454 455 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);456 if (!threads) {457 err = -ENOMEM;458 goto err;459 }460 461 atomic_set(&data.children, 0);462 for (i = 0; i < ncpus; i++) {463 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);464 if (IS_ERR(threads[i])) {465 ncpus = i;466 break;467 }468 atomic_inc(&data.children);469 get_task_struct(threads[i]);470 }471 472 wait_var_event_timeout(&data.children,473 !atomic_read(&data.children),474 5 * HZ);475 476 for (i = 0; i < ncpus; i++) {477 int ret;478 479 ret = kthread_stop_put(threads[i]);480 if (ret && !err)481 err = ret;482 }483 kfree(threads);484 485 count = 0;486 for (i = 0; i < data.fc.chain_length; i++)487 if (dma_fence_is_signaled(data.fc.fences[i]))488 count++;489 pr_info("Completed %lu cycles\n", count);490 491err:492 fence_chains_fini(&data.fc);493 return err;494}495 496static int signal_forward(void *arg)497{498 struct fence_chains fc;499 int err;500 int i;501 502 err = fence_chains_init(&fc, 64, seqno_inc);503 if (err)504 return err;505 506 for (i = 0; i < fc.chain_length; i++) {507 dma_fence_signal(fc.fences[i]);508 509 if (!dma_fence_is_signaled(fc.chains[i])) {510 pr_err("chain[%d] not signaled!\n", i);511 err = -EINVAL;512 goto err;513 }514 515 if (i + 1 < fc.chain_length &&516 dma_fence_is_signaled(fc.chains[i + 1])) {517 pr_err("chain[%d] is signaled!\n", i);518 err = -EINVAL;519 goto err;520 }521 }522 523err:524 fence_chains_fini(&fc);525 return err;526}527 528static int signal_backward(void *arg)529{530 struct fence_chains fc;531 int err;532 int i;533 534 err = fence_chains_init(&fc, 64, seqno_inc);535 if (err)536 return err;537 538 for (i = fc.chain_length; i--; ) {539 dma_fence_signal(fc.fences[i]);540 541 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {542 pr_err("chain[%d] is signaled!\n", i);543 err = -EINVAL;544 goto err;545 }546 }547 548 for (i = 0; i < fc.chain_length; i++) {549 if (!dma_fence_is_signaled(fc.chains[i])) {550 pr_err("chain[%d] was not signaled!\n", i);551 err = -EINVAL;552 goto err;553 }554 }555 556err:557 fence_chains_fini(&fc);558 return err;559}560 561static int __wait_fence_chains(void *arg)562{563 struct fence_chains *fc = arg;564 565 if (dma_fence_wait(fc->tail, false))566 return -EIO;567 568 return 0;569}570 571static int wait_forward(void *arg)572{573 struct fence_chains fc;574 struct task_struct *tsk;575 int err;576 int i;577 578 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);579 if (err)580 return err;581 582 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");583 if (IS_ERR(tsk)) {584 err = PTR_ERR(tsk);585 goto err;586 }587 get_task_struct(tsk);588 yield_to(tsk, true);589 590 for (i = 0; i < fc.chain_length; i++)591 dma_fence_signal(fc.fences[i]);592 593 err = kthread_stop_put(tsk);594 595err:596 fence_chains_fini(&fc);597 return err;598}599 600static int wait_backward(void *arg)601{602 struct fence_chains fc;603 struct task_struct *tsk;604 int err;605 int i;606 607 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);608 if (err)609 return err;610 611 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");612 if (IS_ERR(tsk)) {613 err = PTR_ERR(tsk);614 goto err;615 }616 get_task_struct(tsk);617 yield_to(tsk, true);618 619 for (i = fc.chain_length; i--; )620 dma_fence_signal(fc.fences[i]);621 622 err = kthread_stop_put(tsk);623 624err:625 fence_chains_fini(&fc);626 return err;627}628 629static void randomise_fences(struct fence_chains *fc)630{631 unsigned int count = fc->chain_length;632 633 /* Fisher-Yates shuffle courtesy of Knuth */634 while (--count) {635 unsigned int swp;636 637 swp = get_random_u32_below(count + 1);638 if (swp == count)639 continue;640 641 swap(fc->fences[count], fc->fences[swp]);642 }643}644 645static int wait_random(void *arg)646{647 struct fence_chains fc;648 struct task_struct *tsk;649 int err;650 int i;651 652 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);653 if (err)654 return err;655 656 randomise_fences(&fc);657 658 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");659 if (IS_ERR(tsk)) {660 err = PTR_ERR(tsk);661 goto err;662 }663 get_task_struct(tsk);664 yield_to(tsk, true);665 666 for (i = 0; i < fc.chain_length; i++)667 dma_fence_signal(fc.fences[i]);668 669 err = kthread_stop_put(tsk);670 671err:672 fence_chains_fini(&fc);673 return err;674}675 676int dma_fence_chain(void)677{678 static const struct subtest tests[] = {679 SUBTEST(sanitycheck),680 SUBTEST(find_seqno),681 SUBTEST(find_signaled),682 SUBTEST(find_out_of_order),683 SUBTEST(find_gap),684 SUBTEST(find_race),685 SUBTEST(signal_forward),686 SUBTEST(signal_backward),687 SUBTEST(wait_forward),688 SUBTEST(wait_backward),689 SUBTEST(wait_random),690 };691 int ret;692 693 pr_info("sizeof(dma_fence_chain)=%zu\n",694 sizeof(struct dma_fence_chain));695 696 slab_fences = KMEM_CACHE(mock_fence,697 SLAB_TYPESAFE_BY_RCU |698 SLAB_HWCACHE_ALIGN);699 if (!slab_fences)700 return -ENOMEM;701 702 ret = subtests(tests, NULL);703 704 kmem_cache_destroy(slab_fences);705 return ret;706}707