162 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include <linux/dma-fence.h>7#include <linux/interval_tree_generic.h>8#include <linux/slab.h>9 10#include "xe_macros.h"11#include "xe_range_fence.h"12 13#define XE_RANGE_TREE_START(_node) ((_node)->start)14#define XE_RANGE_TREE_LAST(_node) ((_node)->last)15 16INTERVAL_TREE_DEFINE(struct xe_range_fence, rb, u64, __subtree_last,17 XE_RANGE_TREE_START, XE_RANGE_TREE_LAST, static,18 xe_range_fence_tree);19 20static void21xe_range_fence_signal_notify(struct dma_fence *fence, struct dma_fence_cb *cb)22{23 struct xe_range_fence *rfence = container_of(cb, typeof(*rfence), cb);24 struct xe_range_fence_tree *tree = rfence->tree;25 26 llist_add(&rfence->link, &tree->list);27}28 29static bool __xe_range_fence_tree_cleanup(struct xe_range_fence_tree *tree)30{31 struct llist_node *node = llist_del_all(&tree->list);32 struct xe_range_fence *rfence, *next;33 34 llist_for_each_entry_safe(rfence, next, node, link) {35 xe_range_fence_tree_remove(rfence, &tree->root);36 dma_fence_put(rfence->fence);37 kfree(rfence);38 }39 40 return !!node;41}42 43/**44 * xe_range_fence_insert() - range fence insert45 * @tree: range fence tree to insert intoi46 * @rfence: range fence47 * @ops: range fence ops48 * @start: start address of range fence49 * @last: last address of range fence50 * @fence: dma fence which signals range fence can be removed + freed51 *52 * Return: 0 on success, non-zero on failure53 */54int xe_range_fence_insert(struct xe_range_fence_tree *tree,55 struct xe_range_fence *rfence,56 const struct xe_range_fence_ops *ops,57 u64 start, u64 last, struct dma_fence *fence)58{59 int err = 0;60 61 __xe_range_fence_tree_cleanup(tree);62 63 if (dma_fence_is_signaled(fence))64 goto free;65 66 rfence->ops = ops;67 rfence->start = start;68 rfence->last = last;69 rfence->tree = tree;70 rfence->fence = dma_fence_get(fence);71 err = dma_fence_add_callback(fence, &rfence->cb,72 xe_range_fence_signal_notify);73 if (err == -ENOENT) {74 dma_fence_put(fence);75 err = 0;76 goto free;77 } else if (err == 0) {78 xe_range_fence_tree_insert(rfence, &tree->root);79 return 0;80 }81 82free:83 if (ops->free)84 ops->free(rfence);85 86 return err;87}88 89static void xe_range_fence_tree_remove_all(struct xe_range_fence_tree *tree)90{91 struct xe_range_fence *rfence;92 bool retry = true;93 94 rfence = xe_range_fence_tree_iter_first(&tree->root, 0, U64_MAX);95 while (rfence) {96 /* Should be ok with the minimalistic callback */97 if (dma_fence_remove_callback(rfence->fence, &rfence->cb))98 llist_add(&rfence->link, &tree->list);99 rfence = xe_range_fence_tree_iter_next(rfence, 0, U64_MAX);100 }101 102 while (retry)103 retry = __xe_range_fence_tree_cleanup(tree);104}105 106/**107 * xe_range_fence_tree_init() - Init range fence tree108 * @tree: range fence tree109 */110void xe_range_fence_tree_init(struct xe_range_fence_tree *tree)111{112 memset(tree, 0, sizeof(*tree));113}114 115/**116 * xe_range_fence_tree_fini() - Fini range fence tree117 * @tree: range fence tree118 */119void xe_range_fence_tree_fini(struct xe_range_fence_tree *tree)120{121 xe_range_fence_tree_remove_all(tree);122 XE_WARN_ON(!RB_EMPTY_ROOT(&tree->root.rb_root));123}124 125/**126 * xe_range_fence_tree_first() - range fence tree iterator first127 * @tree: range fence tree128 * @start: start address of range fence129 * @last: last address of range fence130 *131 * Return: first range fence found in range or NULL132 */133struct xe_range_fence *134xe_range_fence_tree_first(struct xe_range_fence_tree *tree, u64 start,135 u64 last)136{137 return xe_range_fence_tree_iter_first(&tree->root, start, last);138}139 140/**141 * xe_range_fence_tree_next() - range fence tree iterator next142 * @rfence: current range fence143 * @start: start address of range fence144 * @last: last address of range fence145 *146 * Return: next range fence found in range or NULL147 */148struct xe_range_fence *149xe_range_fence_tree_next(struct xe_range_fence *rfence, u64 start, u64 last)150{151 return xe_range_fence_tree_iter_next(rfence, start, last);152}153 154static void xe_range_fence_free(struct xe_range_fence *rfence)155{156 kfree(rfence);157}158 159const struct xe_range_fence_ops xe_range_fence_kfree_ops = {160 .free = xe_range_fence_free,161};162