262 lines · c
1/* SPDX-License-Identifier: MIT */2/*3 * Copyright © 2021 Intel Corporation4 */5 6#ifndef __I915_VMA_RESOURCE_H__7#define __I915_VMA_RESOURCE_H__8 9#include <linux/dma-fence.h>10#include <linux/refcount.h>11 12#include "i915_gem.h"13#include "i915_scatterlist.h"14#include "i915_sw_fence.h"15#include "intel_runtime_pm.h"16 17struct intel_memory_region;18 19struct i915_page_sizes {20 /**21 * The sg mask of the pages sg_table. i.e the mask of22 * the lengths for each sg entry.23 */24 unsigned int phys;25 26 /**27 * The gtt page sizes we are allowed to use given the28 * sg mask and the supported page sizes. This will29 * express the smallest unit we can use for the whole30 * object, as well as the larger sizes we may be able31 * to use opportunistically.32 */33 unsigned int sg;34};35 36/**37 * struct i915_vma_bindinfo - Information needed for async bind38 * only but that can be dropped after the bind has taken place.39 * Consider making this a separate argument to the bind_vma40 * op, coalescing with other arguments like vm, stash, cache_level41 * and flags42 * @pages: The pages sg-table.43 * @page_sizes: Page sizes of the pages.44 * @pages_rsgt: Refcounted sg-table when delayed object destruction45 * is supported. May be NULL.46 * @readonly: Whether the vma should be bound read-only.47 * @lmem: Whether the vma points to lmem.48 */49struct i915_vma_bindinfo {50 struct sg_table *pages;51 struct i915_page_sizes page_sizes;52 struct i915_refct_sgt *pages_rsgt;53 bool readonly:1;54 bool lmem:1;55};56 57/**58 * struct i915_vma_resource - Snapshotted unbind information.59 * @unbind_fence: Fence to mark unbinding complete. Note that this fence60 * is not considered published until unbind is scheduled, and as such it61 * is illegal to access this fence before scheduled unbind other than62 * for refcounting.63 * @lock: The @unbind_fence lock.64 * @hold_count: Number of holders blocking the fence from finishing.65 * The vma itself is keeping a hold, which is released when unbind66 * is scheduled.67 * @work: Work struct for deferred unbind work.68 * @chain: Pointer to struct i915_sw_fence used to await dependencies.69 * @rb: Rb node for the vm's pending unbind interval tree.70 * @__subtree_last: Interval tree private member.71 * @wakeref: wakeref.72 * @vm: non-refcounted pointer to the vm. This is for internal use only and73 * this member is cleared after vm_resource unbind.74 * @mr: The memory region of the object pointed to by the vma.75 * @ops: Pointer to the backend i915_vma_ops.76 * @private: Bind backend private info.77 * @start: Offset into the address space of bind range start. Note that78 * this is after any padding that might have been allocated.79 * @node_size: Size of the allocated range manager node with padding80 * subtracted.81 * @vma_size: Bind size.82 * @guard: The size of guard area preceding and trailing the bind.83 * @page_sizes_gtt: Resulting page sizes from the bind operation.84 * @bound_flags: Flags indicating binding status.85 * @allocated: Backend private data. TODO: Should move into @private.86 * @immediate_unbind: Unbind can be done immediately and doesn't need to be87 * deferred to a work item awaiting unsignaled fences. This is a hack.88 * (dma_fence_work uses a fence flag for this, but this seems slightly89 * cleaner).90 * @needs_wakeref: Whether a wakeref is needed during unbind. Since we can't91 * take a wakeref in the dma-fence signalling critical path, it needs to be92 * taken when the unbind is scheduled.93 * @skip_pte_rewrite: During ggtt suspend and vm takedown pte rewriting94 * needs to be skipped for unbind.95 * @tlb: pointer for obj->mm.tlb, if async unbind. Otherwise, NULL96 *97 * The lifetime of a struct i915_vma_resource is from a binding request to98 * the actual possible asynchronous unbind has completed.99 */100struct i915_vma_resource {101 struct dma_fence unbind_fence;102 /* See above for description of the lock. */103 spinlock_t lock;104 refcount_t hold_count;105 struct work_struct work;106 struct i915_sw_fence chain;107 struct rb_node rb;108 u64 __subtree_last;109 struct i915_address_space *vm;110 intel_wakeref_t wakeref;111 112 /**113 * @bi: Information needed for async bind only but that can be dropped114 * after the bind has taken place.115 *116 * Consider making this a separate argument to the bind_vma op,117 * coalescing with other arguments like vm, stash, cache_level and flags118 */119 struct i915_vma_bindinfo bi;120 121#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)122 struct intel_memory_region *mr;123#endif124 const struct i915_vma_ops *ops;125 void *private;126 u64 start;127 u64 node_size;128 u64 vma_size;129 u32 guard;130 u32 page_sizes_gtt;131 132 u32 bound_flags;133 bool allocated:1;134 bool immediate_unbind:1;135 bool needs_wakeref:1;136 bool skip_pte_rewrite:1;137 138 u32 *tlb;139};140 141bool i915_vma_resource_hold(struct i915_vma_resource *vma_res,142 bool *lockdep_cookie);143 144void i915_vma_resource_unhold(struct i915_vma_resource *vma_res,145 bool lockdep_cookie);146 147struct i915_vma_resource *i915_vma_resource_alloc(void);148 149void i915_vma_resource_free(struct i915_vma_resource *vma_res);150 151struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res,152 u32 *tlb);153 154void __i915_vma_resource_init(struct i915_vma_resource *vma_res);155 156/**157 * i915_vma_resource_get - Take a reference on a vma resource158 * @vma_res: The vma resource on which to take a reference.159 *160 * Return: The @vma_res pointer161 */162static inline struct i915_vma_resource163*i915_vma_resource_get(struct i915_vma_resource *vma_res)164{165 dma_fence_get(&vma_res->unbind_fence);166 return vma_res;167}168 169/**170 * i915_vma_resource_put - Release a reference to a struct i915_vma_resource171 * @vma_res: The resource172 */173static inline void i915_vma_resource_put(struct i915_vma_resource *vma_res)174{175 dma_fence_put(&vma_res->unbind_fence);176}177 178/**179 * i915_vma_resource_init - Initialize a vma resource.180 * @vma_res: The vma resource to initialize181 * @vm: Pointer to the vm.182 * @pages: The pages sg-table.183 * @page_sizes: Page sizes of the pages.184 * @pages_rsgt: Pointer to a struct i915_refct_sgt of an object with185 * delayed destruction.186 * @readonly: Whether the vma should be bound read-only.187 * @lmem: Whether the vma points to lmem.188 * @mr: The memory region of the object the vma points to.189 * @ops: The backend ops.190 * @private: Bind backend private info.191 * @start: Offset into the address space of bind range start after padding.192 * @node_size: Size of the allocated range manager node minus padding.193 * @size: Bind size.194 * @guard: The size of the guard area preceding and trailing the bind.195 *196 * Initializes a vma resource allocated using i915_vma_resource_alloc().197 * The reason for having separate allocate and initialize function is that198 * initialization may need to be performed from under a lock where199 * allocation is not allowed.200 */201static inline void i915_vma_resource_init(struct i915_vma_resource *vma_res,202 struct i915_address_space *vm,203 struct sg_table *pages,204 const struct i915_page_sizes *page_sizes,205 struct i915_refct_sgt *pages_rsgt,206 bool readonly,207 bool lmem,208 struct intel_memory_region *mr,209 const struct i915_vma_ops *ops,210 void *private,211 u64 start,212 u64 node_size,213 u64 size,214 u32 guard)215{216 __i915_vma_resource_init(vma_res);217 vma_res->vm = vm;218 vma_res->bi.pages = pages;219 vma_res->bi.page_sizes = *page_sizes;220 if (pages_rsgt)221 vma_res->bi.pages_rsgt = i915_refct_sgt_get(pages_rsgt);222 vma_res->bi.readonly = readonly;223 vma_res->bi.lmem = lmem;224#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)225 vma_res->mr = mr;226#endif227 vma_res->ops = ops;228 vma_res->private = private;229 vma_res->start = start;230 vma_res->node_size = node_size;231 vma_res->vma_size = size;232 vma_res->guard = guard;233}234 235static inline void i915_vma_resource_fini(struct i915_vma_resource *vma_res)236{237 GEM_BUG_ON(refcount_read(&vma_res->hold_count) != 1);238 if (vma_res->bi.pages_rsgt)239 i915_refct_sgt_put(vma_res->bi.pages_rsgt);240 i915_sw_fence_fini(&vma_res->chain);241}242 243int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm,244 u64 first,245 u64 last,246 bool intr);247 248int i915_vma_resource_bind_dep_await(struct i915_address_space *vm,249 struct i915_sw_fence *sw_fence,250 u64 first,251 u64 last,252 bool intr,253 gfp_t gfp);254 255void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm);256 257void i915_vma_resource_module_exit(void);258 259int i915_vma_resource_module_init(void);260 261#endif262