498 lines · c
1/*2 * Copyright © 2016 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 */24 25#ifndef __I915_VMA_H__26#define __I915_VMA_H__27 28#include <linux/io-mapping.h>29#include <linux/rbtree.h>30 31#include <drm/drm_mm.h>32 33#include "gt/intel_ggtt_fencing.h"34#include "gem/i915_gem_object.h"35 36#include "i915_gem_gtt.h"37 38#include "i915_active.h"39#include "i915_request.h"40#include "i915_vma_resource.h"41#include "i915_vma_types.h"42 43struct i915_vma *44i915_vma_instance(struct drm_i915_gem_object *obj,45 struct i915_address_space *vm,46 const struct i915_gtt_view *view);47 48void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);49#define I915_VMA_RELEASE_MAP BIT(0)50 51static inline bool i915_vma_is_active(const struct i915_vma *vma)52{53 return !i915_active_is_idle(&vma->active);54}55 56/* do not reserve memory to prevent deadlocks */57#define __EXEC_OBJECT_NO_RESERVE BIT(31)58#define __EXEC_OBJECT_NO_REQUEST_AWAIT BIT(30)59 60int __must_check _i915_vma_move_to_active(struct i915_vma *vma,61 struct i915_request *rq,62 struct dma_fence *fence,63 unsigned int flags);64static inline int __must_check65i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq,66 unsigned int flags)67{68 return _i915_vma_move_to_active(vma, rq, &rq->fence, flags);69}70 71#define __i915_vma_flags(v) ((unsigned long *)&(v)->flags.counter)72 73static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)74{75 return test_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));76}77 78static inline bool i915_vma_is_dpt(const struct i915_vma *vma)79{80 return i915_is_dpt(vma->vm);81}82 83static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)84{85 return test_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));86}87 88static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)89{90 GEM_BUG_ON(!i915_vma_is_ggtt(vma));91 set_bit(I915_VMA_GGTT_WRITE_BIT, __i915_vma_flags(vma));92}93 94static inline bool i915_vma_unset_ggtt_write(struct i915_vma *vma)95{96 return test_and_clear_bit(I915_VMA_GGTT_WRITE_BIT,97 __i915_vma_flags(vma));98}99 100void i915_vma_flush_writes(struct i915_vma *vma);101 102static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)103{104 return test_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));105}106 107static inline bool i915_vma_set_userfault(struct i915_vma *vma)108{109 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));110 return test_and_set_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));111}112 113static inline void i915_vma_unset_userfault(struct i915_vma *vma)114{115 return clear_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));116}117 118static inline bool i915_vma_has_userfault(const struct i915_vma *vma)119{120 return test_bit(I915_VMA_USERFAULT_BIT, __i915_vma_flags(vma));121}122 123static inline bool i915_vma_is_closed(const struct i915_vma *vma)124{125 return !list_empty(&vma->closed_link);126}127 128/* Internal use only. */129static inline u64 __i915_vma_size(const struct i915_vma *vma)130{131 return vma->node.size - 2 * vma->guard;132}133 134/**135 * i915_vma_size - Obtain the va range size of the vma136 * @vma: The vma137 *138 * GPU virtual address space may be allocated with padding. This139 * function returns the effective virtual address range size140 * with padding subtracted.141 *142 * Return: The effective virtual address range size.143 */144static inline u64 i915_vma_size(const struct i915_vma *vma)145{146 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));147 return __i915_vma_size(vma);148}149 150/* Internal use only. */151static inline u64 __i915_vma_offset(const struct i915_vma *vma)152{153 /* The actual start of the vma->pages is after the guard pages. */154 return vma->node.start + vma->guard;155}156 157/**158 * i915_vma_offset - Obtain the va offset of the vma159 * @vma: The vma160 *161 * GPU virtual address space may be allocated with padding. This162 * function returns the effective virtual address offset the gpu163 * should use to access the bound data.164 *165 * Return: The effective virtual address offset.166 */167static inline u64 i915_vma_offset(const struct i915_vma *vma)168{169 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));170 return __i915_vma_offset(vma);171}172 173static inline u32 i915_ggtt_offset(const struct i915_vma *vma)174{175 GEM_BUG_ON(!i915_vma_is_ggtt(vma));176 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));177 GEM_BUG_ON(upper_32_bits(i915_vma_offset(vma)));178 GEM_BUG_ON(upper_32_bits(i915_vma_offset(vma) +179 i915_vma_size(vma) - 1));180 return lower_32_bits(i915_vma_offset(vma));181}182 183static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)184{185 return i915_vm_to_ggtt(vma->vm)->pin_bias;186}187 188static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)189{190 i915_gem_object_get(vma->obj);191 return vma;192}193 194static inline struct i915_vma *i915_vma_tryget(struct i915_vma *vma)195{196 if (likely(kref_get_unless_zero(&vma->obj->base.refcount)))197 return vma;198 199 return NULL;200}201 202static inline void i915_vma_put(struct i915_vma *vma)203{204 i915_gem_object_put(vma->obj);205}206 207static inline long208i915_vma_compare(struct i915_vma *vma,209 struct i915_address_space *vm,210 const struct i915_gtt_view *view)211{212 ptrdiff_t cmp;213 214 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));215 216 cmp = ptrdiff(vma->vm, vm);217 if (cmp)218 return cmp;219 220 BUILD_BUG_ON(I915_GTT_VIEW_NORMAL != 0);221 cmp = vma->gtt_view.type;222 if (!view)223 return cmp;224 225 cmp -= view->type;226 if (cmp)227 return cmp;228 229 assert_i915_gem_gtt_types();230 231 /* gtt_view.type also encodes its size so that we both distinguish232 * different views using it as a "type" and also use a compact (no233 * accessing of uninitialised padding bytes) memcmp without storing234 * an extra parameter or adding more code.235 *236 * To ensure that the memcmp is valid for all branches of the union,237 * even though the code looks like it is just comparing one branch,238 * we assert above that all branches have the same address, and that239 * each branch has a unique type/size.240 */241 BUILD_BUG_ON(I915_GTT_VIEW_NORMAL >= I915_GTT_VIEW_PARTIAL);242 BUILD_BUG_ON(I915_GTT_VIEW_PARTIAL >= I915_GTT_VIEW_ROTATED);243 BUILD_BUG_ON(I915_GTT_VIEW_ROTATED >= I915_GTT_VIEW_REMAPPED);244 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=245 offsetof(typeof(*view), partial));246 BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=247 offsetof(typeof(*view), remapped));248 return memcmp(&vma->gtt_view.partial, &view->partial, view->type);249}250 251struct i915_vma_work *i915_vma_work(void);252int i915_vma_bind(struct i915_vma *vma,253 unsigned int pat_index,254 u32 flags,255 struct i915_vma_work *work,256 struct i915_vma_resource *vma_res);257 258bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color);259bool i915_vma_misplaced(const struct i915_vma *vma,260 u64 size, u64 alignment, u64 flags);261void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);262void i915_vma_revoke_mmap(struct i915_vma *vma);263void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb);264struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);265int __i915_vma_unbind(struct i915_vma *vma);266int __must_check i915_vma_unbind(struct i915_vma *vma);267int __must_check i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm);268int __must_check i915_vma_unbind_unlocked(struct i915_vma *vma);269void i915_vma_unlink_ctx(struct i915_vma *vma);270void i915_vma_close(struct i915_vma *vma);271void i915_vma_reopen(struct i915_vma *vma);272 273void i915_vma_destroy_locked(struct i915_vma *vma);274void i915_vma_destroy(struct i915_vma *vma);275 276#define assert_vma_held(vma) dma_resv_assert_held((vma)->obj->base.resv)277 278static inline void i915_vma_lock(struct i915_vma *vma)279{280 dma_resv_lock(vma->obj->base.resv, NULL);281}282 283static inline void i915_vma_unlock(struct i915_vma *vma)284{285 dma_resv_unlock(vma->obj->base.resv);286}287 288int __must_check289i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,290 u64 size, u64 alignment, u64 flags);291 292static inline int __must_check293i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)294{295 struct i915_gem_ww_ctx ww;296 int err;297 298 i915_gem_ww_ctx_init(&ww, true);299retry:300 err = i915_gem_object_lock(vma->obj, &ww);301 if (!err)302 err = i915_vma_pin_ww(vma, &ww, size, alignment, flags);303 if (err == -EDEADLK) {304 err = i915_gem_ww_ctx_backoff(&ww);305 if (!err)306 goto retry;307 }308 i915_gem_ww_ctx_fini(&ww);309 310 return err;311}312 313int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,314 u32 align, unsigned int flags);315 316static inline int i915_vma_pin_count(const struct i915_vma *vma)317{318 return atomic_read(&vma->flags) & I915_VMA_PIN_MASK;319}320 321static inline bool i915_vma_is_pinned(const struct i915_vma *vma)322{323 return i915_vma_pin_count(vma);324}325 326static inline void __i915_vma_pin(struct i915_vma *vma)327{328 atomic_inc(&vma->flags);329 GEM_BUG_ON(!i915_vma_is_pinned(vma));330}331 332static inline void __i915_vma_unpin(struct i915_vma *vma)333{334 GEM_BUG_ON(!i915_vma_is_pinned(vma));335 atomic_dec(&vma->flags);336}337 338static inline void i915_vma_unpin(struct i915_vma *vma)339{340 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));341 __i915_vma_unpin(vma);342}343 344static inline bool i915_vma_is_bound(const struct i915_vma *vma,345 unsigned int where)346{347 return atomic_read(&vma->flags) & where;348}349 350static inline bool i915_node_color_differs(const struct drm_mm_node *node,351 unsigned long color)352{353 return drm_mm_node_allocated(node) && node->color != color;354}355 356/**357 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture358 * @vma: VMA to iomap359 *360 * The passed in VMA has to be pinned in the global GTT mappable region.361 * An extra pinning of the VMA is acquired for the return iomapping,362 * the caller must call i915_vma_unpin_iomap to relinquish the pinning363 * after the iomapping is no longer required.364 *365 * Returns a valid iomapped pointer or ERR_PTR.366 */367void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);368 369/**370 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap371 * @vma: VMA to unpin372 *373 * Unpins the previously iomapped VMA from i915_vma_pin_iomap().374 *375 * This function is only valid to be called on a VMA previously376 * iomapped by the caller with i915_vma_pin_iomap().377 */378void i915_vma_unpin_iomap(struct i915_vma *vma);379 380/**381 * i915_vma_pin_fence - pin fencing state382 * @vma: vma to pin fencing for383 *384 * This pins the fencing state (whether tiled or untiled) to make sure the385 * vma (and its object) is ready to be used as a scanout target. Fencing386 * status must be synchronize first by calling i915_vma_get_fence():387 *388 * The resulting fence pin reference must be released again with389 * i915_vma_unpin_fence().390 *391 * Returns:392 * True if the vma has a fence, false otherwise.393 */394int __must_check i915_vma_pin_fence(struct i915_vma *vma);395void i915_vma_revoke_fence(struct i915_vma *vma);396 397int __i915_vma_pin_fence(struct i915_vma *vma);398 399static inline void __i915_vma_unpin_fence(struct i915_vma *vma)400{401 GEM_BUG_ON(atomic_read(&vma->fence->pin_count) <= 0);402 atomic_dec(&vma->fence->pin_count);403}404 405/**406 * i915_vma_unpin_fence - unpin fencing state407 * @vma: vma to unpin fencing for408 *409 * This releases the fence pin reference acquired through410 * i915_vma_pin_fence. It will handle both objects with and without an411 * attached fence correctly, callers do not need to distinguish this.412 */413static inline void414i915_vma_unpin_fence(struct i915_vma *vma)415{416 if (vma->fence)417 __i915_vma_unpin_fence(vma);418}419 420static inline int i915_vma_fence_id(const struct i915_vma *vma)421{422 return vma->fence ? vma->fence->id : -1;423}424 425void i915_vma_parked(struct intel_gt *gt);426 427static inline bool i915_vma_is_scanout(const struct i915_vma *vma)428{429 return test_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));430}431 432static inline void i915_vma_mark_scanout(struct i915_vma *vma)433{434 set_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));435}436 437static inline void i915_vma_clear_scanout(struct i915_vma *vma)438{439 clear_bit(I915_VMA_SCANOUT_BIT, __i915_vma_flags(vma));440}441 442void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj);443 444#define for_each_until(cond) if (cond) break; else445 446/**447 * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object.448 * @V: the #i915_vma iterator449 * @OBJ: the #drm_i915_gem_object450 *451 * GGTT VMA are placed at the being of the object's vma_list, see452 * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA,453 * or the list is empty ofc.454 */455#define for_each_ggtt_vma(V, OBJ) \456 list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \457 for_each_until(!i915_vma_is_ggtt(V))458 459struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma);460void i915_vma_make_shrinkable(struct i915_vma *vma);461void i915_vma_make_purgeable(struct i915_vma *vma);462 463int i915_vma_wait_for_bind(struct i915_vma *vma);464 465static inline int i915_vma_sync(struct i915_vma *vma)466{467 /* Wait for the asynchronous bindings and pending GPU reads */468 return i915_active_wait(&vma->active);469}470 471/**472 * i915_vma_get_current_resource - Get the current resource of the vma473 * @vma: The vma to get the current resource from.474 *475 * It's illegal to call this function if the vma is not bound.476 *477 * Return: A refcounted pointer to the current vma resource478 * of the vma, assuming the vma is bound.479 */480static inline struct i915_vma_resource *481i915_vma_get_current_resource(struct i915_vma *vma)482{483 return i915_vma_resource_get(vma->resource);484}485 486#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)487void i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,488 struct i915_vma *vma);489#endif490 491void i915_vma_module_exit(void);492int i915_vma_module_init(void);493 494I915_SELFTEST_DECLARE(int i915_vma_get_pages(struct i915_vma *vma));495I915_SELFTEST_DECLARE(void i915_vma_put_pages(struct i915_vma *vma));496 497#endif498