974 lines · c
1/*2 * Copyright © 2017 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 */24 25#include <linux/highmem.h>26#include <linux/sched/mm.h>27 28#include <drm/drm_cache.h>29 30#include "display/intel_frontbuffer.h"31#include "pxp/intel_pxp.h"32 33#include "i915_drv.h"34#include "i915_file_private.h"35#include "i915_gem_clflush.h"36#include "i915_gem_context.h"37#include "i915_gem_dmabuf.h"38#include "i915_gem_mman.h"39#include "i915_gem_object.h"40#include "i915_gem_object_frontbuffer.h"41#include "i915_gem_ttm.h"42#include "i915_memcpy.h"43#include "i915_trace.h"44 45static struct kmem_cache *slab_objects;46 47static const struct drm_gem_object_funcs i915_gem_object_funcs;48 49unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915,50 enum i915_cache_level level)51{52 if (drm_WARN_ON(&i915->drm, level >= I915_MAX_CACHE_LEVEL))53 return 0;54 55 return INTEL_INFO(i915)->cachelevel_to_pat[level];56}57 58bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj,59 enum i915_cache_level lvl)60{61 /*62 * In case the pat_index is set by user space, this kernel mode63 * driver should leave the coherency to be managed by user space,64 * simply return true here.65 */66 if (obj->pat_set_by_user)67 return true;68 69 /*70 * Otherwise the pat_index should have been converted from cache_level71 * so that the following comparison is valid.72 */73 return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), lvl);74}75 76struct drm_i915_gem_object *i915_gem_object_alloc(void)77{78 struct drm_i915_gem_object *obj;79 80 obj = kmem_cache_zalloc(slab_objects, GFP_KERNEL);81 if (!obj)82 return NULL;83 obj->base.funcs = &i915_gem_object_funcs;84 85 return obj;86}87 88void i915_gem_object_free(struct drm_i915_gem_object *obj)89{90 return kmem_cache_free(slab_objects, obj);91}92 93void i915_gem_object_init(struct drm_i915_gem_object *obj,94 const struct drm_i915_gem_object_ops *ops,95 struct lock_class_key *key, unsigned flags)96{97 /*98 * A gem object is embedded both in a struct ttm_buffer_object :/ and99 * in a drm_i915_gem_object. Make sure they are aliased.100 */101 BUILD_BUG_ON(offsetof(typeof(*obj), base) !=102 offsetof(typeof(*obj), __do_not_access.base));103 104 spin_lock_init(&obj->vma.lock);105 INIT_LIST_HEAD(&obj->vma.list);106 107 INIT_LIST_HEAD(&obj->mm.link);108 109#ifdef CONFIG_PROC_FS110 INIT_LIST_HEAD(&obj->client_link);111#endif112 113 INIT_LIST_HEAD(&obj->lut_list);114 spin_lock_init(&obj->lut_lock);115 116 spin_lock_init(&obj->mmo.lock);117 obj->mmo.offsets = RB_ROOT;118 119 init_rcu_head(&obj->rcu);120 121 obj->ops = ops;122 GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS);123 obj->flags = flags;124 125 obj->mm.madv = I915_MADV_WILLNEED;126 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);127 mutex_init(&obj->mm.get_page.lock);128 INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN);129 mutex_init(&obj->mm.get_dma_page.lock);130}131 132/**133 * __i915_gem_object_fini - Clean up a GEM object initialization134 * @obj: The gem object to cleanup135 *136 * This function cleans up gem object fields that are set up by137 * drm_gem_private_object_init() and i915_gem_object_init().138 * It's primarily intended as a helper for backends that need to139 * clean up the gem object in separate steps.140 */141void __i915_gem_object_fini(struct drm_i915_gem_object *obj)142{143 mutex_destroy(&obj->mm.get_page.lock);144 mutex_destroy(&obj->mm.get_dma_page.lock);145 dma_resv_fini(&obj->base._resv);146}147 148/**149 * i915_gem_object_set_cache_coherency - Mark up the object's coherency levels150 * for a given cache_level151 * @obj: #drm_i915_gem_object152 * @cache_level: cache level153 */154void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,155 unsigned int cache_level)156{157 struct drm_i915_private *i915 = to_i915(obj->base.dev);158 159 obj->pat_index = i915_gem_get_pat_index(i915, cache_level);160 161 if (cache_level != I915_CACHE_NONE)162 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |163 I915_BO_CACHE_COHERENT_FOR_WRITE);164 else if (HAS_LLC(i915))165 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;166 else167 obj->cache_coherent = 0;168 169 obj->cache_dirty =170 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) &&171 !IS_DGFX(i915);172}173 174/**175 * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode176 * @obj: #drm_i915_gem_object177 * @pat_index: PAT index178 *179 * This is a clone of i915_gem_object_set_cache_coherency taking pat index180 * instead of cache_level as its second argument.181 */182void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj,183 unsigned int pat_index)184{185 struct drm_i915_private *i915 = to_i915(obj->base.dev);186 187 if (obj->pat_index == pat_index)188 return;189 190 obj->pat_index = pat_index;191 192 if (pat_index != i915_gem_get_pat_index(i915, I915_CACHE_NONE))193 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |194 I915_BO_CACHE_COHERENT_FOR_WRITE);195 else if (HAS_LLC(i915))196 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;197 else198 obj->cache_coherent = 0;199 200 obj->cache_dirty =201 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) &&202 !IS_DGFX(i915);203}204 205bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj)206{207 struct drm_i915_private *i915 = to_i915(obj->base.dev);208 209 /*210 * This is purely from a security perspective, so we simply don't care211 * about non-userspace objects being able to bypass the LLC.212 */213 if (!(obj->flags & I915_BO_ALLOC_USER))214 return false;215 216 /*217 * Always flush cache for UMD objects at creation time.218 */219 if (obj->pat_set_by_user)220 return true;221 222 /*223 * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it224 * possible for userspace to bypass the GTT caching bits set by the225 * kernel, as per the given object cache_level. This is troublesome226 * since the heavy flush we apply when first gathering the pages is227 * skipped if the kernel thinks the object is coherent with the GPU. As228 * a result it might be possible to bypass the cache and read the229 * contents of the page directly, which could be stale data. If it's230 * just a case of userspace shooting themselves in the foot then so be231 * it, but since i915 takes the stance of always zeroing memory before232 * handing it to userspace, we need to prevent this.233 */234 return (IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915));235}236 237static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)238{239 struct drm_i915_gem_object *obj = to_intel_bo(gem);240 struct drm_i915_file_private *fpriv = file->driver_priv;241 struct i915_lut_handle bookmark = {};242 struct i915_mmap_offset *mmo, *mn;243 struct i915_lut_handle *lut, *ln;244 LIST_HEAD(close);245 246 spin_lock(&obj->lut_lock);247 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {248 struct i915_gem_context *ctx = lut->ctx;249 250 if (ctx && ctx->file_priv == fpriv) {251 i915_gem_context_get(ctx);252 list_move(&lut->obj_link, &close);253 }254 255 /* Break long locks, and carefully continue on from this spot */256 if (&ln->obj_link != &obj->lut_list) {257 list_add_tail(&bookmark.obj_link, &ln->obj_link);258 if (cond_resched_lock(&obj->lut_lock))259 list_safe_reset_next(&bookmark, ln, obj_link);260 __list_del_entry(&bookmark.obj_link);261 }262 }263 spin_unlock(&obj->lut_lock);264 265 spin_lock(&obj->mmo.lock);266 rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset)267 drm_vma_node_revoke(&mmo->vma_node, file);268 spin_unlock(&obj->mmo.lock);269 270 list_for_each_entry_safe(lut, ln, &close, obj_link) {271 struct i915_gem_context *ctx = lut->ctx;272 struct i915_vma *vma;273 274 /*275 * We allow the process to have multiple handles to the same276 * vma, in the same fd namespace, by virtue of flink/open.277 */278 279 mutex_lock(&ctx->lut_mutex);280 vma = radix_tree_delete(&ctx->handles_vma, lut->handle);281 if (vma) {282 GEM_BUG_ON(vma->obj != obj);283 GEM_BUG_ON(!atomic_read(&vma->open_count));284 i915_vma_close(vma);285 }286 mutex_unlock(&ctx->lut_mutex);287 288 i915_gem_context_put(lut->ctx);289 i915_lut_handle_free(lut);290 i915_gem_object_put(obj);291 }292}293 294void __i915_gem_free_object_rcu(struct rcu_head *head)295{296 struct drm_i915_gem_object *obj =297 container_of(head, typeof(*obj), rcu);298 struct drm_i915_private *i915 = to_i915(obj->base.dev);299 300 /* We need to keep this alive for RCU read access from fdinfo. */301 if (obj->mm.n_placements > 1)302 kfree(obj->mm.placements);303 304 i915_gem_object_free(obj);305 306 GEM_BUG_ON(!atomic_read(&i915->mm.free_count));307 atomic_dec(&i915->mm.free_count);308}309 310static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj)311{312 /* Skip serialisation and waking the device if known to be not used. */313 314 if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev)))315 i915_gem_object_release_mmap_gtt(obj);316 317 if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) {318 struct i915_mmap_offset *mmo, *mn;319 320 i915_gem_object_release_mmap_offset(obj);321 322 rbtree_postorder_for_each_entry_safe(mmo, mn,323 &obj->mmo.offsets,324 offset) {325 drm_vma_offset_remove(obj->base.dev->vma_offset_manager,326 &mmo->vma_node);327 kfree(mmo);328 }329 obj->mmo.offsets = RB_ROOT;330 }331}332 333/**334 * __i915_gem_object_pages_fini - Clean up pages use of a gem object335 * @obj: The gem object to clean up336 *337 * This function cleans up usage of the object mm.pages member. It338 * is intended for backends that need to clean up a gem object in339 * separate steps and needs to be called when the object is idle before340 * the object's backing memory is freed.341 */342void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj)343{344 assert_object_held_shared(obj);345 346 if (!list_empty(&obj->vma.list)) {347 struct i915_vma *vma;348 349 spin_lock(&obj->vma.lock);350 while ((vma = list_first_entry_or_null(&obj->vma.list,351 struct i915_vma,352 obj_link))) {353 GEM_BUG_ON(vma->obj != obj);354 spin_unlock(&obj->vma.lock);355 356 i915_vma_destroy(vma);357 358 spin_lock(&obj->vma.lock);359 }360 spin_unlock(&obj->vma.lock);361 }362 363 __i915_gem_object_free_mmaps(obj);364 365 atomic_set(&obj->mm.pages_pin_count, 0);366 367 /*368 * dma_buf_unmap_attachment() requires reservation to be369 * locked. The imported GEM shouldn't share reservation lock370 * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for371 * dma-buf, so it's safe to take the lock.372 */373 if (obj->base.import_attach)374 i915_gem_object_lock(obj, NULL);375 376 __i915_gem_object_put_pages(obj);377 378 if (obj->base.import_attach)379 i915_gem_object_unlock(obj);380 381 GEM_BUG_ON(i915_gem_object_has_pages(obj));382}383 384void __i915_gem_free_object(struct drm_i915_gem_object *obj)385{386 trace_i915_gem_object_destroy(obj);387 388 GEM_BUG_ON(!list_empty(&obj->lut_list));389 390 bitmap_free(obj->bit_17);391 392 if (obj->base.import_attach)393 drm_prime_gem_destroy(&obj->base, NULL);394 395 drm_gem_free_mmap_offset(&obj->base);396 397 if (obj->ops->release)398 obj->ops->release(obj);399 400 if (obj->shares_resv_from)401 i915_vm_resv_put(obj->shares_resv_from);402 403 __i915_gem_object_fini(obj);404}405 406static void __i915_gem_free_objects(struct drm_i915_private *i915,407 struct llist_node *freed)408{409 struct drm_i915_gem_object *obj, *on;410 411 llist_for_each_entry_safe(obj, on, freed, freed) {412 might_sleep();413 if (obj->ops->delayed_free) {414 obj->ops->delayed_free(obj);415 continue;416 }417 418 __i915_gem_object_pages_fini(obj);419 __i915_gem_free_object(obj);420 421 /* But keep the pointer alive for RCU-protected lookups */422 call_rcu(&obj->rcu, __i915_gem_free_object_rcu);423 cond_resched();424 }425}426 427void i915_gem_flush_free_objects(struct drm_i915_private *i915)428{429 struct llist_node *freed = llist_del_all(&i915->mm.free_list);430 431 if (unlikely(freed))432 __i915_gem_free_objects(i915, freed);433}434 435static void __i915_gem_free_work(struct work_struct *work)436{437 struct drm_i915_private *i915 =438 container_of(work, struct drm_i915_private, mm.free_work);439 440 i915_gem_flush_free_objects(i915);441}442 443static void i915_gem_free_object(struct drm_gem_object *gem_obj)444{445 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);446 struct drm_i915_private *i915 = to_i915(obj->base.dev);447 448 GEM_BUG_ON(i915_gem_object_is_framebuffer(obj));449 450 i915_drm_client_remove_object(obj);451 452 /*453 * Before we free the object, make sure any pure RCU-only454 * read-side critical sections are complete, e.g.455 * i915_gem_busy_ioctl(). For the corresponding synchronized456 * lookup see i915_gem_object_lookup_rcu().457 */458 atomic_inc(&i915->mm.free_count);459 460 /*461 * Since we require blocking on struct_mutex to unbind the freed462 * object from the GPU before releasing resources back to the463 * system, we can not do that directly from the RCU callback (which may464 * be a softirq context), but must instead then defer that work onto a465 * kthread. We use the RCU callback rather than move the freed object466 * directly onto the work queue so that we can mix between using the467 * worker and performing frees directly from subsequent allocations for468 * crude but effective memory throttling.469 */470 471 if (llist_add(&obj->freed, &i915->mm.free_list))472 queue_work(i915->wq, &i915->mm.free_work);473}474 475void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,476 enum fb_op_origin origin)477{478 struct intel_frontbuffer *front;479 480 front = i915_gem_object_get_frontbuffer(obj);481 if (front) {482 intel_frontbuffer_flush(front, origin);483 intel_frontbuffer_put(front);484 }485}486 487void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,488 enum fb_op_origin origin)489{490 struct intel_frontbuffer *front;491 492 front = i915_gem_object_get_frontbuffer(obj);493 if (front) {494 intel_frontbuffer_invalidate(front, origin);495 intel_frontbuffer_put(front);496 }497}498 499static void500i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)501{502 pgoff_t idx = offset >> PAGE_SHIFT;503 void *src_ptr;504 505 src_ptr = kmap_local_page(i915_gem_object_get_page(obj, idx))506 + offset_in_page(offset);507 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))508 drm_clflush_virt_range(src_ptr, size);509 memcpy(dst, src_ptr, size);510 511 kunmap_local(src_ptr);512}513 514static void515i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)516{517 pgoff_t idx = offset >> PAGE_SHIFT;518 dma_addr_t dma = i915_gem_object_get_dma_address(obj, idx);519 void __iomem *src_map;520 void __iomem *src_ptr;521 522 src_map = io_mapping_map_wc(&obj->mm.region->iomap,523 dma - obj->mm.region->region.start,524 PAGE_SIZE);525 526 src_ptr = src_map + offset_in_page(offset);527 if (!i915_memcpy_from_wc(dst, (void __force *)src_ptr, size))528 memcpy_fromio(dst, src_ptr, size);529 530 io_mapping_unmap(src_map);531}532 533static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj)534{535 GEM_BUG_ON(!i915_gem_object_has_iomem(obj));536 537 if (IS_DGFX(to_i915(obj->base.dev)))538 return i915_ttm_resource_mappable(i915_gem_to_ttm(obj)->resource);539 540 return true;541}542 543/**544 * i915_gem_object_read_from_page - read data from the page of a GEM object545 * @obj: GEM object to read from546 * @offset: offset within the object547 * @dst: buffer to store the read data548 * @size: size to read549 *550 * Reads data from @obj at the specified offset. The requested region to read551 * from can't cross a page boundary. The caller must ensure that @obj pages552 * are pinned and that @obj is synced wrt. any related writes.553 *554 * Return: %0 on success or -ENODEV if the type of @obj's backing store is555 * unsupported.556 */557int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)558{559 GEM_BUG_ON(overflows_type(offset >> PAGE_SHIFT, pgoff_t));560 GEM_BUG_ON(offset >= obj->base.size);561 GEM_BUG_ON(offset_in_page(offset) > PAGE_SIZE - size);562 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));563 564 if (i915_gem_object_has_struct_page(obj))565 i915_gem_object_read_from_page_kmap(obj, offset, dst, size);566 else if (i915_gem_object_has_iomem(obj) && object_has_mappable_iomem(obj))567 i915_gem_object_read_from_page_iomap(obj, offset, dst, size);568 else569 return -ENODEV;570 571 return 0;572}573 574/**575 * i915_gem_object_evictable - Whether object is likely evictable after unbind.576 * @obj: The object to check577 *578 * This function checks whether the object is likely unvictable after unbind.579 * If the object is not locked when checking, the result is only advisory.580 * If the object is locked when checking, and the function returns true,581 * then an eviction should indeed be possible. But since unlocked vma582 * unpinning and unbinding is currently possible, the object can actually583 * become evictable even if this function returns false.584 *585 * Return: true if the object may be evictable. False otherwise.586 */587bool i915_gem_object_evictable(struct drm_i915_gem_object *obj)588{589 struct i915_vma *vma;590 int pin_count = atomic_read(&obj->mm.pages_pin_count);591 592 if (!pin_count)593 return true;594 595 spin_lock(&obj->vma.lock);596 list_for_each_entry(vma, &obj->vma.list, obj_link) {597 if (i915_vma_is_pinned(vma)) {598 spin_unlock(&obj->vma.lock);599 return false;600 }601 if (atomic_read(&vma->pages_count))602 pin_count--;603 }604 spin_unlock(&obj->vma.lock);605 GEM_WARN_ON(pin_count < 0);606 607 return pin_count == 0;608}609 610/**611 * i915_gem_object_migratable - Whether the object is migratable out of the612 * current region.613 * @obj: Pointer to the object.614 *615 * Return: Whether the object is allowed to be resident in other616 * regions than the current while pages are present.617 */618bool i915_gem_object_migratable(struct drm_i915_gem_object *obj)619{620 struct intel_memory_region *mr = READ_ONCE(obj->mm.region);621 622 if (!mr)623 return false;624 625 return obj->mm.n_placements > 1;626}627 628/**629 * i915_gem_object_has_struct_page - Whether the object is page-backed630 * @obj: The object to query.631 *632 * This function should only be called while the object is locked or pinned,633 * otherwise the page backing may change under the caller.634 *635 * Return: True if page-backed, false otherwise.636 */637bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)638{639#ifdef CONFIG_LOCKDEP640 if (IS_DGFX(to_i915(obj->base.dev)) &&641 i915_gem_object_evictable((void __force *)obj))642 assert_object_held_shared(obj);643#endif644 return obj->mem_flags & I915_BO_FLAG_STRUCT_PAGE;645}646 647/**648 * i915_gem_object_has_iomem - Whether the object is iomem-backed649 * @obj: The object to query.650 *651 * This function should only be called while the object is locked or pinned,652 * otherwise the iomem backing may change under the caller.653 *654 * Return: True if iomem-backed, false otherwise.655 */656bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj)657{658#ifdef CONFIG_LOCKDEP659 if (IS_DGFX(to_i915(obj->base.dev)) &&660 i915_gem_object_evictable((void __force *)obj))661 assert_object_held_shared(obj);662#endif663 return obj->mem_flags & I915_BO_FLAG_IOMEM;664}665 666/**667 * i915_gem_object_can_migrate - Whether an object likely can be migrated668 *669 * @obj: The object to migrate670 * @id: The region intended to migrate to671 *672 * Check whether the object backend supports migration to the673 * given region. Note that pinning may affect the ability to migrate as674 * returned by this function.675 *676 * This function is primarily intended as a helper for checking the677 * possibility to migrate objects and might be slightly less permissive678 * than i915_gem_object_migrate() when it comes to objects with the679 * I915_BO_ALLOC_USER flag set.680 *681 * Return: true if migration is possible, false otherwise.682 */683bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj,684 enum intel_region_id id)685{686 struct drm_i915_private *i915 = to_i915(obj->base.dev);687 unsigned int num_allowed = obj->mm.n_placements;688 struct intel_memory_region *mr;689 unsigned int i;690 691 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN);692 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED);693 694 mr = i915->mm.regions[id];695 if (!mr)696 return false;697 698 if (!IS_ALIGNED(obj->base.size, mr->min_page_size))699 return false;700 701 if (obj->mm.region == mr)702 return true;703 704 if (!i915_gem_object_evictable(obj))705 return false;706 707 if (!obj->ops->migrate)708 return false;709 710 if (!(obj->flags & I915_BO_ALLOC_USER))711 return true;712 713 if (num_allowed == 0)714 return false;715 716 for (i = 0; i < num_allowed; ++i) {717 if (mr == obj->mm.placements[i])718 return true;719 }720 721 return false;722}723 724/**725 * i915_gem_object_migrate - Migrate an object to the desired region id726 * @obj: The object to migrate.727 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may728 * not be successful in evicting other objects to make room for this object.729 * @id: The region id to migrate to.730 *731 * Attempt to migrate the object to the desired memory region. The732 * object backend must support migration and the object may not be733 * pinned, (explicitly pinned pages or pinned vmas). The object must734 * be locked.735 * On successful completion, the object will have pages pointing to736 * memory in the new region, but an async migration task may not have737 * completed yet, and to accomplish that, i915_gem_object_wait_migration()738 * must be called.739 *740 * Note: the @ww parameter is not used yet, but included to make sure741 * callers put some effort into obtaining a valid ww ctx if one is742 * available.743 *744 * Return: 0 on success. Negative error code on failure. In particular may745 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance746 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and747 * -EBUSY if the object is pinned.748 */749int i915_gem_object_migrate(struct drm_i915_gem_object *obj,750 struct i915_gem_ww_ctx *ww,751 enum intel_region_id id)752{753 return __i915_gem_object_migrate(obj, ww, id, obj->flags);754}755 756/**757 * __i915_gem_object_migrate - Migrate an object to the desired region id, with758 * control of the extra flags759 * @obj: The object to migrate.760 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may761 * not be successful in evicting other objects to make room for this object.762 * @id: The region id to migrate to.763 * @flags: The object flags. Normally just obj->flags.764 *765 * Attempt to migrate the object to the desired memory region. The766 * object backend must support migration and the object may not be767 * pinned, (explicitly pinned pages or pinned vmas). The object must768 * be locked.769 * On successful completion, the object will have pages pointing to770 * memory in the new region, but an async migration task may not have771 * completed yet, and to accomplish that, i915_gem_object_wait_migration()772 * must be called.773 *774 * Note: the @ww parameter is not used yet, but included to make sure775 * callers put some effort into obtaining a valid ww ctx if one is776 * available.777 *778 * Return: 0 on success. Negative error code on failure. In particular may779 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance780 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and781 * -EBUSY if the object is pinned.782 */783int __i915_gem_object_migrate(struct drm_i915_gem_object *obj,784 struct i915_gem_ww_ctx *ww,785 enum intel_region_id id,786 unsigned int flags)787{788 struct drm_i915_private *i915 = to_i915(obj->base.dev);789 struct intel_memory_region *mr;790 791 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN);792 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED);793 assert_object_held(obj);794 795 mr = i915->mm.regions[id];796 GEM_BUG_ON(!mr);797 798 if (!i915_gem_object_can_migrate(obj, id))799 return -EINVAL;800 801 if (!obj->ops->migrate) {802 if (GEM_WARN_ON(obj->mm.region != mr))803 return -EINVAL;804 return 0;805 }806 807 return obj->ops->migrate(obj, mr, flags);808}809 810/**811 * i915_gem_object_placement_possible - Check whether the object can be812 * placed at certain memory type813 * @obj: Pointer to the object814 * @type: The memory type to check815 *816 * Return: True if the object can be placed in @type. False otherwise.817 */818bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj,819 enum intel_memory_type type)820{821 unsigned int i;822 823 if (!obj->mm.n_placements) {824 switch (type) {825 case INTEL_MEMORY_LOCAL:826 return i915_gem_object_has_iomem(obj);827 case INTEL_MEMORY_SYSTEM:828 return i915_gem_object_has_pages(obj);829 default:830 /* Ignore stolen for now */831 GEM_BUG_ON(1);832 return false;833 }834 }835 836 for (i = 0; i < obj->mm.n_placements; i++) {837 if (obj->mm.placements[i]->type == type)838 return true;839 }840 841 return false;842}843 844/**845 * i915_gem_object_needs_ccs_pages - Check whether the object requires extra846 * pages when placed in system-memory, in order to save and later restore the847 * flat-CCS aux state when the object is moved between local-memory and848 * system-memory849 * @obj: Pointer to the object850 *851 * Return: True if the object needs extra ccs pages. False otherwise.852 */853bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj)854{855 bool lmem_placement = false;856 int i;857 858 if (!HAS_FLAT_CCS(to_i915(obj->base.dev)))859 return false;860 861 if (obj->flags & I915_BO_ALLOC_CCS_AUX)862 return true;863 864 for (i = 0; i < obj->mm.n_placements; i++) {865 /* Compression is not allowed for the objects with smem placement */866 if (obj->mm.placements[i]->type == INTEL_MEMORY_SYSTEM)867 return false;868 if (!lmem_placement &&869 obj->mm.placements[i]->type == INTEL_MEMORY_LOCAL)870 lmem_placement = true;871 }872 873 return lmem_placement;874}875 876void i915_gem_init__objects(struct drm_i915_private *i915)877{878 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);879}880 881void i915_objects_module_exit(void)882{883 kmem_cache_destroy(slab_objects);884}885 886int __init i915_objects_module_init(void)887{888 slab_objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);889 if (!slab_objects)890 return -ENOMEM;891 892 return 0;893}894 895static const struct drm_gem_object_funcs i915_gem_object_funcs = {896 .free = i915_gem_free_object,897 .close = i915_gem_close_object,898 .export = i915_gem_prime_export,899};900 901/**902 * i915_gem_object_get_moving_fence - Get the object's moving fence if any903 * @obj: The object whose moving fence to get.904 * @fence: The resulting fence905 *906 * A non-signaled moving fence means that there is an async operation907 * pending on the object that needs to be waited on before setting up908 * any GPU- or CPU PTEs to the object's pages.909 *910 * Return: Negative error code or 0 for success.911 */912int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj,913 struct dma_fence **fence)914{915 return dma_resv_get_singleton(obj->base.resv, DMA_RESV_USAGE_KERNEL,916 fence);917}918 919/**920 * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any921 * @obj: The object whose moving fence to wait for.922 * @intr: Whether to wait interruptible.923 *924 * If the moving fence signaled without an error, it is detached from the925 * object and put.926 *927 * Return: 0 if successful, -ERESTARTSYS if the wait was interrupted,928 * negative error code if the async operation represented by the929 * moving fence failed.930 */931int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj,932 bool intr)933{934 long ret;935 936 assert_object_held(obj);937 938 ret = dma_resv_wait_timeout(obj->base. resv, DMA_RESV_USAGE_KERNEL,939 intr, MAX_SCHEDULE_TIMEOUT);940 if (!ret)941 ret = -ETIME;942 else if (ret > 0 && i915_gem_object_has_unknown_state(obj))943 ret = -EIO;944 945 return ret < 0 ? ret : 0;946}947 948/*949 * i915_gem_object_has_unknown_state - Return true if the object backing pages are950 * in an unknown_state. This means that userspace must NEVER be allowed to touch951 * the pages, with either the GPU or CPU.952 *953 * ONLY valid to be called after ensuring that all kernel fences have signalled954 * (in particular the fence for moving/clearing the object).955 */956bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj)957{958 /*959 * The below barrier pairs with the dma_fence_signal() in960 * __memcpy_work(). We should only sample the unknown_state after all961 * the kernel fences have signalled.962 */963 smp_rmb();964 return obj->mm.unknown_state;965}966 967#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)968#include "selftests/huge_gem_object.c"969#include "selftests/huge_pages.c"970#include "selftests/i915_gem_migrate.c"971#include "selftests/i915_gem_object.c"972#include "selftests/i915_gem_coherency.c"973#endif974