677 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2014-2016 Intel Corporation5 */6 7#include <drm/drm_cache.h>8#include <linux/vmalloc.h>9 10#include "gt/intel_gt.h"11#include "gt/intel_tlb.h"12 13#include "i915_drv.h"14#include "i915_gem_object.h"15#include "i915_scatterlist.h"16#include "i915_gem_lmem.h"17#include "i915_gem_mman.h"18 19void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,20 struct sg_table *pages)21{22 struct drm_i915_private *i915 = to_i915(obj->base.dev);23 unsigned long supported = RUNTIME_INFO(i915)->page_sizes;24 bool shrinkable;25 int i;26 27 assert_object_held_shared(obj);28 29 if (i915_gem_object_is_volatile(obj))30 obj->mm.madv = I915_MADV_DONTNEED;31 32 /* Make the pages coherent with the GPU (flushing any swapin). */33 if (obj->cache_dirty) {34 WARN_ON_ONCE(IS_DGFX(i915));35 obj->write_domain = 0;36 if (i915_gem_object_has_struct_page(obj))37 drm_clflush_sg(pages);38 obj->cache_dirty = false;39 }40 41 obj->mm.get_page.sg_pos = pages->sgl;42 obj->mm.get_page.sg_idx = 0;43 obj->mm.get_dma_page.sg_pos = pages->sgl;44 obj->mm.get_dma_page.sg_idx = 0;45 46 obj->mm.pages = pages;47 48 obj->mm.page_sizes.phys = i915_sg_dma_sizes(pages->sgl);49 GEM_BUG_ON(!obj->mm.page_sizes.phys);50 51 /*52 * Calculate the supported page-sizes which fit into the given53 * sg_page_sizes. This will give us the page-sizes which we may be able54 * to use opportunistically when later inserting into the GTT. For55 * example if phys=2G, then in theory we should be able to use 1G, 2M,56 * 64K or 4K pages, although in practice this will depend on a number of57 * other factors.58 */59 obj->mm.page_sizes.sg = 0;60 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {61 if (obj->mm.page_sizes.phys & ~0u << i)62 obj->mm.page_sizes.sg |= BIT(i);63 }64 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));65 66 shrinkable = i915_gem_object_is_shrinkable(obj);67 68 if (i915_gem_object_is_tiled(obj) &&69 i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {70 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));71 i915_gem_object_set_tiling_quirk(obj);72 GEM_BUG_ON(!list_empty(&obj->mm.link));73 atomic_inc(&obj->mm.shrink_pin);74 shrinkable = false;75 }76 77 if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) {78 struct list_head *list;79 unsigned long flags;80 81 assert_object_held(obj);82 spin_lock_irqsave(&i915->mm.obj_lock, flags);83 84 i915->mm.shrink_count++;85 i915->mm.shrink_memory += obj->base.size;86 87 if (obj->mm.madv != I915_MADV_WILLNEED)88 list = &i915->mm.purge_list;89 else90 list = &i915->mm.shrink_list;91 list_add_tail(&obj->mm.link, list);92 93 atomic_set(&obj->mm.shrink_pin, 0);94 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);95 }96}97 98int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)99{100 struct drm_i915_private *i915 = to_i915(obj->base.dev);101 int err;102 103 assert_object_held_shared(obj);104 105 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {106 drm_dbg(&i915->drm,107 "Attempting to obtain a purgeable object\n");108 return -EFAULT;109 }110 111 err = obj->ops->get_pages(obj);112 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));113 114 return err;115}116 117/* Ensure that the associated pages are gathered from the backing storage118 * and pinned into our object. i915_gem_object_pin_pages() may be called119 * multiple times before they are released by a single call to120 * i915_gem_object_unpin_pages() - once the pages are no longer referenced121 * either as a result of memory pressure (reaping pages under the shrinker)122 * or as the object is itself released.123 */124int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)125{126 int err;127 128 assert_object_held(obj);129 130 assert_object_held_shared(obj);131 132 if (unlikely(!i915_gem_object_has_pages(obj))) {133 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));134 135 err = ____i915_gem_object_get_pages(obj);136 if (err)137 return err;138 139 smp_mb__before_atomic();140 }141 atomic_inc(&obj->mm.pages_pin_count);142 143 return 0;144}145 146int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)147{148 struct i915_gem_ww_ctx ww;149 int err;150 151 i915_gem_ww_ctx_init(&ww, true);152retry:153 err = i915_gem_object_lock(obj, &ww);154 if (!err)155 err = i915_gem_object_pin_pages(obj);156 157 if (err == -EDEADLK) {158 err = i915_gem_ww_ctx_backoff(&ww);159 if (!err)160 goto retry;161 }162 i915_gem_ww_ctx_fini(&ww);163 return err;164}165 166/* Immediately discard the backing storage */167int i915_gem_object_truncate(struct drm_i915_gem_object *obj)168{169 if (obj->ops->truncate)170 return obj->ops->truncate(obj);171 172 return 0;173}174 175static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)176{177 struct radix_tree_iter iter;178 void __rcu **slot;179 180 rcu_read_lock();181 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)182 radix_tree_delete(&obj->mm.get_page.radix, iter.index);183 radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)184 radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);185 rcu_read_unlock();186}187 188static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)189{190 if (is_vmalloc_addr(ptr))191 vunmap(ptr);192}193 194static void flush_tlb_invalidate(struct drm_i915_gem_object *obj)195{196 struct drm_i915_private *i915 = to_i915(obj->base.dev);197 struct intel_gt *gt;198 int id;199 200 for_each_gt(gt, i915, id) {201 if (!obj->mm.tlb[id])202 continue;203 204 intel_gt_invalidate_tlb_full(gt, obj->mm.tlb[id]);205 obj->mm.tlb[id] = 0;206 }207}208 209struct sg_table *210__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)211{212 struct sg_table *pages;213 214 assert_object_held_shared(obj);215 216 pages = fetch_and_zero(&obj->mm.pages);217 if (IS_ERR_OR_NULL(pages))218 return pages;219 220 if (i915_gem_object_is_volatile(obj))221 obj->mm.madv = I915_MADV_WILLNEED;222 223 if (!i915_gem_object_has_self_managed_shrink_list(obj))224 i915_gem_object_make_unshrinkable(obj);225 226 if (obj->mm.mapping) {227 unmap_object(obj, page_mask_bits(obj->mm.mapping));228 obj->mm.mapping = NULL;229 }230 231 __i915_gem_object_reset_page_iter(obj);232 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;233 234 flush_tlb_invalidate(obj);235 236 return pages;237}238 239int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)240{241 struct sg_table *pages;242 243 if (i915_gem_object_has_pinned_pages(obj))244 return -EBUSY;245 246 /* May be called by shrinker from within get_pages() (on another bo) */247 assert_object_held_shared(obj);248 249 i915_gem_object_release_mmap_offset(obj);250 251 /*252 * ->put_pages might need to allocate memory for the bit17 swizzle253 * array, hence protect them from being reaped by removing them from gtt254 * lists early.255 */256 pages = __i915_gem_object_unset_pages(obj);257 258 /*259 * XXX Temporary hijinx to avoid updating all backends to handle260 * NULL pages. In the future, when we have more asynchronous261 * get_pages backends we should be better able to handle the262 * cancellation of the async task in a more uniform manner.263 */264 if (!IS_ERR_OR_NULL(pages))265 obj->ops->put_pages(obj, pages);266 267 return 0;268}269 270/* The 'mapping' part of i915_gem_object_pin_map() below */271static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,272 enum i915_map_type type)273{274 unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;275 struct page *stack[32], **pages = stack, *page;276 struct sgt_iter iter;277 pgprot_t pgprot;278 void *vaddr;279 280 switch (type) {281 default:282 MISSING_CASE(type);283 fallthrough; /* to use PAGE_KERNEL anyway */284 case I915_MAP_WB:285 /*286 * On 32b, highmem using a finite set of indirect PTE (i.e.287 * vmap) to provide virtual mappings of the high pages.288 * As these are finite, map_new_virtual() must wait for some289 * other kmap() to finish when it runs out. If we map a large290 * number of objects, there is no method for it to tell us291 * to release the mappings, and we deadlock.292 *293 * However, if we make an explicit vmap of the page, that294 * uses a larger vmalloc arena, and also has the ability295 * to tell us to release unwanted mappings. Most importantly,296 * it will fail and propagate an error instead of waiting297 * forever.298 *299 * So if the page is beyond the 32b boundary, make an explicit300 * vmap.301 */302 if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))303 return page_address(sg_page(obj->mm.pages->sgl));304 pgprot = PAGE_KERNEL;305 break;306 case I915_MAP_WC:307 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);308 break;309 }310 311 if (n_pages > ARRAY_SIZE(stack)) {312 /* Too big for stack -- allocate temporary array instead */313 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);314 if (!pages)315 return ERR_PTR(-ENOMEM);316 }317 318 i = 0;319 for_each_sgt_page(page, iter, obj->mm.pages)320 pages[i++] = page;321 vaddr = vmap(pages, n_pages, 0, pgprot);322 if (pages != stack)323 kvfree(pages);324 325 return vaddr ?: ERR_PTR(-ENOMEM);326}327 328static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,329 enum i915_map_type type)330{331 resource_size_t iomap = obj->mm.region->iomap.base -332 obj->mm.region->region.start;333 unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;334 unsigned long stack[32], *pfns = stack, i;335 struct sgt_iter iter;336 dma_addr_t addr;337 void *vaddr;338 339 GEM_BUG_ON(type != I915_MAP_WC);340 341 if (n_pfn > ARRAY_SIZE(stack)) {342 /* Too big for stack -- allocate temporary array instead */343 pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);344 if (!pfns)345 return ERR_PTR(-ENOMEM);346 }347 348 i = 0;349 for_each_sgt_daddr(addr, iter, obj->mm.pages)350 pfns[i++] = (iomap + addr) >> PAGE_SHIFT;351 vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));352 if (pfns != stack)353 kvfree(pfns);354 355 return vaddr ?: ERR_PTR(-ENOMEM);356}357 358/* get, pin, and map the pages of the object into kernel space */359void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,360 enum i915_map_type type)361{362 enum i915_map_type has_type;363 bool pinned;364 void *ptr;365 int err;366 367 if (!i915_gem_object_has_struct_page(obj) &&368 !i915_gem_object_has_iomem(obj))369 return ERR_PTR(-ENXIO);370 371 if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY))372 return ERR_PTR(-EINVAL);373 374 assert_object_held(obj);375 376 pinned = !(type & I915_MAP_OVERRIDE);377 type &= ~I915_MAP_OVERRIDE;378 379 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {380 if (unlikely(!i915_gem_object_has_pages(obj))) {381 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));382 383 err = ____i915_gem_object_get_pages(obj);384 if (err)385 return ERR_PTR(err);386 387 smp_mb__before_atomic();388 }389 atomic_inc(&obj->mm.pages_pin_count);390 pinned = false;391 }392 GEM_BUG_ON(!i915_gem_object_has_pages(obj));393 394 /*395 * For discrete our CPU mappings needs to be consistent in order to396 * function correctly on !x86. When mapping things through TTM, we use397 * the same rules to determine the caching type.398 *399 * The caching rules, starting from DG1:400 *401 * - If the object can be placed in device local-memory, then the402 * pages should be allocated and mapped as write-combined only.403 *404 * - Everything else is always allocated and mapped as write-back,405 * with the guarantee that everything is also coherent with the406 * GPU.407 *408 * Internal users of lmem are already expected to get this right, so no409 * fudging needed there.410 */411 if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) {412 if (type != I915_MAP_WC && !obj->mm.n_placements) {413 ptr = ERR_PTR(-ENODEV);414 goto err_unpin;415 }416 417 type = I915_MAP_WC;418 } else if (IS_DGFX(to_i915(obj->base.dev))) {419 type = I915_MAP_WB;420 }421 422 ptr = page_unpack_bits(obj->mm.mapping, &has_type);423 if (ptr && has_type != type) {424 if (pinned) {425 ptr = ERR_PTR(-EBUSY);426 goto err_unpin;427 }428 429 unmap_object(obj, ptr);430 431 ptr = obj->mm.mapping = NULL;432 }433 434 if (!ptr) {435 err = i915_gem_object_wait_moving_fence(obj, true);436 if (err) {437 ptr = ERR_PTR(err);438 goto err_unpin;439 }440 441 if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled()))442 ptr = ERR_PTR(-ENODEV);443 else if (i915_gem_object_has_struct_page(obj))444 ptr = i915_gem_object_map_page(obj, type);445 else446 ptr = i915_gem_object_map_pfn(obj, type);447 if (IS_ERR(ptr))448 goto err_unpin;449 450 obj->mm.mapping = page_pack_bits(ptr, type);451 }452 453 return ptr;454 455err_unpin:456 atomic_dec(&obj->mm.pages_pin_count);457 return ptr;458}459 460void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,461 enum i915_map_type type)462{463 void *ret;464 465 i915_gem_object_lock(obj, NULL);466 ret = i915_gem_object_pin_map(obj, type);467 i915_gem_object_unlock(obj);468 469 return ret;470}471 472void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,473 unsigned long offset,474 unsigned long size)475{476 enum i915_map_type has_type;477 void *ptr;478 479 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));480 GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),481 offset, size, obj->base.size));482 483 wmb(); /* let all previous writes be visible to coherent partners */484 obj->mm.dirty = true;485 486 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)487 return;488 489 ptr = page_unpack_bits(obj->mm.mapping, &has_type);490 if (has_type == I915_MAP_WC)491 return;492 493 drm_clflush_virt_range(ptr + offset, size);494 if (size == obj->base.size) {495 obj->write_domain &= ~I915_GEM_DOMAIN_CPU;496 obj->cache_dirty = false;497 }498}499 500void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)501{502 GEM_BUG_ON(!obj->mm.mapping);503 504 /*505 * We allow removing the mapping from underneath pinned pages!506 *507 * Furthermore, since this is an unsafe operation reserved only508 * for construction time manipulation, we ignore locking prudence.509 */510 unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));511 512 i915_gem_object_unpin_map(obj);513}514 515struct scatterlist *516__i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj,517 struct i915_gem_object_page_iter *iter,518 pgoff_t n,519 unsigned int *offset)520 521{522 const bool dma = iter == &obj->mm.get_dma_page ||523 iter == &obj->ttm.get_io_page;524 unsigned int idx, count;525 struct scatterlist *sg;526 527 might_sleep();528 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);529 if (!i915_gem_object_has_pinned_pages(obj))530 assert_object_held(obj);531 532 /* As we iterate forward through the sg, we record each entry in a533 * radixtree for quick repeated (backwards) lookups. If we have seen534 * this index previously, we will have an entry for it.535 *536 * Initial lookup is O(N), but this is amortized to O(1) for537 * sequential page access (where each new request is consecutive538 * to the previous one). Repeated lookups are O(lg(obj->base.size)),539 * i.e. O(1) with a large constant!540 */541 if (n < READ_ONCE(iter->sg_idx))542 goto lookup;543 544 mutex_lock(&iter->lock);545 546 /* We prefer to reuse the last sg so that repeated lookup of this547 * (or the subsequent) sg are fast - comparing against the last548 * sg is faster than going through the radixtree.549 */550 551 sg = iter->sg_pos;552 idx = iter->sg_idx;553 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);554 555 while (idx + count <= n) {556 void *entry;557 unsigned long i;558 int ret;559 560 /* If we cannot allocate and insert this entry, or the561 * individual pages from this range, cancel updating the562 * sg_idx so that on this lookup we are forced to linearly563 * scan onwards, but on future lookups we will try the564 * insertion again (in which case we need to be careful of565 * the error return reporting that we have already inserted566 * this index).567 */568 ret = radix_tree_insert(&iter->radix, idx, sg);569 if (ret && ret != -EEXIST)570 goto scan;571 572 entry = xa_mk_value(idx);573 for (i = 1; i < count; i++) {574 ret = radix_tree_insert(&iter->radix, idx + i, entry);575 if (ret && ret != -EEXIST)576 goto scan;577 }578 579 idx += count;580 sg = ____sg_next(sg);581 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);582 }583 584scan:585 iter->sg_pos = sg;586 iter->sg_idx = idx;587 588 mutex_unlock(&iter->lock);589 590 if (unlikely(n < idx)) /* insertion completed by another thread */591 goto lookup;592 593 /* In case we failed to insert the entry into the radixtree, we need594 * to look beyond the current sg.595 */596 while (idx + count <= n) {597 idx += count;598 sg = ____sg_next(sg);599 count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);600 }601 602 *offset = n - idx;603 return sg;604 605lookup:606 rcu_read_lock();607 608 sg = radix_tree_lookup(&iter->radix, n);609 GEM_BUG_ON(!sg);610 611 /* If this index is in the middle of multi-page sg entry,612 * the radix tree will contain a value entry that points613 * to the start of that range. We will return the pointer to614 * the base page and the offset of this page within the615 * sg entry's range.616 */617 *offset = 0;618 if (unlikely(xa_is_value(sg))) {619 unsigned long base = xa_to_value(sg);620 621 sg = radix_tree_lookup(&iter->radix, base);622 GEM_BUG_ON(!sg);623 624 *offset = n - base;625 }626 627 rcu_read_unlock();628 629 return sg;630}631 632struct page *633__i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n)634{635 struct scatterlist *sg;636 unsigned int offset;637 638 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));639 640 sg = i915_gem_object_get_sg(obj, n, &offset);641 return nth_page(sg_page(sg), offset);642}643 644/* Like i915_gem_object_get_page(), but mark the returned page dirty */645struct page *646__i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n)647{648 struct page *page;649 650 page = i915_gem_object_get_page(obj, n);651 if (!obj->mm.dirty)652 set_page_dirty(page);653 654 return page;655}656 657dma_addr_t658__i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,659 pgoff_t n, unsigned int *len)660{661 struct scatterlist *sg;662 unsigned int offset;663 664 sg = i915_gem_object_get_sg_dma(obj, n, &offset);665 666 if (len)667 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);668 669 return sg_dma_address(sg) + (offset << PAGE_SHIFT);670}671 672dma_addr_t673__i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n)674{675 return i915_gem_object_get_dma_address_len(obj, n, NULL);676}677