738 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2014-2016 Intel Corporation5 */6 7#include <linux/pagevec.h>8#include <linux/shmem_fs.h>9#include <linux/swap.h>10 11#include <drm/drm_cache.h>12 13#include "gem/i915_gem_region.h"14#include "i915_drv.h"15#include "i915_gem_object.h"16#include "i915_gem_tiling.h"17#include "i915_gemfs.h"18#include "i915_scatterlist.h"19#include "i915_trace.h"20 21/*22 * Move folios to appropriate lru and release the batch, decrementing the23 * ref count of those folios.24 */25static void check_release_folio_batch(struct folio_batch *fbatch)26{27 check_move_unevictable_folios(fbatch);28 __folio_batch_release(fbatch);29 cond_resched();30}31 32void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,33 bool dirty, bool backup)34{35 struct sgt_iter sgt_iter;36 struct folio_batch fbatch;37 struct folio *last = NULL;38 struct page *page;39 40 mapping_clear_unevictable(mapping);41 42 folio_batch_init(&fbatch);43 for_each_sgt_page(page, sgt_iter, st) {44 struct folio *folio = page_folio(page);45 46 if (folio == last)47 continue;48 last = folio;49 if (dirty)50 folio_mark_dirty(folio);51 if (backup)52 folio_mark_accessed(folio);53 54 if (!folio_batch_add(&fbatch, folio))55 check_release_folio_batch(&fbatch);56 }57 if (fbatch.nr)58 check_release_folio_batch(&fbatch);59 60 sg_free_table(st);61}62 63int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,64 size_t size, struct intel_memory_region *mr,65 struct address_space *mapping,66 unsigned int max_segment)67{68 unsigned int page_count; /* restricted by sg_alloc_table */69 unsigned long i;70 struct scatterlist *sg;71 unsigned long next_pfn = 0; /* suppress gcc warning */72 gfp_t noreclaim;73 int ret;74 75 if (overflows_type(size / PAGE_SIZE, page_count))76 return -E2BIG;77 78 page_count = size / PAGE_SIZE;79 /*80 * If there's no chance of allocating enough pages for the whole81 * object, bail early.82 */83 if (size > resource_size(&mr->region))84 return -ENOMEM;85 86 if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN))87 return -ENOMEM;88 89 /*90 * Get the list of pages out of our struct file. They'll be pinned91 * at this point until we release them.92 *93 * Fail silently without starting the shrinker94 */95 mapping_set_unevictable(mapping);96 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);97 noreclaim |= __GFP_NORETRY | __GFP_NOWARN;98 99 sg = st->sgl;100 st->nents = 0;101 for (i = 0; i < page_count; i++) {102 struct folio *folio;103 unsigned long nr_pages;104 const unsigned int shrink[] = {105 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,106 0,107 }, *s = shrink;108 gfp_t gfp = noreclaim;109 110 do {111 cond_resched();112 folio = shmem_read_folio_gfp(mapping, i, gfp);113 if (!IS_ERR(folio))114 break;115 116 if (!*s) {117 ret = PTR_ERR(folio);118 goto err_sg;119 }120 121 i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);122 123 /*124 * We've tried hard to allocate the memory by reaping125 * our own buffer, now let the real VM do its job and126 * go down in flames if truly OOM.127 *128 * However, since graphics tend to be disposable,129 * defer the oom here by reporting the ENOMEM back130 * to userspace.131 */132 if (!*s) {133 /* reclaim and warn, but no oom */134 gfp = mapping_gfp_mask(mapping);135 136 /*137 * Our bo are always dirty and so we require138 * kswapd to reclaim our pages (direct reclaim139 * does not effectively begin pageout of our140 * buffers on its own). However, direct reclaim141 * only waits for kswapd when under allocation142 * congestion. So as a result __GFP_RECLAIM is143 * unreliable and fails to actually reclaim our144 * dirty pages -- unless you try over and over145 * again with !__GFP_NORETRY. However, we still146 * want to fail this allocation rather than147 * trigger the out-of-memory killer and for148 * this we want __GFP_RETRY_MAYFAIL.149 */150 gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;151 }152 } while (1);153 154 nr_pages = min_t(unsigned long,155 folio_nr_pages(folio), page_count - i);156 if (!i ||157 sg->length >= max_segment ||158 folio_pfn(folio) != next_pfn) {159 if (i)160 sg = sg_next(sg);161 162 st->nents++;163 sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0);164 } else {165 /* XXX: could overflow? */166 sg->length += nr_pages * PAGE_SIZE;167 }168 next_pfn = folio_pfn(folio) + nr_pages;169 i += nr_pages - 1;170 171 /* Check that the i965g/gm workaround works. */172 GEM_BUG_ON(gfp & __GFP_DMA32 && next_pfn >= 0x00100000UL);173 }174 if (sg) /* loop terminated early; short sg table */175 sg_mark_end(sg);176 177 /* Trim unused sg entries to avoid wasting memory. */178 i915_sg_trim(st);179 180 return 0;181err_sg:182 sg_mark_end(sg);183 if (sg != st->sgl) {184 shmem_sg_free_table(st, mapping, false, false);185 } else {186 mapping_clear_unevictable(mapping);187 sg_free_table(st);188 }189 190 /*191 * shmemfs first checks if there is enough memory to allocate the page192 * and reports ENOSPC should there be insufficient, along with the usual193 * ENOMEM for a genuine allocation failure.194 *195 * We use ENOSPC in our driver to mean that we have run out of aperture196 * space and so want to translate the error from shmemfs back to our197 * usual understanding of ENOMEM.198 */199 if (ret == -ENOSPC)200 ret = -ENOMEM;201 202 return ret;203}204 205static int shmem_get_pages(struct drm_i915_gem_object *obj)206{207 struct drm_i915_private *i915 = to_i915(obj->base.dev);208 struct intel_memory_region *mem = obj->mm.region;209 struct address_space *mapping = obj->base.filp->f_mapping;210 unsigned int max_segment = i915_sg_segment_size(i915->drm.dev);211 struct sg_table *st;212 struct sgt_iter sgt_iter;213 struct page *page;214 int ret;215 216 /*217 * Assert that the object is not currently in any GPU domain. As it218 * wasn't in the GTT, there shouldn't be any way it could have been in219 * a GPU cache220 */221 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);222 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);223 224rebuild_st:225 st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN);226 if (!st)227 return -ENOMEM;228 229 ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,230 max_segment);231 if (ret)232 goto err_st;233 234 ret = i915_gem_gtt_prepare_pages(obj, st);235 if (ret) {236 /*237 * DMA remapping failed? One possible cause is that238 * it could not reserve enough large entries, asking239 * for PAGE_SIZE chunks instead may be helpful.240 */241 if (max_segment > PAGE_SIZE) {242 for_each_sgt_page(page, sgt_iter, st)243 put_page(page);244 sg_free_table(st);245 kfree(st);246 247 max_segment = PAGE_SIZE;248 goto rebuild_st;249 } else {250 dev_warn(i915->drm.dev,251 "Failed to DMA remap %zu pages\n",252 obj->base.size >> PAGE_SHIFT);253 goto err_pages;254 }255 }256 257 if (i915_gem_object_needs_bit17_swizzle(obj))258 i915_gem_object_do_bit_17_swizzle(obj, st);259 260 if (i915_gem_object_can_bypass_llc(obj))261 obj->cache_dirty = true;262 263 __i915_gem_object_set_pages(obj, st);264 265 return 0;266 267err_pages:268 shmem_sg_free_table(st, mapping, false, false);269 /*270 * shmemfs first checks if there is enough memory to allocate the page271 * and reports ENOSPC should there be insufficient, along with the usual272 * ENOMEM for a genuine allocation failure.273 *274 * We use ENOSPC in our driver to mean that we have run out of aperture275 * space and so want to translate the error from shmemfs back to our276 * usual understanding of ENOMEM.277 */278err_st:279 if (ret == -ENOSPC)280 ret = -ENOMEM;281 282 kfree(st);283 284 return ret;285}286 287static int288shmem_truncate(struct drm_i915_gem_object *obj)289{290 /*291 * Our goal here is to return as much of the memory as292 * is possible back to the system as we are called from OOM.293 * To do this we must instruct the shmfs to drop all of its294 * backing pages, *now*.295 */296 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);297 obj->mm.madv = __I915_MADV_PURGED;298 obj->mm.pages = ERR_PTR(-EFAULT);299 300 return 0;301}302 303void __shmem_writeback(size_t size, struct address_space *mapping)304{305 struct writeback_control wbc = {306 .sync_mode = WB_SYNC_NONE,307 .nr_to_write = SWAP_CLUSTER_MAX,308 .range_start = 0,309 .range_end = LLONG_MAX,310 .for_reclaim = 1,311 };312 unsigned long i;313 314 /*315 * Leave mmapings intact (GTT will have been revoked on unbinding,316 * leaving only CPU mmapings around) and add those pages to the LRU317 * instead of invoking writeback so they are aged and paged out318 * as normal.319 */320 321 /* Begin writeback on each dirty page */322 for (i = 0; i < size >> PAGE_SHIFT; i++) {323 struct page *page;324 325 page = find_lock_page(mapping, i);326 if (!page)327 continue;328 329 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {330 int ret;331 332 SetPageReclaim(page);333 ret = mapping->a_ops->writepage(page, &wbc);334 if (!PageWriteback(page))335 ClearPageReclaim(page);336 if (!ret)337 goto put;338 }339 unlock_page(page);340put:341 put_page(page);342 }343}344 345static void346shmem_writeback(struct drm_i915_gem_object *obj)347{348 __shmem_writeback(obj->base.size, obj->base.filp->f_mapping);349}350 351static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)352{353 switch (obj->mm.madv) {354 case I915_MADV_DONTNEED:355 return i915_gem_object_truncate(obj);356 case __I915_MADV_PURGED:357 return 0;358 }359 360 if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK)361 shmem_writeback(obj);362 363 return 0;364}365 366void367__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,368 struct sg_table *pages,369 bool needs_clflush)370{371 struct drm_i915_private *i915 = to_i915(obj->base.dev);372 373 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);374 375 if (obj->mm.madv == I915_MADV_DONTNEED)376 obj->mm.dirty = false;377 378 if (needs_clflush &&379 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&380 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))381 drm_clflush_sg(pages);382 383 __start_cpu_write(obj);384 /*385 * On non-LLC igfx platforms, force the flush-on-acquire if this is ever386 * swapped-in. Our async flush path is not trust worthy enough yet(and387 * happens in the wrong order), and with some tricks it's conceivable388 * for userspace to change the cache-level to I915_CACHE_NONE after the389 * pages are swapped-in, and since execbuf binds the object before doing390 * the async flush, we have a race window.391 */392 if (!HAS_LLC(i915) && !IS_DGFX(i915))393 obj->cache_dirty = true;394}395 396void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)397{398 __i915_gem_object_release_shmem(obj, pages, true);399 400 i915_gem_gtt_finish_pages(obj, pages);401 402 if (i915_gem_object_needs_bit17_swizzle(obj))403 i915_gem_object_save_bit_17_swizzle(obj, pages);404 405 shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,406 obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED);407 kfree(pages);408 obj->mm.dirty = false;409}410 411static void412shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)413{414 if (likely(i915_gem_object_has_struct_page(obj)))415 i915_gem_object_put_pages_shmem(obj, pages);416 else417 i915_gem_object_put_pages_phys(obj, pages);418}419 420static int421shmem_pwrite(struct drm_i915_gem_object *obj,422 const struct drm_i915_gem_pwrite *arg)423{424 struct address_space *mapping = obj->base.filp->f_mapping;425 const struct address_space_operations *aops = mapping->a_ops;426 char __user *user_data = u64_to_user_ptr(arg->data_ptr);427 u64 remain;428 loff_t pos;429 unsigned int pg;430 431 /* Caller already validated user args */432 GEM_BUG_ON(!access_ok(user_data, arg->size));433 434 if (!i915_gem_object_has_struct_page(obj))435 return i915_gem_object_pwrite_phys(obj, arg);436 437 /*438 * Before we instantiate/pin the backing store for our use, we439 * can prepopulate the shmemfs filp efficiently using a write into440 * the pagecache. We avoid the penalty of instantiating all the441 * pages, important if the user is just writing to a few and never442 * uses the object on the GPU, and using a direct write into shmemfs443 * allows it to avoid the cost of retrieving a page (either swapin444 * or clearing-before-use) before it is overwritten.445 */446 if (i915_gem_object_has_pages(obj))447 return -ENODEV;448 449 if (obj->mm.madv != I915_MADV_WILLNEED)450 return -EFAULT;451 452 /*453 * Before the pages are instantiated the object is treated as being454 * in the CPU domain. The pages will be clflushed as required before455 * use, and we can freely write into the pages directly. If userspace456 * races pwrite with any other operation; corruption will ensue -457 * that is userspace's prerogative!458 */459 460 remain = arg->size;461 pos = arg->offset;462 pg = offset_in_page(pos);463 464 do {465 unsigned int len, unwritten;466 struct folio *folio;467 void *data, *vaddr;468 int err;469 char __maybe_unused c;470 471 len = PAGE_SIZE - pg;472 if (len > remain)473 len = remain;474 475 /* Prefault the user page to reduce potential recursion */476 err = __get_user(c, user_data);477 if (err)478 return err;479 480 err = __get_user(c, user_data + len - 1);481 if (err)482 return err;483 484 err = aops->write_begin(obj->base.filp, mapping, pos, len,485 &folio, &data);486 if (err < 0)487 return err;488 489 vaddr = kmap_local_folio(folio, offset_in_folio(folio, pos));490 pagefault_disable();491 unwritten = __copy_from_user_inatomic(vaddr, user_data, len);492 pagefault_enable();493 kunmap_local(vaddr);494 495 err = aops->write_end(obj->base.filp, mapping, pos, len,496 len - unwritten, folio, data);497 if (err < 0)498 return err;499 500 /* We don't handle -EFAULT, leave it to the caller to check */501 if (unwritten)502 return -ENODEV;503 504 remain -= len;505 user_data += len;506 pos += len;507 pg = 0;508 } while (remain);509 510 return 0;511}512 513static int514shmem_pread(struct drm_i915_gem_object *obj,515 const struct drm_i915_gem_pread *arg)516{517 if (!i915_gem_object_has_struct_page(obj))518 return i915_gem_object_pread_phys(obj, arg);519 520 return -ENODEV;521}522 523static void shmem_release(struct drm_i915_gem_object *obj)524{525 if (i915_gem_object_has_struct_page(obj))526 i915_gem_object_release_memory_region(obj);527 528 fput(obj->base.filp);529}530 531const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {532 .name = "i915_gem_object_shmem",533 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,534 535 .get_pages = shmem_get_pages,536 .put_pages = shmem_put_pages,537 .truncate = shmem_truncate,538 .shrink = shmem_shrink,539 540 .pwrite = shmem_pwrite,541 .pread = shmem_pread,542 543 .release = shmem_release,544};545 546static int __create_shmem(struct drm_i915_private *i915,547 struct drm_gem_object *obj,548 resource_size_t size)549{550 unsigned long flags = VM_NORESERVE;551 struct file *filp;552 553 drm_gem_private_object_init(&i915->drm, obj, size);554 555 /* XXX: The __shmem_file_setup() function returns -EINVAL if size is556 * greater than MAX_LFS_FILESIZE.557 * To handle the same error as other code that returns -E2BIG when558 * the size is too large, we add a code that returns -E2BIG when the559 * size is larger than the size that can be handled.560 * If BITS_PER_LONG is 32, size > MAX_LFS_FILESIZE is always false,561 * so we only needs to check when BITS_PER_LONG is 64.562 * If BITS_PER_LONG is 32, E2BIG checks are processed when563 * i915_gem_object_size_2big() is called before init_object() callback564 * is called.565 */566 if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE)567 return -E2BIG;568 569 if (i915->mm.gemfs)570 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,571 flags);572 else573 filp = shmem_file_setup("i915", size, flags);574 if (IS_ERR(filp))575 return PTR_ERR(filp);576 577 obj->filp = filp;578 return 0;579}580 581static int shmem_object_init(struct intel_memory_region *mem,582 struct drm_i915_gem_object *obj,583 resource_size_t offset,584 resource_size_t size,585 resource_size_t page_size,586 unsigned int flags)587{588 static struct lock_class_key lock_class;589 struct drm_i915_private *i915 = mem->i915;590 struct address_space *mapping;591 unsigned int cache_level;592 gfp_t mask;593 int ret;594 595 ret = __create_shmem(i915, &obj->base, size);596 if (ret)597 return ret;598 599 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;600 if (IS_I965GM(i915) || IS_I965G(i915)) {601 /* 965gm cannot relocate objects above 4GiB. */602 mask &= ~__GFP_HIGHMEM;603 mask |= __GFP_DMA32;604 }605 606 mapping = obj->base.filp->f_mapping;607 mapping_set_gfp_mask(mapping, mask);608 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));609 610 i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags);611 obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;612 obj->write_domain = I915_GEM_DOMAIN_CPU;613 obj->read_domains = I915_GEM_DOMAIN_CPU;614 615 /*616 * MTL doesn't snoop CPU cache by default for GPU access (namely617 * 1-way coherency). However some UMD's are currently depending on618 * that. Make 1-way coherent the default setting for MTL. A follow619 * up patch will extend the GEM_CREATE uAPI to allow UMD's specify620 * caching mode at BO creation time621 */622 if (HAS_LLC(i915) || (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)))623 /* On some devices, we can have the GPU use the LLC (the CPU624 * cache) for about a 10% performance improvement625 * compared to uncached. Graphics requests other than626 * display scanout are coherent with the CPU in627 * accessing this cache. This means in this mode we628 * don't need to clflush on the CPU side, and on the629 * GPU side we only need to flush internal caches to630 * get data visible to the CPU.631 *632 * However, we maintain the display planes as UC, and so633 * need to rebind when first used as such.634 */635 cache_level = I915_CACHE_LLC;636 else637 cache_level = I915_CACHE_NONE;638 639 i915_gem_object_set_cache_coherency(obj, cache_level);640 641 i915_gem_object_init_memory_region(obj, mem);642 643 return 0;644}645 646struct drm_i915_gem_object *647i915_gem_object_create_shmem(struct drm_i915_private *i915,648 resource_size_t size)649{650 return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],651 size, 0, 0);652}653 654/* Allocate a new GEM object and fill it with the supplied data */655struct drm_i915_gem_object *656i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915,657 const void *data, resource_size_t size)658{659 struct drm_i915_gem_object *obj;660 struct file *file;661 const struct address_space_operations *aops;662 loff_t pos;663 int err;664 665 GEM_WARN_ON(IS_DGFX(i915));666 obj = i915_gem_object_create_shmem(i915, round_up(size, PAGE_SIZE));667 if (IS_ERR(obj))668 return obj;669 670 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);671 672 file = obj->base.filp;673 aops = file->f_mapping->a_ops;674 pos = 0;675 do {676 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);677 struct folio *folio;678 void *fsdata;679 680 err = aops->write_begin(file, file->f_mapping, pos, len,681 &folio, &fsdata);682 if (err < 0)683 goto fail;684 685 memcpy_to_folio(folio, offset_in_folio(folio, pos), data, len);686 687 err = aops->write_end(file, file->f_mapping, pos, len, len,688 folio, fsdata);689 if (err < 0)690 goto fail;691 692 size -= len;693 data += len;694 pos += len;695 } while (size);696 697 return obj;698 699fail:700 i915_gem_object_put(obj);701 return ERR_PTR(err);702}703 704static int init_shmem(struct intel_memory_region *mem)705{706 i915_gemfs_init(mem->i915);707 intel_memory_region_set_name(mem, "system");708 709 return 0; /* We have fallback to the kernel mnt if gemfs init failed. */710}711 712static int release_shmem(struct intel_memory_region *mem)713{714 i915_gemfs_fini(mem->i915);715 return 0;716}717 718static const struct intel_memory_region_ops shmem_region_ops = {719 .init = init_shmem,720 .release = release_shmem,721 .init_object = shmem_object_init,722};723 724struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915,725 u16 type, u16 instance)726{727 return intel_memory_region_create(i915, 0,728 totalram_pages() << PAGE_SHIFT,729 PAGE_SIZE, 0, 0,730 type, instance,731 &shmem_region_ops);732}733 734bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)735{736 return obj->ops == &i915_gem_shmem_ops;737}738