1155 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2014-2016 Intel Corporation5 */6 7#include <linux/anon_inodes.h>8#include <linux/mman.h>9#include <linux/pfn_t.h>10#include <linux/sizes.h>11 12#include <drm/drm_cache.h>13 14#include "gt/intel_gt.h"15#include "gt/intel_gt_requests.h"16 17#include "i915_drv.h"18#include "i915_gem_evict.h"19#include "i915_gem_gtt.h"20#include "i915_gem_ioctls.h"21#include "i915_gem_object.h"22#include "i915_gem_mman.h"23#include "i915_mm.h"24#include "i915_trace.h"25#include "i915_user_extensions.h"26#include "i915_gem_ttm.h"27#include "i915_vma.h"28 29static inline bool30__vma_matches(struct vm_area_struct *vma, struct file *filp,31 unsigned long addr, unsigned long size)32{33 if (vma->vm_file != filp)34 return false;35 36 return vma->vm_start == addr &&37 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);38}39 40/**41 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address42 * it is mapped to.43 * @dev: drm device44 * @data: ioctl data blob45 * @file: drm file46 *47 * While the mapping holds a reference on the contents of the object, it doesn't48 * imply a ref on the object itself.49 *50 * IMPORTANT:51 *52 * DRM driver writers who look a this function as an example for how to do GEM53 * mmap support, please don't implement mmap support like here. The modern way54 * to implement DRM mmap support is with an mmap offset ioctl (like55 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.56 * That way debug tooling like valgrind will understand what's going on, hiding57 * the mmap call in a driver private ioctl will break that. The i915 driver only58 * does cpu mmaps this way because we didn't know better.59 */60int61i915_gem_mmap_ioctl(struct drm_device *dev, void *data,62 struct drm_file *file)63{64 struct drm_i915_private *i915 = to_i915(dev);65 struct drm_i915_gem_mmap *args = data;66 struct drm_i915_gem_object *obj;67 unsigned long addr;68 69 /*70 * mmap ioctl is disallowed for all discrete platforms,71 * and for all platforms with GRAPHICS_VER > 12.72 */73 if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0))74 return -EOPNOTSUPP;75 76 if (args->flags & ~(I915_MMAP_WC))77 return -EINVAL;78 79 if (args->flags & I915_MMAP_WC && !pat_enabled())80 return -ENODEV;81 82 obj = i915_gem_object_lookup(file, args->handle);83 if (!obj)84 return -ENOENT;85 86 /* prime objects have no backing filp to GEM mmap87 * pages from.88 */89 if (!obj->base.filp) {90 addr = -ENXIO;91 goto err;92 }93 94 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {95 addr = -EINVAL;96 goto err;97 }98 99 addr = vm_mmap(obj->base.filp, 0, args->size,100 PROT_READ | PROT_WRITE, MAP_SHARED,101 args->offset);102 if (IS_ERR_VALUE(addr))103 goto err;104 105 if (args->flags & I915_MMAP_WC) {106 struct mm_struct *mm = current->mm;107 struct vm_area_struct *vma;108 109 if (mmap_write_lock_killable(mm)) {110 addr = -EINTR;111 goto err;112 }113 vma = find_vma(mm, addr);114 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))115 vma->vm_page_prot =116 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));117 else118 addr = -ENOMEM;119 mmap_write_unlock(mm);120 if (IS_ERR_VALUE(addr))121 goto err;122 }123 i915_gem_object_put(obj);124 125 args->addr_ptr = (u64)addr;126 return 0;127 128err:129 i915_gem_object_put(obj);130 return addr;131}132 133static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)134{135 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;136}137 138/**139 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps140 *141 * A history of the GTT mmap interface:142 *143 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to144 * aligned and suitable for fencing, and still fit into the available145 * mappable space left by the pinned display objects. A classic problem146 * we called the page-fault-of-doom where we would ping-pong between147 * two objects that could not fit inside the GTT and so the memcpy148 * would page one object in at the expense of the other between every149 * single byte.150 *151 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none152 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the153 * object is too large for the available space (or simply too large154 * for the mappable aperture!), a view is created instead and faulted155 * into userspace. (This view is aligned and sized appropriately for156 * fenced access.)157 *158 * 2 - Recognise WC as a separate cache domain so that we can flush the159 * delayed writes via GTT before performing direct access via WC.160 *161 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial162 * pagefault; swapin remains transparent.163 *164 * 4 - Support multiple fault handlers per object depending on object's165 * backing storage (a.k.a. MMAP_OFFSET).166 *167 * Restrictions:168 *169 * * snoopable objects cannot be accessed via the GTT. It can cause machine170 * hangs on some architectures, corruption on others. An attempt to service171 * a GTT page fault from a snoopable object will generate a SIGBUS.172 *173 * * the object must be able to fit into RAM (physical memory, though no174 * limited to the mappable aperture).175 *176 *177 * Caveats:178 *179 * * a new GTT page fault will synchronize rendering from the GPU and flush180 * all data to system memory. Subsequent access will not be synchronized.181 *182 * * all mappings are revoked on runtime device suspend.183 *184 * * there are only 8, 16 or 32 fence registers to share between all users185 * (older machines require fence register for display and blitter access186 * as well). Contention of the fence registers will cause the previous users187 * to be unmapped and any new access will generate new page faults.188 *189 * * running out of memory while servicing a fault may generate a SIGBUS,190 * rather than the expected SIGSEGV.191 */192int i915_gem_mmap_gtt_version(void)193{194 return 4;195}196 197static inline struct i915_gtt_view198compute_partial_view(const struct drm_i915_gem_object *obj,199 pgoff_t page_offset,200 unsigned int chunk)201{202 struct i915_gtt_view view;203 204 if (i915_gem_object_is_tiled(obj))205 chunk = roundup(chunk, tile_row_pages(obj) ?: 1);206 207 view.type = I915_GTT_VIEW_PARTIAL;208 view.partial.offset = rounddown(page_offset, chunk);209 view.partial.size =210 min_t(unsigned int, chunk,211 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);212 213 /* If the partial covers the entire object, just create a normal VMA. */214 if (chunk >= obj->base.size >> PAGE_SHIFT)215 view.type = I915_GTT_VIEW_NORMAL;216 217 return view;218}219 220static vm_fault_t i915_error_to_vmf_fault(int err)221{222 switch (err) {223 default:224 WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);225 fallthrough;226 case -EIO: /* shmemfs failure from swap device */227 case -EFAULT: /* purged object */228 case -ENODEV: /* bad object, how did you get here! */229 case -ENXIO: /* unable to access backing store (on device) */230 return VM_FAULT_SIGBUS;231 232 case -ENOMEM: /* our allocation failure */233 return VM_FAULT_OOM;234 235 case 0:236 case -EAGAIN:237 case -ENOSPC: /* transient failure to evict? */238 case -ENOBUFS: /* temporarily out of fences? */239 case -ERESTARTSYS:240 case -EINTR:241 case -EBUSY:242 /*243 * EBUSY is ok: this just means that another thread244 * already did the job.245 */246 return VM_FAULT_NOPAGE;247 }248}249 250static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)251{252 struct vm_area_struct *area = vmf->vma;253 struct i915_mmap_offset *mmo = area->vm_private_data;254 struct drm_i915_gem_object *obj = mmo->obj;255 unsigned long obj_offset;256 resource_size_t iomap;257 int err;258 259 /* Sanity check that we allow writing into this object */260 if (unlikely(i915_gem_object_is_readonly(obj) &&261 area->vm_flags & VM_WRITE))262 return VM_FAULT_SIGBUS;263 264 if (i915_gem_object_lock_interruptible(obj, NULL))265 return VM_FAULT_NOPAGE;266 267 err = i915_gem_object_pin_pages(obj);268 if (err)269 goto out;270 271 iomap = -1;272 if (!i915_gem_object_has_struct_page(obj)) {273 iomap = obj->mm.region->iomap.base;274 iomap -= obj->mm.region->region.start;275 }276 277 obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node);278 /* PTEs are revoked in obj->ops->put_pages() */279 err = remap_io_sg(area,280 area->vm_start, area->vm_end - area->vm_start,281 obj->mm.pages->sgl, obj_offset, iomap);282 283 if (area->vm_flags & VM_WRITE) {284 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));285 obj->mm.dirty = true;286 }287 288 i915_gem_object_unpin_pages(obj);289 290out:291 i915_gem_object_unlock(obj);292 return i915_error_to_vmf_fault(err);293}294 295static void set_address_limits(struct vm_area_struct *area,296 struct i915_vma *vma,297 unsigned long obj_offset,298 resource_size_t gmadr_start,299 unsigned long *start_vaddr,300 unsigned long *end_vaddr,301 unsigned long *pfn)302{303 unsigned long vm_start, vm_end, vma_size; /* user's memory parameters */304 long start, end; /* memory boundaries */305 306 /*307 * Let's move into the ">> PAGE_SHIFT"308 * domain to be sure not to lose bits309 */310 vm_start = area->vm_start >> PAGE_SHIFT;311 vm_end = area->vm_end >> PAGE_SHIFT;312 vma_size = vma->size >> PAGE_SHIFT;313 314 /*315 * Calculate the memory boundaries by considering the offset316 * provided by the user during memory mapping and the offset317 * provided for the partial mapping.318 */319 start = vm_start;320 start -= obj_offset;321 start += vma->gtt_view.partial.offset;322 end = start + vma_size;323 324 start = max_t(long, start, vm_start);325 end = min_t(long, end, vm_end);326 327 /* Let's move back into the "<< PAGE_SHIFT" domain */328 *start_vaddr = (unsigned long)start << PAGE_SHIFT;329 *end_vaddr = (unsigned long)end << PAGE_SHIFT;330 331 *pfn = (gmadr_start + i915_ggtt_offset(vma)) >> PAGE_SHIFT;332 *pfn += (*start_vaddr - area->vm_start) >> PAGE_SHIFT;333 *pfn += obj_offset - vma->gtt_view.partial.offset;334}335 336static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)337{338#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)339 struct vm_area_struct *area = vmf->vma;340 struct i915_mmap_offset *mmo = area->vm_private_data;341 struct drm_i915_gem_object *obj = mmo->obj;342 struct drm_device *dev = obj->base.dev;343 struct drm_i915_private *i915 = to_i915(dev);344 struct intel_runtime_pm *rpm = &i915->runtime_pm;345 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;346 bool write = area->vm_flags & VM_WRITE;347 struct i915_gem_ww_ctx ww;348 unsigned long obj_offset;349 unsigned long start, end; /* memory boundaries */350 intel_wakeref_t wakeref;351 struct i915_vma *vma;352 pgoff_t page_offset;353 unsigned long pfn;354 int srcu;355 int ret;356 357 obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node);358 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;359 page_offset += obj_offset;360 361 trace_i915_gem_object_fault(obj, page_offset, true, write);362 363 wakeref = intel_runtime_pm_get(rpm);364 365 i915_gem_ww_ctx_init(&ww, true);366retry:367 ret = i915_gem_object_lock(obj, &ww);368 if (ret)369 goto err_rpm;370 371 /* Sanity check that we allow writing into this object */372 if (i915_gem_object_is_readonly(obj) && write) {373 ret = -EFAULT;374 goto err_rpm;375 }376 377 ret = i915_gem_object_pin_pages(obj);378 if (ret)379 goto err_rpm;380 381 ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu);382 if (ret)383 goto err_pages;384 385 /* Now pin it into the GTT as needed */386 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,387 PIN_MAPPABLE |388 PIN_NONBLOCK /* NOWARN */ |389 PIN_NOEVICT);390 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {391 /* Use a partial view if it is bigger than available space */392 struct i915_gtt_view view =393 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);394 unsigned int flags;395 396 flags = PIN_MAPPABLE | PIN_NOSEARCH;397 if (view.type == I915_GTT_VIEW_NORMAL)398 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */399 400 /*401 * Userspace is now writing through an untracked VMA, abandon402 * all hope that the hardware is able to track future writes.403 */404 405 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);406 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {407 flags = PIN_MAPPABLE;408 view.type = I915_GTT_VIEW_PARTIAL;409 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);410 }411 412 /*413 * The entire mappable GGTT is pinned? Unexpected!414 * Try to evict the object we locked too, as normally we skip it415 * due to lack of short term pinning inside execbuf.416 */417 if (vma == ERR_PTR(-ENOSPC)) {418 ret = mutex_lock_interruptible(&ggtt->vm.mutex);419 if (!ret) {420 ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);421 mutex_unlock(&ggtt->vm.mutex);422 }423 if (ret)424 goto err_reset;425 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);426 }427 }428 if (IS_ERR(vma)) {429 ret = PTR_ERR(vma);430 goto err_reset;431 }432 433 /* Access to snoopable pages through the GTT is incoherent. */434 /*435 * For objects created by userspace through GEM_CREATE with pat_index436 * set by set_pat extension, coherency is managed by userspace, make437 * sure we don't fail handling the vm fault by calling438 * i915_gem_object_has_cache_level() which always return true for such439 * objects. Otherwise this helper function would fall back to checking440 * whether the object is un-cached.441 */442 if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||443 HAS_LLC(i915))) {444 ret = -EFAULT;445 goto err_unpin;446 }447 448 ret = i915_vma_pin_fence(vma);449 if (ret)450 goto err_unpin;451 452 /*453 * Dump all the necessary parameters in this function to perform the454 * arithmetic calculation for the virtual address start and end and455 * the PFN (Page Frame Number).456 */457 set_address_limits(area, vma, obj_offset, ggtt->gmadr.start,458 &start, &end, &pfn);459 460 /* Finally, remap it using the new GTT offset */461 ret = remap_io_mapping(area, start, pfn, end - start, &ggtt->iomap);462 if (ret)463 goto err_fence;464 465 assert_rpm_wakelock_held(rpm);466 467 /* Mark as being mmapped into userspace for later revocation */468 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);469 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)470 list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list);471 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);472 473 /* Track the mmo associated with the fenced vma */474 vma->mmo = mmo;475 476 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)477 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref,478 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));479 480 if (write) {481 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));482 i915_vma_set_ggtt_write(vma);483 obj->mm.dirty = true;484 }485 486err_fence:487 i915_vma_unpin_fence(vma);488err_unpin:489 __i915_vma_unpin(vma);490err_reset:491 intel_gt_reset_unlock(ggtt->vm.gt, srcu);492err_pages:493 i915_gem_object_unpin_pages(obj);494err_rpm:495 if (ret == -EDEADLK) {496 ret = i915_gem_ww_ctx_backoff(&ww);497 if (!ret)498 goto retry;499 }500 i915_gem_ww_ctx_fini(&ww);501 intel_runtime_pm_put(rpm, wakeref);502 return i915_error_to_vmf_fault(ret);503}504 505static int506vm_access(struct vm_area_struct *area, unsigned long addr,507 void *buf, int len, int write)508{509 struct i915_mmap_offset *mmo = area->vm_private_data;510 struct drm_i915_gem_object *obj = mmo->obj;511 struct i915_gem_ww_ctx ww;512 void *vaddr;513 int err = 0;514 515 if (i915_gem_object_is_readonly(obj) && write)516 return -EACCES;517 518 addr -= area->vm_start;519 if (range_overflows_t(u64, addr, len, obj->base.size))520 return -EINVAL;521 522 i915_gem_ww_ctx_init(&ww, true);523retry:524 err = i915_gem_object_lock(obj, &ww);525 if (err)526 goto out;527 528 /* As this is primarily for debugging, let's focus on simplicity */529 vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);530 if (IS_ERR(vaddr)) {531 err = PTR_ERR(vaddr);532 goto out;533 }534 535 if (write) {536 memcpy(vaddr + addr, buf, len);537 __i915_gem_object_flush_map(obj, addr, len);538 } else {539 memcpy(buf, vaddr + addr, len);540 }541 542 i915_gem_object_unpin_map(obj);543out:544 if (err == -EDEADLK) {545 err = i915_gem_ww_ctx_backoff(&ww);546 if (!err)547 goto retry;548 }549 i915_gem_ww_ctx_fini(&ww);550 551 if (err)552 return err;553 554 return len;555}556 557void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)558{559 struct i915_vma *vma;560 561 GEM_BUG_ON(!obj->userfault_count);562 563 for_each_ggtt_vma(vma, obj)564 i915_vma_revoke_mmap(vma);565 566 GEM_BUG_ON(obj->userfault_count);567}568 569/*570 * It is vital that we remove the page mapping if we have mapped a tiled571 * object through the GTT and then lose the fence register due to572 * resource pressure. Similarly if the object has been moved out of the573 * aperture, than pages mapped into userspace must be revoked. Removing the574 * mapping will then trigger a page fault on the next user access, allowing575 * fixup by vm_fault_gtt().576 */577void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)578{579 struct drm_i915_private *i915 = to_i915(obj->base.dev);580 intel_wakeref_t wakeref;581 582 /*583 * Serialisation between user GTT access and our code depends upon584 * revoking the CPU's PTE whilst the mutex is held. The next user585 * pagefault then has to wait until we release the mutex.586 *587 * Note that RPM complicates somewhat by adding an additional588 * requirement that operations to the GGTT be made holding the RPM589 * wakeref.590 */591 wakeref = intel_runtime_pm_get(&i915->runtime_pm);592 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);593 594 if (!obj->userfault_count)595 goto out;596 597 __i915_gem_object_release_mmap_gtt(obj);598 599 /*600 * Ensure that the CPU's PTE are revoked and there are not outstanding601 * memory transactions from userspace before we return. The TLB602 * flushing implied above by changing the PTE above *should* be603 * sufficient, an extra barrier here just provides us with a bit604 * of paranoid documentation about our requirement to serialise605 * memory writes before touching registers / GSM.606 */607 wmb();608 609out:610 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);611 intel_runtime_pm_put(&i915->runtime_pm, wakeref);612}613 614void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj)615{616 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);617 struct ttm_device *bdev = bo->bdev;618 619 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);620 621 /*622 * We have exclusive access here via runtime suspend. All other callers623 * must first grab the rpm wakeref.624 */625 GEM_BUG_ON(!obj->userfault_count);626 list_del(&obj->userfault_link);627 obj->userfault_count = 0;628}629 630void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)631{632 struct i915_mmap_offset *mmo, *mn;633 634 if (obj->ops->unmap_virtual)635 obj->ops->unmap_virtual(obj);636 637 spin_lock(&obj->mmo.lock);638 rbtree_postorder_for_each_entry_safe(mmo, mn,639 &obj->mmo.offsets, offset) {640 /*641 * vma_node_unmap for GTT mmaps handled already in642 * __i915_gem_object_release_mmap_gtt643 */644 if (mmo->mmap_type == I915_MMAP_TYPE_GTT)645 continue;646 647 spin_unlock(&obj->mmo.lock);648 drm_vma_node_unmap(&mmo->vma_node,649 obj->base.dev->anon_inode->i_mapping);650 spin_lock(&obj->mmo.lock);651 }652 spin_unlock(&obj->mmo.lock);653}654 655static struct i915_mmap_offset *656lookup_mmo(struct drm_i915_gem_object *obj,657 enum i915_mmap_type mmap_type)658{659 struct rb_node *rb;660 661 spin_lock(&obj->mmo.lock);662 rb = obj->mmo.offsets.rb_node;663 while (rb) {664 struct i915_mmap_offset *mmo =665 rb_entry(rb, typeof(*mmo), offset);666 667 if (mmo->mmap_type == mmap_type) {668 spin_unlock(&obj->mmo.lock);669 return mmo;670 }671 672 if (mmo->mmap_type < mmap_type)673 rb = rb->rb_right;674 else675 rb = rb->rb_left;676 }677 spin_unlock(&obj->mmo.lock);678 679 return NULL;680}681 682static struct i915_mmap_offset *683insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo)684{685 struct rb_node *rb, **p;686 687 spin_lock(&obj->mmo.lock);688 rb = NULL;689 p = &obj->mmo.offsets.rb_node;690 while (*p) {691 struct i915_mmap_offset *pos;692 693 rb = *p;694 pos = rb_entry(rb, typeof(*pos), offset);695 696 if (pos->mmap_type == mmo->mmap_type) {697 spin_unlock(&obj->mmo.lock);698 drm_vma_offset_remove(obj->base.dev->vma_offset_manager,699 &mmo->vma_node);700 kfree(mmo);701 return pos;702 }703 704 if (pos->mmap_type < mmo->mmap_type)705 p = &rb->rb_right;706 else707 p = &rb->rb_left;708 }709 rb_link_node(&mmo->offset, rb, p);710 rb_insert_color(&mmo->offset, &obj->mmo.offsets);711 spin_unlock(&obj->mmo.lock);712 713 return mmo;714}715 716static struct i915_mmap_offset *717mmap_offset_attach(struct drm_i915_gem_object *obj,718 enum i915_mmap_type mmap_type,719 struct drm_file *file)720{721 struct drm_i915_private *i915 = to_i915(obj->base.dev);722 struct i915_mmap_offset *mmo;723 int err;724 725 GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops);726 727 mmo = lookup_mmo(obj, mmap_type);728 if (mmo)729 goto out;730 731 mmo = kmalloc(sizeof(*mmo), GFP_KERNEL);732 if (!mmo)733 return ERR_PTR(-ENOMEM);734 735 mmo->obj = obj;736 mmo->mmap_type = mmap_type;737 drm_vma_node_reset(&mmo->vma_node);738 739 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,740 &mmo->vma_node, obj->base.size / PAGE_SIZE);741 if (likely(!err))742 goto insert;743 744 /* Attempt to reap some mmap space from dead objects */745 err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT,746 NULL);747 if (err)748 goto err;749 750 i915_gem_drain_freed_objects(i915);751 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,752 &mmo->vma_node, obj->base.size / PAGE_SIZE);753 if (err)754 goto err;755 756insert:757 mmo = insert_mmo(obj, mmo);758 GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo);759out:760 if (file)761 drm_vma_node_allow_once(&mmo->vma_node, file);762 return mmo;763 764err:765 kfree(mmo);766 return ERR_PTR(err);767}768 769static int770__assign_mmap_offset(struct drm_i915_gem_object *obj,771 enum i915_mmap_type mmap_type,772 u64 *offset, struct drm_file *file)773{774 struct i915_mmap_offset *mmo;775 776 if (i915_gem_object_never_mmap(obj))777 return -ENODEV;778 779 if (obj->ops->mmap_offset) {780 if (mmap_type != I915_MMAP_TYPE_FIXED)781 return -ENODEV;782 783 *offset = obj->ops->mmap_offset(obj);784 return 0;785 }786 787 if (mmap_type == I915_MMAP_TYPE_FIXED)788 return -ENODEV;789 790 if (mmap_type != I915_MMAP_TYPE_GTT &&791 !i915_gem_object_has_struct_page(obj) &&792 !i915_gem_object_has_iomem(obj))793 return -ENODEV;794 795 mmo = mmap_offset_attach(obj, mmap_type, file);796 if (IS_ERR(mmo))797 return PTR_ERR(mmo);798 799 *offset = drm_vma_node_offset_addr(&mmo->vma_node);800 return 0;801}802 803static int804__assign_mmap_offset_handle(struct drm_file *file,805 u32 handle,806 enum i915_mmap_type mmap_type,807 u64 *offset)808{809 struct drm_i915_gem_object *obj;810 int err;811 812 obj = i915_gem_object_lookup(file, handle);813 if (!obj)814 return -ENOENT;815 816 err = i915_gem_object_lock_interruptible(obj, NULL);817 if (err)818 goto out_put;819 err = __assign_mmap_offset(obj, mmap_type, offset, file);820 i915_gem_object_unlock(obj);821out_put:822 i915_gem_object_put(obj);823 return err;824}825 826int827i915_gem_dumb_mmap_offset(struct drm_file *file,828 struct drm_device *dev,829 u32 handle,830 u64 *offset)831{832 struct drm_i915_private *i915 = to_i915(dev);833 enum i915_mmap_type mmap_type;834 835 if (HAS_LMEM(to_i915(dev)))836 mmap_type = I915_MMAP_TYPE_FIXED;837 else if (pat_enabled())838 mmap_type = I915_MMAP_TYPE_WC;839 else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))840 return -ENODEV;841 else842 mmap_type = I915_MMAP_TYPE_GTT;843 844 return __assign_mmap_offset_handle(file, handle, mmap_type, offset);845}846 847/**848 * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing849 * @dev: DRM device850 * @data: GTT mapping ioctl data851 * @file: GEM object info852 *853 * Simply returns the fake offset to userspace so it can mmap it.854 * The mmap call will end up in drm_gem_mmap(), which will set things855 * up so we can get faults in the handler above.856 *857 * The fault handler will take care of binding the object into the GTT858 * (since it may have been evicted to make room for something), allocating859 * a fence register, and mapping the appropriate aperture address into860 * userspace.861 */862int863i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,864 struct drm_file *file)865{866 struct drm_i915_private *i915 = to_i915(dev);867 struct drm_i915_gem_mmap_offset *args = data;868 enum i915_mmap_type type;869 int err;870 871 /*872 * Historically we failed to check args.pad and args.offset873 * and so we cannot use those fields for user input and we cannot874 * add -EINVAL for them as the ABI is fixed, i.e. old userspace875 * may be feeding in garbage in those fields.876 *877 * if (args->pad) return -EINVAL; is verbotten!878 */879 880 err = i915_user_extensions(u64_to_user_ptr(args->extensions),881 NULL, 0, NULL);882 if (err)883 return err;884 885 switch (args->flags) {886 case I915_MMAP_OFFSET_GTT:887 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))888 return -ENODEV;889 type = I915_MMAP_TYPE_GTT;890 break;891 892 case I915_MMAP_OFFSET_WC:893 if (!pat_enabled())894 return -ENODEV;895 type = I915_MMAP_TYPE_WC;896 break;897 898 case I915_MMAP_OFFSET_WB:899 type = I915_MMAP_TYPE_WB;900 break;901 902 case I915_MMAP_OFFSET_UC:903 if (!pat_enabled())904 return -ENODEV;905 type = I915_MMAP_TYPE_UC;906 break;907 908 case I915_MMAP_OFFSET_FIXED:909 type = I915_MMAP_TYPE_FIXED;910 break;911 912 default:913 return -EINVAL;914 }915 916 return __assign_mmap_offset_handle(file, args->handle, type, &args->offset);917}918 919static void vm_open(struct vm_area_struct *vma)920{921 struct i915_mmap_offset *mmo = vma->vm_private_data;922 struct drm_i915_gem_object *obj = mmo->obj;923 924 GEM_BUG_ON(!obj);925 i915_gem_object_get(obj);926}927 928static void vm_close(struct vm_area_struct *vma)929{930 struct i915_mmap_offset *mmo = vma->vm_private_data;931 struct drm_i915_gem_object *obj = mmo->obj;932 933 GEM_BUG_ON(!obj);934 i915_gem_object_put(obj);935}936 937static const struct vm_operations_struct vm_ops_gtt = {938 .fault = vm_fault_gtt,939 .access = vm_access,940 .open = vm_open,941 .close = vm_close,942};943 944static const struct vm_operations_struct vm_ops_cpu = {945 .fault = vm_fault_cpu,946 .access = vm_access,947 .open = vm_open,948 .close = vm_close,949};950 951static int singleton_release(struct inode *inode, struct file *file)952{953 struct drm_i915_private *i915 = file->private_data;954 955 cmpxchg(&i915->gem.mmap_singleton, file, NULL);956 drm_dev_put(&i915->drm);957 958 return 0;959}960 961static const struct file_operations singleton_fops = {962 .owner = THIS_MODULE,963 .release = singleton_release,964};965 966static struct file *mmap_singleton(struct drm_i915_private *i915)967{968 struct file *file;969 970 file = get_file_active(&i915->gem.mmap_singleton);971 if (file)972 return file;973 974 file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR);975 if (IS_ERR(file))976 return file;977 978 /* Everyone shares a single global address space */979 file->f_mapping = i915->drm.anon_inode->i_mapping;980 981 smp_store_mb(i915->gem.mmap_singleton, file);982 drm_dev_get(&i915->drm);983 984 return file;985}986 987static int988i915_gem_object_mmap(struct drm_i915_gem_object *obj,989 struct i915_mmap_offset *mmo,990 struct vm_area_struct *vma)991{992 struct drm_i915_private *i915 = to_i915(obj->base.dev);993 struct drm_device *dev = &i915->drm;994 struct file *anon;995 996 if (i915_gem_object_is_readonly(obj)) {997 if (vma->vm_flags & VM_WRITE) {998 i915_gem_object_put(obj);999 return -EINVAL;1000 }1001 vm_flags_clear(vma, VM_MAYWRITE);1002 }1003 1004 anon = mmap_singleton(to_i915(dev));1005 if (IS_ERR(anon)) {1006 i915_gem_object_put(obj);1007 return PTR_ERR(anon);1008 }1009 1010 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);1011 1012 /*1013 * We keep the ref on mmo->obj, not vm_file, but we require1014 * vma->vm_file->f_mapping, see vma_link(), for later revocation.1015 * Our userspace is accustomed to having per-file resource cleanup1016 * (i.e. contexts, objects and requests) on their close(fd), which1017 * requires avoiding extraneous references to their filp, hence why1018 * we prefer to use an anonymous file for their mmaps.1019 */1020 vma_set_file(vma, anon);1021 /* Drop the initial creation reference, the vma is now holding one. */1022 fput(anon);1023 1024 if (obj->ops->mmap_ops) {1025 vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));1026 vma->vm_ops = obj->ops->mmap_ops;1027 vma->vm_private_data = obj->base.vma_node.driver_private;1028 return 0;1029 }1030 1031 vma->vm_private_data = mmo;1032 1033 switch (mmo->mmap_type) {1034 case I915_MMAP_TYPE_WC:1035 vma->vm_page_prot =1036 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));1037 vma->vm_ops = &vm_ops_cpu;1038 break;1039 1040 case I915_MMAP_TYPE_FIXED:1041 GEM_WARN_ON(1);1042 fallthrough;1043 case I915_MMAP_TYPE_WB:1044 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);1045 vma->vm_ops = &vm_ops_cpu;1046 break;1047 1048 case I915_MMAP_TYPE_UC:1049 vma->vm_page_prot =1050 pgprot_noncached(vm_get_page_prot(vma->vm_flags));1051 vma->vm_ops = &vm_ops_cpu;1052 break;1053 1054 case I915_MMAP_TYPE_GTT:1055 vma->vm_page_prot =1056 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));1057 vma->vm_ops = &vm_ops_gtt;1058 break;1059 }1060 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);1061 1062 return 0;1063}1064 1065/*1066 * This overcomes the limitation in drm_gem_mmap's assignment of a1067 * drm_gem_object as the vma->vm_private_data. Since we need to1068 * be able to resolve multiple mmap offsets which could be tied1069 * to a single gem object.1070 */1071int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)1072{1073 struct drm_vma_offset_node *node;1074 struct drm_file *priv = filp->private_data;1075 struct drm_device *dev = priv->minor->dev;1076 struct drm_i915_gem_object *obj = NULL;1077 struct i915_mmap_offset *mmo = NULL;1078 1079 if (drm_dev_is_unplugged(dev))1080 return -ENODEV;1081 1082 rcu_read_lock();1083 drm_vma_offset_lock_lookup(dev->vma_offset_manager);1084 node = drm_vma_offset_lookup_locked(dev->vma_offset_manager,1085 vma->vm_pgoff,1086 vma_pages(vma));1087 if (node && drm_vma_node_is_allowed(node, priv)) {1088 /*1089 * Skip 0-refcnted objects as it is in the process of being1090 * destroyed and will be invalid when the vma manager lock1091 * is released.1092 */1093 if (!node->driver_private) {1094 mmo = container_of(node, struct i915_mmap_offset, vma_node);1095 obj = i915_gem_object_get_rcu(mmo->obj);1096 1097 GEM_BUG_ON(obj && obj->ops->mmap_ops);1098 } else {1099 obj = i915_gem_object_get_rcu1100 (container_of(node, struct drm_i915_gem_object,1101 base.vma_node));1102 1103 GEM_BUG_ON(obj && !obj->ops->mmap_ops);1104 }1105 }1106 drm_vma_offset_unlock_lookup(dev->vma_offset_manager);1107 rcu_read_unlock();1108 if (!obj)1109 return node ? -EACCES : -EINVAL;1110 1111 return i915_gem_object_mmap(obj, mmo, vma);1112}1113 1114int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma)1115{1116 struct drm_i915_private *i915 = to_i915(obj->base.dev);1117 struct drm_device *dev = &i915->drm;1118 struct i915_mmap_offset *mmo = NULL;1119 enum i915_mmap_type mmap_type;1120 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;1121 1122 if (drm_dev_is_unplugged(dev))1123 return -ENODEV;1124 1125 /* handle ttm object */1126 if (obj->ops->mmap_ops) {1127 /*1128 * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset1129 * to calculate page offset so set that up.1130 */1131 vma->vm_pgoff += drm_vma_node_start(&obj->base.vma_node);1132 } else {1133 /* handle stolen and smem objects */1134 mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC;1135 mmo = mmap_offset_attach(obj, mmap_type, NULL);1136 if (IS_ERR(mmo))1137 return PTR_ERR(mmo);1138 1139 vma->vm_pgoff += drm_vma_node_start(&mmo->vma_node);1140 }1141 1142 /*1143 * When we install vm_ops for mmap we are too late for1144 * the vm_ops->open() which increases the ref_count of1145 * this obj and then it gets decreased by the vm_ops->close().1146 * To balance this increase the obj ref_count here.1147 */1148 obj = i915_gem_object_get(obj);1149 return i915_gem_object_mmap(obj, mmo, vma);1150}1151 1152#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)1153#include "selftests/i915_gem_mman.c"1154#endif1155