1354 lines · c
1/*2 * Copyright © 2008-2015 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 * Authors:24 * Eric Anholt <eric@anholt.net>25 *26 */27 28#include <linux/dma-fence-array.h>29#include <linux/kthread.h>30#include <linux/dma-resv.h>31#include <linux/shmem_fs.h>32#include <linux/slab.h>33#include <linux/stop_machine.h>34#include <linux/swap.h>35#include <linux/pci.h>36#include <linux/dma-buf.h>37#include <linux/mman.h>38 39#include <drm/drm_cache.h>40#include <drm/drm_vma_manager.h>41 42#include "gem/i915_gem_clflush.h"43#include "gem/i915_gem_context.h"44#include "gem/i915_gem_ioctls.h"45#include "gem/i915_gem_mman.h"46#include "gem/i915_gem_object_frontbuffer.h"47#include "gem/i915_gem_pm.h"48#include "gem/i915_gem_region.h"49#include "gt/intel_engine_user.h"50#include "gt/intel_gt.h"51#include "gt/intel_gt_pm.h"52#include "gt/intel_workarounds.h"53 54#include "i915_drv.h"55#include "i915_file_private.h"56#include "i915_trace.h"57#include "i915_vgpu.h"58#include "intel_clock_gating.h"59 60static int61insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)62{63 int err;64 65 err = mutex_lock_interruptible(&ggtt->vm.mutex);66 if (err)67 return err;68 69 memset(node, 0, sizeof(*node));70 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,71 size, 0, I915_COLOR_UNEVICTABLE,72 0, ggtt->mappable_end,73 DRM_MM_INSERT_LOW);74 75 mutex_unlock(&ggtt->vm.mutex);76 77 return err;78}79 80static void81remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)82{83 mutex_lock(&ggtt->vm.mutex);84 drm_mm_remove_node(node);85 mutex_unlock(&ggtt->vm.mutex);86}87 88int89i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,90 struct drm_file *file)91{92 struct drm_i915_private *i915 = to_i915(dev);93 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;94 struct drm_i915_gem_get_aperture *args = data;95 struct i915_vma *vma;96 u64 pinned;97 98 if (mutex_lock_interruptible(&ggtt->vm.mutex))99 return -EINTR;100 101 pinned = ggtt->vm.reserved;102 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)103 if (i915_vma_is_pinned(vma))104 pinned += vma->node.size;105 106 mutex_unlock(&ggtt->vm.mutex);107 108 args->aper_size = ggtt->vm.total;109 args->aper_available_size = args->aper_size - pinned;110 111 return 0;112}113 114int i915_gem_object_unbind(struct drm_i915_gem_object *obj,115 unsigned long flags)116{117 struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;118 bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);119 LIST_HEAD(still_in_list);120 intel_wakeref_t wakeref;121 struct i915_vma *vma;122 int ret;123 124 assert_object_held(obj);125 126 if (list_empty(&obj->vma.list))127 return 0;128 129 /*130 * As some machines use ACPI to handle runtime-resume callbacks, and131 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex132 * as they are required by the shrinker. Ergo, we wake the device up133 * first just in case.134 */135 wakeref = intel_runtime_pm_get(rpm);136 137try_again:138 ret = 0;139 spin_lock(&obj->vma.lock);140 while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,141 struct i915_vma,142 obj_link))) {143 list_move_tail(&vma->obj_link, &still_in_list);144 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))145 continue;146 147 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {148 ret = -EBUSY;149 break;150 }151 152 /*153 * Requiring the vm destructor to take the object lock154 * before destroying a vma would help us eliminate the155 * i915_vm_tryget() here, AND thus also the barrier stuff156 * at the end. That's an easy fix, but sleeping locks in157 * a kthread should generally be avoided.158 */159 ret = -EAGAIN;160 if (!i915_vm_tryget(vma->vm))161 break;162 163 spin_unlock(&obj->vma.lock);164 165 /*166 * Since i915_vma_parked() takes the object lock167 * before vma destruction, it won't race us here,168 * and destroy the vma from under us.169 */170 171 ret = -EBUSY;172 if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {173 assert_object_held(vma->obj);174 ret = i915_vma_unbind_async(vma, vm_trylock);175 }176 177 if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||178 !i915_vma_is_active(vma))) {179 if (vm_trylock) {180 if (mutex_trylock(&vma->vm->mutex)) {181 ret = __i915_vma_unbind(vma);182 mutex_unlock(&vma->vm->mutex);183 }184 } else {185 ret = i915_vma_unbind(vma);186 }187 }188 189 i915_vm_put(vma->vm);190 spin_lock(&obj->vma.lock);191 }192 list_splice_init(&still_in_list, &obj->vma.list);193 spin_unlock(&obj->vma.lock);194 195 if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {196 rcu_barrier(); /* flush the i915_vm_release() */197 goto try_again;198 }199 200 intel_runtime_pm_put(rpm, wakeref);201 202 return ret;203}204 205static int206shmem_pread(struct page *page, int offset, int len, char __user *user_data,207 bool needs_clflush)208{209 char *vaddr;210 int ret;211 212 vaddr = kmap(page);213 214 if (needs_clflush)215 drm_clflush_virt_range(vaddr + offset, len);216 217 ret = __copy_to_user(user_data, vaddr + offset, len);218 219 kunmap(page);220 221 return ret ? -EFAULT : 0;222}223 224static int225i915_gem_shmem_pread(struct drm_i915_gem_object *obj,226 struct drm_i915_gem_pread *args)227{228 unsigned int needs_clflush;229 char __user *user_data;230 unsigned long offset;231 pgoff_t idx;232 u64 remain;233 int ret;234 235 ret = i915_gem_object_lock_interruptible(obj, NULL);236 if (ret)237 return ret;238 239 ret = i915_gem_object_pin_pages(obj);240 if (ret)241 goto err_unlock;242 243 ret = i915_gem_object_prepare_read(obj, &needs_clflush);244 if (ret)245 goto err_unpin;246 247 i915_gem_object_finish_access(obj);248 i915_gem_object_unlock(obj);249 250 remain = args->size;251 user_data = u64_to_user_ptr(args->data_ptr);252 offset = offset_in_page(args->offset);253 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {254 struct page *page = i915_gem_object_get_page(obj, idx);255 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);256 257 ret = shmem_pread(page, offset, length, user_data,258 needs_clflush);259 if (ret)260 break;261 262 remain -= length;263 user_data += length;264 offset = 0;265 }266 267 i915_gem_object_unpin_pages(obj);268 return ret;269 270err_unpin:271 i915_gem_object_unpin_pages(obj);272err_unlock:273 i915_gem_object_unlock(obj);274 return ret;275}276 277static inline bool278gtt_user_read(struct io_mapping *mapping,279 loff_t base, int offset,280 char __user *user_data, int length)281{282 void __iomem *vaddr;283 unsigned long unwritten;284 285 /* We can use the cpu mem copy function because this is X86. */286 vaddr = io_mapping_map_atomic_wc(mapping, base);287 unwritten = __copy_to_user_inatomic(user_data,288 (void __force *)vaddr + offset,289 length);290 io_mapping_unmap_atomic(vaddr);291 if (unwritten) {292 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);293 unwritten = copy_to_user(user_data,294 (void __force *)vaddr + offset,295 length);296 io_mapping_unmap(vaddr);297 }298 return unwritten;299}300 301static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,302 struct drm_mm_node *node,303 bool write)304{305 struct drm_i915_private *i915 = to_i915(obj->base.dev);306 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;307 struct i915_vma *vma;308 struct i915_gem_ww_ctx ww;309 int ret;310 311 i915_gem_ww_ctx_init(&ww, true);312retry:313 vma = ERR_PTR(-ENODEV);314 ret = i915_gem_object_lock(obj, &ww);315 if (ret)316 goto err_ww;317 318 ret = i915_gem_object_set_to_gtt_domain(obj, write);319 if (ret)320 goto err_ww;321 322 if (!i915_gem_object_is_tiled(obj))323 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,324 PIN_MAPPABLE |325 PIN_NONBLOCK /* NOWARN */ |326 PIN_NOEVICT);327 if (vma == ERR_PTR(-EDEADLK)) {328 ret = -EDEADLK;329 goto err_ww;330 } else if (!IS_ERR(vma)) {331 node->start = i915_ggtt_offset(vma);332 node->flags = 0;333 } else {334 ret = insert_mappable_node(ggtt, node, PAGE_SIZE);335 if (ret)336 goto err_ww;337 GEM_BUG_ON(!drm_mm_node_allocated(node));338 vma = NULL;339 }340 341 ret = i915_gem_object_pin_pages(obj);342 if (ret) {343 if (drm_mm_node_allocated(node)) {344 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);345 remove_mappable_node(ggtt, node);346 } else {347 i915_vma_unpin(vma);348 }349 }350 351err_ww:352 if (ret == -EDEADLK) {353 ret = i915_gem_ww_ctx_backoff(&ww);354 if (!ret)355 goto retry;356 }357 i915_gem_ww_ctx_fini(&ww);358 359 return ret ? ERR_PTR(ret) : vma;360}361 362static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,363 struct drm_mm_node *node,364 struct i915_vma *vma)365{366 struct drm_i915_private *i915 = to_i915(obj->base.dev);367 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;368 369 i915_gem_object_unpin_pages(obj);370 if (drm_mm_node_allocated(node)) {371 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);372 remove_mappable_node(ggtt, node);373 } else {374 i915_vma_unpin(vma);375 }376}377 378static int379i915_gem_gtt_pread(struct drm_i915_gem_object *obj,380 const struct drm_i915_gem_pread *args)381{382 struct drm_i915_private *i915 = to_i915(obj->base.dev);383 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;384 unsigned long remain, offset;385 intel_wakeref_t wakeref;386 struct drm_mm_node node;387 void __user *user_data;388 struct i915_vma *vma;389 int ret = 0;390 391 if (overflows_type(args->size, remain) ||392 overflows_type(args->offset, offset))393 return -EINVAL;394 395 wakeref = intel_runtime_pm_get(&i915->runtime_pm);396 397 vma = i915_gem_gtt_prepare(obj, &node, false);398 if (IS_ERR(vma)) {399 ret = PTR_ERR(vma);400 goto out_rpm;401 }402 403 user_data = u64_to_user_ptr(args->data_ptr);404 remain = args->size;405 offset = args->offset;406 407 while (remain > 0) {408 /* Operation in this page409 *410 * page_base = page offset within aperture411 * page_offset = offset within page412 * page_length = bytes to copy for this page413 */414 u32 page_base = node.start;415 unsigned page_offset = offset_in_page(offset);416 unsigned page_length = PAGE_SIZE - page_offset;417 page_length = remain < page_length ? remain : page_length;418 if (drm_mm_node_allocated(&node)) {419 ggtt->vm.insert_page(&ggtt->vm,420 i915_gem_object_get_dma_address(obj,421 offset >> PAGE_SHIFT),422 node.start,423 i915_gem_get_pat_index(i915,424 I915_CACHE_NONE), 0);425 } else {426 page_base += offset & PAGE_MASK;427 }428 429 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,430 user_data, page_length)) {431 ret = -EFAULT;432 break;433 }434 435 remain -= page_length;436 user_data += page_length;437 offset += page_length;438 }439 440 i915_gem_gtt_cleanup(obj, &node, vma);441out_rpm:442 intel_runtime_pm_put(&i915->runtime_pm, wakeref);443 return ret;444}445 446/**447 * i915_gem_pread_ioctl - Reads data from the object referenced by handle.448 * @dev: drm device pointer449 * @data: ioctl data blob450 * @file: drm file pointer451 *452 * On error, the contents of *data are undefined.453 */454int455i915_gem_pread_ioctl(struct drm_device *dev, void *data,456 struct drm_file *file)457{458 struct drm_i915_private *i915 = to_i915(dev);459 struct drm_i915_gem_pread *args = data;460 struct drm_i915_gem_object *obj;461 int ret;462 463 /* PREAD is disallowed for all platforms after TGL-LP. This also464 * covers all platforms with local memory.465 */466 if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))467 return -EOPNOTSUPP;468 469 if (args->size == 0)470 return 0;471 472 if (!access_ok(u64_to_user_ptr(args->data_ptr),473 args->size))474 return -EFAULT;475 476 obj = i915_gem_object_lookup(file, args->handle);477 if (!obj)478 return -ENOENT;479 480 /* Bounds check source. */481 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {482 ret = -EINVAL;483 goto out;484 }485 486 trace_i915_gem_object_pread(obj, args->offset, args->size);487 ret = -ENODEV;488 if (obj->ops->pread)489 ret = obj->ops->pread(obj, args);490 if (ret != -ENODEV)491 goto out;492 493 ret = i915_gem_object_wait(obj,494 I915_WAIT_INTERRUPTIBLE,495 MAX_SCHEDULE_TIMEOUT);496 if (ret)497 goto out;498 499 ret = i915_gem_shmem_pread(obj, args);500 if (ret == -EFAULT || ret == -ENODEV)501 ret = i915_gem_gtt_pread(obj, args);502 503out:504 i915_gem_object_put(obj);505 return ret;506}507 508/* This is the fast write path which cannot handle509 * page faults in the source data510 */511 512static inline bool513ggtt_write(struct io_mapping *mapping,514 loff_t base, int offset,515 char __user *user_data, int length)516{517 void __iomem *vaddr;518 unsigned long unwritten;519 520 /* We can use the cpu mem copy function because this is X86. */521 vaddr = io_mapping_map_atomic_wc(mapping, base);522 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,523 user_data, length);524 io_mapping_unmap_atomic(vaddr);525 if (unwritten) {526 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);527 unwritten = copy_from_user((void __force *)vaddr + offset,528 user_data, length);529 io_mapping_unmap(vaddr);530 }531 532 return unwritten;533}534 535/**536 * i915_gem_gtt_pwrite_fast - This is the fast pwrite path, where we copy the data directly from the537 * user into the GTT, uncached.538 * @obj: i915 GEM object539 * @args: pwrite arguments structure540 */541static int542i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,543 const struct drm_i915_gem_pwrite *args)544{545 struct drm_i915_private *i915 = to_i915(obj->base.dev);546 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;547 struct intel_runtime_pm *rpm = &i915->runtime_pm;548 unsigned long remain, offset;549 intel_wakeref_t wakeref;550 struct drm_mm_node node;551 struct i915_vma *vma;552 void __user *user_data;553 int ret = 0;554 555 if (overflows_type(args->size, remain) ||556 overflows_type(args->offset, offset))557 return -EINVAL;558 559 if (i915_gem_object_has_struct_page(obj)) {560 /*561 * Avoid waking the device up if we can fallback, as562 * waking/resuming is very slow (worst-case 10-100 ms563 * depending on PCI sleeps and our own resume time).564 * This easily dwarfs any performance advantage from565 * using the cache bypass of indirect GGTT access.566 */567 wakeref = intel_runtime_pm_get_if_in_use(rpm);568 if (!wakeref)569 return -EFAULT;570 } else {571 /* No backing pages, no fallback, we must force GGTT access */572 wakeref = intel_runtime_pm_get(rpm);573 }574 575 vma = i915_gem_gtt_prepare(obj, &node, true);576 if (IS_ERR(vma)) {577 ret = PTR_ERR(vma);578 goto out_rpm;579 }580 581 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);582 583 user_data = u64_to_user_ptr(args->data_ptr);584 offset = args->offset;585 remain = args->size;586 while (remain) {587 /* Operation in this page588 *589 * page_base = page offset within aperture590 * page_offset = offset within page591 * page_length = bytes to copy for this page592 */593 u32 page_base = node.start;594 unsigned int page_offset = offset_in_page(offset);595 unsigned int page_length = PAGE_SIZE - page_offset;596 page_length = remain < page_length ? remain : page_length;597 if (drm_mm_node_allocated(&node)) {598 /* flush the write before we modify the GGTT */599 intel_gt_flush_ggtt_writes(ggtt->vm.gt);600 ggtt->vm.insert_page(&ggtt->vm,601 i915_gem_object_get_dma_address(obj,602 offset >> PAGE_SHIFT),603 node.start,604 i915_gem_get_pat_index(i915,605 I915_CACHE_NONE), 0);606 wmb(); /* flush modifications to the GGTT (insert_page) */607 } else {608 page_base += offset & PAGE_MASK;609 }610 /* If we get a fault while copying data, then (presumably) our611 * source page isn't available. Return the error and we'll612 * retry in the slow path.613 * If the object is non-shmem backed, we retry again with the614 * path that handles page fault.615 */616 if (ggtt_write(&ggtt->iomap, page_base, page_offset,617 user_data, page_length)) {618 ret = -EFAULT;619 break;620 }621 622 remain -= page_length;623 user_data += page_length;624 offset += page_length;625 }626 627 intel_gt_flush_ggtt_writes(ggtt->vm.gt);628 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);629 630 i915_gem_gtt_cleanup(obj, &node, vma);631out_rpm:632 intel_runtime_pm_put(rpm, wakeref);633 return ret;634}635 636/* Per-page copy function for the shmem pwrite fastpath.637 * Flushes invalid cachelines before writing to the target if638 * needs_clflush_before is set and flushes out any written cachelines after639 * writing if needs_clflush is set.640 */641static int642shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,643 bool needs_clflush_before,644 bool needs_clflush_after)645{646 char *vaddr;647 int ret;648 649 vaddr = kmap(page);650 651 if (needs_clflush_before)652 drm_clflush_virt_range(vaddr + offset, len);653 654 ret = __copy_from_user(vaddr + offset, user_data, len);655 if (!ret && needs_clflush_after)656 drm_clflush_virt_range(vaddr + offset, len);657 658 kunmap(page);659 660 return ret ? -EFAULT : 0;661}662 663static int664i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,665 const struct drm_i915_gem_pwrite *args)666{667 unsigned int partial_cacheline_write;668 unsigned int needs_clflush;669 void __user *user_data;670 unsigned long offset;671 pgoff_t idx;672 u64 remain;673 int ret;674 675 ret = i915_gem_object_lock_interruptible(obj, NULL);676 if (ret)677 return ret;678 679 ret = i915_gem_object_pin_pages(obj);680 if (ret)681 goto err_unlock;682 683 ret = i915_gem_object_prepare_write(obj, &needs_clflush);684 if (ret)685 goto err_unpin;686 687 i915_gem_object_finish_access(obj);688 i915_gem_object_unlock(obj);689 690 /* If we don't overwrite a cacheline completely we need to be691 * careful to have up-to-date data by first clflushing. Don't692 * overcomplicate things and flush the entire patch.693 */694 partial_cacheline_write = 0;695 if (needs_clflush & CLFLUSH_BEFORE)696 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;697 698 user_data = u64_to_user_ptr(args->data_ptr);699 remain = args->size;700 offset = offset_in_page(args->offset);701 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {702 struct page *page = i915_gem_object_get_page(obj, idx);703 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);704 705 ret = shmem_pwrite(page, offset, length, user_data,706 (offset | length) & partial_cacheline_write,707 needs_clflush & CLFLUSH_AFTER);708 if (ret)709 break;710 711 remain -= length;712 user_data += length;713 offset = 0;714 }715 716 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);717 718 i915_gem_object_unpin_pages(obj);719 return ret;720 721err_unpin:722 i915_gem_object_unpin_pages(obj);723err_unlock:724 i915_gem_object_unlock(obj);725 return ret;726}727 728/**729 * i915_gem_pwrite_ioctl - Writes data to the object referenced by handle.730 * @dev: drm device731 * @data: ioctl data blob732 * @file: drm file733 *734 * On error, the contents of the buffer that were to be modified are undefined.735 */736int737i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,738 struct drm_file *file)739{740 struct drm_i915_private *i915 = to_i915(dev);741 struct drm_i915_gem_pwrite *args = data;742 struct drm_i915_gem_object *obj;743 int ret;744 745 /* PWRITE is disallowed for all platforms after TGL-LP. This also746 * covers all platforms with local memory.747 */748 if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))749 return -EOPNOTSUPP;750 751 if (args->size == 0)752 return 0;753 754 if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))755 return -EFAULT;756 757 obj = i915_gem_object_lookup(file, args->handle);758 if (!obj)759 return -ENOENT;760 761 /* Bounds check destination. */762 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {763 ret = -EINVAL;764 goto err;765 }766 767 /* Writes not allowed into this read-only object */768 if (i915_gem_object_is_readonly(obj)) {769 ret = -EINVAL;770 goto err;771 }772 773 trace_i915_gem_object_pwrite(obj, args->offset, args->size);774 775 ret = -ENODEV;776 if (obj->ops->pwrite)777 ret = obj->ops->pwrite(obj, args);778 if (ret != -ENODEV)779 goto err;780 781 ret = i915_gem_object_wait(obj,782 I915_WAIT_INTERRUPTIBLE |783 I915_WAIT_ALL,784 MAX_SCHEDULE_TIMEOUT);785 if (ret)786 goto err;787 788 ret = -EFAULT;789 /* We can only do the GTT pwrite on untiled buffers, as otherwise790 * it would end up going through the fenced access, and we'll get791 * different detiling behavior between reading and writing.792 * pread/pwrite currently are reading and writing from the CPU793 * perspective, requiring manual detiling by the client.794 */795 if (!i915_gem_object_has_struct_page(obj) ||796 i915_gem_cpu_write_needs_clflush(obj))797 /* Note that the gtt paths might fail with non-page-backed user798 * pointers (e.g. gtt mappings when moving data between799 * textures). Fallback to the shmem path in that case.800 */801 ret = i915_gem_gtt_pwrite_fast(obj, args);802 803 if (ret == -EFAULT || ret == -ENOSPC) {804 if (i915_gem_object_has_struct_page(obj))805 ret = i915_gem_shmem_pwrite(obj, args);806 }807 808err:809 i915_gem_object_put(obj);810 return ret;811}812 813/**814 * i915_gem_sw_finish_ioctl - Called when user space has done writes to this buffer815 * @dev: drm device816 * @data: ioctl data blob817 * @file: drm file818 */819int820i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,821 struct drm_file *file)822{823 struct drm_i915_gem_sw_finish *args = data;824 struct drm_i915_gem_object *obj;825 826 obj = i915_gem_object_lookup(file, args->handle);827 if (!obj)828 return -ENOENT;829 830 /*831 * Proxy objects are barred from CPU access, so there is no832 * need to ban sw_finish as it is a nop.833 */834 835 /* Pinned buffers may be scanout, so flush the cache */836 i915_gem_object_flush_if_display(obj);837 i915_gem_object_put(obj);838 839 return 0;840}841 842void i915_gem_runtime_suspend(struct drm_i915_private *i915)843{844 struct drm_i915_gem_object *obj, *on;845 int i;846 847 /*848 * Only called during RPM suspend. All users of the userfault_list849 * must be holding an RPM wakeref to ensure that this can not850 * run concurrently with themselves (and use the struct_mutex for851 * protection between themselves).852 */853 854 list_for_each_entry_safe(obj, on,855 &to_gt(i915)->ggtt->userfault_list, userfault_link)856 __i915_gem_object_release_mmap_gtt(obj);857 858 list_for_each_entry_safe(obj, on,859 &i915->runtime_pm.lmem_userfault_list, userfault_link)860 i915_gem_object_runtime_pm_release_mmap_offset(obj);861 862 /*863 * The fence will be lost when the device powers down. If any were864 * in use by hardware (i.e. they are pinned), we should not be powering865 * down! All other fences will be reacquired by the user upon waking.866 */867 for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {868 struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];869 870 /*871 * Ideally we want to assert that the fence register is not872 * live at this point (i.e. that no piece of code will be873 * trying to write through fence + GTT, as that both violates874 * our tracking of activity and associated locking/barriers,875 * but also is illegal given that the hw is powered down).876 *877 * Previously we used reg->pin_count as a "liveness" indicator.878 * That is not sufficient, and we need a more fine-grained879 * tool if we want to have a sanity check here.880 */881 882 if (!reg->vma)883 continue;884 885 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));886 reg->dirty = true;887 }888}889 890static void discard_ggtt_vma(struct i915_vma *vma)891{892 struct drm_i915_gem_object *obj = vma->obj;893 894 spin_lock(&obj->vma.lock);895 if (!RB_EMPTY_NODE(&vma->obj_node)) {896 rb_erase(&vma->obj_node, &obj->vma.tree);897 RB_CLEAR_NODE(&vma->obj_node);898 }899 spin_unlock(&obj->vma.lock);900}901 902struct i915_vma *903i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,904 struct i915_gem_ww_ctx *ww,905 const struct i915_gtt_view *view,906 u64 size, u64 alignment, u64 flags)907{908 struct drm_i915_private *i915 = to_i915(obj->base.dev);909 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;910 struct i915_vma *vma;911 int ret;912 913 GEM_WARN_ON(!ww);914 915 if (flags & PIN_MAPPABLE &&916 (!view || view->type == I915_GTT_VIEW_NORMAL)) {917 /*918 * If the required space is larger than the available919 * aperture, we will not able to find a slot for the920 * object and unbinding the object now will be in921 * vain. Worse, doing so may cause us to ping-pong922 * the object in and out of the Global GTT and923 * waste a lot of cycles under the mutex.924 */925 if (obj->base.size > ggtt->mappable_end)926 return ERR_PTR(-E2BIG);927 928 /*929 * If NONBLOCK is set the caller is optimistically930 * trying to cache the full object within the mappable931 * aperture, and *must* have a fallback in place for932 * situations where we cannot bind the object. We933 * can be a little more lax here and use the fallback934 * more often to avoid costly migrations of ourselves935 * and other objects within the aperture.936 *937 * Half-the-aperture is used as a simple heuristic.938 * More interesting would to do search for a free939 * block prior to making the commitment to unbind.940 * That caters for the self-harm case, and with a941 * little more heuristics (e.g. NOFAULT, NOEVICT)942 * we could try to minimise harm to others.943 */944 if (flags & PIN_NONBLOCK &&945 obj->base.size > ggtt->mappable_end / 2)946 return ERR_PTR(-ENOSPC);947 }948 949new_vma:950 vma = i915_vma_instance(obj, &ggtt->vm, view);951 if (IS_ERR(vma))952 return vma;953 954 if (i915_vma_misplaced(vma, size, alignment, flags)) {955 if (flags & PIN_NONBLOCK) {956 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))957 return ERR_PTR(-ENOSPC);958 959 /*960 * If this misplaced vma is too big (i.e, at-least961 * half the size of aperture) or hasn't been pinned962 * mappable before, we ignore the misplacement when963 * PIN_NONBLOCK is set in order to avoid the ping-pong964 * issue described above. In other words, we try to965 * avoid the costly operation of unbinding this vma966 * from the GGTT and rebinding it back because there967 * may not be enough space for this vma in the aperture.968 */969 if (flags & PIN_MAPPABLE &&970 (vma->fence_size > ggtt->mappable_end / 2 ||971 !i915_vma_is_map_and_fenceable(vma)))972 return ERR_PTR(-ENOSPC);973 }974 975 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {976 discard_ggtt_vma(vma);977 goto new_vma;978 }979 980 ret = i915_vma_unbind(vma);981 if (ret)982 return ERR_PTR(ret);983 }984 985 ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);986 987 if (ret)988 return ERR_PTR(ret);989 990 if (vma->fence && !i915_gem_object_is_tiled(obj)) {991 mutex_lock(&ggtt->vm.mutex);992 i915_vma_revoke_fence(vma);993 mutex_unlock(&ggtt->vm.mutex);994 }995 996 ret = i915_vma_wait_for_bind(vma);997 if (ret) {998 i915_vma_unpin(vma);999 return ERR_PTR(ret);1000 }1001 1002 return vma;1003}1004 1005struct i915_vma * __must_check1006i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,1007 const struct i915_gtt_view *view,1008 u64 size, u64 alignment, u64 flags)1009{1010 struct i915_gem_ww_ctx ww;1011 struct i915_vma *ret;1012 int err;1013 1014 for_i915_gem_ww(&ww, err, true) {1015 err = i915_gem_object_lock(obj, &ww);1016 if (err)1017 continue;1018 1019 ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,1020 alignment, flags);1021 if (IS_ERR(ret))1022 err = PTR_ERR(ret);1023 }1024 1025 return err ? ERR_PTR(err) : ret;1026}1027 1028int1029i915_gem_madvise_ioctl(struct drm_device *dev, void *data,1030 struct drm_file *file_priv)1031{1032 struct drm_i915_private *i915 = to_i915(dev);1033 struct drm_i915_gem_madvise *args = data;1034 struct drm_i915_gem_object *obj;1035 int err;1036 1037 switch (args->madv) {1038 case I915_MADV_DONTNEED:1039 case I915_MADV_WILLNEED:1040 break;1041 default:1042 return -EINVAL;1043 }1044 1045 obj = i915_gem_object_lookup(file_priv, args->handle);1046 if (!obj)1047 return -ENOENT;1048 1049 err = i915_gem_object_lock_interruptible(obj, NULL);1050 if (err)1051 goto out;1052 1053 if (i915_gem_object_has_pages(obj) &&1054 i915_gem_object_is_tiled(obj) &&1055 i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {1056 if (obj->mm.madv == I915_MADV_WILLNEED) {1057 GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));1058 i915_gem_object_clear_tiling_quirk(obj);1059 i915_gem_object_make_shrinkable(obj);1060 }1061 if (args->madv == I915_MADV_WILLNEED) {1062 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));1063 i915_gem_object_make_unshrinkable(obj);1064 i915_gem_object_set_tiling_quirk(obj);1065 }1066 }1067 1068 if (obj->mm.madv != __I915_MADV_PURGED) {1069 obj->mm.madv = args->madv;1070 if (obj->ops->adjust_lru)1071 obj->ops->adjust_lru(obj);1072 }1073 1074 if (i915_gem_object_has_pages(obj) ||1075 i915_gem_object_has_self_managed_shrink_list(obj)) {1076 unsigned long flags;1077 1078 spin_lock_irqsave(&i915->mm.obj_lock, flags);1079 if (!list_empty(&obj->mm.link)) {1080 struct list_head *list;1081 1082 if (obj->mm.madv != I915_MADV_WILLNEED)1083 list = &i915->mm.purge_list;1084 else1085 list = &i915->mm.shrink_list;1086 list_move_tail(&obj->mm.link, list);1087 1088 }1089 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);1090 }1091 1092 /* if the object is no longer attached, discard its backing storage */1093 if (obj->mm.madv == I915_MADV_DONTNEED &&1094 !i915_gem_object_has_pages(obj))1095 i915_gem_object_truncate(obj);1096 1097 args->retained = obj->mm.madv != __I915_MADV_PURGED;1098 1099 i915_gem_object_unlock(obj);1100out:1101 i915_gem_object_put(obj);1102 return err;1103}1104 1105/*1106 * A single pass should suffice to release all the freed objects (along most1107 * call paths), but be a little more paranoid in that freeing the objects does1108 * take a little amount of time, during which the rcu callbacks could have added1109 * new objects into the freed list, and armed the work again.1110 */1111void i915_gem_drain_freed_objects(struct drm_i915_private *i915)1112{1113 while (atomic_read(&i915->mm.free_count)) {1114 flush_work(&i915->mm.free_work);1115 drain_workqueue(i915->bdev.wq);1116 rcu_barrier();1117 }1118}1119 1120/*1121 * Similar to objects above (see i915_gem_drain_freed-objects), in general we1122 * have workers that are armed by RCU and then rearm themselves in their1123 * callbacks. To be paranoid, we need to drain the workqueue a second time after1124 * waiting for the RCU grace period so that we catch work queued via RCU from1125 * the first pass. As neither drain_workqueue() nor flush_workqueue() report a1126 * result, we make an assumption that we only don't require more than 3 passes1127 * to catch all _recursive_ RCU delayed work.1128 */1129void i915_gem_drain_workqueue(struct drm_i915_private *i915)1130{1131 int i;1132 1133 for (i = 0; i < 3; i++) {1134 flush_workqueue(i915->wq);1135 rcu_barrier();1136 i915_gem_drain_freed_objects(i915);1137 }1138 1139 drain_workqueue(i915->wq);1140}1141 1142int i915_gem_init(struct drm_i915_private *dev_priv)1143{1144 struct intel_gt *gt;1145 unsigned int i;1146 int ret;1147 1148 /*1149 * In the proccess of replacing cache_level with pat_index a tricky1150 * dependency is created on the definition of the enum i915_cache_level.1151 * in case this enum is changed, PTE encode would be broken.1152 * Add a WARNING here. And remove when we completely quit using this1153 * enum1154 */1155 BUILD_BUG_ON(I915_CACHE_NONE != 0 ||1156 I915_CACHE_LLC != 1 ||1157 I915_CACHE_L3_LLC != 2 ||1158 I915_CACHE_WT != 3 ||1159 I915_MAX_CACHE_LEVEL != 4);1160 1161 /* We need to fallback to 4K pages if host doesn't support huge gtt. */1162 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))1163 RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K;1164 1165 for_each_gt(gt, dev_priv, i) {1166 intel_uc_fetch_firmwares(>->uc);1167 intel_wopcm_init(>->wopcm);1168 if (GRAPHICS_VER(dev_priv) >= 8)1169 setup_private_pat(gt);1170 }1171 1172 ret = i915_init_ggtt(dev_priv);1173 if (ret) {1174 GEM_BUG_ON(ret == -EIO);1175 goto err_unlock;1176 }1177 1178 /*1179 * Despite its name intel_clock_gating_init applies both display1180 * clock gating workarounds; GT mmio workarounds and the occasional1181 * GT power context workaround. Worse, sometimes it includes a context1182 * register workaround which we need to apply before we record the1183 * default HW state for all contexts.1184 *1185 * FIXME: break up the workarounds and apply them at the right time!1186 */1187 intel_clock_gating_init(dev_priv);1188 1189 for_each_gt(gt, dev_priv, i) {1190 ret = intel_gt_init(gt);1191 if (ret)1192 goto err_unlock;1193 }1194 1195 /*1196 * Register engines early to ensure the engine list is in its final1197 * rb-tree form, lowering the amount of code that has to deal with1198 * the intermediate llist state.1199 */1200 intel_engines_driver_register(dev_priv);1201 1202 return 0;1203 1204 /*1205 * Unwinding is complicated by that we want to handle -EIO to mean1206 * disable GPU submission but keep KMS alive. We want to mark the1207 * HW as irrevisibly wedged, but keep enough state around that the1208 * driver doesn't explode during runtime.1209 */1210err_unlock:1211 i915_gem_drain_workqueue(dev_priv);1212 1213 if (ret != -EIO) {1214 for_each_gt(gt, dev_priv, i) {1215 intel_gt_driver_remove(gt);1216 intel_gt_driver_release(gt);1217 intel_uc_cleanup_firmwares(>->uc);1218 }1219 }1220 1221 if (ret == -EIO) {1222 /*1223 * Allow engines or uC initialisation to fail by marking the GPU1224 * as wedged. But we only want to do this when the GPU is angry,1225 * for all other failure, such as an allocation failure, bail.1226 */1227 for_each_gt(gt, dev_priv, i) {1228 if (!intel_gt_is_wedged(gt)) {1229 i915_probe_error(dev_priv,1230 "Failed to initialize GPU, declaring it wedged!\n");1231 intel_gt_set_wedged(gt);1232 }1233 }1234 1235 /* Minimal basic recovery for KMS */1236 ret = i915_ggtt_enable_hw(dev_priv);1237 i915_ggtt_resume(to_gt(dev_priv)->ggtt);1238 intel_clock_gating_init(dev_priv);1239 }1240 1241 i915_gem_drain_freed_objects(dev_priv);1242 1243 return ret;1244}1245 1246void i915_gem_driver_register(struct drm_i915_private *i915)1247{1248 i915_gem_driver_register__shrinker(i915);1249}1250 1251void i915_gem_driver_unregister(struct drm_i915_private *i915)1252{1253 i915_gem_driver_unregister__shrinker(i915);1254}1255 1256void i915_gem_driver_remove(struct drm_i915_private *dev_priv)1257{1258 struct intel_gt *gt;1259 unsigned int i;1260 1261 i915_gem_suspend_late(dev_priv);1262 for_each_gt(gt, dev_priv, i)1263 intel_gt_driver_remove(gt);1264 dev_priv->uabi_engines = RB_ROOT;1265 1266 /* Flush any outstanding unpin_work. */1267 i915_gem_drain_workqueue(dev_priv);1268}1269 1270void i915_gem_driver_release(struct drm_i915_private *dev_priv)1271{1272 struct intel_gt *gt;1273 unsigned int i;1274 1275 for_each_gt(gt, dev_priv, i) {1276 intel_gt_driver_release(gt);1277 intel_uc_cleanup_firmwares(>->uc);1278 }1279 1280 /* Flush any outstanding work, including i915_gem_context.release_work. */1281 i915_gem_drain_workqueue(dev_priv);1282 1283 drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));1284}1285 1286static void i915_gem_init__mm(struct drm_i915_private *i915)1287{1288 spin_lock_init(&i915->mm.obj_lock);1289 1290 init_llist_head(&i915->mm.free_list);1291 1292 INIT_LIST_HEAD(&i915->mm.purge_list);1293 INIT_LIST_HEAD(&i915->mm.shrink_list);1294 1295 i915_gem_init__objects(i915);1296}1297 1298void i915_gem_init_early(struct drm_i915_private *dev_priv)1299{1300 i915_gem_init__mm(dev_priv);1301 i915_gem_init__contexts(dev_priv);1302}1303 1304void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)1305{1306 i915_gem_drain_workqueue(dev_priv);1307 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));1308 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));1309 drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);1310}1311 1312int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)1313{1314 struct drm_i915_file_private *file_priv;1315 struct i915_drm_client *client;1316 int ret = -ENOMEM;1317 1318 drm_dbg(&i915->drm, "\n");1319 1320 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);1321 if (!file_priv)1322 goto err_alloc;1323 1324 client = i915_drm_client_alloc();1325 if (!client)1326 goto err_client;1327 1328 file->driver_priv = file_priv;1329 file_priv->i915 = i915;1330 file_priv->file = file;1331 file_priv->client = client;1332 1333 file_priv->bsd_engine = -1;1334 file_priv->hang_timestamp = jiffies;1335 1336 ret = i915_gem_context_open(i915, file);1337 if (ret)1338 goto err_context;1339 1340 return 0;1341 1342err_context:1343 i915_drm_client_put(client);1344err_client:1345 kfree(file_priv);1346err_alloc:1347 return ret;1348}1349 1350#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)1351#include "selftests/mock_gem_device.c"1352#include "selftests/i915_gem.c"1353#endif1354