922 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */2/**************************************************************************3 *4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA5 * All Rights Reserved.6 *7 * Permission is hereby granted, free of charge, to any person obtaining a8 * copy of this software and associated documentation files (the9 * "Software"), to deal in the Software without restriction, including10 * without limitation the rights to use, copy, modify, merge, publish,11 * distribute, sub license, and/or sell copies of the Software, and to12 * permit persons to whom the Software is furnished to do so, subject to13 * the following conditions:14 *15 * The above copyright notice and this permission notice (including the16 * next paragraph) shall be included in all copies or substantial portions17 * of the Software.18 *19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE25 * USE OR OTHER DEALINGS IN THE SOFTWARE.26 *27 **************************************************************************/28/*29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>30 */31 32#include <linux/vmalloc.h>33 34#include <drm/ttm/ttm_bo.h>35#include <drm/ttm/ttm_placement.h>36#include <drm/ttm/ttm_tt.h>37 38#include <drm/drm_cache.h>39 40struct ttm_transfer_obj {41 struct ttm_buffer_object base;42 struct ttm_buffer_object *bo;43};44 45int ttm_mem_io_reserve(struct ttm_device *bdev,46 struct ttm_resource *mem)47{48 if (mem->bus.offset || mem->bus.addr)49 return 0;50 51 mem->bus.is_iomem = false;52 if (!bdev->funcs->io_mem_reserve)53 return 0;54 55 return bdev->funcs->io_mem_reserve(bdev, mem);56}57 58void ttm_mem_io_free(struct ttm_device *bdev,59 struct ttm_resource *mem)60{61 if (!mem)62 return;63 64 if (!mem->bus.offset && !mem->bus.addr)65 return;66 67 if (bdev->funcs->io_mem_free)68 bdev->funcs->io_mem_free(bdev, mem);69 70 mem->bus.offset = 0;71 mem->bus.addr = NULL;72}73 74/**75 * ttm_move_memcpy - Helper to perform a memcpy ttm move operation.76 * @clear: Whether to clear rather than copy.77 * @num_pages: Number of pages of the operation.78 * @dst_iter: A struct ttm_kmap_iter representing the destination resource.79 * @src_iter: A struct ttm_kmap_iter representing the source resource.80 *81 * This function is intended to be able to move out async under a82 * dma-fence if desired.83 */84void ttm_move_memcpy(bool clear,85 u32 num_pages,86 struct ttm_kmap_iter *dst_iter,87 struct ttm_kmap_iter *src_iter)88{89 const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops;90 const struct ttm_kmap_iter_ops *src_ops = src_iter->ops;91 struct iosys_map src_map, dst_map;92 pgoff_t i;93 94 /* Single TTM move. NOP */95 if (dst_ops->maps_tt && src_ops->maps_tt)96 return;97 98 /* Don't move nonexistent data. Clear destination instead. */99 if (clear) {100 for (i = 0; i < num_pages; ++i) {101 dst_ops->map_local(dst_iter, &dst_map, i);102 if (dst_map.is_iomem)103 memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE);104 else105 memset(dst_map.vaddr, 0, PAGE_SIZE);106 if (dst_ops->unmap_local)107 dst_ops->unmap_local(dst_iter, &dst_map);108 }109 return;110 }111 112 for (i = 0; i < num_pages; ++i) {113 dst_ops->map_local(dst_iter, &dst_map, i);114 src_ops->map_local(src_iter, &src_map, i);115 116 drm_memcpy_from_wc(&dst_map, &src_map, PAGE_SIZE);117 118 if (src_ops->unmap_local)119 src_ops->unmap_local(src_iter, &src_map);120 if (dst_ops->unmap_local)121 dst_ops->unmap_local(dst_iter, &dst_map);122 }123}124EXPORT_SYMBOL(ttm_move_memcpy);125 126/**127 * ttm_bo_move_memcpy128 *129 * @bo: A pointer to a struct ttm_buffer_object.130 * @ctx: operation context131 * @dst_mem: struct ttm_resource indicating where to move.132 *133 * Fallback move function for a mappable buffer object in mappable memory.134 * The function will, if successful,135 * free any old aperture space, and set (@new_mem)->mm_node to NULL,136 * and update the (@bo)->mem placement flags. If unsuccessful, the old137 * data remains untouched, and it's up to the caller to free the138 * memory space indicated by @new_mem.139 * Returns:140 * !0: Failure.141 */142int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,143 struct ttm_operation_ctx *ctx,144 struct ttm_resource *dst_mem)145{146 struct ttm_device *bdev = bo->bdev;147 struct ttm_resource_manager *dst_man =148 ttm_manager_type(bo->bdev, dst_mem->mem_type);149 struct ttm_tt *ttm = bo->ttm;150 struct ttm_resource *src_mem = bo->resource;151 struct ttm_resource_manager *src_man;152 union {153 struct ttm_kmap_iter_tt tt;154 struct ttm_kmap_iter_linear_io io;155 } _dst_iter, _src_iter;156 struct ttm_kmap_iter *dst_iter, *src_iter;157 bool clear;158 int ret = 0;159 160 if (WARN_ON(!src_mem))161 return -EINVAL;162 163 src_man = ttm_manager_type(bdev, src_mem->mem_type);164 if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) ||165 dst_man->use_tt)) {166 ret = ttm_tt_populate(bdev, ttm, ctx);167 if (ret)168 return ret;169 }170 171 dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem);172 if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt)173 dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm);174 if (IS_ERR(dst_iter))175 return PTR_ERR(dst_iter);176 177 src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem);178 if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt)179 src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm);180 if (IS_ERR(src_iter)) {181 ret = PTR_ERR(src_iter);182 goto out_src_iter;183 }184 185 clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm));186 if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC)))187 ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter);188 189 if (!src_iter->ops->maps_tt)190 ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem);191 ttm_bo_move_sync_cleanup(bo, dst_mem);192 193out_src_iter:194 if (!dst_iter->ops->maps_tt)195 ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem);196 197 return ret;198}199EXPORT_SYMBOL(ttm_bo_move_memcpy);200 201static void ttm_transfered_destroy(struct ttm_buffer_object *bo)202{203 struct ttm_transfer_obj *fbo;204 205 fbo = container_of(bo, struct ttm_transfer_obj, base);206 dma_resv_fini(&fbo->base.base._resv);207 ttm_bo_put(fbo->bo);208 kfree(fbo);209}210 211/**212 * ttm_buffer_object_transfer213 *214 * @bo: A pointer to a struct ttm_buffer_object.215 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,216 * holding the data of @bo with the old placement.217 *218 * This is a utility function that may be called after an accelerated move219 * has been scheduled. A new buffer object is created as a placeholder for220 * the old data while it's being copied. When that buffer object is idle,221 * it can be destroyed, releasing the space of the old placement.222 * Returns:223 * !0: Failure.224 */225 226static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,227 struct ttm_buffer_object **new_obj)228{229 struct ttm_transfer_obj *fbo;230 int ret;231 232 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);233 if (!fbo)234 return -ENOMEM;235 236 fbo->base = *bo;237 238 /**239 * Fix up members that we shouldn't copy directly:240 * TODO: Explicit member copy would probably be better here.241 */242 243 atomic_inc(&ttm_glob.bo_count);244 drm_vma_node_reset(&fbo->base.base.vma_node);245 246 kref_init(&fbo->base.kref);247 fbo->base.destroy = &ttm_transfered_destroy;248 fbo->base.pin_count = 0;249 if (bo->type != ttm_bo_type_sg)250 fbo->base.base.resv = &fbo->base.base._resv;251 252 dma_resv_init(&fbo->base.base._resv);253 fbo->base.base.dev = NULL;254 ret = dma_resv_trylock(&fbo->base.base._resv);255 WARN_ON(!ret);256 257 if (fbo->base.resource) {258 ttm_resource_set_bo(fbo->base.resource, &fbo->base);259 bo->resource = NULL;260 ttm_bo_set_bulk_move(&fbo->base, NULL);261 } else {262 fbo->base.bulk_move = NULL;263 }264 265 ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1);266 if (ret) {267 kfree(fbo);268 return ret;269 }270 271 ttm_bo_get(bo);272 fbo->bo = bo;273 274 ttm_bo_move_to_lru_tail_unlocked(&fbo->base);275 276 *new_obj = &fbo->base;277 return 0;278}279 280/**281 * ttm_io_prot282 *283 * @bo: ttm buffer object284 * @res: ttm resource object285 * @tmp: Page protection flag for a normal, cached mapping.286 *287 * Utility function that returns the pgprot_t that should be used for288 * setting up a PTE with the caching model indicated by @c_state.289 */290pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,291 pgprot_t tmp)292{293 struct ttm_resource_manager *man;294 enum ttm_caching caching;295 296 man = ttm_manager_type(bo->bdev, res->mem_type);297 if (man->use_tt) {298 caching = bo->ttm->caching;299 if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED)300 tmp = pgprot_decrypted(tmp);301 } else {302 caching = res->bus.caching;303 }304 305 return ttm_prot_from_caching(caching, tmp);306}307EXPORT_SYMBOL(ttm_io_prot);308 309static int ttm_bo_ioremap(struct ttm_buffer_object *bo,310 unsigned long offset,311 unsigned long size,312 struct ttm_bo_kmap_obj *map)313{314 struct ttm_resource *mem = bo->resource;315 316 if (bo->resource->bus.addr) {317 map->bo_kmap_type = ttm_bo_map_premapped;318 map->virtual = ((u8 *)bo->resource->bus.addr) + offset;319 } else {320 resource_size_t res = bo->resource->bus.offset + offset;321 322 map->bo_kmap_type = ttm_bo_map_iomap;323 if (mem->bus.caching == ttm_write_combined)324 map->virtual = ioremap_wc(res, size);325#ifdef CONFIG_X86326 else if (mem->bus.caching == ttm_cached)327 map->virtual = ioremap_cache(res, size);328#endif329 else330 map->virtual = ioremap(res, size);331 }332 return (!map->virtual) ? -ENOMEM : 0;333}334 335static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,336 unsigned long start_page,337 unsigned long num_pages,338 struct ttm_bo_kmap_obj *map)339{340 struct ttm_resource *mem = bo->resource;341 struct ttm_operation_ctx ctx = {342 .interruptible = false,343 .no_wait_gpu = false344 };345 struct ttm_tt *ttm = bo->ttm;346 struct ttm_resource_manager *man =347 ttm_manager_type(bo->bdev, bo->resource->mem_type);348 pgprot_t prot;349 int ret;350 351 BUG_ON(!ttm);352 353 ret = ttm_tt_populate(bo->bdev, ttm, &ctx);354 if (ret)355 return ret;356 357 if (num_pages == 1 && ttm->caching == ttm_cached &&358 !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) {359 /*360 * We're mapping a single page, and the desired361 * page protection is consistent with the bo.362 */363 364 map->bo_kmap_type = ttm_bo_map_kmap;365 map->page = ttm->pages[start_page];366 map->virtual = kmap(map->page);367 } else {368 /*369 * We need to use vmap to get the desired page protection370 * or to make the buffer object look contiguous.371 */372 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);373 map->bo_kmap_type = ttm_bo_map_vmap;374 map->virtual = vmap(ttm->pages + start_page, num_pages,375 0, prot);376 }377 return (!map->virtual) ? -ENOMEM : 0;378}379 380/**381 * ttm_bo_kmap382 *383 * @bo: The buffer object.384 * @start_page: The first page to map.385 * @num_pages: Number of pages to map.386 * @map: pointer to a struct ttm_bo_kmap_obj representing the map.387 *388 * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the389 * data in the buffer object. The ttm_kmap_obj_virtual function can then be390 * used to obtain a virtual address to the data.391 *392 * Returns393 * -ENOMEM: Out of memory.394 * -EINVAL: Invalid range.395 */396int ttm_bo_kmap(struct ttm_buffer_object *bo,397 unsigned long start_page, unsigned long num_pages,398 struct ttm_bo_kmap_obj *map)399{400 unsigned long offset, size;401 int ret;402 403 map->virtual = NULL;404 map->bo = bo;405 if (num_pages > PFN_UP(bo->resource->size))406 return -EINVAL;407 if ((start_page + num_pages) > PFN_UP(bo->resource->size))408 return -EINVAL;409 410 ret = ttm_mem_io_reserve(bo->bdev, bo->resource);411 if (ret)412 return ret;413 if (!bo->resource->bus.is_iomem) {414 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);415 } else {416 offset = start_page << PAGE_SHIFT;417 size = num_pages << PAGE_SHIFT;418 return ttm_bo_ioremap(bo, offset, size, map);419 }420}421EXPORT_SYMBOL(ttm_bo_kmap);422 423/**424 * ttm_bo_kunmap425 *426 * @map: Object describing the map to unmap.427 *428 * Unmaps a kernel map set up by ttm_bo_kmap.429 */430void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)431{432 if (!map->virtual)433 return;434 switch (map->bo_kmap_type) {435 case ttm_bo_map_iomap:436 iounmap(map->virtual);437 break;438 case ttm_bo_map_vmap:439 vunmap(map->virtual);440 break;441 case ttm_bo_map_kmap:442 kunmap(map->page);443 break;444 case ttm_bo_map_premapped:445 break;446 default:447 BUG();448 }449 ttm_mem_io_free(map->bo->bdev, map->bo->resource);450 map->virtual = NULL;451 map->page = NULL;452}453EXPORT_SYMBOL(ttm_bo_kunmap);454 455/**456 * ttm_bo_vmap457 *458 * @bo: The buffer object.459 * @map: pointer to a struct iosys_map representing the map.460 *461 * Sets up a kernel virtual mapping, using ioremap or vmap to the462 * data in the buffer object. The parameter @map returns the virtual463 * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap().464 *465 * Returns466 * -ENOMEM: Out of memory.467 * -EINVAL: Invalid range.468 */469int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map)470{471 struct ttm_resource *mem = bo->resource;472 int ret;473 474 dma_resv_assert_held(bo->base.resv);475 476 ret = ttm_mem_io_reserve(bo->bdev, mem);477 if (ret)478 return ret;479 480 if (mem->bus.is_iomem) {481 void __iomem *vaddr_iomem;482 483 if (mem->bus.addr)484 vaddr_iomem = (void __iomem *)mem->bus.addr;485 else if (mem->bus.caching == ttm_write_combined)486 vaddr_iomem = ioremap_wc(mem->bus.offset,487 bo->base.size);488#ifdef CONFIG_X86489 else if (mem->bus.caching == ttm_cached)490 vaddr_iomem = ioremap_cache(mem->bus.offset,491 bo->base.size);492#endif493 else494 vaddr_iomem = ioremap(mem->bus.offset, bo->base.size);495 496 if (!vaddr_iomem)497 return -ENOMEM;498 499 iosys_map_set_vaddr_iomem(map, vaddr_iomem);500 501 } else {502 struct ttm_operation_ctx ctx = {503 .interruptible = false,504 .no_wait_gpu = false505 };506 struct ttm_tt *ttm = bo->ttm;507 pgprot_t prot;508 void *vaddr;509 510 ret = ttm_tt_populate(bo->bdev, ttm, &ctx);511 if (ret)512 return ret;513 514 /*515 * We need to use vmap to get the desired page protection516 * or to make the buffer object look contiguous.517 */518 prot = ttm_io_prot(bo, mem, PAGE_KERNEL);519 vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot);520 if (!vaddr)521 return -ENOMEM;522 523 iosys_map_set_vaddr(map, vaddr);524 }525 526 return 0;527}528EXPORT_SYMBOL(ttm_bo_vmap);529 530/**531 * ttm_bo_vunmap532 *533 * @bo: The buffer object.534 * @map: Object describing the map to unmap.535 *536 * Unmaps a kernel map set up by ttm_bo_vmap().537 */538void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map)539{540 struct ttm_resource *mem = bo->resource;541 542 dma_resv_assert_held(bo->base.resv);543 544 if (iosys_map_is_null(map))545 return;546 547 if (!map->is_iomem)548 vunmap(map->vaddr);549 else if (!mem->bus.addr)550 iounmap(map->vaddr_iomem);551 iosys_map_clear(map);552 553 ttm_mem_io_free(bo->bdev, bo->resource);554}555EXPORT_SYMBOL(ttm_bo_vunmap);556 557static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo,558 bool dst_use_tt)559{560 long ret;561 562 ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,563 false, 15 * HZ);564 if (ret == 0)565 return -EBUSY;566 if (ret < 0)567 return ret;568 569 if (!dst_use_tt)570 ttm_bo_tt_destroy(bo);571 ttm_resource_free(bo, &bo->resource);572 return 0;573}574 575static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo,576 struct dma_fence *fence,577 bool dst_use_tt)578{579 struct ttm_buffer_object *ghost_obj;580 int ret;581 582 /**583 * This should help pipeline ordinary buffer moves.584 *585 * Hang old buffer memory on a new buffer object,586 * and leave it to be released when the GPU587 * operation has completed.588 */589 590 ret = ttm_buffer_object_transfer(bo, &ghost_obj);591 if (ret)592 return ret;593 594 dma_resv_add_fence(&ghost_obj->base._resv, fence,595 DMA_RESV_USAGE_KERNEL);596 597 /**598 * If we're not moving to fixed memory, the TTM object599 * needs to stay alive. Otherwhise hang it on the ghost600 * bo to be unbound and destroyed.601 */602 603 if (dst_use_tt)604 ghost_obj->ttm = NULL;605 else606 bo->ttm = NULL;607 608 dma_resv_unlock(&ghost_obj->base._resv);609 ttm_bo_put(ghost_obj);610 return 0;611}612 613static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo,614 struct dma_fence *fence)615{616 struct ttm_device *bdev = bo->bdev;617 struct ttm_resource_manager *from;618 619 from = ttm_manager_type(bdev, bo->resource->mem_type);620 621 /**622 * BO doesn't have a TTM we need to bind/unbind. Just remember623 * this eviction and free up the allocation624 */625 spin_lock(&from->move_lock);626 if (!from->move || dma_fence_is_later(fence, from->move)) {627 dma_fence_put(from->move);628 from->move = dma_fence_get(fence);629 }630 spin_unlock(&from->move_lock);631 632 ttm_resource_free(bo, &bo->resource);633}634 635/**636 * ttm_bo_move_accel_cleanup - cleanup helper for hw copies637 *638 * @bo: A pointer to a struct ttm_buffer_object.639 * @fence: A fence object that signals when moving is complete.640 * @evict: This is an evict move. Don't return until the buffer is idle.641 * @pipeline: evictions are to be pipelined.642 * @new_mem: struct ttm_resource indicating where to move.643 *644 * Accelerated move function to be called when an accelerated move645 * has been scheduled. The function will create a new temporary buffer object646 * representing the old placement, and put the sync object on both buffer647 * objects. After that the newly created buffer object is unref'd to be648 * destroyed when the move is complete. This will help pipeline649 * buffer moves.650 */651int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,652 struct dma_fence *fence,653 bool evict,654 bool pipeline,655 struct ttm_resource *new_mem)656{657 struct ttm_device *bdev = bo->bdev;658 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->resource->mem_type);659 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);660 int ret = 0;661 662 dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);663 if (!evict)664 ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt);665 else if (!from->use_tt && pipeline)666 ttm_bo_move_pipeline_evict(bo, fence);667 else668 ret = ttm_bo_wait_free_node(bo, man->use_tt);669 670 if (ret)671 return ret;672 673 ttm_bo_assign_mem(bo, new_mem);674 675 return 0;676}677EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);678 679/**680 * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish681 *682 * @bo: A pointer to a struct ttm_buffer_object.683 * @new_mem: struct ttm_resource indicating where to move.684 *685 * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed686 * by the caller to be idle. Typically used after memcpy buffer moves.687 */688void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo,689 struct ttm_resource *new_mem)690{691 struct ttm_device *bdev = bo->bdev;692 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type);693 int ret;694 695 ret = ttm_bo_wait_free_node(bo, man->use_tt);696 if (WARN_ON(ret))697 return;698 699 ttm_bo_assign_mem(bo, new_mem);700}701EXPORT_SYMBOL(ttm_bo_move_sync_cleanup);702 703/**704 * ttm_bo_pipeline_gutting - purge the contents of a bo705 * @bo: The buffer object706 *707 * Purge the contents of a bo, async if the bo is not idle.708 * After a successful call, the bo is left unpopulated in709 * system placement. The function may wait uninterruptible710 * for idle on OOM.711 *712 * Return: 0 if successful, negative error code on failure.713 */714int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)715{716 struct ttm_buffer_object *ghost;717 struct ttm_tt *ttm;718 int ret;719 720 /* If already idle, no need for ghost object dance. */721 if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) {722 if (!bo->ttm) {723 /* See comment below about clearing. */724 ret = ttm_tt_create(bo, true);725 if (ret)726 return ret;727 } else {728 ttm_tt_unpopulate(bo->bdev, bo->ttm);729 if (bo->type == ttm_bo_type_device)730 ttm_tt_mark_for_clear(bo->ttm);731 }732 ttm_resource_free(bo, &bo->resource);733 return 0;734 }735 736 /*737 * We need an unpopulated ttm_tt after giving our current one,738 * if any, to the ghost object. And we can't afford to fail739 * creating one *after* the operation. If the bo subsequently gets740 * resurrected, make sure it's cleared (if ttm_bo_type_device)741 * to avoid leaking sensitive information to user-space.742 */743 744 ttm = bo->ttm;745 bo->ttm = NULL;746 ret = ttm_tt_create(bo, true);747 swap(bo->ttm, ttm);748 if (ret)749 return ret;750 751 ret = ttm_buffer_object_transfer(bo, &ghost);752 if (ret)753 goto error_destroy_tt;754 755 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);756 /* Last resort, wait for the BO to be idle when we are OOM */757 if (ret) {758 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,759 false, MAX_SCHEDULE_TIMEOUT);760 }761 762 dma_resv_unlock(&ghost->base._resv);763 ttm_bo_put(ghost);764 bo->ttm = ttm;765 return 0;766 767error_destroy_tt:768 ttm_tt_destroy(bo->bdev, ttm);769 return ret;770}771 772static bool ttm_lru_walk_trylock(struct ttm_lru_walk *walk,773 struct ttm_buffer_object *bo,774 bool *needs_unlock)775{776 struct ttm_operation_ctx *ctx = walk->ctx;777 778 *needs_unlock = false;779 780 if (dma_resv_trylock(bo->base.resv)) {781 *needs_unlock = true;782 return true;783 }784 785 if (bo->base.resv == ctx->resv && ctx->allow_res_evict) {786 dma_resv_assert_held(bo->base.resv);787 return true;788 }789 790 return false;791}792 793static int ttm_lru_walk_ticketlock(struct ttm_lru_walk *walk,794 struct ttm_buffer_object *bo,795 bool *needs_unlock)796{797 struct dma_resv *resv = bo->base.resv;798 int ret;799 800 if (walk->ctx->interruptible)801 ret = dma_resv_lock_interruptible(resv, walk->ticket);802 else803 ret = dma_resv_lock(resv, walk->ticket);804 805 if (!ret) {806 *needs_unlock = true;807 /*808 * Only a single ticketlock per loop. Ticketlocks are prone809 * to return -EDEADLK causing the eviction to fail, so810 * after waiting for the ticketlock, revert back to811 * trylocking for this walk.812 */813 walk->ticket = NULL;814 } else if (ret == -EDEADLK) {815 /* Caller needs to exit the ww transaction. */816 ret = -ENOSPC;817 }818 819 return ret;820}821 822static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked)823{824 if (locked)825 dma_resv_unlock(bo->base.resv);826}827 828/**829 * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on830 * valid items.831 * @walk: describe the walks and actions taken832 * @bdev: The TTM device.833 * @man: The struct ttm_resource manager whose LRU lists we're walking.834 * @target: The end condition for the walk.835 *836 * The LRU lists of @man are walk, and for each struct ttm_resource encountered,837 * the corresponding ttm_buffer_object is locked and taken a reference on, and838 * the LRU lock is dropped. the LRU lock may be dropped before locking and, in839 * that case, it's verified that the item actually remains on the LRU list after840 * the lock, and that the buffer object didn't switch resource in between.841 *842 * With a locked object, the actions indicated by @walk->process_bo are843 * performed, and after that, the bo is unlocked, the refcount dropped and the844 * next struct ttm_resource is processed. Here, the walker relies on845 * TTM's restartable LRU list implementation.846 *847 * Typically @walk->process_bo() would return the number of pages evicted,848 * swapped or shrunken, so that when the total exceeds @target, or when the849 * LRU list has been walked in full, iteration is terminated. It's also terminated850 * on error. Note that the definition of @target is done by the caller, it851 * could have a different meaning than the number of pages.852 *853 * Note that the way dma_resv individualization is done, locking needs to be done854 * either with the LRU lock held (trylocking only) or with a reference on the855 * object.856 *857 * Return: The progress made towards target or negative error code on error.858 */859s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev,860 struct ttm_resource_manager *man, s64 target)861{862 struct ttm_resource_cursor cursor;863 struct ttm_resource *res;864 s64 progress = 0;865 s64 lret;866 867 spin_lock(&bdev->lru_lock);868 ttm_resource_manager_for_each_res(man, &cursor, res) {869 struct ttm_buffer_object *bo = res->bo;870 bool bo_needs_unlock = false;871 bool bo_locked = false;872 int mem_type;873 874 /*875 * Attempt a trylock before taking a reference on the bo,876 * since if we do it the other way around, and the trylock fails,877 * we need to drop the lru lock to put the bo.878 */879 if (ttm_lru_walk_trylock(walk, bo, &bo_needs_unlock))880 bo_locked = true;881 else if (!walk->ticket || walk->ctx->no_wait_gpu ||882 walk->trylock_only)883 continue;884 885 if (!ttm_bo_get_unless_zero(bo)) {886 ttm_lru_walk_unlock(bo, bo_needs_unlock);887 continue;888 }889 890 mem_type = res->mem_type;891 spin_unlock(&bdev->lru_lock);892 893 lret = 0;894 if (!bo_locked)895 lret = ttm_lru_walk_ticketlock(walk, bo, &bo_needs_unlock);896 897 /*898 * Note that in between the release of the lru lock and the899 * ticketlock, the bo may have switched resource,900 * and also memory type, since the resource may have been901 * freed and allocated again with a different memory type.902 * In that case, just skip it.903 */904 if (!lret && bo->resource && bo->resource->mem_type == mem_type)905 lret = walk->ops->process_bo(walk, bo);906 907 ttm_lru_walk_unlock(bo, bo_needs_unlock);908 ttm_bo_put(bo);909 if (lret == -EBUSY || lret == -EALREADY)910 lret = 0;911 progress = (lret < 0) ? lret : progress + lret;912 913 spin_lock(&bdev->lru_lock);914 if (progress < 0 || progress >= target)915 break;916 }917 ttm_resource_cursor_fini(&cursor);918 spin_unlock(&bdev->lru_lock);919 920 return progress;921}922