879 lines · c
1/*2 * Copyright 2020 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Christian König23 */24 25#include <linux/debugfs.h>26#include <linux/io-mapping.h>27#include <linux/iosys-map.h>28#include <linux/scatterlist.h>29 30#include <drm/ttm/ttm_bo.h>31#include <drm/ttm/ttm_placement.h>32#include <drm/ttm/ttm_resource.h>33 34#include <drm/drm_util.h>35 36/* Detach the cursor from the bulk move list*/37static void38ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)39{40 lockdep_assert_held(&cursor->man->bdev->lru_lock);41 42 cursor->bulk = NULL;43 list_del_init(&cursor->bulk_link);44}45 46/* Move the cursor to the end of the bulk move list it's in */47static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,48 struct ttm_resource_cursor *cursor)49{50 struct ttm_lru_bulk_move_pos *pos;51 52 lockdep_assert_held(&cursor->man->bdev->lru_lock);53 54 if (WARN_ON_ONCE(bulk != cursor->bulk)) {55 list_del_init(&cursor->bulk_link);56 return;57 }58 59 pos = &bulk->pos[cursor->mem_type][cursor->priority];60 if (pos->last)61 list_move(&cursor->hitch.link, &pos->last->lru.link);62 ttm_resource_cursor_clear_bulk(cursor);63}64 65/* Move all cursors attached to a bulk move to its end */66static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)67{68 struct ttm_resource_cursor *cursor, *next;69 70 list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)71 ttm_resource_cursor_move_bulk_tail(bulk, cursor);72}73 74/* Remove a cursor from an empty bulk move list */75static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)76{77 struct ttm_resource_cursor *cursor, *next;78 79 list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)80 ttm_resource_cursor_clear_bulk(cursor);81}82 83/**84 * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage85 * @cursor: The struct ttm_resource_cursor to finalize.86 *87 * The function pulls the LRU list cursor off any lists it was previusly88 * attached to. Needs to be called with the LRU lock held. The function89 * can be called multiple times after eachother.90 */91void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)92{93 lockdep_assert_held(&cursor->man->bdev->lru_lock);94 list_del_init(&cursor->hitch.link);95 ttm_resource_cursor_clear_bulk(cursor);96}97 98/**99 * ttm_lru_bulk_move_init - initialize a bulk move structure100 * @bulk: the structure to init101 *102 * For now just memset the structure to zero.103 */104void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)105{106 memset(bulk, 0, sizeof(*bulk));107 INIT_LIST_HEAD(&bulk->cursor_list);108}109EXPORT_SYMBOL(ttm_lru_bulk_move_init);110 111/**112 * ttm_lru_bulk_move_fini - finalize a bulk move structure113 * @bdev: The struct ttm_device114 * @bulk: the structure to finalize115 *116 * Sanity checks that bulk moves don't have any117 * resources left and hence no cursors attached.118 */119void ttm_lru_bulk_move_fini(struct ttm_device *bdev,120 struct ttm_lru_bulk_move *bulk)121{122 spin_lock(&bdev->lru_lock);123 ttm_bulk_move_drop_cursors(bulk);124 spin_unlock(&bdev->lru_lock);125}126EXPORT_SYMBOL(ttm_lru_bulk_move_fini);127 128/**129 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.130 *131 * @bulk: bulk move structure132 *133 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that134 * resource order never changes. Should be called with &ttm_device.lru_lock held.135 */136void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)137{138 unsigned i, j;139 140 ttm_bulk_move_adjust_cursors(bulk);141 for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {142 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {143 struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];144 struct ttm_resource_manager *man;145 146 if (!pos->first)147 continue;148 149 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);150 dma_resv_assert_held(pos->first->bo->base.resv);151 dma_resv_assert_held(pos->last->bo->base.resv);152 153 man = ttm_manager_type(pos->first->bo->bdev, i);154 list_bulk_move_tail(&man->lru[j], &pos->first->lru.link,155 &pos->last->lru.link);156 }157 }158}159EXPORT_SYMBOL(ttm_lru_bulk_move_tail);160 161/* Return the bulk move pos object for this resource */162static struct ttm_lru_bulk_move_pos *163ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)164{165 return &bulk->pos[res->mem_type][res->bo->priority];166}167 168/* Return the previous resource on the list (skip over non-resource list items) */169static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)170{171 struct ttm_lru_item *lru = &cur->lru;172 173 do {174 lru = list_prev_entry(lru, link);175 } while (!ttm_lru_item_is_res(lru));176 177 return ttm_lru_item_to_res(lru);178}179 180/* Return the next resource on the list (skip over non-resource list items) */181static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)182{183 struct ttm_lru_item *lru = &cur->lru;184 185 do {186 lru = list_next_entry(lru, link);187 } while (!ttm_lru_item_is_res(lru));188 189 return ttm_lru_item_to_res(lru);190}191 192/* Move the resource to the tail of the bulk move range */193static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,194 struct ttm_resource *res)195{196 if (pos->last != res) {197 if (pos->first == res)198 pos->first = ttm_lru_next_res(res);199 list_move(&res->lru.link, &pos->last->lru.link);200 pos->last = res;201 }202}203 204/* Add the resource to a bulk_move cursor */205static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,206 struct ttm_resource *res)207{208 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);209 210 if (!pos->first) {211 pos->first = res;212 pos->last = res;213 } else {214 WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);215 ttm_lru_bulk_move_pos_tail(pos, res);216 }217}218 219/* Remove the resource from a bulk_move range */220static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,221 struct ttm_resource *res)222{223 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);224 225 if (unlikely(WARN_ON(!pos->first || !pos->last) ||226 (pos->first == res && pos->last == res))) {227 pos->first = NULL;228 pos->last = NULL;229 } else if (pos->first == res) {230 pos->first = ttm_lru_next_res(res);231 } else if (pos->last == res) {232 pos->last = ttm_lru_prev_res(res);233 } else {234 list_move(&res->lru.link, &pos->last->lru.link);235 }236}237 238/* Add the resource to a bulk move if the BO is configured for it */239void ttm_resource_add_bulk_move(struct ttm_resource *res,240 struct ttm_buffer_object *bo)241{242 if (bo->bulk_move && !bo->pin_count)243 ttm_lru_bulk_move_add(bo->bulk_move, res);244}245 246/* Remove the resource from a bulk move if the BO is configured for it */247void ttm_resource_del_bulk_move(struct ttm_resource *res,248 struct ttm_buffer_object *bo)249{250 if (bo->bulk_move && !bo->pin_count)251 ttm_lru_bulk_move_del(bo->bulk_move, res);252}253 254/* Move a resource to the LRU or bulk tail */255void ttm_resource_move_to_lru_tail(struct ttm_resource *res)256{257 struct ttm_buffer_object *bo = res->bo;258 struct ttm_device *bdev = bo->bdev;259 260 lockdep_assert_held(&bo->bdev->lru_lock);261 262 if (bo->pin_count) {263 list_move_tail(&res->lru.link, &bdev->pinned);264 265 } else if (bo->bulk_move) {266 struct ttm_lru_bulk_move_pos *pos =267 ttm_lru_bulk_move_pos(bo->bulk_move, res);268 269 ttm_lru_bulk_move_pos_tail(pos, res);270 } else {271 struct ttm_resource_manager *man;272 273 man = ttm_manager_type(bdev, res->mem_type);274 list_move_tail(&res->lru.link, &man->lru[bo->priority]);275 }276}277 278/**279 * ttm_resource_init - resource object constructure280 * @bo: buffer object this resources is allocated for281 * @place: placement of the resource282 * @res: the resource object to inistilize283 *284 * Initialize a new resource object. Counterpart of ttm_resource_fini().285 */286void ttm_resource_init(struct ttm_buffer_object *bo,287 const struct ttm_place *place,288 struct ttm_resource *res)289{290 struct ttm_resource_manager *man;291 292 res->start = 0;293 res->size = bo->base.size;294 res->mem_type = place->mem_type;295 res->placement = place->flags;296 res->bus.addr = NULL;297 res->bus.offset = 0;298 res->bus.is_iomem = false;299 res->bus.caching = ttm_cached;300 res->bo = bo;301 302 man = ttm_manager_type(bo->bdev, place->mem_type);303 spin_lock(&bo->bdev->lru_lock);304 if (bo->pin_count)305 list_add_tail(&res->lru.link, &bo->bdev->pinned);306 else307 list_add_tail(&res->lru.link, &man->lru[bo->priority]);308 man->usage += res->size;309 spin_unlock(&bo->bdev->lru_lock);310}311EXPORT_SYMBOL(ttm_resource_init);312 313/**314 * ttm_resource_fini - resource destructor315 * @man: the resource manager this resource belongs to316 * @res: the resource to clean up317 *318 * Should be used by resource manager backends to clean up the TTM resource319 * objects before freeing the underlying structure. Makes sure the resource is320 * removed from the LRU before destruction.321 * Counterpart of ttm_resource_init().322 */323void ttm_resource_fini(struct ttm_resource_manager *man,324 struct ttm_resource *res)325{326 struct ttm_device *bdev = man->bdev;327 328 spin_lock(&bdev->lru_lock);329 list_del_init(&res->lru.link);330 man->usage -= res->size;331 spin_unlock(&bdev->lru_lock);332}333EXPORT_SYMBOL(ttm_resource_fini);334 335int ttm_resource_alloc(struct ttm_buffer_object *bo,336 const struct ttm_place *place,337 struct ttm_resource **res_ptr)338{339 struct ttm_resource_manager *man =340 ttm_manager_type(bo->bdev, place->mem_type);341 int ret;342 343 ret = man->func->alloc(man, bo, place, res_ptr);344 if (ret)345 return ret;346 347 spin_lock(&bo->bdev->lru_lock);348 ttm_resource_add_bulk_move(*res_ptr, bo);349 spin_unlock(&bo->bdev->lru_lock);350 return 0;351}352EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);353 354void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)355{356 struct ttm_resource_manager *man;357 358 if (!*res)359 return;360 361 spin_lock(&bo->bdev->lru_lock);362 ttm_resource_del_bulk_move(*res, bo);363 spin_unlock(&bo->bdev->lru_lock);364 man = ttm_manager_type(bo->bdev, (*res)->mem_type);365 man->func->free(man, *res);366 *res = NULL;367}368EXPORT_SYMBOL(ttm_resource_free);369 370/**371 * ttm_resource_intersects - test for intersection372 *373 * @bdev: TTM device structure374 * @res: The resource to test375 * @place: The placement to test376 * @size: How many bytes the new allocation needs.377 *378 * Test if @res intersects with @place and @size. Used for testing if evictions379 * are valueable or not.380 *381 * Returns true if the res placement intersects with @place and @size.382 */383bool ttm_resource_intersects(struct ttm_device *bdev,384 struct ttm_resource *res,385 const struct ttm_place *place,386 size_t size)387{388 struct ttm_resource_manager *man;389 390 if (!res)391 return false;392 393 man = ttm_manager_type(bdev, res->mem_type);394 if (!place || !man->func->intersects)395 return true;396 397 return man->func->intersects(man, res, place, size);398}399 400/**401 * ttm_resource_compatible - check if resource is compatible with placement402 *403 * @res: the resource to check404 * @placement: the placement to check against405 * @evicting: true if the caller is doing evictions406 *407 * Returns true if the placement is compatible.408 */409bool ttm_resource_compatible(struct ttm_resource *res,410 struct ttm_placement *placement,411 bool evicting)412{413 struct ttm_buffer_object *bo = res->bo;414 struct ttm_device *bdev = bo->bdev;415 unsigned i;416 417 if (res->placement & TTM_PL_FLAG_TEMPORARY)418 return false;419 420 for (i = 0; i < placement->num_placement; i++) {421 const struct ttm_place *place = &placement->placement[i];422 struct ttm_resource_manager *man;423 424 if (res->mem_type != place->mem_type)425 continue;426 427 if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :428 TTM_PL_FLAG_FALLBACK))429 continue;430 431 if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&432 !(res->placement & TTM_PL_FLAG_CONTIGUOUS))433 continue;434 435 man = ttm_manager_type(bdev, res->mem_type);436 if (man->func->compatible &&437 !man->func->compatible(man, res, place, bo->base.size))438 continue;439 440 return true;441 }442 return false;443}444 445void ttm_resource_set_bo(struct ttm_resource *res,446 struct ttm_buffer_object *bo)447{448 spin_lock(&bo->bdev->lru_lock);449 res->bo = bo;450 spin_unlock(&bo->bdev->lru_lock);451}452 453/**454 * ttm_resource_manager_init455 *456 * @man: memory manager object to init457 * @bdev: ttm device this manager belongs to458 * @size: size of managed resources in arbitrary units459 *460 * Initialise core parts of a manager object.461 */462void ttm_resource_manager_init(struct ttm_resource_manager *man,463 struct ttm_device *bdev,464 uint64_t size)465{466 unsigned i;467 468 spin_lock_init(&man->move_lock);469 man->bdev = bdev;470 man->size = size;471 man->usage = 0;472 473 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)474 INIT_LIST_HEAD(&man->lru[i]);475 man->move = NULL;476}477EXPORT_SYMBOL(ttm_resource_manager_init);478 479/*480 * ttm_resource_manager_evict_all481 *482 * @bdev - device to use483 * @man - manager to use484 *485 * Evict all the objects out of a memory manager until it is empty.486 * Part of memory manager cleanup sequence.487 */488int ttm_resource_manager_evict_all(struct ttm_device *bdev,489 struct ttm_resource_manager *man)490{491 struct ttm_operation_ctx ctx = {492 .interruptible = false,493 .no_wait_gpu = false,494 .force_alloc = true495 };496 struct dma_fence *fence;497 int ret;498 499 do {500 ret = ttm_bo_evict_first(bdev, man, &ctx);501 cond_resched();502 } while (!ret);503 504 spin_lock(&man->move_lock);505 fence = dma_fence_get(man->move);506 spin_unlock(&man->move_lock);507 508 if (fence) {509 ret = dma_fence_wait(fence, false);510 dma_fence_put(fence);511 if (ret)512 return ret;513 }514 515 return 0;516}517EXPORT_SYMBOL(ttm_resource_manager_evict_all);518 519/**520 * ttm_resource_manager_usage521 *522 * @man: A memory manager object.523 *524 * Return how many resources are currently used.525 */526uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)527{528 uint64_t usage;529 530 spin_lock(&man->bdev->lru_lock);531 usage = man->usage;532 spin_unlock(&man->bdev->lru_lock);533 return usage;534}535EXPORT_SYMBOL(ttm_resource_manager_usage);536 537/**538 * ttm_resource_manager_debug539 *540 * @man: manager type to dump.541 * @p: printer to use for debug.542 */543void ttm_resource_manager_debug(struct ttm_resource_manager *man,544 struct drm_printer *p)545{546 drm_printf(p, " use_type: %d\n", man->use_type);547 drm_printf(p, " use_tt: %d\n", man->use_tt);548 drm_printf(p, " size: %llu\n", man->size);549 drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man));550 if (man->func->debug)551 man->func->debug(man, p);552}553EXPORT_SYMBOL(ttm_resource_manager_debug);554 555static void556ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,557 struct ttm_lru_item *next_lru)558{559 struct ttm_resource *next = ttm_lru_item_to_res(next_lru);560 struct ttm_lru_bulk_move *bulk = NULL;561 struct ttm_buffer_object *bo = next->bo;562 563 lockdep_assert_held(&cursor->man->bdev->lru_lock);564 bulk = bo->bulk_move;565 566 if (cursor->bulk != bulk) {567 if (bulk) {568 list_move_tail(&cursor->bulk_link, &bulk->cursor_list);569 cursor->mem_type = next->mem_type;570 } else {571 list_del_init(&cursor->bulk_link);572 }573 cursor->bulk = bulk;574 }575}576 577/**578 * ttm_resource_manager_first() - Start iterating over the resources579 * of a resource manager580 * @man: resource manager to iterate over581 * @cursor: cursor to record the position582 *583 * Initializes the cursor and starts iterating. When done iterating,584 * the caller must explicitly call ttm_resource_cursor_fini().585 *586 * Return: The first resource from the resource manager.587 */588struct ttm_resource *589ttm_resource_manager_first(struct ttm_resource_manager *man,590 struct ttm_resource_cursor *cursor)591{592 lockdep_assert_held(&man->bdev->lru_lock);593 594 cursor->priority = 0;595 cursor->man = man;596 ttm_lru_item_init(&cursor->hitch, TTM_LRU_HITCH);597 INIT_LIST_HEAD(&cursor->bulk_link);598 list_add(&cursor->hitch.link, &man->lru[cursor->priority]);599 600 return ttm_resource_manager_next(cursor);601}602 603/**604 * ttm_resource_manager_next() - Continue iterating over the resource manager605 * resources606 * @cursor: cursor to record the position607 *608 * Return: the next resource from the resource manager.609 */610struct ttm_resource *611ttm_resource_manager_next(struct ttm_resource_cursor *cursor)612{613 struct ttm_resource_manager *man = cursor->man;614 struct ttm_lru_item *lru;615 616 lockdep_assert_held(&man->bdev->lru_lock);617 618 for (;;) {619 lru = &cursor->hitch;620 list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {621 if (ttm_lru_item_is_res(lru)) {622 ttm_resource_cursor_check_bulk(cursor, lru);623 list_move(&cursor->hitch.link, &lru->link);624 return ttm_lru_item_to_res(lru);625 }626 }627 628 if (++cursor->priority >= TTM_MAX_BO_PRIORITY)629 break;630 631 list_move(&cursor->hitch.link, &man->lru[cursor->priority]);632 ttm_resource_cursor_clear_bulk(cursor);633 }634 635 ttm_resource_cursor_fini(cursor);636 637 return NULL;638}639 640/**641 * ttm_lru_first_res_or_null() - Return the first resource on an lru list642 * @head: The list head of the lru list.643 *644 * Return: Pointer to the first resource on the lru list or NULL if645 * there is none.646 */647struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)648{649 struct ttm_lru_item *lru;650 651 list_for_each_entry(lru, head, link) {652 if (ttm_lru_item_is_res(lru))653 return ttm_lru_item_to_res(lru);654 }655 656 return NULL;657}658 659static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,660 struct iosys_map *dmap,661 pgoff_t i)662{663 struct ttm_kmap_iter_iomap *iter_io =664 container_of(iter, typeof(*iter_io), base);665 void __iomem *addr;666 667retry:668 while (i >= iter_io->cache.end) {669 iter_io->cache.sg = iter_io->cache.sg ?670 sg_next(iter_io->cache.sg) : iter_io->st->sgl;671 iter_io->cache.i = iter_io->cache.end;672 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>673 PAGE_SHIFT;674 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -675 iter_io->start;676 }677 678 if (i < iter_io->cache.i) {679 iter_io->cache.end = 0;680 iter_io->cache.sg = NULL;681 goto retry;682 }683 684 addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +685 (((resource_size_t)i - iter_io->cache.i)686 << PAGE_SHIFT));687 iosys_map_set_vaddr_iomem(dmap, addr);688}689 690static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,691 struct iosys_map *map)692{693 io_mapping_unmap_local(map->vaddr_iomem);694}695 696static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {697 .map_local = ttm_kmap_iter_iomap_map_local,698 .unmap_local = ttm_kmap_iter_iomap_unmap_local,699 .maps_tt = false,700};701 702/**703 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap704 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.705 * @iomap: The struct io_mapping representing the underlying linear io_memory.706 * @st: sg_table into @iomap, representing the memory of the struct707 * ttm_resource.708 * @start: Offset that needs to be subtracted from @st to make709 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.710 *711 * Return: Pointer to the embedded struct ttm_kmap_iter.712 */713struct ttm_kmap_iter *714ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,715 struct io_mapping *iomap,716 struct sg_table *st,717 resource_size_t start)718{719 iter_io->base.ops = &ttm_kmap_iter_io_ops;720 iter_io->iomap = iomap;721 iter_io->st = st;722 iter_io->start = start;723 memset(&iter_io->cache, 0, sizeof(iter_io->cache));724 725 return &iter_io->base;726}727EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);728 729/**730 * DOC: Linear io iterator731 *732 * This code should die in the not too near future. Best would be if we could733 * make io-mapping use memremap for all io memory, and have memremap734 * implement a kmap_local functionality. We could then strip a huge amount of735 * code. These linear io iterators are implemented to mimic old functionality,736 * and they don't use kmap_local semantics at all internally. Rather ioremap or737 * friends, and at least on 32-bit they add global TLB flushes and points738 * of failure.739 */740 741static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,742 struct iosys_map *dmap,743 pgoff_t i)744{745 struct ttm_kmap_iter_linear_io *iter_io =746 container_of(iter, typeof(*iter_io), base);747 748 *dmap = iter_io->dmap;749 iosys_map_incr(dmap, i * PAGE_SIZE);750}751 752static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {753 .map_local = ttm_kmap_iter_linear_io_map_local,754 .maps_tt = false,755};756 757/**758 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory759 * @iter_io: The iterator to initialize760 * @bdev: The TTM device761 * @mem: The ttm resource representing the iomap.762 *763 * This function is for internal TTM use only. It sets up a memcpy kmap iterator764 * pointing at a linear chunk of io memory.765 *766 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on767 * failure.768 */769struct ttm_kmap_iter *770ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,771 struct ttm_device *bdev,772 struct ttm_resource *mem)773{774 int ret;775 776 ret = ttm_mem_io_reserve(bdev, mem);777 if (ret)778 goto out_err;779 if (!mem->bus.is_iomem) {780 ret = -EINVAL;781 goto out_io_free;782 }783 784 if (mem->bus.addr) {785 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);786 iter_io->needs_unmap = false;787 } else {788 iter_io->needs_unmap = true;789 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));790 if (mem->bus.caching == ttm_write_combined)791 iosys_map_set_vaddr_iomem(&iter_io->dmap,792 ioremap_wc(mem->bus.offset,793 mem->size));794 else if (mem->bus.caching == ttm_cached)795 iosys_map_set_vaddr(&iter_io->dmap,796 memremap(mem->bus.offset, mem->size,797 MEMREMAP_WB |798 MEMREMAP_WT |799 MEMREMAP_WC));800 801 /* If uncached requested or if mapping cached or wc failed */802 if (iosys_map_is_null(&iter_io->dmap))803 iosys_map_set_vaddr_iomem(&iter_io->dmap,804 ioremap(mem->bus.offset,805 mem->size));806 807 if (iosys_map_is_null(&iter_io->dmap)) {808 ret = -ENOMEM;809 goto out_io_free;810 }811 }812 813 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;814 return &iter_io->base;815 816out_io_free:817 ttm_mem_io_free(bdev, mem);818out_err:819 return ERR_PTR(ret);820}821 822/**823 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory824 * @iter_io: The iterator to initialize825 * @bdev: The TTM device826 * @mem: The ttm resource representing the iomap.827 *828 * This function is for internal TTM use only. It cleans up a memcpy kmap829 * iterator initialized by ttm_kmap_iter_linear_io_init.830 */831void832ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,833 struct ttm_device *bdev,834 struct ttm_resource *mem)835{836 if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {837 if (iter_io->dmap.is_iomem)838 iounmap(iter_io->dmap.vaddr_iomem);839 else840 memunmap(iter_io->dmap.vaddr);841 }842 843 ttm_mem_io_free(bdev, mem);844}845 846#if defined(CONFIG_DEBUG_FS)847 848static int ttm_resource_manager_show(struct seq_file *m, void *unused)849{850 struct ttm_resource_manager *man =851 (struct ttm_resource_manager *)m->private;852 struct drm_printer p = drm_seq_file_printer(m);853 ttm_resource_manager_debug(man, &p);854 return 0;855}856DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);857 858#endif859 860/**861 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified862 * resource manager.863 * @man: The TTM resource manager for which the debugfs stats file be creates864 * @parent: debugfs directory in which the file will reside865 * @name: The filename to create.866 *867 * This function setups up a debugfs file that can be used to look868 * at debug statistics of the specified ttm_resource_manager.869 */870void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,871 struct dentry * parent,872 const char *name)873{874#if defined(CONFIG_DEBUG_FS)875 debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);876#endif877}878EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);879