848 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2020 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 * Authors: Christian König24 */25 26/* Pooling of allocated pages is necessary because changing the caching27 * attributes on x86 of the linear mapping requires a costly cross CPU TLB28 * invalidate for those addresses.29 *30 * Additional to that allocations from the DMA coherent API are pooled as well31 * cause they are rather slow compared to alloc_pages+map.32 */33 34#include <linux/module.h>35#include <linux/dma-mapping.h>36#include <linux/debugfs.h>37#include <linux/highmem.h>38#include <linux/sched/mm.h>39 40#ifdef CONFIG_X8641#include <asm/set_memory.h>42#endif43 44#include <drm/ttm/ttm_pool.h>45#include <drm/ttm/ttm_tt.h>46#include <drm/ttm/ttm_bo.h>47 48#include "ttm_module.h"49 50/**51 * struct ttm_pool_dma - Helper object for coherent DMA mappings52 *53 * @addr: original DMA address returned for the mapping54 * @vaddr: original vaddr return for the mapping and order in the lower bits55 */56struct ttm_pool_dma {57 dma_addr_t addr;58 unsigned long vaddr;59};60 61static unsigned long page_pool_size;62 63MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");64module_param(page_pool_size, ulong, 0644);65 66static atomic_long_t allocated_pages;67 68static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS];69static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS];70 71static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS];72static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS];73 74static spinlock_t shrinker_lock;75static struct list_head shrinker_list;76static struct shrinker *mm_shrinker;77static DECLARE_RWSEM(pool_shrink_rwsem);78 79/* Allocate pages of size 1 << order with the given gfp_flags */80static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,81 unsigned int order)82{83 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;84 struct ttm_pool_dma *dma;85 struct page *p;86 void *vaddr;87 88 /* Don't set the __GFP_COMP flag for higher order allocations.89 * Mapping pages directly into an userspace process and calling90 * put_page() on a TTM allocated page is illegal.91 */92 if (order)93 gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN |94 __GFP_THISNODE;95 96 if (!pool->use_dma_alloc) {97 p = alloc_pages_node(pool->nid, gfp_flags, order);98 if (p)99 p->private = order;100 return p;101 }102 103 dma = kmalloc(sizeof(*dma), GFP_KERNEL);104 if (!dma)105 return NULL;106 107 if (order)108 attr |= DMA_ATTR_NO_WARN;109 110 vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE,111 &dma->addr, gfp_flags, attr);112 if (!vaddr)113 goto error_free;114 115 /* TODO: This is an illegal abuse of the DMA API, but we need to rework116 * TTM page fault handling and extend the DMA API to clean this up.117 */118 if (is_vmalloc_addr(vaddr))119 p = vmalloc_to_page(vaddr);120 else121 p = virt_to_page(vaddr);122 123 dma->vaddr = (unsigned long)vaddr | order;124 p->private = (unsigned long)dma;125 return p;126 127error_free:128 kfree(dma);129 return NULL;130}131 132/* Reset the caching and pages of size 1 << order */133static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,134 unsigned int order, struct page *p)135{136 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;137 struct ttm_pool_dma *dma;138 void *vaddr;139 140#ifdef CONFIG_X86141 /* We don't care that set_pages_wb is inefficient here. This is only142 * used when we have to shrink and CPU overhead is irrelevant then.143 */144 if (caching != ttm_cached && !PageHighMem(p))145 set_pages_wb(p, 1 << order);146#endif147 148 if (!pool || !pool->use_dma_alloc) {149 __free_pages(p, order);150 return;151 }152 153 if (order)154 attr |= DMA_ATTR_NO_WARN;155 156 dma = (void *)p->private;157 vaddr = (void *)(dma->vaddr & PAGE_MASK);158 dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr,159 attr);160 kfree(dma);161}162 163/* Apply a new caching to an array of pages */164static int ttm_pool_apply_caching(struct page **first, struct page **last,165 enum ttm_caching caching)166{167#ifdef CONFIG_X86168 unsigned int num_pages = last - first;169 170 if (!num_pages)171 return 0;172 173 switch (caching) {174 case ttm_cached:175 break;176 case ttm_write_combined:177 return set_pages_array_wc(first, num_pages);178 case ttm_uncached:179 return set_pages_array_uc(first, num_pages);180 }181#endif182 return 0;183}184 185/* Map pages of 1 << order size and fill the DMA address array */186static int ttm_pool_map(struct ttm_pool *pool, unsigned int order,187 struct page *p, dma_addr_t **dma_addr)188{189 dma_addr_t addr;190 unsigned int i;191 192 if (pool->use_dma_alloc) {193 struct ttm_pool_dma *dma = (void *)p->private;194 195 addr = dma->addr;196 } else {197 size_t size = (1ULL << order) * PAGE_SIZE;198 199 addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL);200 if (dma_mapping_error(pool->dev, addr))201 return -EFAULT;202 }203 204 for (i = 1 << order; i ; --i) {205 *(*dma_addr)++ = addr;206 addr += PAGE_SIZE;207 }208 209 return 0;210}211 212/* Unmap pages of 1 << order size */213static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr,214 unsigned int num_pages)215{216 /* Unmapped while freeing the page */217 if (pool->use_dma_alloc)218 return;219 220 dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT,221 DMA_BIDIRECTIONAL);222}223 224/* Give pages into a specific pool_type */225static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p)226{227 unsigned int i, num_pages = 1 << pt->order;228 229 for (i = 0; i < num_pages; ++i) {230 if (PageHighMem(p))231 clear_highpage(p + i);232 else233 clear_page(page_address(p + i));234 }235 236 spin_lock(&pt->lock);237 list_add(&p->lru, &pt->pages);238 spin_unlock(&pt->lock);239 atomic_long_add(1 << pt->order, &allocated_pages);240}241 242/* Take pages from a specific pool_type, return NULL when nothing available */243static struct page *ttm_pool_type_take(struct ttm_pool_type *pt)244{245 struct page *p;246 247 spin_lock(&pt->lock);248 p = list_first_entry_or_null(&pt->pages, typeof(*p), lru);249 if (p) {250 atomic_long_sub(1 << pt->order, &allocated_pages);251 list_del(&p->lru);252 }253 spin_unlock(&pt->lock);254 255 return p;256}257 258/* Initialize and add a pool type to the global shrinker list */259static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool,260 enum ttm_caching caching, unsigned int order)261{262 pt->pool = pool;263 pt->caching = caching;264 pt->order = order;265 spin_lock_init(&pt->lock);266 INIT_LIST_HEAD(&pt->pages);267 268 spin_lock(&shrinker_lock);269 list_add_tail(&pt->shrinker_list, &shrinker_list);270 spin_unlock(&shrinker_lock);271}272 273/* Remove a pool_type from the global shrinker list and free all pages */274static void ttm_pool_type_fini(struct ttm_pool_type *pt)275{276 struct page *p;277 278 spin_lock(&shrinker_lock);279 list_del(&pt->shrinker_list);280 spin_unlock(&shrinker_lock);281 282 while ((p = ttm_pool_type_take(pt)))283 ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);284}285 286/* Return the pool_type to use for the given caching and order */287static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,288 enum ttm_caching caching,289 unsigned int order)290{291 if (pool->use_dma_alloc)292 return &pool->caching[caching].orders[order];293 294#ifdef CONFIG_X86295 switch (caching) {296 case ttm_write_combined:297 if (pool->nid != NUMA_NO_NODE)298 return &pool->caching[caching].orders[order];299 300 if (pool->use_dma32)301 return &global_dma32_write_combined[order];302 303 return &global_write_combined[order];304 case ttm_uncached:305 if (pool->nid != NUMA_NO_NODE)306 return &pool->caching[caching].orders[order];307 308 if (pool->use_dma32)309 return &global_dma32_uncached[order];310 311 return &global_uncached[order];312 default:313 break;314 }315#endif316 317 return NULL;318}319 320/* Free pages using the global shrinker list */321static unsigned int ttm_pool_shrink(void)322{323 struct ttm_pool_type *pt;324 unsigned int num_pages;325 struct page *p;326 327 down_read(&pool_shrink_rwsem);328 spin_lock(&shrinker_lock);329 pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list);330 list_move_tail(&pt->shrinker_list, &shrinker_list);331 spin_unlock(&shrinker_lock);332 333 p = ttm_pool_type_take(pt);334 if (p) {335 ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);336 num_pages = 1 << pt->order;337 } else {338 num_pages = 0;339 }340 up_read(&pool_shrink_rwsem);341 342 return num_pages;343}344 345/* Return the allocation order based for a page */346static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p)347{348 if (pool->use_dma_alloc) {349 struct ttm_pool_dma *dma = (void *)p->private;350 351 return dma->vaddr & ~PAGE_MASK;352 }353 354 return p->private;355}356 357/* Called when we got a page, either from a pool or newly allocated */358static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,359 struct page *p, dma_addr_t **dma_addr,360 unsigned long *num_pages,361 struct page ***pages)362{363 unsigned int i;364 int r;365 366 if (*dma_addr) {367 r = ttm_pool_map(pool, order, p, dma_addr);368 if (r)369 return r;370 }371 372 *num_pages -= 1 << order;373 for (i = 1 << order; i; --i, ++(*pages), ++p)374 **pages = p;375 376 return 0;377}378 379/**380 * ttm_pool_free_range() - Free a range of TTM pages381 * @pool: The pool used for allocating.382 * @tt: The struct ttm_tt holding the page pointers.383 * @caching: The page caching mode used by the range.384 * @start_page: index for first page to free.385 * @end_page: index for last page to free + 1.386 *387 * During allocation the ttm_tt page-vector may be populated with ranges of388 * pages with different attributes if allocation hit an error without being389 * able to completely fulfill the allocation. This function can be used390 * to free these individual ranges.391 */392static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,393 enum ttm_caching caching,394 pgoff_t start_page, pgoff_t end_page)395{396 struct page **pages = &tt->pages[start_page];397 unsigned int order;398 pgoff_t i, nr;399 400 for (i = start_page; i < end_page; i += nr, pages += nr) {401 struct ttm_pool_type *pt = NULL;402 403 order = ttm_pool_page_order(pool, *pages);404 nr = (1UL << order);405 if (tt->dma_address)406 ttm_pool_unmap(pool, tt->dma_address[i], nr);407 408 pt = ttm_pool_select_type(pool, caching, order);409 if (pt)410 ttm_pool_type_give(pt, *pages);411 else412 ttm_pool_free_page(pool, caching, order, *pages);413 }414}415 416/**417 * ttm_pool_alloc - Fill a ttm_tt object418 *419 * @pool: ttm_pool to use420 * @tt: ttm_tt object to fill421 * @ctx: operation context422 *423 * Fill the ttm_tt object with pages and also make sure to DMA map them when424 * necessary.425 *426 * Returns: 0 on successe, negative error code otherwise.427 */428int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,429 struct ttm_operation_ctx *ctx)430{431 pgoff_t num_pages = tt->num_pages;432 dma_addr_t *dma_addr = tt->dma_address;433 struct page **caching = tt->pages;434 struct page **pages = tt->pages;435 enum ttm_caching page_caching;436 gfp_t gfp_flags = GFP_USER;437 pgoff_t caching_divide;438 unsigned int order;439 struct page *p;440 int r;441 442 WARN_ON(!num_pages || ttm_tt_is_populated(tt));443 WARN_ON(dma_addr && !pool->dev);444 445 if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC)446 gfp_flags |= __GFP_ZERO;447 448 if (ctx->gfp_retry_mayfail)449 gfp_flags |= __GFP_RETRY_MAYFAIL;450 451 if (pool->use_dma32)452 gfp_flags |= GFP_DMA32;453 else454 gfp_flags |= GFP_HIGHUSER;455 456 for (order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages));457 num_pages;458 order = min_t(unsigned int, order, __fls(num_pages))) {459 struct ttm_pool_type *pt;460 461 page_caching = tt->caching;462 pt = ttm_pool_select_type(pool, tt->caching, order);463 p = pt ? ttm_pool_type_take(pt) : NULL;464 if (p) {465 r = ttm_pool_apply_caching(caching, pages,466 tt->caching);467 if (r)468 goto error_free_page;469 470 caching = pages;471 do {472 r = ttm_pool_page_allocated(pool, order, p,473 &dma_addr,474 &num_pages,475 &pages);476 if (r)477 goto error_free_page;478 479 caching = pages;480 if (num_pages < (1 << order))481 break;482 483 p = ttm_pool_type_take(pt);484 } while (p);485 }486 487 page_caching = ttm_cached;488 while (num_pages >= (1 << order) &&489 (p = ttm_pool_alloc_page(pool, gfp_flags, order))) {490 491 if (PageHighMem(p)) {492 r = ttm_pool_apply_caching(caching, pages,493 tt->caching);494 if (r)495 goto error_free_page;496 caching = pages;497 }498 r = ttm_pool_page_allocated(pool, order, p, &dma_addr,499 &num_pages, &pages);500 if (r)501 goto error_free_page;502 if (PageHighMem(p))503 caching = pages;504 }505 506 if (!p) {507 if (order) {508 --order;509 continue;510 }511 r = -ENOMEM;512 goto error_free_all;513 }514 }515 516 r = ttm_pool_apply_caching(caching, pages, tt->caching);517 if (r)518 goto error_free_all;519 520 return 0;521 522error_free_page:523 ttm_pool_free_page(pool, page_caching, order, p);524 525error_free_all:526 num_pages = tt->num_pages - num_pages;527 caching_divide = caching - tt->pages;528 ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide);529 ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, num_pages);530 531 return r;532}533EXPORT_SYMBOL(ttm_pool_alloc);534 535/**536 * ttm_pool_free - Free the backing pages from a ttm_tt object537 *538 * @pool: Pool to give pages back to.539 * @tt: ttm_tt object to unpopulate540 *541 * Give the packing pages back to a pool or free them542 */543void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)544{545 ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages);546 547 while (atomic_long_read(&allocated_pages) > page_pool_size)548 ttm_pool_shrink();549}550EXPORT_SYMBOL(ttm_pool_free);551 552/**553 * ttm_pool_init - Initialize a pool554 *555 * @pool: the pool to initialize556 * @dev: device for DMA allocations and mappings557 * @nid: NUMA node to use for allocations558 * @use_dma_alloc: true if coherent DMA alloc should be used559 * @use_dma32: true if GFP_DMA32 should be used560 *561 * Initialize the pool and its pool types.562 */563void ttm_pool_init(struct ttm_pool *pool, struct device *dev,564 int nid, bool use_dma_alloc, bool use_dma32)565{566 unsigned int i, j;567 568 WARN_ON(!dev && use_dma_alloc);569 570 pool->dev = dev;571 pool->nid = nid;572 pool->use_dma_alloc = use_dma_alloc;573 pool->use_dma32 = use_dma32;574 575 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {576 for (j = 0; j < NR_PAGE_ORDERS; ++j) {577 struct ttm_pool_type *pt;578 579 /* Initialize only pool types which are actually used */580 pt = ttm_pool_select_type(pool, i, j);581 if (pt != &pool->caching[i].orders[j])582 continue;583 584 ttm_pool_type_init(pt, pool, i, j);585 }586 }587}588EXPORT_SYMBOL(ttm_pool_init);589 590/**591 * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete.592 *593 * This is useful to guarantee that all shrinker invocations have seen an594 * update, before freeing memory, similar to rcu.595 */596static void ttm_pool_synchronize_shrinkers(void)597{598 down_write(&pool_shrink_rwsem);599 up_write(&pool_shrink_rwsem);600}601 602/**603 * ttm_pool_fini - Cleanup a pool604 *605 * @pool: the pool to clean up606 *607 * Free all pages in the pool and unregister the types from the global608 * shrinker.609 */610void ttm_pool_fini(struct ttm_pool *pool)611{612 unsigned int i, j;613 614 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {615 for (j = 0; j < NR_PAGE_ORDERS; ++j) {616 struct ttm_pool_type *pt;617 618 pt = ttm_pool_select_type(pool, i, j);619 if (pt != &pool->caching[i].orders[j])620 continue;621 622 ttm_pool_type_fini(pt);623 }624 }625 626 /* We removed the pool types from the LRU, but we need to also make sure627 * that no shrinker is concurrently freeing pages from the pool.628 */629 ttm_pool_synchronize_shrinkers();630}631EXPORT_SYMBOL(ttm_pool_fini);632 633/* As long as pages are available make sure to release at least one */634static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink,635 struct shrink_control *sc)636{637 unsigned long num_freed = 0;638 639 do640 num_freed += ttm_pool_shrink();641 while (!num_freed && atomic_long_read(&allocated_pages));642 643 return num_freed;644}645 646/* Return the number of pages available or SHRINK_EMPTY if we have none */647static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink,648 struct shrink_control *sc)649{650 unsigned long num_pages = atomic_long_read(&allocated_pages);651 652 return num_pages ? num_pages : SHRINK_EMPTY;653}654 655#ifdef CONFIG_DEBUG_FS656/* Count the number of pages available in a pool_type */657static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt)658{659 unsigned int count = 0;660 struct page *p;661 662 spin_lock(&pt->lock);663 /* Only used for debugfs, the overhead doesn't matter */664 list_for_each_entry(p, &pt->pages, lru)665 ++count;666 spin_unlock(&pt->lock);667 668 return count;669}670 671/* Print a nice header for the order */672static void ttm_pool_debugfs_header(struct seq_file *m)673{674 unsigned int i;675 676 seq_puts(m, "\t ");677 for (i = 0; i < NR_PAGE_ORDERS; ++i)678 seq_printf(m, " ---%2u---", i);679 seq_puts(m, "\n");680}681 682/* Dump information about the different pool types */683static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt,684 struct seq_file *m)685{686 unsigned int i;687 688 for (i = 0; i < NR_PAGE_ORDERS; ++i)689 seq_printf(m, " %8u", ttm_pool_type_count(&pt[i]));690 seq_puts(m, "\n");691}692 693/* Dump the total amount of allocated pages */694static void ttm_pool_debugfs_footer(struct seq_file *m)695{696 seq_printf(m, "\ntotal\t: %8lu of %8lu\n",697 atomic_long_read(&allocated_pages), page_pool_size);698}699 700/* Dump the information for the global pools */701static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data)702{703 ttm_pool_debugfs_header(m);704 705 spin_lock(&shrinker_lock);706 seq_puts(m, "wc\t:");707 ttm_pool_debugfs_orders(global_write_combined, m);708 seq_puts(m, "uc\t:");709 ttm_pool_debugfs_orders(global_uncached, m);710 seq_puts(m, "wc 32\t:");711 ttm_pool_debugfs_orders(global_dma32_write_combined, m);712 seq_puts(m, "uc 32\t:");713 ttm_pool_debugfs_orders(global_dma32_uncached, m);714 spin_unlock(&shrinker_lock);715 716 ttm_pool_debugfs_footer(m);717 718 return 0;719}720DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals);721 722/**723 * ttm_pool_debugfs - Debugfs dump function for a pool724 *725 * @pool: the pool to dump the information for726 * @m: seq_file to dump to727 *728 * Make a debugfs dump with the per pool and global information.729 */730int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m)731{732 unsigned int i;733 734 if (!pool->use_dma_alloc) {735 seq_puts(m, "unused\n");736 return 0;737 }738 739 ttm_pool_debugfs_header(m);740 741 spin_lock(&shrinker_lock);742 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {743 seq_puts(m, "DMA ");744 switch (i) {745 case ttm_cached:746 seq_puts(m, "\t:");747 break;748 case ttm_write_combined:749 seq_puts(m, "wc\t:");750 break;751 case ttm_uncached:752 seq_puts(m, "uc\t:");753 break;754 }755 ttm_pool_debugfs_orders(pool->caching[i].orders, m);756 }757 spin_unlock(&shrinker_lock);758 759 ttm_pool_debugfs_footer(m);760 return 0;761}762EXPORT_SYMBOL(ttm_pool_debugfs);763 764/* Test the shrinker functions and dump the result */765static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data)766{767 struct shrink_control sc = { .gfp_mask = GFP_NOFS };768 769 fs_reclaim_acquire(GFP_KERNEL);770 seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(mm_shrinker, &sc),771 ttm_pool_shrinker_scan(mm_shrinker, &sc));772 fs_reclaim_release(GFP_KERNEL);773 774 return 0;775}776DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink);777 778#endif779 780/**781 * ttm_pool_mgr_init - Initialize globals782 *783 * @num_pages: default number of pages784 *785 * Initialize the global locks and lists for the MM shrinker.786 */787int ttm_pool_mgr_init(unsigned long num_pages)788{789 unsigned int i;790 791 if (!page_pool_size)792 page_pool_size = num_pages;793 794 spin_lock_init(&shrinker_lock);795 INIT_LIST_HEAD(&shrinker_list);796 797 for (i = 0; i < NR_PAGE_ORDERS; ++i) {798 ttm_pool_type_init(&global_write_combined[i], NULL,799 ttm_write_combined, i);800 ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i);801 802 ttm_pool_type_init(&global_dma32_write_combined[i], NULL,803 ttm_write_combined, i);804 ttm_pool_type_init(&global_dma32_uncached[i], NULL,805 ttm_uncached, i);806 }807 808#ifdef CONFIG_DEBUG_FS809 debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL,810 &ttm_pool_debugfs_globals_fops);811 debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL,812 &ttm_pool_debugfs_shrink_fops);813#endif814 815 mm_shrinker = shrinker_alloc(0, "drm-ttm_pool");816 if (!mm_shrinker)817 return -ENOMEM;818 819 mm_shrinker->count_objects = ttm_pool_shrinker_count;820 mm_shrinker->scan_objects = ttm_pool_shrinker_scan;821 mm_shrinker->seeks = 1;822 823 shrinker_register(mm_shrinker);824 825 return 0;826}827 828/**829 * ttm_pool_mgr_fini - Finalize globals830 *831 * Cleanup the global pools and unregister the MM shrinker.832 */833void ttm_pool_mgr_fini(void)834{835 unsigned int i;836 837 for (i = 0; i < NR_PAGE_ORDERS; ++i) {838 ttm_pool_type_fini(&global_write_combined[i]);839 ttm_pool_type_fini(&global_uncached[i]);840 841 ttm_pool_type_fini(&global_dma32_write_combined[i]);842 ttm_pool_type_fini(&global_dma32_uncached[i]);843 }844 845 shrinker_free(mm_shrinker);846 WARN_ON(!list_empty(&shrinker_list));847}848