971 lines · c
1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Christian König23 */24 25#include <linux/dma-mapping.h>26#include <drm/ttm/ttm_range_manager.h>27 28#include "amdgpu.h"29#include "amdgpu_vm.h"30#include "amdgpu_res_cursor.h"31#include "amdgpu_atomfirmware.h"32#include "atom.h"33 34#define AMDGPU_MAX_SG_SEGMENT_SIZE (2UL << 30)35 36struct amdgpu_vram_reservation {37 u64 start;38 u64 size;39 struct list_head allocated;40 struct list_head blocks;41};42 43static inline struct amdgpu_vram_mgr *44to_vram_mgr(struct ttm_resource_manager *man)45{46 return container_of(man, struct amdgpu_vram_mgr, manager);47}48 49static inline struct amdgpu_device *50to_amdgpu_device(struct amdgpu_vram_mgr *mgr)51{52 return container_of(mgr, struct amdgpu_device, mman.vram_mgr);53}54 55static inline struct drm_buddy_block *56amdgpu_vram_mgr_first_block(struct list_head *list)57{58 return list_first_entry_or_null(list, struct drm_buddy_block, link);59}60 61static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)62{63 struct drm_buddy_block *block;64 u64 start, size;65 66 block = amdgpu_vram_mgr_first_block(head);67 if (!block)68 return false;69 70 while (head != block->link.next) {71 start = amdgpu_vram_mgr_block_start(block);72 size = amdgpu_vram_mgr_block_size(block);73 74 block = list_entry(block->link.next, struct drm_buddy_block, link);75 if (start + size != amdgpu_vram_mgr_block_start(block))76 return false;77 }78 79 return true;80}81 82static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head)83{84 struct drm_buddy_block *block;85 u64 size = 0;86 87 list_for_each_entry(block, head, link)88 size += amdgpu_vram_mgr_block_size(block);89 90 return size;91}92 93/**94 * DOC: mem_info_vram_total95 *96 * The amdgpu driver provides a sysfs API for reporting current total VRAM97 * available on the device98 * The file mem_info_vram_total is used for this and returns the total99 * amount of VRAM in bytes100 */101static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,102 struct device_attribute *attr, char *buf)103{104 struct drm_device *ddev = dev_get_drvdata(dev);105 struct amdgpu_device *adev = drm_to_adev(ddev);106 107 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);108}109 110/**111 * DOC: mem_info_vis_vram_total112 *113 * The amdgpu driver provides a sysfs API for reporting current total114 * visible VRAM available on the device115 * The file mem_info_vis_vram_total is used for this and returns the total116 * amount of visible VRAM in bytes117 */118static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,119 struct device_attribute *attr, char *buf)120{121 struct drm_device *ddev = dev_get_drvdata(dev);122 struct amdgpu_device *adev = drm_to_adev(ddev);123 124 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);125}126 127/**128 * DOC: mem_info_vram_used129 *130 * The amdgpu driver provides a sysfs API for reporting current total VRAM131 * available on the device132 * The file mem_info_vram_used is used for this and returns the total133 * amount of currently used VRAM in bytes134 */135static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,136 struct device_attribute *attr,137 char *buf)138{139 struct drm_device *ddev = dev_get_drvdata(dev);140 struct amdgpu_device *adev = drm_to_adev(ddev);141 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;142 143 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man));144}145 146/**147 * DOC: mem_info_vis_vram_used148 *149 * The amdgpu driver provides a sysfs API for reporting current total of150 * used visible VRAM151 * The file mem_info_vis_vram_used is used for this and returns the total152 * amount of currently used visible VRAM in bytes153 */154static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,155 struct device_attribute *attr,156 char *buf)157{158 struct drm_device *ddev = dev_get_drvdata(dev);159 struct amdgpu_device *adev = drm_to_adev(ddev);160 161 return sysfs_emit(buf, "%llu\n",162 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));163}164 165/**166 * DOC: mem_info_vram_vendor167 *168 * The amdgpu driver provides a sysfs API for reporting the vendor of the169 * installed VRAM170 * The file mem_info_vram_vendor is used for this and returns the name of the171 * vendor.172 */173static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,174 struct device_attribute *attr,175 char *buf)176{177 struct drm_device *ddev = dev_get_drvdata(dev);178 struct amdgpu_device *adev = drm_to_adev(ddev);179 180 switch (adev->gmc.vram_vendor) {181 case SAMSUNG:182 return sysfs_emit(buf, "samsung\n");183 case INFINEON:184 return sysfs_emit(buf, "infineon\n");185 case ELPIDA:186 return sysfs_emit(buf, "elpida\n");187 case ETRON:188 return sysfs_emit(buf, "etron\n");189 case NANYA:190 return sysfs_emit(buf, "nanya\n");191 case HYNIX:192 return sysfs_emit(buf, "hynix\n");193 case MOSEL:194 return sysfs_emit(buf, "mosel\n");195 case WINBOND:196 return sysfs_emit(buf, "winbond\n");197 case ESMT:198 return sysfs_emit(buf, "esmt\n");199 case MICRON:200 return sysfs_emit(buf, "micron\n");201 default:202 return sysfs_emit(buf, "unknown\n");203 }204}205 206static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,207 amdgpu_mem_info_vram_total_show, NULL);208static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,209 amdgpu_mem_info_vis_vram_total_show,NULL);210static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,211 amdgpu_mem_info_vram_used_show, NULL);212static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,213 amdgpu_mem_info_vis_vram_used_show, NULL);214static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,215 amdgpu_mem_info_vram_vendor, NULL);216 217static struct attribute *amdgpu_vram_mgr_attributes[] = {218 &dev_attr_mem_info_vram_total.attr,219 &dev_attr_mem_info_vis_vram_total.attr,220 &dev_attr_mem_info_vram_used.attr,221 &dev_attr_mem_info_vis_vram_used.attr,222 &dev_attr_mem_info_vram_vendor.attr,223 NULL224};225 226static umode_t amdgpu_vram_attrs_is_visible(struct kobject *kobj,227 struct attribute *attr, int i)228{229 struct device *dev = kobj_to_dev(kobj);230 struct drm_device *ddev = dev_get_drvdata(dev);231 struct amdgpu_device *adev = drm_to_adev(ddev);232 233 if (attr == &dev_attr_mem_info_vram_vendor.attr &&234 !adev->gmc.vram_vendor)235 return 0;236 237 return attr->mode;238}239 240const struct attribute_group amdgpu_vram_mgr_attr_group = {241 .attrs = amdgpu_vram_mgr_attributes,242 .is_visible = amdgpu_vram_attrs_is_visible243};244 245/**246 * amdgpu_vram_mgr_vis_size - Calculate visible block size247 *248 * @adev: amdgpu_device pointer249 * @block: DRM BUDDY block structure250 *251 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM252 */253static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,254 struct drm_buddy_block *block)255{256 u64 start = amdgpu_vram_mgr_block_start(block);257 u64 end = start + amdgpu_vram_mgr_block_size(block);258 259 if (start >= adev->gmc.visible_vram_size)260 return 0;261 262 return (end > adev->gmc.visible_vram_size ?263 adev->gmc.visible_vram_size : end) - start;264}265 266/**267 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size268 *269 * @bo: &amdgpu_bo buffer object (must be in VRAM)270 *271 * Returns:272 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.273 */274u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)275{276 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);277 struct ttm_resource *res = bo->tbo.resource;278 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);279 struct drm_buddy_block *block;280 u64 usage = 0;281 282 if (amdgpu_gmc_vram_full_visible(&adev->gmc))283 return amdgpu_bo_size(bo);284 285 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)286 return 0;287 288 list_for_each_entry(block, &vres->blocks, link)289 usage += amdgpu_vram_mgr_vis_size(adev, block);290 291 return usage;292}293 294/* Commit the reservation of VRAM pages */295static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)296{297 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);298 struct amdgpu_device *adev = to_amdgpu_device(mgr);299 struct drm_buddy *mm = &mgr->mm;300 struct amdgpu_vram_reservation *rsv, *temp;301 struct drm_buddy_block *block;302 uint64_t vis_usage;303 304 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {305 if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,306 rsv->size, mm->chunk_size, &rsv->allocated,307 DRM_BUDDY_RANGE_ALLOCATION))308 continue;309 310 block = amdgpu_vram_mgr_first_block(&rsv->allocated);311 if (!block)312 continue;313 314 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",315 rsv->start, rsv->size);316 317 vis_usage = amdgpu_vram_mgr_vis_size(adev, block);318 atomic64_add(vis_usage, &mgr->vis_usage);319 spin_lock(&man->bdev->lru_lock);320 man->usage += rsv->size;321 spin_unlock(&man->bdev->lru_lock);322 list_move(&rsv->blocks, &mgr->reserved_pages);323 }324}325 326/**327 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM328 *329 * @mgr: amdgpu_vram_mgr pointer330 * @start: start address of the range in VRAM331 * @size: size of the range332 *333 * Reserve memory from start address with the specified size in VRAM334 */335int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,336 uint64_t start, uint64_t size)337{338 struct amdgpu_vram_reservation *rsv;339 340 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);341 if (!rsv)342 return -ENOMEM;343 344 INIT_LIST_HEAD(&rsv->allocated);345 INIT_LIST_HEAD(&rsv->blocks);346 347 rsv->start = start;348 rsv->size = size;349 350 mutex_lock(&mgr->lock);351 list_add_tail(&rsv->blocks, &mgr->reservations_pending);352 amdgpu_vram_mgr_do_reserve(&mgr->manager);353 mutex_unlock(&mgr->lock);354 355 return 0;356}357 358/**359 * amdgpu_vram_mgr_query_page_status - query the reservation status360 *361 * @mgr: amdgpu_vram_mgr pointer362 * @start: start address of a page in VRAM363 *364 * Returns:365 * -EBUSY: the page is still hold and in pending list366 * 0: the page has been reserved367 * -ENOENT: the input page is not a reservation368 */369int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,370 uint64_t start)371{372 struct amdgpu_vram_reservation *rsv;373 int ret;374 375 mutex_lock(&mgr->lock);376 377 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {378 if (rsv->start <= start &&379 (start < (rsv->start + rsv->size))) {380 ret = -EBUSY;381 goto out;382 }383 }384 385 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {386 if (rsv->start <= start &&387 (start < (rsv->start + rsv->size))) {388 ret = 0;389 goto out;390 }391 }392 393 ret = -ENOENT;394out:395 mutex_unlock(&mgr->lock);396 return ret;397}398 399static void amdgpu_dummy_vram_mgr_debug(struct ttm_resource_manager *man,400 struct drm_printer *printer)401{402 DRM_DEBUG_DRIVER("Dummy vram mgr debug\n");403}404 405static bool amdgpu_dummy_vram_mgr_compatible(struct ttm_resource_manager *man,406 struct ttm_resource *res,407 const struct ttm_place *place,408 size_t size)409{410 DRM_DEBUG_DRIVER("Dummy vram mgr compatible\n");411 return false;412}413 414static bool amdgpu_dummy_vram_mgr_intersects(struct ttm_resource_manager *man,415 struct ttm_resource *res,416 const struct ttm_place *place,417 size_t size)418{419 DRM_DEBUG_DRIVER("Dummy vram mgr intersects\n");420 return true;421}422 423static void amdgpu_dummy_vram_mgr_del(struct ttm_resource_manager *man,424 struct ttm_resource *res)425{426 DRM_DEBUG_DRIVER("Dummy vram mgr deleted\n");427}428 429static int amdgpu_dummy_vram_mgr_new(struct ttm_resource_manager *man,430 struct ttm_buffer_object *tbo,431 const struct ttm_place *place,432 struct ttm_resource **res)433{434 DRM_DEBUG_DRIVER("Dummy vram mgr new\n");435 return -ENOSPC;436}437 438/**439 * amdgpu_vram_mgr_new - allocate new ranges440 *441 * @man: TTM memory type manager442 * @tbo: TTM BO we need this range for443 * @place: placement flags and restrictions444 * @res: the resulting mem object445 *446 * Allocate VRAM for the given BO.447 */448static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,449 struct ttm_buffer_object *tbo,450 const struct ttm_place *place,451 struct ttm_resource **res)452{453 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);454 struct amdgpu_device *adev = to_amdgpu_device(mgr);455 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);456 u64 vis_usage = 0, max_bytes, min_block_size;457 struct amdgpu_vram_mgr_resource *vres;458 u64 size, remaining_size, lpfn, fpfn;459 unsigned int adjust_dcc_size = 0;460 struct drm_buddy *mm = &mgr->mm;461 struct drm_buddy_block *block;462 unsigned long pages_per_block;463 int r;464 465 lpfn = (u64)place->lpfn << PAGE_SHIFT;466 if (!lpfn)467 lpfn = man->size;468 469 fpfn = (u64)place->fpfn << PAGE_SHIFT;470 471 max_bytes = adev->gmc.mc_vram_size;472 if (tbo->type != ttm_bo_type_kernel)473 max_bytes -= AMDGPU_VM_RESERVED_VRAM;474 475 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {476 pages_per_block = ~0ul;477 } else {478#ifdef CONFIG_TRANSPARENT_HUGEPAGE479 pages_per_block = HPAGE_PMD_NR;480#else481 /* default to 2MB */482 pages_per_block = 2UL << (20UL - PAGE_SHIFT);483#endif484 pages_per_block = max_t(u32, pages_per_block,485 tbo->page_alignment);486 }487 488 vres = kzalloc(sizeof(*vres), GFP_KERNEL);489 if (!vres)490 return -ENOMEM;491 492 ttm_resource_init(tbo, place, &vres->base);493 494 /* bail out quickly if there's likely not enough VRAM for this BO */495 if (ttm_resource_manager_usage(man) > max_bytes) {496 r = -ENOSPC;497 goto error_fini;498 }499 500 INIT_LIST_HEAD(&vres->blocks);501 502 if (place->flags & TTM_PL_FLAG_TOPDOWN)503 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;504 505 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)506 vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION;507 508 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED)509 vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION;510 511 if (fpfn || lpfn != mgr->mm.size)512 /* Allocate blocks in desired range */513 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;514 515 if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC &&516 adev->gmc.gmc_funcs->get_dcc_alignment)517 adjust_dcc_size = amdgpu_gmc_get_dcc_alignment(adev);518 519 remaining_size = (u64)vres->base.size;520 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {521 unsigned int dcc_size;522 523 dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size);524 remaining_size = (u64)dcc_size;525 526 vres->flags |= DRM_BUDDY_TRIM_DISABLE;527 }528 529 mutex_lock(&mgr->lock);530 while (remaining_size) {531 if (tbo->page_alignment)532 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT;533 else534 min_block_size = mgr->default_page_size;535 536 size = remaining_size;537 538 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size)539 min_block_size = size;540 else if ((size >= (u64)pages_per_block << PAGE_SHIFT) &&541 !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))542 min_block_size = (u64)pages_per_block << PAGE_SHIFT;543 544 BUG_ON(min_block_size < mm->chunk_size);545 546 r = drm_buddy_alloc_blocks(mm, fpfn,547 lpfn,548 size,549 min_block_size,550 &vres->blocks,551 vres->flags);552 553 if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul &&554 !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) {555 vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION;556 pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT),557 tbo->page_alignment);558 559 continue;560 }561 562 if (unlikely(r))563 goto error_free_blocks;564 565 if (size > remaining_size)566 remaining_size = 0;567 else568 remaining_size -= size;569 }570 mutex_unlock(&mgr->lock);571 572 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {573 struct drm_buddy_block *dcc_block;574 unsigned long dcc_start;575 u64 trim_start;576 577 dcc_block = amdgpu_vram_mgr_first_block(&vres->blocks);578 /* Adjust the start address for DCC buffers only */579 dcc_start =580 roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block),581 adjust_dcc_size);582 trim_start = (u64)dcc_start;583 drm_buddy_block_trim(mm, &trim_start,584 (u64)vres->base.size,585 &vres->blocks);586 }587 588 vres->base.start = 0;589 size = max_t(u64, amdgpu_vram_mgr_blocks_size(&vres->blocks),590 vres->base.size);591 list_for_each_entry(block, &vres->blocks, link) {592 unsigned long start;593 594 start = amdgpu_vram_mgr_block_start(block) +595 amdgpu_vram_mgr_block_size(block);596 start >>= PAGE_SHIFT;597 598 if (start > PFN_UP(size))599 start -= PFN_UP(size);600 else601 start = 0;602 vres->base.start = max(vres->base.start, start);603 604 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);605 }606 607 if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks))608 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;609 610 if (adev->gmc.xgmi.connected_to_cpu)611 vres->base.bus.caching = ttm_cached;612 else613 vres->base.bus.caching = ttm_write_combined;614 615 atomic64_add(vis_usage, &mgr->vis_usage);616 *res = &vres->base;617 return 0;618 619error_free_blocks:620 drm_buddy_free_list(mm, &vres->blocks, 0);621 mutex_unlock(&mgr->lock);622error_fini:623 ttm_resource_fini(man, &vres->base);624 kfree(vres);625 626 return r;627}628 629/**630 * amdgpu_vram_mgr_del - free ranges631 *632 * @man: TTM memory type manager633 * @res: TTM memory object634 *635 * Free the allocated VRAM again.636 */637static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,638 struct ttm_resource *res)639{640 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);641 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);642 struct amdgpu_device *adev = to_amdgpu_device(mgr);643 struct drm_buddy *mm = &mgr->mm;644 struct drm_buddy_block *block;645 uint64_t vis_usage = 0;646 647 mutex_lock(&mgr->lock);648 list_for_each_entry(block, &vres->blocks, link)649 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);650 651 amdgpu_vram_mgr_do_reserve(man);652 653 drm_buddy_free_list(mm, &vres->blocks, vres->flags);654 mutex_unlock(&mgr->lock);655 656 atomic64_sub(vis_usage, &mgr->vis_usage);657 658 ttm_resource_fini(man, res);659 kfree(vres);660}661 662/**663 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table664 *665 * @adev: amdgpu device pointer666 * @res: TTM memory object667 * @offset: byte offset from the base of VRAM BO668 * @length: number of bytes to export in sg_table669 * @dev: the other device670 * @dir: dma direction671 * @sgt: resulting sg table672 *673 * Allocate and fill a sg table from a VRAM allocation.674 */675int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,676 struct ttm_resource *res,677 u64 offset, u64 length,678 struct device *dev,679 enum dma_data_direction dir,680 struct sg_table **sgt)681{682 struct amdgpu_res_cursor cursor;683 struct scatterlist *sg;684 int num_entries = 0;685 int i, r;686 687 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);688 if (!*sgt)689 return -ENOMEM;690 691 /* Determine the number of DRM_BUDDY blocks to export */692 amdgpu_res_first(res, offset, length, &cursor);693 while (cursor.remaining) {694 num_entries++;695 amdgpu_res_next(&cursor, min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE));696 }697 698 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);699 if (r)700 goto error_free;701 702 /* Initialize scatterlist nodes of sg_table */703 for_each_sgtable_sg((*sgt), sg, i)704 sg->length = 0;705 706 /*707 * Walk down DRM_BUDDY blocks to populate scatterlist nodes708 * @note: Use iterator api to get first the DRM_BUDDY block709 * and the number of bytes from it. Access the following710 * DRM_BUDDY block(s) if more buffer needs to exported711 */712 amdgpu_res_first(res, offset, length, &cursor);713 for_each_sgtable_sg((*sgt), sg, i) {714 phys_addr_t phys = cursor.start + adev->gmc.aper_base;715 unsigned long size = min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE);716 dma_addr_t addr;717 718 addr = dma_map_resource(dev, phys, size, dir,719 DMA_ATTR_SKIP_CPU_SYNC);720 r = dma_mapping_error(dev, addr);721 if (r)722 goto error_unmap;723 724 sg_set_page(sg, NULL, size, 0);725 sg_dma_address(sg) = addr;726 sg_dma_len(sg) = size;727 728 amdgpu_res_next(&cursor, size);729 }730 731 return 0;732 733error_unmap:734 for_each_sgtable_sg((*sgt), sg, i) {735 if (!sg->length)736 continue;737 738 dma_unmap_resource(dev, sg->dma_address,739 sg->length, dir,740 DMA_ATTR_SKIP_CPU_SYNC);741 }742 sg_free_table(*sgt);743 744error_free:745 kfree(*sgt);746 return r;747}748 749/**750 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table751 *752 * @dev: device pointer753 * @dir: data direction of resource to unmap754 * @sgt: sg table to free755 *756 * Free a previously allocate sg table.757 */758void amdgpu_vram_mgr_free_sgt(struct device *dev,759 enum dma_data_direction dir,760 struct sg_table *sgt)761{762 struct scatterlist *sg;763 int i;764 765 for_each_sgtable_sg(sgt, sg, i)766 dma_unmap_resource(dev, sg->dma_address,767 sg->length, dir,768 DMA_ATTR_SKIP_CPU_SYNC);769 sg_free_table(sgt);770 kfree(sgt);771}772 773/**774 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part775 *776 * @mgr: amdgpu_vram_mgr pointer777 *778 * Returns how many bytes are used in the visible part of VRAM779 */780uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)781{782 return atomic64_read(&mgr->vis_usage);783}784 785/**786 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection787 *788 * @man: TTM memory type manager789 * @res: The resource to test790 * @place: The place to test against791 * @size: Size of the new allocation792 *793 * Test each drm buddy block for intersection for eviction decision.794 */795static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,796 struct ttm_resource *res,797 const struct ttm_place *place,798 size_t size)799{800 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);801 struct drm_buddy_block *block;802 803 /* Check each drm buddy block individually */804 list_for_each_entry(block, &mgr->blocks, link) {805 unsigned long fpfn =806 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;807 unsigned long lpfn = fpfn +808 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);809 810 if (place->fpfn < lpfn &&811 (!place->lpfn || place->lpfn > fpfn))812 return true;813 }814 815 return false;816}817 818/**819 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility820 *821 * @man: TTM memory type manager822 * @res: The resource to test823 * @place: The place to test against824 * @size: Size of the new allocation825 *826 * Test each drm buddy block for placement compatibility.827 */828static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,829 struct ttm_resource *res,830 const struct ttm_place *place,831 size_t size)832{833 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);834 struct drm_buddy_block *block;835 836 /* Check each drm buddy block individually */837 list_for_each_entry(block, &mgr->blocks, link) {838 unsigned long fpfn =839 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;840 unsigned long lpfn = fpfn +841 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);842 843 if (fpfn < place->fpfn ||844 (place->lpfn && lpfn > place->lpfn))845 return false;846 }847 848 return true;849}850 851/**852 * amdgpu_vram_mgr_debug - dump VRAM table853 *854 * @man: TTM memory type manager855 * @printer: DRM printer to use856 *857 * Dump the table content using printk.858 */859static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,860 struct drm_printer *printer)861{862 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);863 struct drm_buddy *mm = &mgr->mm;864 struct amdgpu_vram_reservation *rsv;865 866 drm_printf(printer, " vis usage:%llu\n",867 amdgpu_vram_mgr_vis_usage(mgr));868 869 mutex_lock(&mgr->lock);870 drm_printf(printer, "default_page_size: %lluKiB\n",871 mgr->default_page_size >> 10);872 873 drm_buddy_print(mm, printer);874 875 drm_printf(printer, "reserved:\n");876 list_for_each_entry(rsv, &mgr->reserved_pages, blocks)877 drm_printf(printer, "%#018llx-%#018llx: %llu\n",878 rsv->start, rsv->start + rsv->size, rsv->size);879 mutex_unlock(&mgr->lock);880}881 882static const struct ttm_resource_manager_func amdgpu_dummy_vram_mgr_func = {883 .alloc = amdgpu_dummy_vram_mgr_new,884 .free = amdgpu_dummy_vram_mgr_del,885 .intersects = amdgpu_dummy_vram_mgr_intersects,886 .compatible = amdgpu_dummy_vram_mgr_compatible,887 .debug = amdgpu_dummy_vram_mgr_debug888};889 890static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {891 .alloc = amdgpu_vram_mgr_new,892 .free = amdgpu_vram_mgr_del,893 .intersects = amdgpu_vram_mgr_intersects,894 .compatible = amdgpu_vram_mgr_compatible,895 .debug = amdgpu_vram_mgr_debug896};897 898/**899 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM900 *901 * @adev: amdgpu_device pointer902 *903 * Allocate and initialize the VRAM manager.904 */905int amdgpu_vram_mgr_init(struct amdgpu_device *adev)906{907 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;908 struct ttm_resource_manager *man = &mgr->manager;909 int err;910 911 ttm_resource_manager_init(man, &adev->mman.bdev,912 adev->gmc.real_vram_size);913 914 mutex_init(&mgr->lock);915 INIT_LIST_HEAD(&mgr->reservations_pending);916 INIT_LIST_HEAD(&mgr->reserved_pages);917 mgr->default_page_size = PAGE_SIZE;918 919 if (!adev->gmc.is_app_apu) {920 man->func = &amdgpu_vram_mgr_func;921 922 err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);923 if (err)924 return err;925 } else {926 man->func = &amdgpu_dummy_vram_mgr_func;927 DRM_INFO("Setup dummy vram mgr\n");928 }929 930 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);931 ttm_resource_manager_set_used(man, true);932 return 0;933}934 935/**936 * amdgpu_vram_mgr_fini - free and destroy VRAM manager937 *938 * @adev: amdgpu_device pointer939 *940 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still941 * allocated inside it.942 */943void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)944{945 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;946 struct ttm_resource_manager *man = &mgr->manager;947 int ret;948 struct amdgpu_vram_reservation *rsv, *temp;949 950 ttm_resource_manager_set_used(man, false);951 952 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);953 if (ret)954 return;955 956 mutex_lock(&mgr->lock);957 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)958 kfree(rsv);959 960 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {961 drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0);962 kfree(rsv);963 }964 if (!adev->gmc.is_app_apu)965 drm_buddy_fini(&mgr->mm);966 mutex_unlock(&mgr->lock);967 968 ttm_resource_manager_cleanup(man);969 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);970}971