588 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Copyright (C) 2023 Loongson Technology Corporation Limited4 */5 6#include <drm/drm_drv.h>7#include <drm/drm_file.h>8#include <drm/drm_gem.h>9#include <drm/drm_managed.h>10#include <drm/drm_prime.h>11 12#include "lsdc_drv.h"13#include "lsdc_ttm.h"14 15const char *lsdc_mem_type_to_str(uint32_t mem_type)16{17 switch (mem_type) {18 case TTM_PL_VRAM:19 return "VRAM";20 case TTM_PL_TT:21 return "GTT";22 case TTM_PL_SYSTEM:23 return "SYSTEM";24 default:25 break;26 }27 28 return "Unknown";29}30 31const char *lsdc_domain_to_str(u32 domain)32{33 switch (domain) {34 case LSDC_GEM_DOMAIN_VRAM:35 return "VRAM";36 case LSDC_GEM_DOMAIN_GTT:37 return "GTT";38 case LSDC_GEM_DOMAIN_SYSTEM:39 return "SYSTEM";40 default:41 break;42 }43 44 return "Unknown";45}46 47static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)48{49 u32 c = 0;50 u32 pflags = 0;51 u32 i;52 53 if (lbo->tbo.base.size <= PAGE_SIZE)54 pflags |= TTM_PL_FLAG_TOPDOWN;55 56 lbo->placement.placement = lbo->placements;57 58 if (domain & LSDC_GEM_DOMAIN_VRAM) {59 lbo->placements[c].mem_type = TTM_PL_VRAM;60 lbo->placements[c++].flags = pflags;61 }62 63 if (domain & LSDC_GEM_DOMAIN_GTT) {64 lbo->placements[c].mem_type = TTM_PL_TT;65 lbo->placements[c++].flags = pflags;66 }67 68 if (domain & LSDC_GEM_DOMAIN_SYSTEM) {69 lbo->placements[c].mem_type = TTM_PL_SYSTEM;70 lbo->placements[c++].flags = 0;71 }72 73 if (!c) {74 lbo->placements[c].mem_type = TTM_PL_SYSTEM;75 lbo->placements[c++].flags = 0;76 }77 78 lbo->placement.num_placement = c;79 80 for (i = 0; i < c; ++i) {81 lbo->placements[i].fpfn = 0;82 lbo->placements[i].lpfn = 0;83 }84}85 86static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)87{88 ttm_tt_fini(tt);89 kfree(tt);90}91 92static struct ttm_tt *93lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)94{95 struct ttm_tt *tt;96 int ret;97 98 tt = kzalloc(sizeof(*tt), GFP_KERNEL);99 if (!tt)100 return NULL;101 102 ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);103 if (ret < 0) {104 kfree(tt);105 return NULL;106 }107 108 return tt;109}110 111static int lsdc_ttm_tt_populate(struct ttm_device *bdev,112 struct ttm_tt *ttm,113 struct ttm_operation_ctx *ctx)114{115 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);116 117 if (slave && ttm->sg) {118 drm_prime_sg_to_dma_addr_array(ttm->sg,119 ttm->dma_address,120 ttm->num_pages);121 122 return 0;123 }124 125 return ttm_pool_alloc(&bdev->pool, ttm, ctx);126}127 128static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,129 struct ttm_tt *ttm)130{131 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);132 133 if (slave)134 return;135 136 return ttm_pool_free(&bdev->pool, ttm);137}138 139static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,140 struct ttm_placement *tplacement)141{142 struct ttm_resource *resource = tbo->resource;143 struct lsdc_bo *lbo = to_lsdc_bo(tbo);144 145 switch (resource->mem_type) {146 case TTM_PL_VRAM:147 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);148 break;149 case TTM_PL_TT:150 default:151 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);152 break;153 }154 155 *tplacement = lbo->placement;156}157 158static int lsdc_bo_move(struct ttm_buffer_object *tbo,159 bool evict,160 struct ttm_operation_ctx *ctx,161 struct ttm_resource *new_mem,162 struct ttm_place *hop)163{164 struct drm_device *ddev = tbo->base.dev;165 struct ttm_resource *old_mem = tbo->resource;166 struct lsdc_bo *lbo = to_lsdc_bo(tbo);167 int ret;168 169 if (unlikely(tbo->pin_count > 0)) {170 drm_warn(ddev, "Can't move a pinned BO\n");171 return -EINVAL;172 }173 174 ret = ttm_bo_wait_ctx(tbo, ctx);175 if (ret)176 return ret;177 178 if (!old_mem) {179 drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",180 lbo, lsdc_mem_type_to_str(new_mem->mem_type),181 lsdc_bo_size(lbo));182 ttm_bo_move_null(tbo, new_mem);183 return 0;184 }185 186 if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {187 ttm_bo_move_null(tbo, new_mem);188 drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",189 lbo, lsdc_bo_size(lbo));190 return 0;191 }192 193 if (old_mem->mem_type == TTM_PL_SYSTEM &&194 new_mem->mem_type == TTM_PL_TT) {195 drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",196 lbo, lsdc_bo_size(lbo));197 ttm_bo_move_null(tbo, new_mem);198 return 0;199 }200 201 if (old_mem->mem_type == TTM_PL_TT &&202 new_mem->mem_type == TTM_PL_SYSTEM) {203 drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",204 lbo, lsdc_bo_size(lbo));205 ttm_resource_free(tbo, &tbo->resource);206 ttm_bo_assign_mem(tbo, new_mem);207 return 0;208 }209 210 drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",211 lbo,212 lsdc_mem_type_to_str(old_mem->mem_type),213 lsdc_mem_type_to_str(new_mem->mem_type),214 lsdc_bo_size(lbo));215 216 return ttm_bo_move_memcpy(tbo, ctx, new_mem);217}218 219static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,220 struct ttm_resource *mem)221{222 struct lsdc_device *ldev = tdev_to_ldev(bdev);223 224 switch (mem->mem_type) {225 case TTM_PL_SYSTEM:226 break;227 case TTM_PL_TT:228 break;229 case TTM_PL_VRAM:230 mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;231 mem->bus.is_iomem = true;232 mem->bus.caching = ttm_write_combined;233 break;234 default:235 return -EINVAL;236 }237 238 return 0;239}240 241static struct ttm_device_funcs lsdc_bo_driver = {242 .ttm_tt_create = lsdc_ttm_tt_create,243 .ttm_tt_populate = lsdc_ttm_tt_populate,244 .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,245 .ttm_tt_destroy = lsdc_ttm_tt_destroy,246 .eviction_valuable = ttm_bo_eviction_valuable,247 .evict_flags = lsdc_bo_evict_flags,248 .move = lsdc_bo_move,249 .io_mem_reserve = lsdc_bo_reserve_io_mem,250};251 252u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)253{254 struct ttm_buffer_object *tbo = &lbo->tbo;255 struct drm_device *ddev = tbo->base.dev;256 struct ttm_resource *resource = tbo->resource;257 258 if (unlikely(!tbo->pin_count)) {259 drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");260 return 0;261 }262 263 if (unlikely(resource->mem_type == TTM_PL_SYSTEM))264 return 0;265 266 return resource->start << PAGE_SHIFT;267}268 269size_t lsdc_bo_size(struct lsdc_bo *lbo)270{271 struct ttm_buffer_object *tbo = &lbo->tbo;272 273 return tbo->base.size;274}275 276int lsdc_bo_reserve(struct lsdc_bo *lbo)277{278 return ttm_bo_reserve(&lbo->tbo, true, false, NULL);279}280 281void lsdc_bo_unreserve(struct lsdc_bo *lbo)282{283 return ttm_bo_unreserve(&lbo->tbo);284}285 286int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)287{288 struct ttm_operation_ctx ctx = { false, false };289 struct ttm_buffer_object *tbo = &lbo->tbo;290 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);291 int ret;292 293 if (tbo->pin_count)294 goto bo_pinned;295 296 if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)297 return -EINVAL;298 299 if (domain)300 lsdc_bo_set_placement(lbo, domain);301 302 ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);303 if (unlikely(ret)) {304 drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);305 return ret;306 }307 308 if (domain == LSDC_GEM_DOMAIN_VRAM)309 ldev->vram_pinned_size += lsdc_bo_size(lbo);310 else if (domain == LSDC_GEM_DOMAIN_GTT)311 ldev->gtt_pinned_size += lsdc_bo_size(lbo);312 313bo_pinned:314 ttm_bo_pin(tbo);315 316 if (gpu_addr)317 *gpu_addr = lsdc_bo_gpu_offset(lbo);318 319 return 0;320}321 322void lsdc_bo_unpin(struct lsdc_bo *lbo)323{324 struct ttm_buffer_object *tbo = &lbo->tbo;325 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);326 327 if (unlikely(!tbo->pin_count)) {328 drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);329 return;330 }331 332 ttm_bo_unpin(tbo);333 334 if (!tbo->pin_count) {335 if (tbo->resource->mem_type == TTM_PL_VRAM)336 ldev->vram_pinned_size -= lsdc_bo_size(lbo);337 else if (tbo->resource->mem_type == TTM_PL_TT)338 ldev->gtt_pinned_size -= lsdc_bo_size(lbo);339 }340}341 342void lsdc_bo_ref(struct lsdc_bo *lbo)343{344 drm_gem_object_get(&lbo->tbo.base);345}346 347void lsdc_bo_unref(struct lsdc_bo *lbo)348{349 drm_gem_object_put(&lbo->tbo.base);350}351 352int lsdc_bo_kmap(struct lsdc_bo *lbo)353{354 struct ttm_buffer_object *tbo = &lbo->tbo;355 struct drm_gem_object *gem = &tbo->base;356 struct drm_device *ddev = gem->dev;357 long ret;358 int err;359 360 ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,361 MAX_SCHEDULE_TIMEOUT);362 if (ret < 0) {363 drm_warn(ddev, "wait fence timeout\n");364 return ret;365 }366 367 if (lbo->kptr)368 return 0;369 370 err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);371 if (err) {372 drm_err(ddev, "kmap %p failed: %d\n", lbo, err);373 return err;374 }375 376 lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);377 378 return 0;379}380 381void lsdc_bo_kunmap(struct lsdc_bo *lbo)382{383 if (!lbo->kptr)384 return;385 386 lbo->kptr = NULL;387 ttm_bo_kunmap(&lbo->kmap);388}389 390void lsdc_bo_clear(struct lsdc_bo *lbo)391{392 lsdc_bo_kmap(lbo);393 394 if (lbo->is_iomem)395 memset_io((void __iomem *)lbo->kptr, 0, lbo->size);396 else397 memset(lbo->kptr, 0, lbo->size);398 399 lsdc_bo_kunmap(lbo);400}401 402int lsdc_bo_evict_vram(struct drm_device *ddev)403{404 struct lsdc_device *ldev = to_lsdc(ddev);405 struct ttm_device *bdev = &ldev->bdev;406 struct ttm_resource_manager *man;407 408 man = ttm_manager_type(bdev, TTM_PL_VRAM);409 if (unlikely(!man))410 return 0;411 412 return ttm_resource_manager_evict_all(bdev, man);413}414 415static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)416{417 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);418 struct lsdc_bo *lbo = to_lsdc_bo(tbo);419 420 mutex_lock(&ldev->gem.mutex);421 list_del_init(&lbo->list);422 mutex_unlock(&ldev->gem.mutex);423 424 drm_gem_object_release(&tbo->base);425 426 kfree(lbo);427}428 429struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,430 u32 domain,431 size_t size,432 bool kernel,433 struct sg_table *sg,434 struct dma_resv *resv)435{436 struct lsdc_device *ldev = to_lsdc(ddev);437 struct ttm_device *bdev = &ldev->bdev;438 struct ttm_buffer_object *tbo;439 struct lsdc_bo *lbo;440 enum ttm_bo_type bo_type;441 int ret;442 443 lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);444 if (!lbo)445 return ERR_PTR(-ENOMEM);446 447 INIT_LIST_HEAD(&lbo->list);448 449 lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |450 LSDC_GEM_DOMAIN_GTT |451 LSDC_GEM_DOMAIN_SYSTEM);452 453 tbo = &lbo->tbo;454 455 size = ALIGN(size, PAGE_SIZE);456 457 ret = drm_gem_object_init(ddev, &tbo->base, size);458 if (ret) {459 kfree(lbo);460 return ERR_PTR(ret);461 }462 463 tbo->bdev = bdev;464 465 if (kernel)466 bo_type = ttm_bo_type_kernel;467 else if (sg)468 bo_type = ttm_bo_type_sg;469 else470 bo_type = ttm_bo_type_device;471 472 lsdc_bo_set_placement(lbo, domain);473 lbo->size = size;474 475 ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,476 false, sg, resv, lsdc_bo_destroy);477 if (ret) {478 kfree(lbo);479 return ERR_PTR(ret);480 }481 482 return lbo;483}484 485struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,486 u32 domain,487 size_t size)488{489 struct lsdc_bo *lbo;490 int ret;491 492 lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);493 if (IS_ERR(lbo))494 return ERR_CAST(lbo);495 496 ret = lsdc_bo_reserve(lbo);497 if (unlikely(ret)) {498 lsdc_bo_unref(lbo);499 return ERR_PTR(ret);500 }501 502 ret = lsdc_bo_pin(lbo, domain, NULL);503 lsdc_bo_unreserve(lbo);504 if (unlikely(ret)) {505 lsdc_bo_unref(lbo);506 return ERR_PTR(ret);507 }508 509 return lbo;510}511 512void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)513{514 int ret;515 516 ret = lsdc_bo_reserve(lbo);517 if (unlikely(ret))518 return;519 520 lsdc_bo_unpin(lbo);521 lsdc_bo_unreserve(lbo);522 523 lsdc_bo_unref(lbo);524}525 526static void lsdc_ttm_fini(struct drm_device *ddev, void *data)527{528 struct lsdc_device *ldev = (struct lsdc_device *)data;529 530 ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);531 ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);532 533 ttm_device_fini(&ldev->bdev);534 535 drm_dbg(ddev, "ttm finished\n");536}537 538int lsdc_ttm_init(struct lsdc_device *ldev)539{540 struct drm_device *ddev = &ldev->base;541 unsigned long num_vram_pages;542 unsigned long num_gtt_pages;543 int ret;544 545 ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,546 ddev->anon_inode->i_mapping,547 ddev->vma_offset_manager, false, true);548 if (ret)549 return ret;550 551 num_vram_pages = ldev->vram_size >> PAGE_SHIFT;552 553 ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);554 if (unlikely(ret))555 return ret;556 557 drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);558 559 /* 512M is far enough for us now */560 ldev->gtt_size = 512 << 20;561 562 num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;563 564 ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);565 if (unlikely(ret))566 return ret;567 568 drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);569 570 return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);571}572 573void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)574{575 struct ttm_device *bdev = &ldev->bdev;576 struct drm_device *ddev = &ldev->base;577 struct drm_minor *minor = ddev->primary;578 struct dentry *root = minor->debugfs_root;579 struct ttm_resource_manager *vram_man;580 struct ttm_resource_manager *gtt_man;581 582 vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);583 gtt_man = ttm_manager_type(bdev, TTM_PL_TT);584 585 ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");586 ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");587}588