364 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright (c) 2007-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA,4 * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA,5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sub license,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice (including the14 * next paragraph) shall be included in all copies or substantial portions15 * of the Software.16 *17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL20 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE23 * USE OR OTHER DEALINGS IN THE SOFTWARE.24 */25 26#include <linux/limits.h>27 28#include <drm/ttm/ttm_range_manager.h>29#include <drm/drm_cache.h>30 31#include "nouveau_drv.h"32#include "nouveau_gem.h"33#include "nouveau_mem.h"34#include "nouveau_ttm.h"35 36#include <core/tegra.h>37 38static void39nouveau_manager_del(struct ttm_resource_manager *man,40 struct ttm_resource *reg)41{42 nouveau_mem_del(man, reg);43}44 45static bool46nouveau_manager_intersects(struct ttm_resource_manager *man,47 struct ttm_resource *res,48 const struct ttm_place *place,49 size_t size)50{51 return nouveau_mem_intersects(res, place, size);52}53 54static bool55nouveau_manager_compatible(struct ttm_resource_manager *man,56 struct ttm_resource *res,57 const struct ttm_place *place,58 size_t size)59{60 return nouveau_mem_compatible(res, place, size);61}62 63static int64nouveau_vram_manager_new(struct ttm_resource_manager *man,65 struct ttm_buffer_object *bo,66 const struct ttm_place *place,67 struct ttm_resource **res)68{69 struct nouveau_bo *nvbo = nouveau_bo(bo);70 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);71 int ret;72 73 if (drm->client.device.info.ram_size == 0)74 return -ENOMEM;75 76 ret = nouveau_mem_new(drm, nvbo->kind, nvbo->comp, res);77 if (ret)78 return ret;79 80 ttm_resource_init(bo, place, *res);81 82 ret = nouveau_mem_vram(*res, nvbo->contig, nvbo->page);83 if (ret) {84 nouveau_mem_del(man, *res);85 return ret;86 }87 88 return 0;89}90 91const struct ttm_resource_manager_func nouveau_vram_manager = {92 .alloc = nouveau_vram_manager_new,93 .free = nouveau_manager_del,94 .intersects = nouveau_manager_intersects,95 .compatible = nouveau_manager_compatible,96};97 98static int99nouveau_gart_manager_new(struct ttm_resource_manager *man,100 struct ttm_buffer_object *bo,101 const struct ttm_place *place,102 struct ttm_resource **res)103{104 struct nouveau_bo *nvbo = nouveau_bo(bo);105 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);106 int ret;107 108 ret = nouveau_mem_new(drm, nvbo->kind, nvbo->comp, res);109 if (ret)110 return ret;111 112 ttm_resource_init(bo, place, *res);113 (*res)->start = 0;114 return 0;115}116 117const struct ttm_resource_manager_func nouveau_gart_manager = {118 .alloc = nouveau_gart_manager_new,119 .free = nouveau_manager_del,120 .intersects = nouveau_manager_intersects,121 .compatible = nouveau_manager_compatible,122};123 124static int125nv04_gart_manager_new(struct ttm_resource_manager *man,126 struct ttm_buffer_object *bo,127 const struct ttm_place *place,128 struct ttm_resource **res)129{130 struct nouveau_bo *nvbo = nouveau_bo(bo);131 struct nouveau_drm *drm = nouveau_bdev(bo->bdev);132 struct nouveau_mem *mem;133 int ret;134 135 ret = nouveau_mem_new(drm, nvbo->kind, nvbo->comp, res);136 if (ret)137 return ret;138 139 mem = nouveau_mem(*res);140 ttm_resource_init(bo, place, *res);141 ret = nvif_vmm_get(&drm->client.vmm.vmm, PTES, false, 12, 0,142 (long)(*res)->size, &mem->vma[0]);143 if (ret) {144 nouveau_mem_del(man, *res);145 return ret;146 }147 148 (*res)->start = mem->vma[0].addr >> PAGE_SHIFT;149 return 0;150}151 152const struct ttm_resource_manager_func nv04_gart_manager = {153 .alloc = nv04_gart_manager_new,154 .free = nouveau_manager_del,155 .intersects = nouveau_manager_intersects,156 .compatible = nouveau_manager_compatible,157};158 159static int160nouveau_ttm_init_host(struct nouveau_drm *drm, u8 kind)161{162 struct nvif_mmu *mmu = &drm->client.mmu;163 int typei;164 165 typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE |166 kind | NVIF_MEM_COHERENT);167 if (typei < 0)168 return -ENOSYS;169 170 drm->ttm.type_host[!!kind] = typei;171 172 typei = nvif_mmu_type(mmu, NVIF_MEM_HOST | NVIF_MEM_MAPPABLE | kind);173 if (typei < 0)174 return -ENOSYS;175 176 drm->ttm.type_ncoh[!!kind] = typei;177 return 0;178}179 180static int181nouveau_ttm_init_vram(struct nouveau_drm *drm)182{183 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {184 struct ttm_resource_manager *man = kzalloc(sizeof(*man), GFP_KERNEL);185 186 if (!man)187 return -ENOMEM;188 189 man->func = &nouveau_vram_manager;190 191 ttm_resource_manager_init(man, &drm->ttm.bdev,192 drm->gem.vram_available >> PAGE_SHIFT);193 ttm_set_driver_manager(&drm->ttm.bdev, TTM_PL_VRAM, man);194 ttm_resource_manager_set_used(man, true);195 return 0;196 } else {197 return ttm_range_man_init(&drm->ttm.bdev, TTM_PL_VRAM, false,198 drm->gem.vram_available >> PAGE_SHIFT);199 }200}201 202static void203nouveau_ttm_fini_vram(struct nouveau_drm *drm)204{205 struct ttm_resource_manager *man = ttm_manager_type(&drm->ttm.bdev, TTM_PL_VRAM);206 207 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {208 ttm_resource_manager_set_used(man, false);209 ttm_resource_manager_evict_all(&drm->ttm.bdev, man);210 ttm_resource_manager_cleanup(man);211 ttm_set_driver_manager(&drm->ttm.bdev, TTM_PL_VRAM, NULL);212 kfree(man);213 } else214 ttm_range_man_fini(&drm->ttm.bdev, TTM_PL_VRAM);215}216 217static int218nouveau_ttm_init_gtt(struct nouveau_drm *drm)219{220 struct ttm_resource_manager *man;221 unsigned long size_pages = drm->gem.gart_available >> PAGE_SHIFT;222 const struct ttm_resource_manager_func *func = NULL;223 224 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)225 func = &nouveau_gart_manager;226 else if (!drm->agp.bridge)227 func = &nv04_gart_manager;228 else229 return ttm_range_man_init(&drm->ttm.bdev, TTM_PL_TT, true,230 size_pages);231 232 man = kzalloc(sizeof(*man), GFP_KERNEL);233 if (!man)234 return -ENOMEM;235 236 man->func = func;237 man->use_tt = true;238 ttm_resource_manager_init(man, &drm->ttm.bdev, size_pages);239 ttm_set_driver_manager(&drm->ttm.bdev, TTM_PL_TT, man);240 ttm_resource_manager_set_used(man, true);241 return 0;242}243 244static void245nouveau_ttm_fini_gtt(struct nouveau_drm *drm)246{247 struct ttm_resource_manager *man = ttm_manager_type(&drm->ttm.bdev, TTM_PL_TT);248 249 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA &&250 drm->agp.bridge)251 ttm_range_man_fini(&drm->ttm.bdev, TTM_PL_TT);252 else {253 ttm_resource_manager_set_used(man, false);254 ttm_resource_manager_evict_all(&drm->ttm.bdev, man);255 ttm_resource_manager_cleanup(man);256 ttm_set_driver_manager(&drm->ttm.bdev, TTM_PL_TT, NULL);257 kfree(man);258 }259}260 261int262nouveau_ttm_init(struct nouveau_drm *drm)263{264 struct nvkm_device *device = nvxx_device(drm);265 struct nvkm_pci *pci = device->pci;266 struct nvif_mmu *mmu = &drm->client.mmu;267 struct drm_device *dev = drm->dev;268 int typei, ret;269 270 ret = nouveau_ttm_init_host(drm, 0);271 if (ret)272 return ret;273 274 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&275 drm->client.device.info.chipset != 0x50) {276 ret = nouveau_ttm_init_host(drm, NVIF_MEM_KIND);277 if (ret)278 return ret;279 }280 281 if (drm->client.device.info.platform != NV_DEVICE_INFO_V0_SOC &&282 drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {283 typei = nvif_mmu_type(mmu, NVIF_MEM_VRAM | NVIF_MEM_MAPPABLE |284 NVIF_MEM_KIND |285 NVIF_MEM_COMP |286 NVIF_MEM_DISP);287 if (typei < 0)288 return -ENOSYS;289 290 drm->ttm.type_vram = typei;291 } else {292 drm->ttm.type_vram = -1;293 }294 295 if (pci && pci->agp.bridge) {296 drm->agp.bridge = pci->agp.bridge;297 drm->agp.base = pci->agp.base;298 drm->agp.size = pci->agp.size;299 drm->agp.cma = pci->agp.cma;300 }301 302 ret = ttm_device_init(&drm->ttm.bdev, &nouveau_bo_driver, drm->dev->dev,303 dev->anon_inode->i_mapping,304 dev->vma_offset_manager,305 drm_need_swiotlb(drm->client.mmu.dmabits),306 drm->client.mmu.dmabits <= 32);307 if (ret) {308 NV_ERROR(drm, "error initialising bo driver, %d\n", ret);309 return ret;310 }311 312 /* VRAM init */313 drm->gem.vram_available = drm->client.device.info.ram_user;314 315 arch_io_reserve_memtype_wc(device->func->resource_addr(device, 1),316 device->func->resource_size(device, 1));317 318 ret = nouveau_ttm_init_vram(drm);319 if (ret) {320 NV_ERROR(drm, "VRAM mm init failed, %d\n", ret);321 return ret;322 }323 324 drm->ttm.mtrr = arch_phys_wc_add(device->func->resource_addr(device, 1),325 device->func->resource_size(device, 1));326 327 /* GART init */328 if (!drm->agp.bridge) {329 drm->gem.gart_available = drm->client.vmm.vmm.limit;330 } else {331 drm->gem.gart_available = drm->agp.size;332 }333 334 ret = nouveau_ttm_init_gtt(drm);335 if (ret) {336 NV_ERROR(drm, "GART mm init failed, %d\n", ret);337 return ret;338 }339 340 mutex_init(&drm->ttm.io_reserve_mutex);341 INIT_LIST_HEAD(&drm->ttm.io_reserve_lru);342 343 NV_INFO(drm, "VRAM: %d MiB\n", (u32)(drm->gem.vram_available >> 20));344 NV_INFO(drm, "GART: %d MiB\n", (u32)(drm->gem.gart_available >> 20));345 return 0;346}347 348void349nouveau_ttm_fini(struct nouveau_drm *drm)350{351 struct nvkm_device *device = nvxx_device(drm);352 353 nouveau_ttm_fini_vram(drm);354 nouveau_ttm_fini_gtt(drm);355 356 ttm_device_fini(&drm->ttm.bdev);357 358 arch_phys_wc_del(drm->ttm.mtrr);359 drm->ttm.mtrr = 0;360 arch_io_free_memtype_wc(device->func->resource_addr(device, 1),361 device->func->resource_size(device, 1));362 363}364