388 lines · c
1/*2 * Copyright 2008 Advanced Micro Devices, Inc.3 * Copyright 2008 Red Hat Inc.4 * Copyright 2009 Jerome Glisse.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice shall be included in14 * all copies or substantial portions of the Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR22 * OTHER DEALINGS IN THE SOFTWARE.23 *24 * Authors: Dave Airlie25 * Alex Deucher26 * Jerome Glisse27 */28 29#include <linux/pci.h>30#include <linux/vmalloc.h>31 32#include <drm/radeon_drm.h>33#ifdef CONFIG_X8634#include <asm/set_memory.h>35#endif36#include "radeon.h"37 38/*39 * GART40 * The GART (Graphics Aperture Remapping Table) is an aperture41 * in the GPU's address space. System pages can be mapped into42 * the aperture and look like contiguous pages from the GPU's43 * perspective. A page table maps the pages in the aperture44 * to the actual backing pages in system memory.45 *46 * Radeon GPUs support both an internal GART, as described above,47 * and AGP. AGP works similarly, but the GART table is configured48 * and maintained by the northbridge rather than the driver.49 * Radeon hw has a separate AGP aperture that is programmed to50 * point to the AGP aperture provided by the northbridge and the51 * requests are passed through to the northbridge aperture.52 * Both AGP and internal GART can be used at the same time, however53 * that is not currently supported by the driver.54 *55 * This file handles the common internal GART management.56 */57 58/*59 * Common GART table functions.60 */61/**62 * radeon_gart_table_ram_alloc - allocate system ram for gart page table63 *64 * @rdev: radeon_device pointer65 *66 * Allocate system memory for GART page table67 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the68 * gart table to be in system memory.69 * Returns 0 for success, -ENOMEM for failure.70 */71int radeon_gart_table_ram_alloc(struct radeon_device *rdev)72{73 void *ptr;74 75 ptr = dma_alloc_coherent(&rdev->pdev->dev, rdev->gart.table_size,76 &rdev->gart.table_addr, GFP_KERNEL);77 if (!ptr)78 return -ENOMEM;79 80#ifdef CONFIG_X8681 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||82 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {83 set_memory_uc((unsigned long)ptr,84 rdev->gart.table_size >> PAGE_SHIFT);85 }86#endif87 rdev->gart.ptr = ptr;88 return 0;89}90 91/**92 * radeon_gart_table_ram_free - free system ram for gart page table93 *94 * @rdev: radeon_device pointer95 *96 * Free system memory for GART page table97 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the98 * gart table to be in system memory.99 */100void radeon_gart_table_ram_free(struct radeon_device *rdev)101{102 if (!rdev->gart.ptr)103 return;104 105#ifdef CONFIG_X86106 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||107 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {108 set_memory_wb((unsigned long)rdev->gart.ptr,109 rdev->gart.table_size >> PAGE_SHIFT);110 }111#endif112 dma_free_coherent(&rdev->pdev->dev, rdev->gart.table_size,113 (void *)rdev->gart.ptr, rdev->gart.table_addr);114 rdev->gart.ptr = NULL;115 rdev->gart.table_addr = 0;116}117 118/**119 * radeon_gart_table_vram_alloc - allocate vram for gart page table120 *121 * @rdev: radeon_device pointer122 *123 * Allocate video memory for GART page table124 * (pcie r4xx, r5xx+). These asics require the125 * gart table to be in video memory.126 * Returns 0 for success, error for failure.127 */128int radeon_gart_table_vram_alloc(struct radeon_device *rdev)129{130 int r;131 132 if (rdev->gart.robj == NULL) {133 r = radeon_bo_create(rdev, rdev->gart.table_size,134 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,135 0, NULL, NULL, &rdev->gart.robj);136 if (r)137 return r;138 }139 return 0;140}141 142/**143 * radeon_gart_table_vram_pin - pin gart page table in vram144 *145 * @rdev: radeon_device pointer146 *147 * Pin the GART page table in vram so it will not be moved148 * by the memory manager (pcie r4xx, r5xx+). These asics require the149 * gart table to be in video memory.150 * Returns 0 for success, error for failure.151 */152int radeon_gart_table_vram_pin(struct radeon_device *rdev)153{154 uint64_t gpu_addr;155 int r;156 157 r = radeon_bo_reserve(rdev->gart.robj, false);158 if (unlikely(r != 0))159 return r;160 r = radeon_bo_pin(rdev->gart.robj,161 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);162 if (r) {163 radeon_bo_unreserve(rdev->gart.robj);164 return r;165 }166 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);167 if (r)168 radeon_bo_unpin(rdev->gart.robj);169 radeon_bo_unreserve(rdev->gart.robj);170 rdev->gart.table_addr = gpu_addr;171 172 if (!r) {173 int i;174 175 /* We might have dropped some GART table updates while it wasn't176 * mapped, restore all entries177 */178 for (i = 0; i < rdev->gart.num_gpu_pages; i++)179 radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]);180 mb();181 radeon_gart_tlb_flush(rdev);182 }183 184 return r;185}186 187/**188 * radeon_gart_table_vram_unpin - unpin gart page table in vram189 *190 * @rdev: radeon_device pointer191 *192 * Unpin the GART page table in vram (pcie r4xx, r5xx+).193 * These asics require the gart table to be in video memory.194 */195void radeon_gart_table_vram_unpin(struct radeon_device *rdev)196{197 int r;198 199 if (!rdev->gart.robj)200 return;201 202 r = radeon_bo_reserve(rdev->gart.robj, false);203 if (likely(r == 0)) {204 radeon_bo_kunmap(rdev->gart.robj);205 radeon_bo_unpin(rdev->gart.robj);206 radeon_bo_unreserve(rdev->gart.robj);207 rdev->gart.ptr = NULL;208 }209}210 211/**212 * radeon_gart_table_vram_free - free gart page table vram213 *214 * @rdev: radeon_device pointer215 *216 * Free the video memory used for the GART page table217 * (pcie r4xx, r5xx+). These asics require the gart table to218 * be in video memory.219 */220void radeon_gart_table_vram_free(struct radeon_device *rdev)221{222 if (!rdev->gart.robj)223 return;224 225 radeon_bo_unref(&rdev->gart.robj);226}227 228/*229 * Common gart functions.230 */231/**232 * radeon_gart_unbind - unbind pages from the gart page table233 *234 * @rdev: radeon_device pointer235 * @offset: offset into the GPU's gart aperture236 * @pages: number of pages to unbind237 *238 * Unbinds the requested pages from the gart page table and239 * replaces them with the dummy page (all asics).240 */241void radeon_gart_unbind(struct radeon_device *rdev, unsigned int offset,242 int pages)243{244 unsigned int t, p;245 int i, j;246 247 if (!rdev->gart.ready) {248 WARN(1, "trying to unbind memory from uninitialized GART !\n");249 return;250 }251 t = offset / RADEON_GPU_PAGE_SIZE;252 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);253 for (i = 0; i < pages; i++, p++) {254 if (rdev->gart.pages[p]) {255 rdev->gart.pages[p] = NULL;256 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {257 rdev->gart.pages_entry[t] = rdev->dummy_page.entry;258 if (rdev->gart.ptr) {259 radeon_gart_set_page(rdev, t,260 rdev->dummy_page.entry);261 }262 }263 }264 }265 if (rdev->gart.ptr) {266 mb();267 radeon_gart_tlb_flush(rdev);268 }269}270 271/**272 * radeon_gart_bind - bind pages into the gart page table273 *274 * @rdev: radeon_device pointer275 * @offset: offset into the GPU's gart aperture276 * @pages: number of pages to bind277 * @pagelist: pages to bind278 * @dma_addr: DMA addresses of pages279 * @flags: RADEON_GART_PAGE_* flags280 *281 * Binds the requested pages to the gart page table282 * (all asics).283 * Returns 0 for success, -EINVAL for failure.284 */285int radeon_gart_bind(struct radeon_device *rdev, unsigned int offset,286 int pages, struct page **pagelist, dma_addr_t *dma_addr,287 uint32_t flags)288{289 unsigned int t, p;290 uint64_t page_base, page_entry;291 int i, j;292 293 if (!rdev->gart.ready) {294 WARN(1, "trying to bind memory to uninitialized GART !\n");295 return -EINVAL;296 }297 t = offset / RADEON_GPU_PAGE_SIZE;298 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);299 300 for (i = 0; i < pages; i++, p++) {301 rdev->gart.pages[p] = pagelist ? pagelist[i] :302 rdev->dummy_page.page;303 page_base = dma_addr[i];304 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {305 page_entry = radeon_gart_get_page_entry(page_base, flags);306 rdev->gart.pages_entry[t] = page_entry;307 if (rdev->gart.ptr)308 radeon_gart_set_page(rdev, t, page_entry);309 310 page_base += RADEON_GPU_PAGE_SIZE;311 }312 }313 if (rdev->gart.ptr) {314 mb();315 radeon_gart_tlb_flush(rdev);316 }317 return 0;318}319 320/**321 * radeon_gart_init - init the driver info for managing the gart322 *323 * @rdev: radeon_device pointer324 *325 * Allocate the dummy page and init the gart driver info (all asics).326 * Returns 0 for success, error for failure.327 */328int radeon_gart_init(struct radeon_device *rdev)329{330 int r, i;331 332 if (rdev->gart.pages)333 return 0;334 335 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */336 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {337 DRM_ERROR("Page size is smaller than GPU page size!\n");338 return -EINVAL;339 }340 r = radeon_dummy_page_init(rdev);341 if (r)342 return r;343 /* Compute table size */344 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;345 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;346 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",347 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);348 /* Allocate pages table */349 rdev->gart.pages = vzalloc(array_size(sizeof(void *),350 rdev->gart.num_cpu_pages));351 if (rdev->gart.pages == NULL) {352 radeon_gart_fini(rdev);353 return -ENOMEM;354 }355 rdev->gart.pages_entry = vmalloc(array_size(sizeof(uint64_t),356 rdev->gart.num_gpu_pages));357 if (rdev->gart.pages_entry == NULL) {358 radeon_gart_fini(rdev);359 return -ENOMEM;360 }361 /* set GART entry to point to the dummy page by default */362 for (i = 0; i < rdev->gart.num_gpu_pages; i++)363 rdev->gart.pages_entry[i] = rdev->dummy_page.entry;364 return 0;365}366 367/**368 * radeon_gart_fini - tear down the driver info for managing the gart369 *370 * @rdev: radeon_device pointer371 *372 * Tear down the gart driver info and free the dummy page (all asics).373 */374void radeon_gart_fini(struct radeon_device *rdev)375{376 if (rdev->gart.ready) {377 /* unbind pages */378 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);379 }380 rdev->gart.ready = false;381 vfree(rdev->gart.pages);382 vfree(rdev->gart.pages_entry);383 rdev->gart.pages = NULL;384 rdev->gart.pages_entry = NULL;385 386 radeon_dummy_page_fini(rdev);387}388