231 lines · c
1/*2 * Copyright 2017 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "vmm.h"23 24#include <subdev/timer.h>25 26static void27nv44_vmm_pgt_fill(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,28 dma_addr_t *list, u32 ptei, u32 ptes)29{30 u32 pteo = (ptei << 2) & ~0x0000000f;31 u32 tmp[4];32 33 tmp[0] = nvkm_ro32(pt->memory, pteo + 0x0);34 tmp[1] = nvkm_ro32(pt->memory, pteo + 0x4);35 tmp[2] = nvkm_ro32(pt->memory, pteo + 0x8);36 tmp[3] = nvkm_ro32(pt->memory, pteo + 0xc);37 38 while (ptes--) {39 u32 addr = (list ? *list++ : vmm->null) >> 12;40 switch (ptei++ & 0x3) {41 case 0:42 tmp[0] &= ~0x07ffffff;43 tmp[0] |= addr;44 break;45 case 1:46 tmp[0] &= ~0xf8000000;47 tmp[0] |= addr << 27;48 tmp[1] &= ~0x003fffff;49 tmp[1] |= addr >> 5;50 break;51 case 2:52 tmp[1] &= ~0xffc00000;53 tmp[1] |= addr << 22;54 tmp[2] &= ~0x0001ffff;55 tmp[2] |= addr >> 10;56 break;57 case 3:58 tmp[2] &= ~0xfffe0000;59 tmp[2] |= addr << 17;60 tmp[3] &= ~0x00000fff;61 tmp[3] |= addr >> 15;62 break;63 }64 }65 66 VMM_WO032(pt, vmm, pteo + 0x0, tmp[0]);67 VMM_WO032(pt, vmm, pteo + 0x4, tmp[1]);68 VMM_WO032(pt, vmm, pteo + 0x8, tmp[2]);69 VMM_WO032(pt, vmm, pteo + 0xc, tmp[3] | 0x40000000);70}71 72static void73nv44_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,74 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)75{76 dma_addr_t tmp[4], i;77 78 if (ptei & 3) {79 const u32 pten = min(ptes, 4 - (ptei & 3));80 for (i = 0; i < pten; i++, addr += 0x1000)81 tmp[i] = addr;82 nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, pten);83 ptei += pten;84 ptes -= pten;85 }86 87 while (ptes >= 4) {88 for (i = 0; i < 4; i++, addr += 0x1000)89 tmp[i] = addr >> 12;90 VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);91 VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);92 VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);93 VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);94 ptes -= 4;95 }96 97 if (ptes) {98 for (i = 0; i < ptes; i++, addr += 0x1000)99 tmp[i] = addr;100 nv44_vmm_pgt_fill(vmm, pt, tmp, ptei, ptes);101 }102}103 104static void105nv44_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,106 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)107{108 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);109}110 111static void112nv44_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,113 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)114{115#if PAGE_SHIFT == 12116 nvkm_kmap(pt->memory);117 if (ptei & 3) {118 const u32 pten = min(ptes, 4 - (ptei & 3));119 nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, pten);120 ptei += pten;121 ptes -= pten;122 map->dma += pten;123 }124 125 while (ptes >= 4) {126 u32 tmp[4], i;127 for (i = 0; i < 4; i++)128 tmp[i] = *map->dma++ >> 12;129 VMM_WO032(pt, vmm, ptei++ * 4, tmp[0] >> 0 | tmp[1] << 27);130 VMM_WO032(pt, vmm, ptei++ * 4, tmp[1] >> 5 | tmp[2] << 22);131 VMM_WO032(pt, vmm, ptei++ * 4, tmp[2] >> 10 | tmp[3] << 17);132 VMM_WO032(pt, vmm, ptei++ * 4, tmp[3] >> 15 | 0x40000000);133 ptes -= 4;134 }135 136 if (ptes) {137 nv44_vmm_pgt_fill(vmm, pt, map->dma, ptei, ptes);138 map->dma += ptes;139 }140 nvkm_done(pt->memory);141#else142 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, nv44_vmm_pgt_pte);143#endif144}145 146static void147nv44_vmm_pgt_unmap(struct nvkm_vmm *vmm,148 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)149{150 nvkm_kmap(pt->memory);151 if (ptei & 3) {152 const u32 pten = min(ptes, 4 - (ptei & 3));153 nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, pten);154 ptei += pten;155 ptes -= pten;156 }157 158 while (ptes > 4) {159 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);160 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);161 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);162 VMM_WO032(pt, vmm, ptei++ * 4, 0x00000000);163 ptes -= 4;164 }165 166 if (ptes)167 nv44_vmm_pgt_fill(vmm, pt, NULL, ptei, ptes);168 nvkm_done(pt->memory);169}170 171static const struct nvkm_vmm_desc_func172nv44_vmm_desc_pgt = {173 .unmap = nv44_vmm_pgt_unmap,174 .dma = nv44_vmm_pgt_dma,175 .sgl = nv44_vmm_pgt_sgl,176};177 178static const struct nvkm_vmm_desc179nv44_vmm_desc_12[] = {180 { PGT, 17, 4, 0x80000, &nv44_vmm_desc_pgt },181 {}182};183 184static void185nv44_vmm_flush(struct nvkm_vmm *vmm, int level)186{187 struct nvkm_device *device = vmm->mmu->subdev.device;188 nvkm_wr32(device, 0x100814, vmm->limit - 4096);189 nvkm_wr32(device, 0x100808, 0x000000020);190 nvkm_msec(device, 2000,191 if (nvkm_rd32(device, 0x100808) & 0x00000001)192 break;193 );194 nvkm_wr32(device, 0x100808, 0x00000000);195}196 197static const struct nvkm_vmm_func198nv44_vmm = {199 .valid = nv04_vmm_valid,200 .flush = nv44_vmm_flush,201 .page = {202 { 12, &nv44_vmm_desc_12[0], NVKM_VMM_PAGE_HOST },203 {}204 }205};206 207int208nv44_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,209 void *argv, u32 argc, struct lock_class_key *key, const char *name,210 struct nvkm_vmm **pvmm)211{212 struct nvkm_subdev *subdev = &mmu->subdev;213 struct nvkm_vmm *vmm;214 int ret;215 216 ret = nv04_vmm_new_(&nv44_vmm, mmu, 0, managed, addr, size,217 argv, argc, key, name, &vmm);218 *pvmm = vmm;219 if (ret)220 return ret;221 222 vmm->nullp = dma_alloc_coherent(subdev->device->dev, 16 * 1024,223 &vmm->null, GFP_KERNEL);224 if (!vmm->nullp) {225 nvkm_warn(subdev, "unable to allocate dummy pages\n");226 vmm->null = 0;227 }228 229 return 0;230}231