381 lines · c
1#ifndef __NVKM_VMM_H__2#define __NVKM_VMM_H__3#include "priv.h"4#include <core/memory.h>5enum nvkm_memory_target;6 7struct nvkm_vmm_pt {8 /* Some GPUs have a mapping level with a dual page tables to9 * support large and small pages in the same address-range.10 *11 * We track the state of both page tables in one place, which12 * is why there's multiple PT pointers/refcounts here.13 */14 struct nvkm_mmu_pt *pt[2];15 u32 refs[2];16 17 /* Page size handled by this PT.18 *19 * Tesla backend needs to know this when writinge PDEs,20 * otherwise unnecessary.21 */22 u8 page;23 24 /* Entire page table sparse.25 *26 * Used to propagate sparseness to child page tables.27 */28 bool sparse:1;29 30 /* Tracking for page directories.31 *32 * The array is indexed by PDE, and will either point to the33 * child page table, or indicate the PDE is marked as sparse.34 **/35#define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)36#define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)37#define NVKM_VMM_PDE_SPARSE ERR_PTR(-EBUSY)38 struct nvkm_vmm_pt **pde;39 40 /* Tracking for dual page tables.41 *42 * There's one entry for each LPTE, keeping track of whether43 * there are valid SPTEs in the same address-range.44 *45 * This information is used to manage LPTE state transitions.46 */47#define NVKM_VMM_PTE_SPARSE 0x8048#define NVKM_VMM_PTE_VALID 0x4049#define NVKM_VMM_PTE_SPTES 0x3f50 u8 pte[];51};52 53typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,54 struct nvkm_mmu_pt *, u32 ptei, u32 ptes);55typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,56 struct nvkm_vmm_pt *, u32 pdei);57typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,58 u32 ptei, u32 ptes, struct nvkm_vmm_map *);59 60struct nvkm_vmm_desc_func {61 nvkm_vmm_pxe_func invalid;62 nvkm_vmm_pxe_func unmap;63 nvkm_vmm_pxe_func sparse;64 65 nvkm_vmm_pde_func pde;66 67 nvkm_vmm_pte_func mem;68 nvkm_vmm_pte_func dma;69 nvkm_vmm_pte_func sgl;70 71 nvkm_vmm_pte_func pfn;72 bool (*pfn_clear)(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32 ptei, u32 ptes);73 nvkm_vmm_pxe_func pfn_unmap;74};75 76extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;77void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);78extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;79void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);80void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,81 struct nvkm_vmm_map *);82void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,83 struct nvkm_vmm_map *);84void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,85 struct nvkm_vmm_map *);86 87void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);88 89struct nvkm_vmm_desc {90 enum {91 PGD,92 PGT,93 SPT,94 LPT,95 } type;96 u8 bits; /* VMA bits covered by PT. */97 u8 size; /* Bytes-per-PTE. */98 u32 align; /* PT address alignment. */99 const struct nvkm_vmm_desc_func *func;100};101 102extern const struct nvkm_vmm_desc nv50_vmm_desc_12[];103extern const struct nvkm_vmm_desc nv50_vmm_desc_16[];104 105extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];106extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];107extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];108extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];109 110extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];111extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];112extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];113extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];114 115extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];116extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];117 118struct nvkm_vmm_page {119 u8 shift;120 const struct nvkm_vmm_desc *desc;121#define NVKM_VMM_PAGE_SPARSE 0x01122#define NVKM_VMM_PAGE_VRAM 0x02123#define NVKM_VMM_PAGE_HOST 0x04124#define NVKM_VMM_PAGE_COMP 0x08125#define NVKM_VMM_PAGE_Sxxx (NVKM_VMM_PAGE_SPARSE)126#define NVKM_VMM_PAGE_xVxx (NVKM_VMM_PAGE_VRAM)127#define NVKM_VMM_PAGE_SVxx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)128#define NVKM_VMM_PAGE_xxHx (NVKM_VMM_PAGE_HOST)129#define NVKM_VMM_PAGE_SxHx (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)130#define NVKM_VMM_PAGE_xVHx (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)131#define NVKM_VMM_PAGE_SVHx (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)132#define NVKM_VMM_PAGE_xVxC (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)133#define NVKM_VMM_PAGE_SVxC (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)134#define NVKM_VMM_PAGE_xxHC (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)135#define NVKM_VMM_PAGE_SxHC (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)136 u8 type;137};138 139struct nvkm_vmm_func {140 int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);141 void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);142 143 int (*aper)(enum nvkm_memory_target);144 int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,145 struct nvkm_vmm_map *);146 void (*flush)(struct nvkm_vmm *, int depth);147 148 int (*mthd)(struct nvkm_vmm *, struct nvkm_client *,149 u32 mthd, void *argv, u32 argc);150 151 void (*invalidate_pdb)(struct nvkm_vmm *, u64 addr);152 153 u64 page_block;154 const struct nvkm_vmm_page page[];155};156 157struct nvkm_vmm_join {158 struct nvkm_memory *inst;159 struct list_head head;160};161 162int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,163 u32 pd_header, bool managed, u64 addr, u64 size,164 struct lock_class_key *, const char *name,165 struct nvkm_vmm **);166struct nvkm_vma *nvkm_vma_new(u64 addr, u64 size);167struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);168struct nvkm_vma *nvkm_vmm_node_split(struct nvkm_vmm *, struct nvkm_vma *,169 u64 addr, u64 size);170int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,171 bool sparse, u8 page, u8 align, u64 size,172 struct nvkm_vma **pvma);173void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);174void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *, bool pfn);175void nvkm_vmm_unmap_region(struct nvkm_vmm *, struct nvkm_vma *);176 177int nvkm_vmm_raw_get(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd);178void nvkm_vmm_raw_put(struct nvkm_vmm *vmm, u64 addr, u64 size, u8 refd);179void nvkm_vmm_raw_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size,180 bool sparse, u8 refd);181int nvkm_vmm_raw_sparse(struct nvkm_vmm *, u64 addr, u64 size, bool ref);182 183static inline bool184nvkm_vmm_in_managed_range(struct nvkm_vmm *vmm, u64 start, u64 size)185{186 u64 p_start = vmm->managed.p.addr;187 u64 p_end = p_start + vmm->managed.p.size;188 u64 n_start = vmm->managed.n.addr;189 u64 n_end = n_start + vmm->managed.n.size;190 u64 end = start + size;191 192 if (start >= p_start && end <= p_end)193 return true;194 195 if (start >= n_start && end <= n_end)196 return true;197 198 return false;199}200 201#define NVKM_VMM_PFN_ADDR 0xfffffffffffff000ULL202#define NVKM_VMM_PFN_ADDR_SHIFT 12203#define NVKM_VMM_PFN_APER 0x00000000000000f0ULL204#define NVKM_VMM_PFN_HOST 0x0000000000000000ULL205#define NVKM_VMM_PFN_VRAM 0x0000000000000010ULL206#define NVKM_VMM_PFN_A 0x0000000000000004ULL207#define NVKM_VMM_PFN_W 0x0000000000000002ULL208#define NVKM_VMM_PFN_V 0x0000000000000001ULL209#define NVKM_VMM_PFN_NONE 0x0000000000000000ULL210 211int nvkm_vmm_pfn_map(struct nvkm_vmm *, u8 page, u64 addr, u64 size, u64 *pfn);212int nvkm_vmm_pfn_unmap(struct nvkm_vmm *, u64 addr, u64 size);213 214struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);215 216int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,217 bool, u64, u64, void *, u32, struct lock_class_key *,218 const char *, struct nvkm_vmm **);219int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);220 221int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);222void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);223int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);224void nv50_vmm_flush(struct nvkm_vmm *, int);225 226int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,227 struct nvkm_mmu *, bool, u64, u64, void *, u32,228 struct lock_class_key *, const char *, struct nvkm_vmm **);229int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);230int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);231void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);232int gf100_vmm_aper(enum nvkm_memory_target);233int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);234void gf100_vmm_flush(struct nvkm_vmm *, int);235void gf100_vmm_invalidate(struct nvkm_vmm *, u32 type);236void gf100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);237 238int gk20a_vmm_aper(enum nvkm_memory_target);239 240int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,241 struct nvkm_mmu *, bool, u64, u64, void *, u32,242 struct lock_class_key *, const char *, struct nvkm_vmm **);243int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);244int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);245 246int gp100_vmm_new_(const struct nvkm_vmm_func *,247 struct nvkm_mmu *, bool, u64, u64, void *, u32,248 struct lock_class_key *, const char *, struct nvkm_vmm **);249int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);250int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);251void gp100_vmm_flush(struct nvkm_vmm *, int);252int gp100_vmm_mthd(struct nvkm_vmm *, struct nvkm_client *, u32, void *, u32);253void gp100_vmm_invalidate_pdb(struct nvkm_vmm *, u64 addr);254 255int gv100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);256 257int nv04_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,258 struct lock_class_key *, const char *, struct nvkm_vmm **);259int nv41_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,260 struct lock_class_key *, const char *, struct nvkm_vmm **);261int nv44_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,262 struct lock_class_key *, const char *, struct nvkm_vmm **);263int nv50_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,264 struct lock_class_key *, const char *, struct nvkm_vmm **);265int mcp77_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,266 struct lock_class_key *, const char *, struct nvkm_vmm **);267int g84_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,268 struct lock_class_key *, const char *, struct nvkm_vmm **);269int gf100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,270 struct lock_class_key *, const char *, struct nvkm_vmm **);271int gk104_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,272 struct lock_class_key *, const char *, struct nvkm_vmm **);273int gk20a_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,274 struct lock_class_key *, const char *, struct nvkm_vmm **);275int gm200_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,276 struct lock_class_key *, const char *,277 struct nvkm_vmm **);278int gm200_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,279 struct lock_class_key *, const char *,280 struct nvkm_vmm **);281int gm20b_vmm_new_fixed(struct nvkm_mmu *, bool, u64, u64, void *, u32,282 struct lock_class_key *, const char *,283 struct nvkm_vmm **);284int gm20b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,285 struct lock_class_key *, const char *,286 struct nvkm_vmm **);287int gp100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,288 struct lock_class_key *, const char *,289 struct nvkm_vmm **);290int gp10b_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,291 struct lock_class_key *, const char *,292 struct nvkm_vmm **);293int gv100_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,294 struct lock_class_key *, const char *,295 struct nvkm_vmm **);296int tu102_vmm_new(struct nvkm_mmu *, bool, u64, u64, void *, u32,297 struct lock_class_key *, const char *,298 struct nvkm_vmm **);299 300#define VMM_PRINT(l,v,p,f,a...) do { \301 struct nvkm_vmm *_vmm = (v); \302 if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) { \303 nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n", \304 _vmm->name, ##a); \305 } \306} while(0)307#define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)308#define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)309#define VMM_SPAM(v,f,a...) VMM_PRINT(NV_DBG_SPAM , (v), dbg, f, ##a)310 311#define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do { \312 nvkm_kmap((PT)->memory); \313 while (PTEN) { \314 u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift; \315 u64 _addr = ((BASE) + MAP->off); \316 \317 if (_ptes > PTEN) { \318 MAP->off += PTEN << MAP->page->shift; \319 _ptes = PTEN; \320 } else { \321 MAP->off = 0; \322 NEXT; \323 } \324 \325 VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes); \326 \327 FILL(VMM, PT, PTEI, _ptes, MAP, _addr); \328 PTEI += _ptes; \329 PTEN -= _ptes; \330 } \331 nvkm_done((PT)->memory); \332} while(0)333 334#define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL) \335 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \336 ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT), \337 ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT), \338 (MAP->mem = MAP->mem->next))339#define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL) \340 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \341 *MAP->dma, PAGE_SIZE, MAP->dma++)342#define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL) \343 VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL, \344 sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl), \345 (MAP->sgl = sg_next(MAP->sgl)))346 347#define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))348#define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))349#define VMM_XO(m,v,o,d,c,b,fn,f,a...) do { \350 const u32 _pteo = (o); u##b _data = (d); \351 VMM_SPAM((v), " %010llx "f, (m)->addr + _pteo, _data, ##a); \352 VMM_##fn((m), (m)->base + _pteo, _data, (c), b); \353} while(0)354 355#define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 32, WO, "%08x")356#define VMM_FO032(m,v,o,d,c) \357 VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))358 359#define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d), 1, 64, WO, "%016llx")360#define VMM_FO064(m,v,o,d,c) \361 VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c))362 363#define VMM_XO128(m,v,o,lo,hi,c,f,a...) do { \364 u32 _pteo = (o), _ptes = (c); \365 const u64 _addr = (m)->addr + _pteo; \366 VMM_SPAM((v), " %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a); \367 while (_ptes--) { \368 nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo)); \369 nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi)); \370 _pteo += 0x10; \371 } \372} while(0)373 374#define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")375#define VMM_FO128(m,v,o,lo,hi,c) do { \376 nvkm_kmap((m)->memory); \377 VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c)); \378 nvkm_done((m)->memory); \379} while(0)380#endif381