470 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2020 Intel Corporation4 */5 6#include <linux/log2.h>7 8#include "gem/i915_gem_internal.h"9 10#include "gen6_ppgtt.h"11#include "i915_scatterlist.h"12#include "i915_trace.h"13#include "i915_vgpu.h"14#include "intel_gt_regs.h"15#include "intel_engine_regs.h"16#include "intel_gt.h"17 18/* Write pde (index) from the page directory @pd to the page table @pt */19static void gen6_write_pde(const struct gen6_ppgtt *ppgtt,20 const unsigned int pde,21 const struct i915_page_table *pt)22{23 dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);24 25 /* Caller needs to make sure the write completes if necessary */26 iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID,27 ppgtt->pd_addr + pde);28}29 30void gen7_ppgtt_enable(struct intel_gt *gt)31{32 struct drm_i915_private *i915 = gt->i915;33 struct intel_uncore *uncore = gt->uncore;34 u32 ecochk;35 36 intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);37 38 ecochk = intel_uncore_read(uncore, GAM_ECOCHK);39 if (IS_HASWELL(i915)) {40 ecochk |= ECOCHK_PPGTT_WB_HSW;41 } else {42 ecochk |= ECOCHK_PPGTT_LLC_IVB;43 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;44 }45 intel_uncore_write(uncore, GAM_ECOCHK, ecochk);46}47 48void gen6_ppgtt_enable(struct intel_gt *gt)49{50 struct intel_uncore *uncore = gt->uncore;51 52 intel_uncore_rmw(uncore,53 GAC_ECO_BITS,54 0,55 ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B);56 57 intel_uncore_rmw(uncore,58 GAB_CTL,59 0,60 GAB_CTL_CONT_AFTER_PAGEFAULT);61 62 intel_uncore_rmw(uncore,63 GAM_ECOCHK,64 0,65 ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);66 67 if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */68 intel_uncore_write(uncore,69 GFX_MODE,70 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));71}72 73/* PPGTT support for Sandybdrige/Gen6 and later */74static void gen6_ppgtt_clear_range(struct i915_address_space *vm,75 u64 start, u64 length)76{77 struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));78 const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;79 const gen6_pte_t scratch_pte = vm->scratch[0]->encode;80 unsigned int pde = first_entry / GEN6_PTES;81 unsigned int pte = first_entry % GEN6_PTES;82 unsigned int num_entries = length / I915_GTT_PAGE_SIZE;83 84 while (num_entries) {85 struct i915_page_table * const pt =86 i915_pt_entry(ppgtt->base.pd, pde++);87 const unsigned int count = min(num_entries, GEN6_PTES - pte);88 gen6_pte_t *vaddr;89 90 num_entries -= count;91 92 GEM_BUG_ON(count > atomic_read(&pt->used));93 if (!atomic_sub_return(count, &pt->used))94 ppgtt->scan_for_unused_pt = true;95 96 /*97 * Note that the hw doesn't support removing PDE on the fly98 * (they are cached inside the context with no means to99 * invalidate the cache), so we can only reset the PTE100 * entries back to scratch.101 */102 103 vaddr = px_vaddr(pt);104 memset32(vaddr + pte, scratch_pte, count);105 106 pte = 0;107 }108}109 110static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,111 struct i915_vma_resource *vma_res,112 unsigned int pat_index,113 u32 flags)114{115 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);116 struct i915_page_directory * const pd = ppgtt->pd;117 unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE;118 unsigned int act_pt = first_entry / GEN6_PTES;119 unsigned int act_pte = first_entry % GEN6_PTES;120 const u32 pte_encode = vm->pte_encode(0, pat_index, flags);121 struct sgt_dma iter = sgt_dma(vma_res);122 gen6_pte_t *vaddr;123 124 GEM_BUG_ON(!pd->entry[act_pt]);125 126 vaddr = px_vaddr(i915_pt_entry(pd, act_pt));127 do {128 GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);129 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);130 131 iter.dma += I915_GTT_PAGE_SIZE;132 if (iter.dma == iter.max) {133 iter.sg = __sg_next(iter.sg);134 if (!iter.sg || sg_dma_len(iter.sg) == 0)135 break;136 137 iter.dma = sg_dma_address(iter.sg);138 iter.max = iter.dma + sg_dma_len(iter.sg);139 }140 141 if (++act_pte == GEN6_PTES) {142 vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt));143 act_pte = 0;144 }145 } while (1);146 147 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;148}149 150static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end)151{152 struct i915_page_directory * const pd = ppgtt->base.pd;153 struct i915_page_table *pt;154 unsigned int pde;155 156 start = round_down(start, SZ_64K);157 end = round_up(end, SZ_64K) - start;158 159 mutex_lock(&ppgtt->flush);160 161 gen6_for_each_pde(pt, pd, start, end, pde)162 gen6_write_pde(ppgtt, pde, pt);163 164 mb();165 ioread32(ppgtt->pd_addr + pde - 1);166 gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);167 mb();168 169 mutex_unlock(&ppgtt->flush);170}171 172static void gen6_alloc_va_range(struct i915_address_space *vm,173 struct i915_vm_pt_stash *stash,174 u64 start, u64 length)175{176 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));177 struct i915_page_directory * const pd = ppgtt->base.pd;178 struct i915_page_table *pt;179 bool flush = false;180 u64 from = start;181 unsigned int pde;182 183 spin_lock(&pd->lock);184 gen6_for_each_pde(pt, pd, start, length, pde) {185 const unsigned int count = gen6_pte_count(start, length);186 187 if (!pt) {188 spin_unlock(&pd->lock);189 190 pt = stash->pt[0];191 __i915_gem_object_pin_pages(pt->base);192 193 fill32_px(pt, vm->scratch[0]->encode);194 195 spin_lock(&pd->lock);196 if (!pd->entry[pde]) {197 stash->pt[0] = pt->stash;198 atomic_set(&pt->used, 0);199 pd->entry[pde] = pt;200 } else {201 pt = pd->entry[pde];202 }203 204 flush = true;205 }206 207 atomic_add(count, &pt->used);208 }209 spin_unlock(&pd->lock);210 211 if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {212 intel_wakeref_t wakeref;213 214 with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)215 gen6_flush_pd(ppgtt, from, start);216 }217}218 219static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)220{221 struct i915_address_space * const vm = &ppgtt->base.vm;222 int ret;223 224 ret = setup_scratch_page(vm);225 if (ret)226 return ret;227 228 vm->scratch[0]->encode =229 vm->pte_encode(px_dma(vm->scratch[0]),230 i915_gem_get_pat_index(vm->i915,231 I915_CACHE_NONE),232 PTE_READ_ONLY);233 234 vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);235 if (IS_ERR(vm->scratch[1])) {236 ret = PTR_ERR(vm->scratch[1]);237 goto err_scratch0;238 }239 240 ret = map_pt_dma(vm, vm->scratch[1]);241 if (ret)242 goto err_scratch1;243 244 fill32_px(vm->scratch[1], vm->scratch[0]->encode);245 246 return 0;247 248err_scratch1:249 i915_gem_object_put(vm->scratch[1]);250err_scratch0:251 i915_gem_object_put(vm->scratch[0]);252 vm->scratch[0] = NULL;253 return ret;254}255 256static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)257{258 struct i915_page_directory * const pd = ppgtt->base.pd;259 struct i915_page_table *pt;260 u32 pde;261 262 gen6_for_all_pdes(pt, pd, pde)263 if (pt)264 free_pt(&ppgtt->base.vm, pt);265}266 267static void gen6_ppgtt_cleanup(struct i915_address_space *vm)268{269 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));270 271 gen6_ppgtt_free_pd(ppgtt);272 free_scratch(vm);273 274 if (ppgtt->base.pd)275 free_pd(&ppgtt->base.vm, ppgtt->base.pd);276 277 mutex_destroy(&ppgtt->flush);278}279 280static void pd_vma_bind(struct i915_address_space *vm,281 struct i915_vm_pt_stash *stash,282 struct i915_vma_resource *vma_res,283 unsigned int pat_index,284 u32 unused)285{286 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);287 struct gen6_ppgtt *ppgtt = vma_res->private;288 u32 ggtt_offset = vma_res->start / I915_GTT_PAGE_SIZE;289 290 ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10;291 ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;292 293 gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);294}295 296static void pd_vma_unbind(struct i915_address_space *vm,297 struct i915_vma_resource *vma_res)298{299 struct gen6_ppgtt *ppgtt = vma_res->private;300 struct i915_page_directory * const pd = ppgtt->base.pd;301 struct i915_page_table *pt;302 unsigned int pde;303 304 if (!ppgtt->scan_for_unused_pt)305 return;306 307 /* Free all no longer used page tables */308 gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {309 if (!pt || atomic_read(&pt->used))310 continue;311 312 free_pt(&ppgtt->base.vm, pt);313 pd->entry[pde] = NULL;314 }315 316 ppgtt->scan_for_unused_pt = false;317}318 319static const struct i915_vma_ops pd_vma_ops = {320 .bind_vma = pd_vma_bind,321 .unbind_vma = pd_vma_unbind,322};323 324int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww)325{326 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);327 int err;328 329 GEM_BUG_ON(!kref_read(&ppgtt->base.vm.ref));330 331 /*332 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt333 * which will be pinned into every active context.334 * (When vma->pin_count becomes atomic, I expect we will naturally335 * need a larger, unpacked, type and kill this redundancy.)336 */337 if (atomic_add_unless(&ppgtt->pin_count, 1, 0))338 return 0;339 340 /* grab the ppgtt resv to pin the object */341 err = i915_vm_lock_objects(&ppgtt->base.vm, ww);342 if (err)343 return err;344 345 /*346 * PPGTT PDEs reside in the GGTT and consists of 512 entries. The347 * allocator works in address space sizes, so it's multiplied by page348 * size. We allocate at the top of the GTT to avoid fragmentation.349 */350 if (!atomic_read(&ppgtt->pin_count)) {351 err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH);352 353 GEM_BUG_ON(ppgtt->vma->fence);354 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(ppgtt->vma));355 }356 if (!err)357 atomic_inc(&ppgtt->pin_count);358 359 return err;360}361 362static int pd_dummy_obj_get_pages(struct drm_i915_gem_object *obj)363{364 obj->mm.pages = ZERO_SIZE_PTR;365 return 0;366}367 368static void pd_dummy_obj_put_pages(struct drm_i915_gem_object *obj,369 struct sg_table *pages)370{371}372 373static const struct drm_i915_gem_object_ops pd_dummy_obj_ops = {374 .name = "pd_dummy_obj",375 .get_pages = pd_dummy_obj_get_pages,376 .put_pages = pd_dummy_obj_put_pages,377};378 379static struct i915_page_directory *380gen6_alloc_top_pd(struct gen6_ppgtt *ppgtt)381{382 struct i915_ggtt * const ggtt = ppgtt->base.vm.gt->ggtt;383 struct i915_page_directory *pd;384 int err;385 386 pd = __alloc_pd(I915_PDES);387 if (unlikely(!pd))388 return ERR_PTR(-ENOMEM);389 390 pd->pt.base = __i915_gem_object_create_internal(ppgtt->base.vm.gt->i915,391 &pd_dummy_obj_ops,392 I915_PDES * SZ_4K);393 if (IS_ERR(pd->pt.base)) {394 err = PTR_ERR(pd->pt.base);395 pd->pt.base = NULL;396 goto err_pd;397 }398 399 pd->pt.base->base.resv = i915_vm_resv_get(&ppgtt->base.vm);400 pd->pt.base->shares_resv_from = &ppgtt->base.vm;401 402 ppgtt->vma = i915_vma_instance(pd->pt.base, &ggtt->vm, NULL);403 if (IS_ERR(ppgtt->vma)) {404 err = PTR_ERR(ppgtt->vma);405 ppgtt->vma = NULL;406 goto err_pd;407 }408 409 /* The dummy object we create is special, override ops.. */410 ppgtt->vma->ops = &pd_vma_ops;411 ppgtt->vma->private = ppgtt;412 return pd;413 414err_pd:415 free_pd(&ppgtt->base.vm, pd);416 return ERR_PTR(err);417}418 419void gen6_ppgtt_unpin(struct i915_ppgtt *base)420{421 struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);422 423 GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));424 if (atomic_dec_and_test(&ppgtt->pin_count))425 i915_vma_unpin(ppgtt->vma);426}427 428struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt)429{430 struct i915_ggtt * const ggtt = gt->ggtt;431 struct gen6_ppgtt *ppgtt;432 int err;433 434 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);435 if (!ppgtt)436 return ERR_PTR(-ENOMEM);437 438 mutex_init(&ppgtt->flush);439 440 ppgtt_init(&ppgtt->base, gt, 0);441 ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));442 ppgtt->base.vm.top = 1;443 444 ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;445 ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;446 ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;447 ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;448 ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;449 450 ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;451 ppgtt->base.vm.alloc_scratch_dma = alloc_pt_dma;452 ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;453 454 err = gen6_ppgtt_init_scratch(ppgtt);455 if (err)456 goto err_put;457 458 ppgtt->base.pd = gen6_alloc_top_pd(ppgtt);459 if (IS_ERR(ppgtt->base.pd)) {460 err = PTR_ERR(ppgtt->base.pd);461 goto err_put;462 }463 464 return &ppgtt->base;465 466err_put:467 i915_vm_put(&ppgtt->base.vm);468 return ERR_PTR(err);469}470