1617 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2020 Intel Corporation4 */5 6#include <asm/set_memory.h>7#include <asm/smp.h>8#include <linux/types.h>9#include <linux/stop_machine.h>10 11#include <drm/drm_managed.h>12#include <drm/intel/i915_drm.h>13#include <drm/intel/intel-gtt.h>14 15#include "gem/i915_gem_lmem.h"16 17#include "intel_context.h"18#include "intel_ggtt_gmch.h"19#include "intel_gpu_commands.h"20#include "intel_gt.h"21#include "intel_gt_regs.h"22#include "intel_pci_config.h"23#include "intel_ring.h"24#include "i915_drv.h"25#include "i915_pci.h"26#include "i915_reg.h"27#include "i915_request.h"28#include "i915_scatterlist.h"29#include "i915_utils.h"30#include "i915_vgpu.h"31 32#include "intel_gtt.h"33#include "gen8_ppgtt.h"34#include "intel_engine_pm.h"35 36static void i915_ggtt_color_adjust(const struct drm_mm_node *node,37 unsigned long color,38 u64 *start,39 u64 *end)40{41 if (i915_node_color_differs(node, color))42 *start += I915_GTT_PAGE_SIZE;43 44 /*45 * Also leave a space between the unallocated reserved node after the46 * GTT and any objects within the GTT, i.e. we use the color adjustment47 * to insert a guard page to prevent prefetches crossing over the48 * GTT boundary.49 */50 node = list_next_entry(node, node_list);51 if (node->color != color)52 *end -= I915_GTT_PAGE_SIZE;53}54 55static int ggtt_init_hw(struct i915_ggtt *ggtt)56{57 struct drm_i915_private *i915 = ggtt->vm.i915;58 59 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);60 61 ggtt->vm.is_ggtt = true;62 63 /* Only VLV supports read-only GGTT mappings */64 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915);65 66 if (!HAS_LLC(i915) && !HAS_PPGTT(i915))67 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust;68 69 if (ggtt->mappable_end) {70 if (!io_mapping_init_wc(&ggtt->iomap,71 ggtt->gmadr.start,72 ggtt->mappable_end)) {73 ggtt->vm.cleanup(&ggtt->vm);74 return -EIO;75 }76 77 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start,78 ggtt->mappable_end);79 }80 81 intel_ggtt_init_fences(ggtt);82 83 return 0;84}85 86/**87 * i915_ggtt_init_hw - Initialize GGTT hardware88 * @i915: i915 device89 */90int i915_ggtt_init_hw(struct drm_i915_private *i915)91{92 int ret;93 94 /*95 * Note that we use page colouring to enforce a guard page at the96 * end of the address space. This is required as the CS may prefetch97 * beyond the end of the batch buffer, across the page boundary,98 * and beyond the end of the GTT if we do not provide a guard.99 */100 ret = ggtt_init_hw(to_gt(i915)->ggtt);101 if (ret)102 return ret;103 104 return 0;105}106 107/**108 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM109 * @vm: The VM to suspend the mappings for110 *111 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a112 * DPT page table.113 */114void i915_ggtt_suspend_vm(struct i915_address_space *vm)115{116 struct i915_vma *vma, *vn;117 int save_skip_rewrite;118 119 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);120 121retry:122 i915_gem_drain_freed_objects(vm->i915);123 124 mutex_lock(&vm->mutex);125 126 /*127 * Skip rewriting PTE on VMA unbind.128 * FIXME: Use an argument to i915_vma_unbind() instead?129 */130 save_skip_rewrite = vm->skip_pte_rewrite;131 vm->skip_pte_rewrite = true;132 133 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {134 struct drm_i915_gem_object *obj = vma->obj;135 136 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));137 138 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))139 continue;140 141 /* unlikely to race when GPU is idle, so no worry about slowpath.. */142 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) {143 /*144 * No dead objects should appear here, GPU should be145 * completely idle, and userspace suspended146 */147 i915_gem_object_get(obj);148 149 mutex_unlock(&vm->mutex);150 151 i915_gem_object_lock(obj, NULL);152 GEM_WARN_ON(i915_vma_unbind(vma));153 i915_gem_object_unlock(obj);154 i915_gem_object_put(obj);155 156 vm->skip_pte_rewrite = save_skip_rewrite;157 goto retry;158 }159 160 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) {161 i915_vma_wait_for_bind(vma);162 163 __i915_vma_evict(vma, false);164 drm_mm_remove_node(&vma->node);165 }166 167 i915_gem_object_unlock(obj);168 }169 170 vm->clear_range(vm, 0, vm->total);171 172 vm->skip_pte_rewrite = save_skip_rewrite;173 174 mutex_unlock(&vm->mutex);175}176 177void i915_ggtt_suspend(struct i915_ggtt *ggtt)178{179 struct intel_gt *gt;180 181 i915_ggtt_suspend_vm(&ggtt->vm);182 ggtt->invalidate(ggtt);183 184 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)185 intel_gt_check_and_clear_faults(gt);186}187 188void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)189{190 struct intel_uncore *uncore = ggtt->vm.gt->uncore;191 192 spin_lock_irq(&uncore->lock);193 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);194 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6);195 spin_unlock_irq(&uncore->lock);196}197 198static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915)199{200 /*201 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range202 * will be dropped. For WC mappings in general we have 64 byte burst203 * writes when the WC buffer is flushed, so we can't use it, but have to204 * resort to an uncached mapping. The WC issue is easily caught by the205 * readback check when writing GTT PTE entries.206 */207 if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11)208 return true;209 210 return false;211}212 213static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)214{215 struct intel_uncore *uncore = ggtt->vm.gt->uncore;216 217 /*218 * Note that as an uncached mmio write, this will flush the219 * WCB of the writes into the GGTT before it triggers the invalidate.220 *221 * Only perform this when GGTT is mapped as WC, see ggtt_probe_common().222 */223 if (needs_wc_ggtt_mapping(ggtt->vm.i915))224 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6,225 GFX_FLSH_CNTL_EN);226}227 228static void guc_ggtt_ct_invalidate(struct intel_gt *gt)229{230 struct intel_uncore *uncore = gt->uncore;231 intel_wakeref_t wakeref;232 233 with_intel_runtime_pm_if_active(uncore->rpm, wakeref)234 intel_guc_invalidate_tlb_guc(gt_to_guc(gt));235}236 237static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)238{239 struct drm_i915_private *i915 = ggtt->vm.i915;240 struct intel_gt *gt;241 242 gen8_ggtt_invalidate(ggtt);243 244 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) {245 if (intel_guc_tlb_invalidation_is_available(gt_to_guc(gt)))246 guc_ggtt_ct_invalidate(gt);247 else if (GRAPHICS_VER(i915) >= 12)248 intel_uncore_write_fw(gt->uncore,249 GEN12_GUC_TLB_INV_CR,250 GEN12_GUC_TLB_INV_CR_INVALIDATE);251 else252 intel_uncore_write_fw(gt->uncore,253 GEN8_GTCR, GEN8_GTCR_INVALIDATE);254 }255}256 257static u64 mtl_ggtt_pte_encode(dma_addr_t addr,258 unsigned int pat_index,259 u32 flags)260{261 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;262 263 WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK);264 265 if (flags & PTE_LM)266 pte |= GEN12_GGTT_PTE_LM;267 268 if (pat_index & BIT(0))269 pte |= MTL_GGTT_PTE_PAT0;270 271 if (pat_index & BIT(1))272 pte |= MTL_GGTT_PTE_PAT1;273 274 return pte;275}276 277u64 gen8_ggtt_pte_encode(dma_addr_t addr,278 unsigned int pat_index,279 u32 flags)280{281 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT;282 283 if (flags & PTE_LM)284 pte |= GEN12_GGTT_PTE_LM;285 286 return pte;287}288 289static bool should_update_ggtt_with_bind(struct i915_ggtt *ggtt)290{291 struct intel_gt *gt = ggtt->vm.gt;292 293 return intel_gt_is_bind_context_ready(gt);294}295 296static struct intel_context *gen8_ggtt_bind_get_ce(struct i915_ggtt *ggtt, intel_wakeref_t *wakeref)297{298 struct intel_context *ce;299 struct intel_gt *gt = ggtt->vm.gt;300 301 if (intel_gt_is_wedged(gt))302 return NULL;303 304 ce = gt->engine[BCS0]->bind_context;305 GEM_BUG_ON(!ce);306 307 /*308 * If the GT is not awake already at this stage then fallback309 * to pci based GGTT update otherwise __intel_wakeref_get_first()310 * would conflict with fs_reclaim trying to allocate memory while311 * doing rpm_resume().312 */313 *wakeref = intel_gt_pm_get_if_awake(gt);314 if (!*wakeref)315 return NULL;316 317 intel_engine_pm_get(ce->engine);318 319 return ce;320}321 322static void gen8_ggtt_bind_put_ce(struct intel_context *ce, intel_wakeref_t wakeref)323{324 intel_engine_pm_put(ce->engine);325 intel_gt_pm_put(ce->engine->gt, wakeref);326}327 328static bool gen8_ggtt_bind_ptes(struct i915_ggtt *ggtt, u32 offset,329 struct sg_table *pages, u32 num_entries,330 const gen8_pte_t pte)331{332 struct i915_sched_attr attr = {};333 struct intel_gt *gt = ggtt->vm.gt;334 const gen8_pte_t scratch_pte = ggtt->vm.scratch[0]->encode;335 struct sgt_iter iter;336 struct i915_request *rq;337 struct intel_context *ce;338 intel_wakeref_t wakeref;339 u32 *cs;340 341 if (!num_entries)342 return true;343 344 ce = gen8_ggtt_bind_get_ce(ggtt, &wakeref);345 if (!ce)346 return false;347 348 if (pages)349 iter = __sgt_iter(pages->sgl, true);350 351 while (num_entries) {352 int count = 0;353 dma_addr_t addr;354 /*355 * MI_UPDATE_GTT can update 512 entries in a single command but356 * that end up with engine reset, 511 works.357 */358 u32 n_ptes = min_t(u32, 511, num_entries);359 360 if (mutex_lock_interruptible(&ce->timeline->mutex))361 goto put_ce;362 363 intel_context_enter(ce);364 rq = __i915_request_create(ce, GFP_NOWAIT | GFP_ATOMIC);365 intel_context_exit(ce);366 if (IS_ERR(rq)) {367 GT_TRACE(gt, "Failed to get bind request\n");368 mutex_unlock(&ce->timeline->mutex);369 goto put_ce;370 }371 372 cs = intel_ring_begin(rq, 2 * n_ptes + 2);373 if (IS_ERR(cs)) {374 GT_TRACE(gt, "Failed to ring space for GGTT bind\n");375 i915_request_set_error_once(rq, PTR_ERR(cs));376 /* once a request is created, it must be queued */377 goto queue_err_rq;378 }379 380 *cs++ = MI_UPDATE_GTT | (2 * n_ptes);381 *cs++ = offset << 12;382 383 if (pages) {384 for_each_sgt_daddr_next(addr, iter) {385 if (count == n_ptes)386 break;387 *cs++ = lower_32_bits(pte | addr);388 *cs++ = upper_32_bits(pte | addr);389 count++;390 }391 /* fill remaining with scratch pte, if any */392 if (count < n_ptes) {393 memset64((u64 *)cs, scratch_pte,394 n_ptes - count);395 cs += (n_ptes - count) * 2;396 }397 } else {398 memset64((u64 *)cs, pte, n_ptes);399 cs += n_ptes * 2;400 }401 402 intel_ring_advance(rq, cs);403queue_err_rq:404 i915_request_get(rq);405 __i915_request_commit(rq);406 __i915_request_queue(rq, &attr);407 408 mutex_unlock(&ce->timeline->mutex);409 /* This will break if the request is complete or after engine reset */410 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);411 if (rq->fence.error)412 goto err_rq;413 414 i915_request_put(rq);415 416 num_entries -= n_ptes;417 offset += n_ptes;418 }419 420 gen8_ggtt_bind_put_ce(ce, wakeref);421 return true;422 423err_rq:424 i915_request_put(rq);425put_ce:426 gen8_ggtt_bind_put_ce(ce, wakeref);427 return false;428}429 430static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)431{432 writeq(pte, addr);433}434 435static void gen8_ggtt_insert_page(struct i915_address_space *vm,436 dma_addr_t addr,437 u64 offset,438 unsigned int pat_index,439 u32 flags)440{441 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);442 gen8_pte_t __iomem *pte =443 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;444 445 gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags));446 447 ggtt->invalidate(ggtt);448}449 450static void gen8_ggtt_insert_page_bind(struct i915_address_space *vm,451 dma_addr_t addr, u64 offset,452 unsigned int pat_index, u32 flags)453{454 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);455 gen8_pte_t pte;456 457 pte = ggtt->vm.pte_encode(addr, pat_index, flags);458 if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) &&459 gen8_ggtt_bind_ptes(ggtt, offset, NULL, 1, pte))460 return ggtt->invalidate(ggtt);461 462 gen8_ggtt_insert_page(vm, addr, offset, pat_index, flags);463}464 465static void gen8_ggtt_insert_entries(struct i915_address_space *vm,466 struct i915_vma_resource *vma_res,467 unsigned int pat_index,468 u32 flags)469{470 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);471 const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);472 gen8_pte_t __iomem *gte;473 gen8_pte_t __iomem *end;474 struct sgt_iter iter;475 dma_addr_t addr;476 477 /*478 * Note that we ignore PTE_READ_ONLY here. The caller must be careful479 * not to allow the user to override access to a read only page.480 */481 482 gte = (gen8_pte_t __iomem *)ggtt->gsm;483 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;484 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;485 while (gte < end)486 gen8_set_pte(gte++, vm->scratch[0]->encode);487 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;488 489 for_each_sgt_daddr(addr, iter, vma_res->bi.pages)490 gen8_set_pte(gte++, pte_encode | addr);491 GEM_BUG_ON(gte > end);492 493 /* Fill the allocated but "unused" space beyond the end of the buffer */494 while (gte < end)495 gen8_set_pte(gte++, vm->scratch[0]->encode);496 497 /*498 * We want to flush the TLBs only after we're certain all the PTE499 * updates have finished.500 */501 ggtt->invalidate(ggtt);502}503 504static bool __gen8_ggtt_insert_entries_bind(struct i915_address_space *vm,505 struct i915_vma_resource *vma_res,506 unsigned int pat_index, u32 flags)507{508 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);509 gen8_pte_t scratch_pte = vm->scratch[0]->encode;510 gen8_pte_t pte_encode;511 u64 start, end;512 513 pte_encode = ggtt->vm.pte_encode(0, pat_index, flags);514 start = (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;515 end = start + vma_res->guard / I915_GTT_PAGE_SIZE;516 if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte))517 goto err;518 519 start = end;520 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;521 if (!gen8_ggtt_bind_ptes(ggtt, start, vma_res->bi.pages,522 vma_res->node_size / I915_GTT_PAGE_SIZE, pte_encode))523 goto err;524 525 start += vma_res->node_size / I915_GTT_PAGE_SIZE;526 if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte))527 goto err;528 529 return true;530 531err:532 return false;533}534 535static void gen8_ggtt_insert_entries_bind(struct i915_address_space *vm,536 struct i915_vma_resource *vma_res,537 unsigned int pat_index, u32 flags)538{539 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);540 541 if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) &&542 __gen8_ggtt_insert_entries_bind(vm, vma_res, pat_index, flags))543 return ggtt->invalidate(ggtt);544 545 gen8_ggtt_insert_entries(vm, vma_res, pat_index, flags);546}547 548static void gen8_ggtt_clear_range(struct i915_address_space *vm,549 u64 start, u64 length)550{551 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);552 unsigned int first_entry = start / I915_GTT_PAGE_SIZE;553 unsigned int num_entries = length / I915_GTT_PAGE_SIZE;554 const gen8_pte_t scratch_pte = vm->scratch[0]->encode;555 gen8_pte_t __iomem *gtt_base =556 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;557 const int max_entries = ggtt_total_entries(ggtt) - first_entry;558 int i;559 560 if (WARN(num_entries > max_entries,561 "First entry = %d; Num entries = %d (max=%d)\n",562 first_entry, num_entries, max_entries))563 num_entries = max_entries;564 565 for (i = 0; i < num_entries; i++)566 gen8_set_pte(>t_base[i], scratch_pte);567}568 569static void gen8_ggtt_scratch_range_bind(struct i915_address_space *vm,570 u64 start, u64 length)571{572 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);573 unsigned int first_entry = start / I915_GTT_PAGE_SIZE;574 unsigned int num_entries = length / I915_GTT_PAGE_SIZE;575 const gen8_pte_t scratch_pte = vm->scratch[0]->encode;576 const int max_entries = ggtt_total_entries(ggtt) - first_entry;577 578 if (WARN(num_entries > max_entries,579 "First entry = %d; Num entries = %d (max=%d)\n",580 first_entry, num_entries, max_entries))581 num_entries = max_entries;582 583 if (should_update_ggtt_with_bind(ggtt) && gen8_ggtt_bind_ptes(ggtt, first_entry,584 NULL, num_entries, scratch_pte))585 return ggtt->invalidate(ggtt);586 587 gen8_ggtt_clear_range(vm, start, length);588}589 590static void gen6_ggtt_insert_page(struct i915_address_space *vm,591 dma_addr_t addr,592 u64 offset,593 unsigned int pat_index,594 u32 flags)595{596 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);597 gen6_pte_t __iomem *pte =598 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;599 600 iowrite32(vm->pte_encode(addr, pat_index, flags), pte);601 602 ggtt->invalidate(ggtt);603}604 605/*606 * Binds an object into the global gtt with the specified cache level.607 * The object will be accessible to the GPU via commands whose operands608 * reference offsets within the global GTT as well as accessible by the GPU609 * through the GMADR mapped BAR (i915->mm.gtt->gtt).610 */611static void gen6_ggtt_insert_entries(struct i915_address_space *vm,612 struct i915_vma_resource *vma_res,613 unsigned int pat_index,614 u32 flags)615{616 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);617 gen6_pte_t __iomem *gte;618 gen6_pte_t __iomem *end;619 struct sgt_iter iter;620 dma_addr_t addr;621 622 gte = (gen6_pte_t __iomem *)ggtt->gsm;623 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE;624 625 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE;626 while (gte < end)627 iowrite32(vm->scratch[0]->encode, gte++);628 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE;629 for_each_sgt_daddr(addr, iter, vma_res->bi.pages)630 iowrite32(vm->pte_encode(addr, pat_index, flags), gte++);631 GEM_BUG_ON(gte > end);632 633 /* Fill the allocated but "unused" space beyond the end of the buffer */634 while (gte < end)635 iowrite32(vm->scratch[0]->encode, gte++);636 637 /*638 * We want to flush the TLBs only after we're certain all the PTE639 * updates have finished.640 */641 ggtt->invalidate(ggtt);642}643 644static void nop_clear_range(struct i915_address_space *vm,645 u64 start, u64 length)646{647}648 649static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)650{651 /*652 * Make sure the internal GAM fifo has been cleared of all GTT653 * writes before exiting stop_machine(). This guarantees that654 * any aperture accesses waiting to start in another process655 * cannot back up behind the GTT writes causing a hang.656 * The register can be any arbitrary GAM register.657 */658 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6);659}660 661struct insert_page {662 struct i915_address_space *vm;663 dma_addr_t addr;664 u64 offset;665 unsigned int pat_index;666};667 668static int bxt_vtd_ggtt_insert_page__cb(void *_arg)669{670 struct insert_page *arg = _arg;671 672 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset,673 arg->pat_index, 0);674 bxt_vtd_ggtt_wa(arg->vm);675 676 return 0;677}678 679static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,680 dma_addr_t addr,681 u64 offset,682 unsigned int pat_index,683 u32 unused)684{685 struct insert_page arg = { vm, addr, offset, pat_index };686 687 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);688}689 690struct insert_entries {691 struct i915_address_space *vm;692 struct i915_vma_resource *vma_res;693 unsigned int pat_index;694 u32 flags;695};696 697static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)698{699 struct insert_entries *arg = _arg;700 701 gen8_ggtt_insert_entries(arg->vm, arg->vma_res,702 arg->pat_index, arg->flags);703 bxt_vtd_ggtt_wa(arg->vm);704 705 return 0;706}707 708static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,709 struct i915_vma_resource *vma_res,710 unsigned int pat_index,711 u32 flags)712{713 struct insert_entries arg = { vm, vma_res, pat_index, flags };714 715 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);716}717 718static void gen6_ggtt_clear_range(struct i915_address_space *vm,719 u64 start, u64 length)720{721 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);722 unsigned int first_entry = start / I915_GTT_PAGE_SIZE;723 unsigned int num_entries = length / I915_GTT_PAGE_SIZE;724 gen6_pte_t scratch_pte, __iomem *gtt_base =725 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;726 const int max_entries = ggtt_total_entries(ggtt) - first_entry;727 int i;728 729 if (WARN(num_entries > max_entries,730 "First entry = %d; Num entries = %d (max=%d)\n",731 first_entry, num_entries, max_entries))732 num_entries = max_entries;733 734 scratch_pte = vm->scratch[0]->encode;735 for (i = 0; i < num_entries; i++)736 iowrite32(scratch_pte, >t_base[i]);737}738 739void intel_ggtt_bind_vma(struct i915_address_space *vm,740 struct i915_vm_pt_stash *stash,741 struct i915_vma_resource *vma_res,742 unsigned int pat_index,743 u32 flags)744{745 u32 pte_flags;746 747 if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK))748 return;749 750 vma_res->bound_flags |= flags;751 752 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */753 pte_flags = 0;754 if (vma_res->bi.readonly)755 pte_flags |= PTE_READ_ONLY;756 if (vma_res->bi.lmem)757 pte_flags |= PTE_LM;758 759 vm->insert_entries(vm, vma_res, pat_index, pte_flags);760 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;761}762 763void intel_ggtt_unbind_vma(struct i915_address_space *vm,764 struct i915_vma_resource *vma_res)765{766 vm->clear_range(vm, vma_res->start, vma_res->vma_size);767}768 769/*770 * Reserve the top of the GuC address space for firmware images. Addresses771 * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC,772 * which makes for a suitable range to hold GuC/HuC firmware images if the773 * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT774 * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk775 * of the same size anyway, which is far more than needed, to keep the logic776 * in uc_fw_ggtt_offset() simple.777 */778#define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP)779 780static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)781{782 u64 offset;783 int ret;784 785 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc))786 return 0;787 788 GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE);789 offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE;790 791 ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw,792 GUC_TOP_RESERVE_SIZE, offset,793 I915_COLOR_UNEVICTABLE, PIN_NOEVICT);794 if (ret)795 drm_dbg(&ggtt->vm.i915->drm,796 "Failed to reserve top of GGTT for GuC\n");797 798 return ret;799}800 801static void ggtt_release_guc_top(struct i915_ggtt *ggtt)802{803 if (drm_mm_node_allocated(&ggtt->uc_fw))804 drm_mm_remove_node(&ggtt->uc_fw);805}806 807static void cleanup_init_ggtt(struct i915_ggtt *ggtt)808{809 ggtt_release_guc_top(ggtt);810 if (drm_mm_node_allocated(&ggtt->error_capture))811 drm_mm_remove_node(&ggtt->error_capture);812 mutex_destroy(&ggtt->error_mutex);813}814 815static int init_ggtt(struct i915_ggtt *ggtt)816{817 /*818 * Let GEM Manage all of the aperture.819 *820 * However, leave one page at the end still bound to the scratch page.821 * There are a number of places where the hardware apparently prefetches822 * past the end of the object, and we've seen multiple hangs with the823 * GPU head pointer stuck in a batchbuffer bound at the last page of the824 * aperture. One page should be enough to keep any prefetching inside825 * of the aperture.826 */827 unsigned long hole_start, hole_end;828 struct drm_mm_node *entry;829 int ret;830 831 /*832 * GuC requires all resources that we're sharing with it to be placed in833 * non-WOPCM memory. If GuC is not present or not in use we still need a834 * small bias as ring wraparound at offset 0 sometimes hangs. No idea835 * why.836 */837 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,838 intel_wopcm_guc_size(&ggtt->vm.gt->wopcm));839 840 ret = intel_vgt_balloon(ggtt);841 if (ret)842 return ret;843 844 mutex_init(&ggtt->error_mutex);845 if (ggtt->mappable_end) {846 /*847 * Reserve a mappable slot for our lockless error capture.848 *849 * We strongly prefer taking address 0x0 in order to protect850 * other critical buffers against accidental overwrites,851 * as writing to address 0 is a very common mistake.852 *853 * Since 0 may already be in use by the system (e.g. the BIOS854 * framebuffer), we let the reservation fail quietly and hope855 * 0 remains reserved always.856 *857 * If we fail to reserve 0, and then fail to find any space858 * for an error-capture, remain silent. We can afford not859 * to reserve an error_capture node as we have fallback860 * paths, and we trust that 0 will remain reserved. However,861 * the only likely reason for failure to insert is a driver862 * bug, which we expect to cause other failures...863 *864 * Since CPU can perform speculative reads on error capture865 * (write-combining allows it) add scratch page after error866 * capture to avoid DMAR errors.867 */868 ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE;869 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE;870 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture))871 drm_mm_insert_node_in_range(&ggtt->vm.mm,872 &ggtt->error_capture,873 ggtt->error_capture.size, 0,874 ggtt->error_capture.color,875 0, ggtt->mappable_end,876 DRM_MM_INSERT_LOW);877 }878 if (drm_mm_node_allocated(&ggtt->error_capture)) {879 u64 start = ggtt->error_capture.start;880 u64 size = ggtt->error_capture.size;881 882 ggtt->vm.scratch_range(&ggtt->vm, start, size);883 drm_dbg(&ggtt->vm.i915->drm,884 "Reserved GGTT:[%llx, %llx] for use by error capture\n",885 start, start + size);886 }887 888 /*889 * The upper portion of the GuC address space has a sizeable hole890 * (several MB) that is inaccessible by GuC. Reserve this range within891 * GGTT as it can comfortably hold GuC/HuC firmware images.892 */893 ret = ggtt_reserve_guc_top(ggtt);894 if (ret)895 goto err;896 897 /* Clear any non-preallocated blocks */898 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {899 drm_dbg(&ggtt->vm.i915->drm,900 "clearing unused GTT space: [%lx, %lx]\n",901 hole_start, hole_end);902 ggtt->vm.clear_range(&ggtt->vm, hole_start,903 hole_end - hole_start);904 }905 906 /* And finally clear the reserved guard page */907 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);908 909 return 0;910 911err:912 cleanup_init_ggtt(ggtt);913 return ret;914}915 916static void aliasing_gtt_bind_vma(struct i915_address_space *vm,917 struct i915_vm_pt_stash *stash,918 struct i915_vma_resource *vma_res,919 unsigned int pat_index,920 u32 flags)921{922 u32 pte_flags;923 924 /* Currently applicable only to VLV */925 pte_flags = 0;926 if (vma_res->bi.readonly)927 pte_flags |= PTE_READ_ONLY;928 929 if (flags & I915_VMA_LOCAL_BIND)930 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm,931 stash, vma_res, pat_index, flags);932 933 if (flags & I915_VMA_GLOBAL_BIND)934 vm->insert_entries(vm, vma_res, pat_index, pte_flags);935 936 vma_res->bound_flags |= flags;937}938 939static void aliasing_gtt_unbind_vma(struct i915_address_space *vm,940 struct i915_vma_resource *vma_res)941{942 if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND)943 vm->clear_range(vm, vma_res->start, vma_res->vma_size);944 945 if (vma_res->bound_flags & I915_VMA_LOCAL_BIND)946 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res);947}948 949static int init_aliasing_ppgtt(struct i915_ggtt *ggtt)950{951 struct i915_vm_pt_stash stash = {};952 struct i915_ppgtt *ppgtt;953 int err;954 955 ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0);956 if (IS_ERR(ppgtt))957 return PTR_ERR(ppgtt);958 959 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {960 err = -ENODEV;961 goto err_ppgtt;962 }963 964 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total);965 if (err)966 goto err_ppgtt;967 968 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);969 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);970 i915_gem_object_unlock(ppgtt->vm.scratch[0]);971 if (err)972 goto err_stash;973 974 /*975 * Note we only pre-allocate as far as the end of the global976 * GTT. On 48b / 4-level page-tables, the difference is very,977 * very significant! We have to preallocate as GVT/vgpu does978 * not like the page directory disappearing.979 */980 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total);981 982 ggtt->alias = ppgtt;983 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags;984 985 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma);986 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;987 988 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma);989 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;990 991 i915_vm_free_pt_stash(&ppgtt->vm, &stash);992 return 0;993 994err_stash:995 i915_vm_free_pt_stash(&ppgtt->vm, &stash);996err_ppgtt:997 i915_vm_put(&ppgtt->vm);998 return err;999}1000 1001static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt)1002{1003 struct i915_ppgtt *ppgtt;1004 1005 ppgtt = fetch_and_zero(&ggtt->alias);1006 if (!ppgtt)1007 return;1008 1009 i915_vm_put(&ppgtt->vm);1010 1011 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;1012 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;1013}1014 1015int i915_init_ggtt(struct drm_i915_private *i915)1016{1017 int ret;1018 1019 ret = init_ggtt(to_gt(i915)->ggtt);1020 if (ret)1021 return ret;1022 1023 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) {1024 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt);1025 if (ret)1026 cleanup_init_ggtt(to_gt(i915)->ggtt);1027 }1028 1029 return 0;1030}1031 1032static void ggtt_cleanup_hw(struct i915_ggtt *ggtt)1033{1034 struct i915_vma *vma, *vn;1035 1036 flush_workqueue(ggtt->vm.i915->wq);1037 i915_gem_drain_freed_objects(ggtt->vm.i915);1038 1039 mutex_lock(&ggtt->vm.mutex);1040 1041 ggtt->vm.skip_pte_rewrite = true;1042 1043 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {1044 struct drm_i915_gem_object *obj = vma->obj;1045 bool trylock;1046 1047 trylock = i915_gem_object_trylock(obj, NULL);1048 WARN_ON(!trylock);1049 1050 WARN_ON(__i915_vma_unbind(vma));1051 if (trylock)1052 i915_gem_object_unlock(obj);1053 }1054 1055 if (drm_mm_node_allocated(&ggtt->error_capture))1056 drm_mm_remove_node(&ggtt->error_capture);1057 mutex_destroy(&ggtt->error_mutex);1058 1059 ggtt_release_guc_top(ggtt);1060 intel_vgt_deballoon(ggtt);1061 1062 ggtt->vm.cleanup(&ggtt->vm);1063 1064 mutex_unlock(&ggtt->vm.mutex);1065 i915_address_space_fini(&ggtt->vm);1066 1067 arch_phys_wc_del(ggtt->mtrr);1068 1069 if (ggtt->iomap.size)1070 io_mapping_fini(&ggtt->iomap);1071}1072 1073/**1074 * i915_ggtt_driver_release - Clean up GGTT hardware initialization1075 * @i915: i915 device1076 */1077void i915_ggtt_driver_release(struct drm_i915_private *i915)1078{1079 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;1080 1081 fini_aliasing_ppgtt(ggtt);1082 1083 intel_ggtt_fini_fences(ggtt);1084 ggtt_cleanup_hw(ggtt);1085}1086 1087/**1088 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after1089 * all free objects have been drained.1090 * @i915: i915 device1091 */1092void i915_ggtt_driver_late_release(struct drm_i915_private *i915)1093{1094 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;1095 1096 GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1);1097 dma_resv_fini(&ggtt->vm._resv);1098}1099 1100static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)1101{1102 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;1103 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;1104 return snb_gmch_ctl << 20;1105}1106 1107static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)1108{1109 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;1110 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;1111 if (bdw_gmch_ctl)1112 bdw_gmch_ctl = 1 << bdw_gmch_ctl;1113 1114#ifdef CONFIG_X86_321115 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */1116 if (bdw_gmch_ctl > 4)1117 bdw_gmch_ctl = 4;1118#endif1119 1120 return bdw_gmch_ctl << 20;1121}1122 1123static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)1124{1125 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;1126 gmch_ctrl &= SNB_GMCH_GGMS_MASK;1127 1128 if (gmch_ctrl)1129 return 1 << (20 + gmch_ctrl);1130 1131 return 0;1132}1133 1134static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915)1135{1136 /*1137 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset1138 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset1139 */1140 GEM_BUG_ON(GRAPHICS_VER(i915) < 6);1141 return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M;1142}1143 1144static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915)1145{1146 return gen6_gttmmadr_size(i915) / 2;1147}1148 1149static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)1150{1151 struct drm_i915_private *i915 = ggtt->vm.i915;1152 struct intel_uncore *uncore = ggtt->vm.gt->uncore;1153 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);1154 phys_addr_t phys_addr;1155 u32 pte_flags;1156 int ret;1157 1158 GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915));1159 1160 if (i915_direct_stolen_access(i915)) {1161 drm_dbg(&i915->drm, "Using direct GSM access\n");1162 phys_addr = intel_uncore_read64(uncore, GEN6_GSMBASE) & GEN11_BDSM_MASK;1163 } else {1164 phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915);1165 }1166 1167 if (needs_wc_ggtt_mapping(i915))1168 ggtt->gsm = ioremap_wc(phys_addr, size);1169 else1170 ggtt->gsm = ioremap(phys_addr, size);1171 1172 if (!ggtt->gsm) {1173 drm_err(&i915->drm, "Failed to map the ggtt page table\n");1174 return -ENOMEM;1175 }1176 1177 kref_init(&ggtt->vm.resv_ref);1178 ret = setup_scratch_page(&ggtt->vm);1179 if (ret) {1180 drm_err(&i915->drm, "Scratch setup failed\n");1181 /* iounmap will also get called at remove, but meh */1182 iounmap(ggtt->gsm);1183 return ret;1184 }1185 1186 pte_flags = 0;1187 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0]))1188 pte_flags |= PTE_LM;1189 1190 ggtt->vm.scratch[0]->encode =1191 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]),1192 i915_gem_get_pat_index(i915,1193 I915_CACHE_NONE),1194 pte_flags);1195 1196 return 0;1197}1198 1199static void gen6_gmch_remove(struct i915_address_space *vm)1200{1201 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);1202 1203 iounmap(ggtt->gsm);1204 free_scratch(vm);1205}1206 1207static struct resource pci_resource(struct pci_dev *pdev, int bar)1208{1209 return DEFINE_RES_MEM(pci_resource_start(pdev, bar),1210 pci_resource_len(pdev, bar));1211}1212 1213static int gen8_gmch_probe(struct i915_ggtt *ggtt)1214{1215 struct drm_i915_private *i915 = ggtt->vm.i915;1216 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);1217 unsigned int size;1218 u16 snb_gmch_ctl;1219 1220 if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) {1221 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))1222 return -ENXIO;1223 1224 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);1225 ggtt->mappable_end = resource_size(&ggtt->gmadr);1226 }1227 1228 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);1229 if (IS_CHERRYVIEW(i915))1230 size = chv_get_total_gtt_size(snb_gmch_ctl);1231 else1232 size = gen8_get_total_gtt_size(snb_gmch_ctl);1233 1234 ggtt->vm.alloc_pt_dma = alloc_pt_dma;1235 ggtt->vm.alloc_scratch_dma = alloc_pt_dma;1236 ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY;1237 1238 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;1239 ggtt->vm.cleanup = gen6_gmch_remove;1240 ggtt->vm.insert_page = gen8_ggtt_insert_page;1241 ggtt->vm.clear_range = nop_clear_range;1242 ggtt->vm.scratch_range = gen8_ggtt_clear_range;1243 1244 ggtt->vm.insert_entries = gen8_ggtt_insert_entries;1245 1246 /*1247 * Serialize GTT updates with aperture access on BXT if VT-d is on,1248 * and always on CHV.1249 */1250 if (intel_vm_no_concurrent_access_wa(i915)) {1251 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;1252 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL;1253 1254 /*1255 * Calling stop_machine() version of GGTT update function1256 * at error capture/reset path will raise lockdep warning.1257 * Allow calling gen8_ggtt_insert_* directly at reset path1258 * which is safe from parallel GGTT updates.1259 */1260 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;1261 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries;1262 1263 ggtt->vm.bind_async_flags =1264 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;1265 }1266 1267 if (i915_ggtt_require_binder(i915)) {1268 ggtt->vm.scratch_range = gen8_ggtt_scratch_range_bind;1269 ggtt->vm.insert_page = gen8_ggtt_insert_page_bind;1270 ggtt->vm.insert_entries = gen8_ggtt_insert_entries_bind;1271 /*1272 * On GPU is hung, we might bind VMAs for error capture.1273 * Fallback to CPU GGTT updates in that case.1274 */1275 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;1276 }1277 1278 if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc))1279 ggtt->invalidate = guc_ggtt_invalidate;1280 else1281 ggtt->invalidate = gen8_ggtt_invalidate;1282 1283 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;1284 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;1285 1286 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))1287 ggtt->vm.pte_encode = mtl_ggtt_pte_encode;1288 else1289 ggtt->vm.pte_encode = gen8_ggtt_pte_encode;1290 1291 return ggtt_probe_common(ggtt, size);1292}1293 1294/*1295 * For pre-gen8 platforms pat_index is the same as enum i915_cache_level,1296 * so the switch-case statements in these PTE encode functions are still valid.1297 * See translation table LEGACY_CACHELEVEL.1298 */1299static u64 snb_pte_encode(dma_addr_t addr,1300 unsigned int pat_index,1301 u32 flags)1302{1303 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;1304 1305 switch (pat_index) {1306 case I915_CACHE_L3_LLC:1307 case I915_CACHE_LLC:1308 pte |= GEN6_PTE_CACHE_LLC;1309 break;1310 case I915_CACHE_NONE:1311 pte |= GEN6_PTE_UNCACHED;1312 break;1313 default:1314 MISSING_CASE(pat_index);1315 }1316 1317 return pte;1318}1319 1320static u64 ivb_pte_encode(dma_addr_t addr,1321 unsigned int pat_index,1322 u32 flags)1323{1324 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;1325 1326 switch (pat_index) {1327 case I915_CACHE_L3_LLC:1328 pte |= GEN7_PTE_CACHE_L3_LLC;1329 break;1330 case I915_CACHE_LLC:1331 pte |= GEN6_PTE_CACHE_LLC;1332 break;1333 case I915_CACHE_NONE:1334 pte |= GEN6_PTE_UNCACHED;1335 break;1336 default:1337 MISSING_CASE(pat_index);1338 }1339 1340 return pte;1341}1342 1343static u64 byt_pte_encode(dma_addr_t addr,1344 unsigned int pat_index,1345 u32 flags)1346{1347 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;1348 1349 if (!(flags & PTE_READ_ONLY))1350 pte |= BYT_PTE_WRITEABLE;1351 1352 if (pat_index != I915_CACHE_NONE)1353 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;1354 1355 return pte;1356}1357 1358static u64 hsw_pte_encode(dma_addr_t addr,1359 unsigned int pat_index,1360 u32 flags)1361{1362 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;1363 1364 if (pat_index != I915_CACHE_NONE)1365 pte |= HSW_WB_LLC_AGE3;1366 1367 return pte;1368}1369 1370static u64 iris_pte_encode(dma_addr_t addr,1371 unsigned int pat_index,1372 u32 flags)1373{1374 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID;1375 1376 switch (pat_index) {1377 case I915_CACHE_NONE:1378 break;1379 case I915_CACHE_WT:1380 pte |= HSW_WT_ELLC_LLC_AGE3;1381 break;1382 default:1383 pte |= HSW_WB_ELLC_LLC_AGE3;1384 break;1385 }1386 1387 return pte;1388}1389 1390static int gen6_gmch_probe(struct i915_ggtt *ggtt)1391{1392 struct drm_i915_private *i915 = ggtt->vm.i915;1393 struct pci_dev *pdev = to_pci_dev(i915->drm.dev);1394 unsigned int size;1395 u16 snb_gmch_ctl;1396 1397 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR))1398 return -ENXIO;1399 1400 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR);1401 ggtt->mappable_end = resource_size(&ggtt->gmadr);1402 1403 /*1404 * 64/512MB is the current min/max we actually know of, but this is1405 * just a coarse sanity check.1406 */1407 if (ggtt->mappable_end < (64 << 20) ||1408 ggtt->mappable_end > (512 << 20)) {1409 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n",1410 &ggtt->mappable_end);1411 return -ENXIO;1412 }1413 1414 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);1415 1416 size = gen6_get_total_gtt_size(snb_gmch_ctl);1417 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;1418 1419 ggtt->vm.alloc_pt_dma = alloc_pt_dma;1420 ggtt->vm.alloc_scratch_dma = alloc_pt_dma;1421 1422 ggtt->vm.clear_range = nop_clear_range;1423 if (!HAS_FULL_PPGTT(i915))1424 ggtt->vm.clear_range = gen6_ggtt_clear_range;1425 ggtt->vm.scratch_range = gen6_ggtt_clear_range;1426 ggtt->vm.insert_page = gen6_ggtt_insert_page;1427 ggtt->vm.insert_entries = gen6_ggtt_insert_entries;1428 ggtt->vm.cleanup = gen6_gmch_remove;1429 1430 ggtt->invalidate = gen6_ggtt_invalidate;1431 1432 if (HAS_EDRAM(i915))1433 ggtt->vm.pte_encode = iris_pte_encode;1434 else if (IS_HASWELL(i915))1435 ggtt->vm.pte_encode = hsw_pte_encode;1436 else if (IS_VALLEYVIEW(i915))1437 ggtt->vm.pte_encode = byt_pte_encode;1438 else if (GRAPHICS_VER(i915) >= 7)1439 ggtt->vm.pte_encode = ivb_pte_encode;1440 else1441 ggtt->vm.pte_encode = snb_pte_encode;1442 1443 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma;1444 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma;1445 1446 return ggtt_probe_common(ggtt, size);1447}1448 1449static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt)1450{1451 struct drm_i915_private *i915 = gt->i915;1452 int ret;1453 1454 ggtt->vm.gt = gt;1455 ggtt->vm.i915 = i915;1456 ggtt->vm.dma = i915->drm.dev;1457 dma_resv_init(&ggtt->vm._resv);1458 1459 if (GRAPHICS_VER(i915) >= 8)1460 ret = gen8_gmch_probe(ggtt);1461 else if (GRAPHICS_VER(i915) >= 6)1462 ret = gen6_gmch_probe(ggtt);1463 else1464 ret = intel_ggtt_gmch_probe(ggtt);1465 1466 if (ret) {1467 dma_resv_fini(&ggtt->vm._resv);1468 return ret;1469 }1470 1471 if ((ggtt->vm.total - 1) >> 32) {1472 drm_err(&i915->drm,1473 "We never expected a Global GTT with more than 32bits"1474 " of address space! Found %lldM!\n",1475 ggtt->vm.total >> 20);1476 ggtt->vm.total = 1ULL << 32;1477 ggtt->mappable_end =1478 min_t(u64, ggtt->mappable_end, ggtt->vm.total);1479 }1480 1481 if (ggtt->mappable_end > ggtt->vm.total) {1482 drm_err(&i915->drm,1483 "mappable aperture extends past end of GGTT,"1484 " aperture=%pa, total=%llx\n",1485 &ggtt->mappable_end, ggtt->vm.total);1486 ggtt->mappable_end = ggtt->vm.total;1487 }1488 1489 /* GMADR is the PCI mmio aperture into the global GTT. */1490 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20);1491 drm_dbg(&i915->drm, "GMADR size = %lluM\n",1492 (u64)ggtt->mappable_end >> 20);1493 drm_dbg(&i915->drm, "DSM size = %lluM\n",1494 (u64)resource_size(&intel_graphics_stolen_res) >> 20);1495 1496 return 0;1497}1498 1499/**1500 * i915_ggtt_probe_hw - Probe GGTT hardware location1501 * @i915: i915 device1502 */1503int i915_ggtt_probe_hw(struct drm_i915_private *i915)1504{1505 struct intel_gt *gt;1506 int ret, i;1507 1508 for_each_gt(gt, i915, i) {1509 ret = intel_gt_assign_ggtt(gt);1510 if (ret)1511 return ret;1512 }1513 1514 ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));1515 if (ret)1516 return ret;1517 1518 if (i915_vtd_active(i915))1519 drm_info(&i915->drm, "VT-d active for gfx access\n");1520 1521 return 0;1522}1523 1524struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)1525{1526 struct i915_ggtt *ggtt;1527 1528 ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);1529 if (!ggtt)1530 return ERR_PTR(-ENOMEM);1531 1532 INIT_LIST_HEAD(&ggtt->gt_list);1533 1534 return ggtt;1535}1536 1537int i915_ggtt_enable_hw(struct drm_i915_private *i915)1538{1539 if (GRAPHICS_VER(i915) < 6)1540 return intel_ggtt_gmch_enable_hw(i915);1541 1542 return 0;1543}1544 1545/**1546 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM1547 * @vm: The VM to restore the mappings for1548 *1549 * Restore the memory mappings for all objects mapped to HW via the GGTT or a1550 * DPT page table.1551 *1552 * Returns %true if restoring the mapping for any object that was in a write1553 * domain before suspend.1554 */1555bool i915_ggtt_resume_vm(struct i915_address_space *vm)1556{1557 struct i915_vma *vma;1558 bool write_domain_objs = false;1559 1560 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt);1561 1562 /* First fill our portion of the GTT with scratch pages */1563 vm->clear_range(vm, 0, vm->total);1564 1565 /* clflush objects bound into the GGTT and rebind them. */1566 list_for_each_entry(vma, &vm->bound_list, vm_link) {1567 struct drm_i915_gem_object *obj = vma->obj;1568 unsigned int was_bound =1569 atomic_read(&vma->flags) & I915_VMA_BIND_MASK;1570 1571 GEM_BUG_ON(!was_bound);1572 1573 /*1574 * Clear the bound flags of the vma resource to allow1575 * ptes to be repopulated.1576 */1577 vma->resource->bound_flags = 0;1578 vma->ops->bind_vma(vm, NULL, vma->resource,1579 obj ? obj->pat_index :1580 i915_gem_get_pat_index(vm->i915,1581 I915_CACHE_NONE),1582 was_bound);1583 1584 if (obj) { /* only used during resume => exclusive access */1585 write_domain_objs |= fetch_and_zero(&obj->write_domain);1586 obj->read_domains |= I915_GEM_DOMAIN_GTT;1587 }1588 }1589 1590 return write_domain_objs;1591}1592 1593void i915_ggtt_resume(struct i915_ggtt *ggtt)1594{1595 struct intel_gt *gt;1596 bool flush;1597 1598 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)1599 intel_gt_check_and_clear_faults(gt);1600 1601 flush = i915_ggtt_resume_vm(&ggtt->vm);1602 1603 if (drm_mm_node_allocated(&ggtt->error_capture))1604 ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start,1605 ggtt->error_capture.size);1606 1607 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)1608 intel_uc_resume_mappings(>->uc);1609 1610 ggtt->invalidate(ggtt);1611 1612 if (flush)1613 wbinvd_on_all_cpus();1614 1615 intel_ggtt_restore_fences(ggtt);1616}1617