324 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include "gem/i915_gem_domain.h"7#include "gem/i915_gem_internal.h"8#include "gem/i915_gem_lmem.h"9#include "gt/gen8_ppgtt.h"10 11#include "i915_drv.h"12#include "intel_display_types.h"13#include "intel_dpt.h"14#include "intel_fb.h"15 16struct i915_dpt {17 struct i915_address_space vm;18 19 struct drm_i915_gem_object *obj;20 struct i915_vma *vma;21 void __iomem *iomem;22};23 24#define i915_is_dpt(vm) ((vm)->is_dpt)25 26static inline struct i915_dpt *27i915_vm_to_dpt(struct i915_address_space *vm)28{29 BUILD_BUG_ON(offsetof(struct i915_dpt, vm));30 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));31 return container_of(vm, struct i915_dpt, vm);32}33 34#define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)35 36static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)37{38 writeq(pte, addr);39}40 41static void dpt_insert_page(struct i915_address_space *vm,42 dma_addr_t addr,43 u64 offset,44 unsigned int pat_index,45 u32 flags)46{47 struct i915_dpt *dpt = i915_vm_to_dpt(vm);48 gen8_pte_t __iomem *base = dpt->iomem;49 50 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,51 vm->pte_encode(addr, pat_index, flags));52}53 54static void dpt_insert_entries(struct i915_address_space *vm,55 struct i915_vma_resource *vma_res,56 unsigned int pat_index,57 u32 flags)58{59 struct i915_dpt *dpt = i915_vm_to_dpt(vm);60 gen8_pte_t __iomem *base = dpt->iomem;61 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);62 struct sgt_iter sgt_iter;63 dma_addr_t addr;64 int i;65 66 /*67 * Note that we ignore PTE_READ_ONLY here. The caller must be careful68 * not to allow the user to override access to a read only page.69 */70 71 i = vma_res->start / I915_GTT_PAGE_SIZE;72 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)73 gen8_set_pte(&base[i++], pte_encode | addr);74}75 76static void dpt_clear_range(struct i915_address_space *vm,77 u64 start, u64 length)78{79}80 81static void dpt_bind_vma(struct i915_address_space *vm,82 struct i915_vm_pt_stash *stash,83 struct i915_vma_resource *vma_res,84 unsigned int pat_index,85 u32 flags)86{87 u32 pte_flags;88 89 if (vma_res->bound_flags)90 return;91 92 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */93 pte_flags = 0;94 if (vm->has_read_only && vma_res->bi.readonly)95 pte_flags |= PTE_READ_ONLY;96 if (vma_res->bi.lmem)97 pte_flags |= PTE_LM;98 99 vm->insert_entries(vm, vma_res, pat_index, pte_flags);100 101 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;102 103 /*104 * Without aliasing PPGTT there's no difference between105 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally106 * upgrade to both bound if we bind either to avoid double-binding.107 */108 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;109}110 111static void dpt_unbind_vma(struct i915_address_space *vm,112 struct i915_vma_resource *vma_res)113{114 vm->clear_range(vm, vma_res->start, vma_res->vma_size);115}116 117static void dpt_cleanup(struct i915_address_space *vm)118{119 struct i915_dpt *dpt = i915_vm_to_dpt(vm);120 121 i915_gem_object_put(dpt->obj);122}123 124struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm,125 unsigned int alignment)126{127 struct drm_i915_private *i915 = vm->i915;128 struct i915_dpt *dpt = i915_vm_to_dpt(vm);129 intel_wakeref_t wakeref;130 struct i915_vma *vma;131 void __iomem *iomem;132 struct i915_gem_ww_ctx ww;133 u64 pin_flags = 0;134 int err;135 136 if (i915_gem_object_is_stolen(dpt->obj))137 pin_flags |= PIN_MAPPABLE;138 139 wakeref = intel_runtime_pm_get(&i915->runtime_pm);140 atomic_inc(&i915->gpu_error.pending_fb_pin);141 142 for_i915_gem_ww(&ww, err, true) {143 err = i915_gem_object_lock(dpt->obj, &ww);144 if (err)145 continue;146 147 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0,148 alignment, pin_flags);149 if (IS_ERR(vma)) {150 err = PTR_ERR(vma);151 continue;152 }153 154 iomem = i915_vma_pin_iomap(vma);155 i915_vma_unpin(vma);156 157 if (IS_ERR(iomem)) {158 err = PTR_ERR(iomem);159 continue;160 }161 162 dpt->vma = vma;163 dpt->iomem = iomem;164 165 i915_vma_get(vma);166 }167 168 dpt->obj->mm.dirty = true;169 170 atomic_dec(&i915->gpu_error.pending_fb_pin);171 intel_runtime_pm_put(&i915->runtime_pm, wakeref);172 173 return err ? ERR_PTR(err) : vma;174}175 176void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm)177{178 struct i915_dpt *dpt = i915_vm_to_dpt(vm);179 180 i915_vma_unpin_iomap(dpt->vma);181 i915_vma_put(dpt->vma);182}183 184/**185 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume186 * @i915: device instance187 *188 * Restore the memory mapping during system resume for all framebuffers which189 * are mapped to HW via a GGTT->DPT page table. The content of these page190 * tables are not stored in the hibernation image during S4 and S3RST->S4191 * transitions, so here we reprogram the PTE entries in those tables.192 *193 * This function must be called after the mappings in GGTT have been restored calling194 * i915_ggtt_resume().195 */196void intel_dpt_resume(struct drm_i915_private *i915)197{198 struct drm_framebuffer *drm_fb;199 200 if (!HAS_DISPLAY(i915))201 return;202 203 mutex_lock(&i915->drm.mode_config.fb_lock);204 drm_for_each_fb(drm_fb, &i915->drm) {205 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);206 207 if (fb->dpt_vm)208 i915_ggtt_resume_vm(fb->dpt_vm);209 }210 mutex_unlock(&i915->drm.mode_config.fb_lock);211}212 213/**214 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend215 * @i915: device instance216 *217 * Suspend the memory mapping during system suspend for all framebuffers which218 * are mapped to HW via a GGTT->DPT page table.219 *220 * This function must be called before the mappings in GGTT are suspended calling221 * i915_ggtt_suspend().222 */223void intel_dpt_suspend(struct drm_i915_private *i915)224{225 struct drm_framebuffer *drm_fb;226 227 if (!HAS_DISPLAY(i915))228 return;229 230 mutex_lock(&i915->drm.mode_config.fb_lock);231 232 drm_for_each_fb(drm_fb, &i915->drm) {233 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);234 235 if (fb->dpt_vm)236 i915_ggtt_suspend_vm(fb->dpt_vm);237 }238 239 mutex_unlock(&i915->drm.mode_config.fb_lock);240}241 242struct i915_address_space *243intel_dpt_create(struct intel_framebuffer *fb)244{245 struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;246 struct drm_i915_private *i915 = to_i915(obj->dev);247 struct drm_i915_gem_object *dpt_obj;248 struct i915_address_space *vm;249 struct i915_dpt *dpt;250 size_t size;251 int ret;252 253 if (intel_fb_needs_pot_stride_remap(fb))254 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);255 else256 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);257 258 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);259 260 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);261 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))262 dpt_obj = i915_gem_object_create_stolen(i915, size);263 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {264 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");265 dpt_obj = i915_gem_object_create_shmem(i915, size);266 }267 if (IS_ERR(dpt_obj))268 return ERR_CAST(dpt_obj);269 270 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);271 if (!ret) {272 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);273 i915_gem_object_unlock(dpt_obj);274 }275 if (ret) {276 i915_gem_object_put(dpt_obj);277 return ERR_PTR(ret);278 }279 280 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);281 if (!dpt) {282 i915_gem_object_put(dpt_obj);283 return ERR_PTR(-ENOMEM);284 }285 286 vm = &dpt->vm;287 288 vm->gt = to_gt(i915);289 vm->i915 = i915;290 vm->dma = i915->drm.dev;291 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;292 vm->is_dpt = true;293 294 i915_address_space_init(vm, VM_CLASS_DPT);295 296 vm->insert_page = dpt_insert_page;297 vm->clear_range = dpt_clear_range;298 vm->insert_entries = dpt_insert_entries;299 vm->cleanup = dpt_cleanup;300 301 vm->vma_ops.bind_vma = dpt_bind_vma;302 vm->vma_ops.unbind_vma = dpt_unbind_vma;303 304 vm->pte_encode = vm->gt->ggtt->vm.pte_encode;305 306 dpt->obj = dpt_obj;307 dpt->obj->is_dpt = true;308 309 return &dpt->vm;310}311 312void intel_dpt_destroy(struct i915_address_space *vm)313{314 struct i915_dpt *dpt = i915_vm_to_dpt(vm);315 316 dpt->obj->is_dpt = false;317 i915_vm_put(&dpt->vm);318}319 320u64 intel_dpt_offset(struct i915_vma *dpt_vma)321{322 return dpt_vma->node.start;323}324