407 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2021 Intel Corporation4 */5 6#include <drm/ttm/ttm_bo.h>7 8#include "intel_display_types.h"9#include "intel_dpt.h"10#include "intel_fb.h"11#include "intel_fb_pin.h"12#include "xe_bo.h"13#include "xe_device.h"14#include "xe_ggtt.h"15#include "xe_pm.h"16 17static void18write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,19 u32 width, u32 height, u32 src_stride, u32 dst_stride)20{21 struct xe_device *xe = xe_bo_device(bo);22 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;23 u32 column, row;24 25 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,26 * by writing dpt/ggtt in a different order?27 */28 29 for (column = 0; column < width; column++) {30 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;31 32 for (row = 0; row < height; row++) {33 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,34 xe->pat.idx[XE_CACHE_NONE]);35 36 iosys_map_wr(map, *dpt_ofs, u64, pte);37 *dpt_ofs += 8;38 src_idx -= src_stride;39 }40 41 /* The DE ignores the PTEs for the padding tiles */42 *dpt_ofs += (dst_stride - height) * 8;43 }44 45 /* Align to next page */46 *dpt_ofs = ALIGN(*dpt_ofs, 4096);47}48 49static void50write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,51 u32 bo_ofs, u32 width, u32 height, u32 src_stride,52 u32 dst_stride)53{54 struct xe_device *xe = xe_bo_device(bo);55 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;56 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index)57 = ggtt->pt_ops->pte_encode_bo;58 u32 column, row;59 60 for (row = 0; row < height; row++) {61 u32 src_idx = src_stride * row + bo_ofs;62 63 for (column = 0; column < width; column++) {64 iosys_map_wr(map, *dpt_ofs, u64,65 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,66 xe->pat.idx[XE_CACHE_NONE]));67 68 *dpt_ofs += 8;69 src_idx++;70 }71 72 /* The DE ignores the PTEs for the padding tiles */73 *dpt_ofs += (dst_stride - width) * 8;74 }75 76 /* Align to next page */77 *dpt_ofs = ALIGN(*dpt_ofs, 4096);78}79 80static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,81 const struct i915_gtt_view *view,82 struct i915_vma *vma)83{84 struct xe_device *xe = to_xe_device(fb->base.dev);85 struct xe_tile *tile0 = xe_device_get_root_tile(xe);86 struct xe_ggtt *ggtt = tile0->mem.ggtt;87 struct xe_bo *bo = intel_fb_obj(&fb->base), *dpt;88 u32 dpt_size, size = bo->ttm.base.size;89 90 if (view->type == I915_GTT_VIEW_NORMAL)91 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);92 else if (view->type == I915_GTT_VIEW_REMAPPED)93 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,94 XE_PAGE_SIZE);95 else96 /* display uses 4K tiles instead of bytes here, convert to entries.. */97 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,98 XE_PAGE_SIZE);99 100 if (IS_DGFX(xe))101 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,102 ttm_bo_type_kernel,103 XE_BO_FLAG_VRAM0 |104 XE_BO_FLAG_GGTT |105 XE_BO_FLAG_PAGETABLE);106 else107 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,108 ttm_bo_type_kernel,109 XE_BO_FLAG_STOLEN |110 XE_BO_FLAG_GGTT |111 XE_BO_FLAG_PAGETABLE);112 if (IS_ERR(dpt))113 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size,114 ttm_bo_type_kernel,115 XE_BO_FLAG_SYSTEM |116 XE_BO_FLAG_GGTT |117 XE_BO_FLAG_PAGETABLE);118 if (IS_ERR(dpt))119 return PTR_ERR(dpt);120 121 if (view->type == I915_GTT_VIEW_NORMAL) {122 u32 x;123 124 for (x = 0; x < size / XE_PAGE_SIZE; x++) {125 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE,126 xe->pat.idx[XE_CACHE_NONE]);127 128 iosys_map_wr(&dpt->vmap, x * 8, u64, pte);129 }130 } else if (view->type == I915_GTT_VIEW_REMAPPED) {131 const struct intel_remapped_info *remap_info = &view->remapped;132 u32 i, dpt_ofs = 0;133 134 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)135 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,136 remap_info->plane[i].offset,137 remap_info->plane[i].width,138 remap_info->plane[i].height,139 remap_info->plane[i].src_stride,140 remap_info->plane[i].dst_stride);141 142 } else {143 const struct intel_rotation_info *rot_info = &view->rotated;144 u32 i, dpt_ofs = 0;145 146 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)147 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,148 rot_info->plane[i].offset,149 rot_info->plane[i].width,150 rot_info->plane[i].height,151 rot_info->plane[i].src_stride,152 rot_info->plane[i].dst_stride);153 }154 155 vma->dpt = dpt;156 vma->node = dpt->ggtt_node;157 return 0;158}159 160static void161write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,162 u32 width, u32 height, u32 src_stride, u32 dst_stride)163{164 struct xe_device *xe = xe_bo_device(bo);165 u32 column, row;166 167 for (column = 0; column < width; column++) {168 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;169 170 for (row = 0; row < height; row++) {171 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,172 xe->pat.idx[XE_CACHE_NONE]);173 174 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte);175 *ggtt_ofs += XE_PAGE_SIZE;176 src_idx -= src_stride;177 }178 179 /* The DE ignores the PTEs for the padding tiles */180 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;181 }182}183 184static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,185 const struct i915_gtt_view *view,186 struct i915_vma *vma)187{188 struct xe_bo *bo = intel_fb_obj(&fb->base);189 struct xe_device *xe = to_xe_device(fb->base.dev);190 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;191 u32 align;192 int ret;193 194 /* TODO: Consider sharing framebuffer mapping?195 * embed i915_vma inside intel_framebuffer196 */197 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));198 ret = mutex_lock_interruptible(&ggtt->lock);199 if (ret)200 goto out;201 202 align = XE_PAGE_SIZE;203 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)204 align = max_t(u32, align, SZ_64K);205 206 if (bo->ggtt_node && view->type == I915_GTT_VIEW_NORMAL) {207 vma->node = bo->ggtt_node;208 } else if (view->type == I915_GTT_VIEW_NORMAL) {209 u32 x, size = bo->ttm.base.size;210 211 vma->node = xe_ggtt_node_init(ggtt);212 if (IS_ERR(vma->node)) {213 ret = PTR_ERR(vma->node);214 goto out_unlock;215 }216 217 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);218 if (ret) {219 xe_ggtt_node_fini(vma->node);220 goto out_unlock;221 }222 223 for (x = 0; x < size; x += XE_PAGE_SIZE) {224 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x,225 xe->pat.idx[XE_CACHE_NONE]);226 227 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte);228 }229 } else {230 u32 i, ggtt_ofs;231 const struct intel_rotation_info *rot_info = &view->rotated;232 233 /* display seems to use tiles instead of bytes here, so convert it back.. */234 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;235 236 vma->node = xe_ggtt_node_init(ggtt);237 if (IS_ERR(vma->node)) {238 ret = PTR_ERR(vma->node);239 goto out_unlock;240 }241 242 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);243 if (ret) {244 xe_ggtt_node_fini(vma->node);245 goto out_unlock;246 }247 248 ggtt_ofs = vma->node->base.start;249 250 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)251 write_ggtt_rotated(bo, ggtt, &ggtt_ofs,252 rot_info->plane[i].offset,253 rot_info->plane[i].width,254 rot_info->plane[i].height,255 rot_info->plane[i].src_stride,256 rot_info->plane[i].dst_stride);257 }258 259out_unlock:260 mutex_unlock(&ggtt->lock);261out:262 xe_pm_runtime_put(tile_to_xe(ggtt->tile));263 return ret;264}265 266static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,267 const struct i915_gtt_view *view)268{269 struct drm_device *dev = fb->base.dev;270 struct xe_device *xe = to_xe_device(dev);271 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);272 struct xe_bo *bo = intel_fb_obj(&fb->base);273 int ret;274 275 if (!vma)276 return ERR_PTR(-ENODEV);277 278 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&279 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&280 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {281 struct xe_tile *tile = xe_device_get_root_tile(xe);282 283 /*284 * If we need to able to access the clear-color value stored in285 * the buffer, then we require that such buffers are also CPU286 * accessible. This is important on small-bar systems where287 * only some subset of VRAM is CPU accessible.288 */289 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {290 ret = -EINVAL;291 goto err;292 }293 }294 295 /*296 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the297 * assumptions are incorrect for framebuffers298 */299 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL);300 if (ret)301 goto err;302 303 if (IS_DGFX(xe))304 ret = xe_bo_migrate(bo, XE_PL_VRAM0);305 else306 ret = xe_bo_validate(bo, NULL, true);307 if (!ret)308 ttm_bo_pin(&bo->ttm);309 ttm_bo_unreserve(&bo->ttm);310 if (ret)311 goto err;312 313 vma->bo = bo;314 if (intel_fb_uses_dpt(&fb->base))315 ret = __xe_pin_fb_vma_dpt(fb, view, vma);316 else317 ret = __xe_pin_fb_vma_ggtt(fb, view, vma);318 if (ret)319 goto err_unpin;320 321 /* Ensure DPT writes are flushed */322 xe_device_l2_flush(xe);323 return vma;324 325err_unpin:326 ttm_bo_reserve(&bo->ttm, false, false, NULL);327 ttm_bo_unpin(&bo->ttm);328 ttm_bo_unreserve(&bo->ttm);329err:330 kfree(vma);331 return ERR_PTR(ret);332}333 334static void __xe_unpin_fb_vma(struct i915_vma *vma)335{336 if (vma->dpt)337 xe_bo_unpin_map_no_vm(vma->dpt);338 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node) ||339 vma->bo->ggtt_node->base.start != vma->node->base.start)340 xe_ggtt_node_remove(vma->node, false);341 342 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);343 ttm_bo_unpin(&vma->bo->ttm);344 ttm_bo_unreserve(&vma->bo->ttm);345 kfree(vma);346}347 348struct i915_vma *349intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,350 const struct i915_gtt_view *view,351 unsigned int alignment,352 unsigned int phys_alignment,353 bool uses_fence,354 unsigned long *out_flags)355{356 *out_flags = 0;357 358 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view);359}360 361void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)362{363 __xe_unpin_fb_vma(vma);364}365 366int intel_plane_pin_fb(struct intel_plane_state *plane_state)367{368 struct drm_framebuffer *fb = plane_state->hw.fb;369 struct xe_bo *bo = intel_fb_obj(fb);370 struct i915_vma *vma;371 372 /* We reject creating !SCANOUT fb's, so this is weird.. */373 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));374 375 vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt);376 if (IS_ERR(vma))377 return PTR_ERR(vma);378 379 plane_state->ggtt_vma = vma;380 return 0;381}382 383void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)384{385 __xe_unpin_fb_vma(old_plane_state->ggtt_vma);386 old_plane_state->ggtt_vma = NULL;387}388 389/*390 * For Xe introduce dummy intel_dpt_create which just return NULL,391 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;392 */393struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)394{395 return NULL;396}397 398void intel_dpt_destroy(struct i915_address_space *vm)399{400 return;401}402 403u64 intel_dpt_offset(struct i915_vma *dpt_vma)404{405 return 0;406}407