446 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020-2023 Intel Corporation4 */5 6#include <linux/dma-buf.h>7#include <linux/highmem.h>8#include <linux/module.h>9#include <linux/set_memory.h>10#include <linux/xarray.h>11 12#include <drm/drm_cache.h>13#include <drm/drm_debugfs.h>14#include <drm/drm_file.h>15#include <drm/drm_utils.h>16 17#include "ivpu_drv.h"18#include "ivpu_gem.h"19#include "ivpu_hw.h"20#include "ivpu_mmu.h"21#include "ivpu_mmu_context.h"22 23static const struct drm_gem_object_funcs ivpu_gem_funcs;24 25static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)26{27 ivpu_dbg(vdev, BO,28 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",29 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0,30 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,31 (bool)bo->base.base.import_attach);32}33 34/*35 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.36 *37 * This function pins physical memory pages, then maps the physical pages38 * to IOMMU address space and finally updates the VPU MMU page tables39 * to allow the VPU to translate VPU address to IOMMU address.40 */41int __must_check ivpu_bo_pin(struct ivpu_bo *bo)42{43 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);44 int ret = 0;45 46 mutex_lock(&bo->lock);47 48 ivpu_dbg_bo(vdev, bo, "pin");49 drm_WARN_ON(&vdev->drm, !bo->ctx);50 51 if (!bo->mmu_mapped) {52 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);53 54 if (IS_ERR(sgt)) {55 ret = PTR_ERR(sgt);56 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);57 goto unlock;58 }59 60 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,61 ivpu_bo_is_snooped(bo));62 if (ret) {63 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);64 goto unlock;65 }66 bo->mmu_mapped = true;67 }68 69unlock:70 mutex_unlock(&bo->lock);71 72 return ret;73}74 75static int76ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,77 const struct ivpu_addr_range *range)78{79 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);80 int idx, ret;81 82 if (!drm_dev_enter(&vdev->drm, &idx))83 return -ENODEV;84 85 mutex_lock(&bo->lock);86 87 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);88 if (!ret) {89 bo->ctx = ctx;90 bo->vpu_addr = bo->mm_node.start;91 } else {92 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);93 }94 95 ivpu_dbg_bo(vdev, bo, "alloc");96 97 mutex_unlock(&bo->lock);98 99 drm_dev_exit(idx);100 101 return ret;102}103 104static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)105{106 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);107 108 lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount));109 110 if (bo->mmu_mapped) {111 drm_WARN_ON(&vdev->drm, !bo->ctx);112 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);113 drm_WARN_ON(&vdev->drm, !bo->base.sgt);114 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);115 bo->mmu_mapped = false;116 }117 118 if (bo->ctx) {119 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);120 bo->ctx = NULL;121 }122 123 if (bo->base.base.import_attach)124 return;125 126 dma_resv_lock(bo->base.base.resv, NULL);127 if (bo->base.sgt) {128 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);129 sg_free_table(bo->base.sgt);130 kfree(bo->base.sgt);131 bo->base.sgt = NULL;132 }133 dma_resv_unlock(bo->base.base.resv);134}135 136void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)137{138 struct ivpu_bo *bo;139 140 if (drm_WARN_ON(&vdev->drm, !ctx))141 return;142 143 mutex_lock(&vdev->bo_list_lock);144 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {145 mutex_lock(&bo->lock);146 if (bo->ctx == ctx) {147 ivpu_dbg_bo(vdev, bo, "unbind");148 ivpu_bo_unbind_locked(bo);149 }150 mutex_unlock(&bo->lock);151 }152 mutex_unlock(&vdev->bo_list_lock);153}154 155struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)156{157 struct ivpu_bo *bo;158 159 if (size == 0 || !PAGE_ALIGNED(size))160 return ERR_PTR(-EINVAL);161 162 bo = kzalloc(sizeof(*bo), GFP_KERNEL);163 if (!bo)164 return ERR_PTR(-ENOMEM);165 166 bo->base.base.funcs = &ivpu_gem_funcs;167 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */168 169 INIT_LIST_HEAD(&bo->bo_list_node);170 mutex_init(&bo->lock);171 172 return &bo->base.base;173}174 175static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)176{177 struct drm_gem_shmem_object *shmem;178 struct ivpu_bo *bo;179 180 switch (flags & DRM_IVPU_BO_CACHE_MASK) {181 case DRM_IVPU_BO_CACHED:182 case DRM_IVPU_BO_WC:183 break;184 default:185 return ERR_PTR(-EINVAL);186 }187 188 shmem = drm_gem_shmem_create(&vdev->drm, size);189 if (IS_ERR(shmem))190 return ERR_CAST(shmem);191 192 bo = to_ivpu_bo(&shmem->base);193 bo->base.map_wc = flags & DRM_IVPU_BO_WC;194 bo->flags = flags;195 196 mutex_lock(&vdev->bo_list_lock);197 list_add_tail(&bo->bo_list_node, &vdev->bo_list);198 mutex_unlock(&vdev->bo_list_lock);199 200 return bo;201}202 203static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)204{205 struct ivpu_file_priv *file_priv = file->driver_priv;206 struct ivpu_device *vdev = file_priv->vdev;207 struct ivpu_bo *bo = to_ivpu_bo(obj);208 struct ivpu_addr_range *range;209 210 if (bo->ctx) {211 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",212 file_priv->ctx.id, bo->ctx->id);213 return -EALREADY;214 }215 216 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)217 range = &vdev->hw->ranges.shave;218 else if (bo->flags & DRM_IVPU_BO_DMA_MEM)219 range = &vdev->hw->ranges.dma;220 else221 range = &vdev->hw->ranges.user;222 223 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);224}225 226static void ivpu_gem_bo_free(struct drm_gem_object *obj)227{228 struct ivpu_device *vdev = to_ivpu_device(obj->dev);229 struct ivpu_bo *bo = to_ivpu_bo(obj);230 231 ivpu_dbg_bo(vdev, bo, "free");232 233 mutex_lock(&vdev->bo_list_lock);234 list_del(&bo->bo_list_node);235 mutex_unlock(&vdev->bo_list_lock);236 237 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));238 239 ivpu_bo_unbind_locked(bo);240 mutex_destroy(&bo->lock);241 242 drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);243 drm_gem_shmem_free(&bo->base);244}245 246static const struct drm_gem_object_funcs ivpu_gem_funcs = {247 .free = ivpu_gem_bo_free,248 .open = ivpu_gem_bo_open,249 .print_info = drm_gem_shmem_object_print_info,250 .pin = drm_gem_shmem_object_pin,251 .unpin = drm_gem_shmem_object_unpin,252 .get_sg_table = drm_gem_shmem_object_get_sg_table,253 .vmap = drm_gem_shmem_object_vmap,254 .vunmap = drm_gem_shmem_object_vunmap,255 .mmap = drm_gem_shmem_object_mmap,256 .vm_ops = &drm_gem_shmem_vm_ops,257};258 259int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)260{261 struct ivpu_file_priv *file_priv = file->driver_priv;262 struct ivpu_device *vdev = file_priv->vdev;263 struct drm_ivpu_bo_create *args = data;264 u64 size = PAGE_ALIGN(args->size);265 struct ivpu_bo *bo;266 int ret;267 268 if (args->flags & ~DRM_IVPU_BO_FLAGS)269 return -EINVAL;270 271 if (size == 0)272 return -EINVAL;273 274 bo = ivpu_bo_alloc(vdev, size, args->flags);275 if (IS_ERR(bo)) {276 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",277 bo, file_priv->ctx.id, args->size, args->flags);278 return PTR_ERR(bo);279 }280 281 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);282 if (!ret)283 args->vpu_addr = bo->vpu_addr;284 285 drm_gem_object_put(&bo->base.base);286 287 return ret;288}289 290struct ivpu_bo *291ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,292 struct ivpu_addr_range *range, u64 size, u32 flags)293{294 struct iosys_map map;295 struct ivpu_bo *bo;296 int ret;297 298 if (drm_WARN_ON(&vdev->drm, !range))299 return NULL;300 301 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));302 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));303 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));304 305 bo = ivpu_bo_alloc(vdev, size, flags);306 if (IS_ERR(bo)) {307 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",308 bo, range->start, size, flags);309 return NULL;310 }311 312 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);313 if (ret)314 goto err_put;315 316 ret = ivpu_bo_pin(bo);317 if (ret)318 goto err_put;319 320 if (flags & DRM_IVPU_BO_MAPPABLE) {321 dma_resv_lock(bo->base.base.resv, NULL);322 ret = drm_gem_shmem_vmap(&bo->base, &map);323 dma_resv_unlock(bo->base.base.resv);324 325 if (ret)326 goto err_put;327 }328 329 return bo;330 331err_put:332 drm_gem_object_put(&bo->base.base);333 return NULL;334}335 336struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)337{338 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);339}340 341void ivpu_bo_free(struct ivpu_bo *bo)342{343 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);344 345 if (bo->flags & DRM_IVPU_BO_MAPPABLE) {346 dma_resv_lock(bo->base.base.resv, NULL);347 drm_gem_shmem_vunmap(&bo->base, &map);348 dma_resv_unlock(bo->base.base.resv);349 }350 351 drm_gem_object_put(&bo->base.base);352}353 354int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)355{356 struct drm_ivpu_bo_info *args = data;357 struct drm_gem_object *obj;358 struct ivpu_bo *bo;359 int ret = 0;360 361 obj = drm_gem_object_lookup(file, args->handle);362 if (!obj)363 return -ENOENT;364 365 bo = to_ivpu_bo(obj);366 367 mutex_lock(&bo->lock);368 args->flags = bo->flags;369 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);370 args->vpu_addr = bo->vpu_addr;371 args->size = obj->size;372 mutex_unlock(&bo->lock);373 374 drm_gem_object_put(obj);375 return ret;376}377 378int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)379{380 struct drm_ivpu_bo_wait *args = data;381 struct drm_gem_object *obj;382 unsigned long timeout;383 long ret;384 385 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);386 387 obj = drm_gem_object_lookup(file, args->handle);388 if (!obj)389 return -EINVAL;390 391 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);392 if (ret == 0) {393 ret = -ETIMEDOUT;394 } else if (ret > 0) {395 ret = 0;396 args->job_status = to_ivpu_bo(obj)->job_status;397 }398 399 drm_gem_object_put(obj);400 401 return ret;402}403 404static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)405{406 mutex_lock(&bo->lock);407 408 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",409 bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size,410 bo->flags, kref_read(&bo->base.base.refcount));411 412 if (bo->base.pages)413 drm_printf(p, " has_pages");414 415 if (bo->mmu_mapped)416 drm_printf(p, " mmu_mapped");417 418 if (bo->base.base.import_attach)419 drm_printf(p, " imported");420 421 drm_printf(p, "\n");422 423 mutex_unlock(&bo->lock);424}425 426void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)427{428 struct ivpu_device *vdev = to_ivpu_device(dev);429 struct ivpu_bo *bo;430 431 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",432 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");433 434 mutex_lock(&vdev->bo_list_lock);435 list_for_each_entry(bo, &vdev->bo_list, bo_list_node)436 ivpu_bo_print_info(bo, p);437 mutex_unlock(&vdev->bo_list_lock);438}439 440void ivpu_bo_list_print(struct drm_device *dev)441{442 struct drm_printer p = drm_info_printer(dev->dev);443 444 ivpu_bo_list(dev, &p);445}446