2812 lines · c
1// SPDX-License-Identifier: GPL-2.0 or MIT2/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */3/* Copyright 2023 Collabora ltd. */4 5#include <drm/drm_debugfs.h>6#include <drm/drm_drv.h>7#include <drm/drm_exec.h>8#include <drm/drm_gpuvm.h>9#include <drm/drm_managed.h>10#include <drm/gpu_scheduler.h>11#include <drm/panthor_drm.h>12 13#include <linux/atomic.h>14#include <linux/bitfield.h>15#include <linux/delay.h>16#include <linux/dma-mapping.h>17#include <linux/interrupt.h>18#include <linux/io.h>19#include <linux/iopoll.h>20#include <linux/io-pgtable.h>21#include <linux/iommu.h>22#include <linux/kmemleak.h>23#include <linux/platform_device.h>24#include <linux/pm_runtime.h>25#include <linux/rwsem.h>26#include <linux/sched.h>27#include <linux/shmem_fs.h>28#include <linux/sizes.h>29 30#include "panthor_device.h"31#include "panthor_gem.h"32#include "panthor_heap.h"33#include "panthor_mmu.h"34#include "panthor_regs.h"35#include "panthor_sched.h"36 37#define MAX_AS_SLOTS 3238 39struct panthor_vm;40 41/**42 * struct panthor_as_slot - Address space slot43 */44struct panthor_as_slot {45 /** @vm: VM bound to this slot. NULL is no VM is bound. */46 struct panthor_vm *vm;47};48 49/**50 * struct panthor_mmu - MMU related data51 */52struct panthor_mmu {53 /** @irq: The MMU irq. */54 struct panthor_irq irq;55 56 /** @as: Address space related fields.57 *58 * The GPU has a limited number of address spaces (AS) slots, forcing59 * us to re-assign them to re-assign slots on-demand.60 */61 struct {62 /** @slots_lock: Lock protecting access to all other AS fields. */63 struct mutex slots_lock;64 65 /** @alloc_mask: Bitmask encoding the allocated slots. */66 unsigned long alloc_mask;67 68 /** @faulty_mask: Bitmask encoding the faulty slots. */69 unsigned long faulty_mask;70 71 /** @slots: VMs currently bound to the AS slots. */72 struct panthor_as_slot slots[MAX_AS_SLOTS];73 74 /**75 * @lru_list: List of least recently used VMs.76 *77 * We use this list to pick a VM to evict when all slots are78 * used.79 *80 * There should be no more active VMs than there are AS slots,81 * so this LRU is just here to keep VMs bound until there's82 * a need to release a slot, thus avoid unnecessary TLB/cache83 * flushes.84 */85 struct list_head lru_list;86 } as;87 88 /** @vm: VMs management fields */89 struct {90 /** @lock: Lock protecting access to list. */91 struct mutex lock;92 93 /** @list: List containing all VMs. */94 struct list_head list;95 96 /** @reset_in_progress: True if a reset is in progress. */97 bool reset_in_progress;98 99 /** @wq: Workqueue used for the VM_BIND queues. */100 struct workqueue_struct *wq;101 } vm;102};103 104/**105 * struct panthor_vm_pool - VM pool object106 */107struct panthor_vm_pool {108 /** @xa: Array used for VM handle tracking. */109 struct xarray xa;110};111 112/**113 * struct panthor_vma - GPU mapping object114 *115 * This is used to track GEM mappings in GPU space.116 */117struct panthor_vma {118 /** @base: Inherits from drm_gpuva. */119 struct drm_gpuva base;120 121 /** @node: Used to implement deferred release of VMAs. */122 struct list_head node;123 124 /**125 * @flags: Combination of drm_panthor_vm_bind_op_flags.126 *127 * Only map related flags are accepted.128 */129 u32 flags;130};131 132/**133 * struct panthor_vm_op_ctx - VM operation context134 *135 * With VM operations potentially taking place in a dma-signaling path, we136 * need to make sure everything that might require resource allocation is137 * pre-allocated upfront. This is what this operation context is far.138 *139 * We also collect resources that have been freed, so we can release them140 * asynchronously, and let the VM_BIND scheduler process the next VM_BIND141 * request.142 */143struct panthor_vm_op_ctx {144 /** @rsvd_page_tables: Pages reserved for the MMU page table update. */145 struct {146 /** @count: Number of pages reserved. */147 u32 count;148 149 /** @ptr: Point to the first unused page in the @pages table. */150 u32 ptr;151 152 /**153 * @page: Array of pages that can be used for an MMU page table update.154 *155 * After an VM operation, there might be free pages left in this array.156 * They should be returned to the pt_cache as part of the op_ctx cleanup.157 */158 void **pages;159 } rsvd_page_tables;160 161 /**162 * @preallocated_vmas: Pre-allocated VMAs to handle the remap case.163 *164 * Partial unmap requests or map requests overlapping existing mappings will165 * trigger a remap call, which need to register up to three panthor_vma objects166 * (one for the new mapping, and two for the previous and next mappings).167 */168 struct panthor_vma *preallocated_vmas[3];169 170 /** @flags: Combination of drm_panthor_vm_bind_op_flags. */171 u32 flags;172 173 /** @va: Virtual range targeted by the VM operation. */174 struct {175 /** @addr: Start address. */176 u64 addr;177 178 /** @range: Range size. */179 u64 range;180 } va;181 182 /**183 * @returned_vmas: List of panthor_vma objects returned after a VM operation.184 *185 * For unmap operations, this will contain all VMAs that were covered by the186 * specified VA range.187 *188 * For map operations, this will contain all VMAs that previously mapped to189 * the specified VA range.190 *191 * Those VMAs, and the resources they point to will be released as part of192 * the op_ctx cleanup operation.193 */194 struct list_head returned_vmas;195 196 /** @map: Fields specific to a map operation. */197 struct {198 /** @vm_bo: Buffer object to map. */199 struct drm_gpuvm_bo *vm_bo;200 201 /** @bo_offset: Offset in the buffer object. */202 u64 bo_offset;203 204 /**205 * @sgt: sg-table pointing to pages backing the GEM object.206 *207 * This is gathered at job creation time, such that we don't have208 * to allocate in ::run_job().209 */210 struct sg_table *sgt;211 212 /**213 * @new_vma: The new VMA object that will be inserted to the VA tree.214 */215 struct panthor_vma *new_vma;216 } map;217};218 219/**220 * struct panthor_vm - VM object221 *222 * A VM is an object representing a GPU (or MCU) virtual address space.223 * It embeds the MMU page table for this address space, a tree containing224 * all the virtual mappings of GEM objects, and other things needed to manage225 * the VM.226 *227 * Except for the MCU VM, which is managed by the kernel, all other VMs are228 * created by userspace and mostly managed by userspace, using the229 * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.230 *231 * A portion of the virtual address space is reserved for kernel objects,232 * like heap chunks, and userspace gets to decide how much of the virtual233 * address space is left to the kernel (half of the virtual address space234 * by default).235 */236struct panthor_vm {237 /**238 * @base: Inherit from drm_gpuvm.239 *240 * We delegate all the VA management to the common drm_gpuvm framework241 * and only implement hooks to update the MMU page table.242 */243 struct drm_gpuvm base;244 245 /**246 * @sched: Scheduler used for asynchronous VM_BIND request.247 *248 * We use a 1:1 scheduler here.249 */250 struct drm_gpu_scheduler sched;251 252 /**253 * @entity: Scheduling entity representing the VM_BIND queue.254 *255 * There's currently one bind queue per VM. It doesn't make sense to256 * allow more given the VM operations are serialized anyway.257 */258 struct drm_sched_entity entity;259 260 /** @ptdev: Device. */261 struct panthor_device *ptdev;262 263 /** @memattr: Value to program to the AS_MEMATTR register. */264 u64 memattr;265 266 /** @pgtbl_ops: Page table operations. */267 struct io_pgtable_ops *pgtbl_ops;268 269 /** @root_page_table: Stores the root page table pointer. */270 void *root_page_table;271 272 /**273 * @op_lock: Lock used to serialize operations on a VM.274 *275 * The serialization of jobs queued to the VM_BIND queue is already276 * taken care of by drm_sched, but we need to serialize synchronous277 * and asynchronous VM_BIND request. This is what this lock is for.278 */279 struct mutex op_lock;280 281 /**282 * @op_ctx: The context attached to the currently executing VM operation.283 *284 * NULL when no operation is in progress.285 */286 struct panthor_vm_op_ctx *op_ctx;287 288 /**289 * @mm: Memory management object representing the auto-VA/kernel-VA.290 *291 * Used to auto-allocate VA space for kernel-managed objects (tiler292 * heaps, ...).293 *294 * For the MCU VM, this is managing the VA range that's used to map295 * all shared interfaces.296 *297 * For user VMs, the range is specified by userspace, and must not298 * exceed half of the VA space addressable.299 */300 struct drm_mm mm;301 302 /** @mm_lock: Lock protecting the @mm field. */303 struct mutex mm_lock;304 305 /** @kernel_auto_va: Automatic VA-range for kernel BOs. */306 struct {307 /** @start: Start of the automatic VA-range for kernel BOs. */308 u64 start;309 310 /** @size: Size of the automatic VA-range for kernel BOs. */311 u64 end;312 } kernel_auto_va;313 314 /** @as: Address space related fields. */315 struct {316 /**317 * @id: ID of the address space this VM is bound to.318 *319 * A value of -1 means the VM is inactive/not bound.320 */321 int id;322 323 /** @active_cnt: Number of active users of this VM. */324 refcount_t active_cnt;325 326 /**327 * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.328 *329 * Active VMs should not be inserted in the LRU list.330 */331 struct list_head lru_node;332 } as;333 334 /**335 * @heaps: Tiler heap related fields.336 */337 struct {338 /**339 * @pool: The heap pool attached to this VM.340 *341 * Will stay NULL until someone creates a heap context on this VM.342 */343 struct panthor_heap_pool *pool;344 345 /** @lock: Lock used to protect access to @pool. */346 struct mutex lock;347 } heaps;348 349 /** @node: Used to insert the VM in the panthor_mmu::vm::list. */350 struct list_head node;351 352 /** @for_mcu: True if this is the MCU VM. */353 bool for_mcu;354 355 /**356 * @destroyed: True if the VM was destroyed.357 *358 * No further bind requests should be queued to a destroyed VM.359 */360 bool destroyed;361 362 /**363 * @unusable: True if the VM has turned unusable because something364 * bad happened during an asynchronous request.365 *366 * We don't try to recover from such failures, because this implies367 * informing userspace about the specific operation that failed, and368 * hoping the userspace driver can replay things from there. This all369 * sounds very complicated for little gain.370 *371 * Instead, we should just flag the VM as unusable, and fail any372 * further request targeting this VM.373 *374 * We also provide a way to query a VM state, so userspace can destroy375 * it and create a new one.376 *377 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST378 * situation, where the logical device needs to be re-created.379 */380 bool unusable;381 382 /**383 * @unhandled_fault: Unhandled fault happened.384 *385 * This should be reported to the scheduler, and the queue/group be386 * flagged as faulty as a result.387 */388 bool unhandled_fault;389};390 391/**392 * struct panthor_vm_bind_job - VM bind job393 */394struct panthor_vm_bind_job {395 /** @base: Inherit from drm_sched_job. */396 struct drm_sched_job base;397 398 /** @refcount: Reference count. */399 struct kref refcount;400 401 /** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */402 struct work_struct cleanup_op_ctx_work;403 404 /** @vm: VM targeted by the VM operation. */405 struct panthor_vm *vm;406 407 /** @ctx: Operation context. */408 struct panthor_vm_op_ctx ctx;409};410 411/**412 * @pt_cache: Cache used to allocate MMU page tables.413 *414 * The pre-allocation pattern forces us to over-allocate to plan for415 * the worst case scenario, and return the pages we didn't use.416 *417 * Having a kmem_cache allows us to speed allocations.418 */419static struct kmem_cache *pt_cache;420 421/**422 * alloc_pt() - Custom page table allocator423 * @cookie: Cookie passed at page table allocation time.424 * @size: Size of the page table. This size should be fixed,425 * and determined at creation time based on the granule size.426 * @gfp: GFP flags.427 *428 * We want a custom allocator so we can use a cache for page table429 * allocations and amortize the cost of the over-reservation that's430 * done to allow asynchronous VM operations.431 *432 * Return: non-NULL on success, NULL if the allocation failed for any433 * reason.434 */435static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)436{437 struct panthor_vm *vm = cookie;438 void *page;439 440 /* Allocation of the root page table happening during init. */441 if (unlikely(!vm->root_page_table)) {442 struct page *p;443 444 drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);445 p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev),446 gfp | __GFP_ZERO, get_order(size));447 page = p ? page_address(p) : NULL;448 vm->root_page_table = page;449 return page;450 }451 452 /* We're not supposed to have anything bigger than 4k here, because we picked a453 * 4k granule size at init time.454 */455 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))456 return NULL;457 458 /* We must have some op_ctx attached to the VM and it must have at least one459 * free page.460 */461 if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||462 drm_WARN_ON(&vm->ptdev->base,463 vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))464 return NULL;465 466 page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];467 memset(page, 0, SZ_4K);468 469 /* Page table entries don't use virtual addresses, which trips out470 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses471 * are mixed with other fields, and I fear kmemleak won't detect that472 * either.473 *474 * Let's just ignore memory passed to the page-table driver for now.475 */476 kmemleak_ignore(page);477 return page;478}479 480/**481 * @free_pt() - Custom page table free function482 * @cookie: Cookie passed at page table allocation time.483 * @data: Page table to free.484 * @size: Size of the page table. This size should be fixed,485 * and determined at creation time based on the granule size.486 */487static void free_pt(void *cookie, void *data, size_t size)488{489 struct panthor_vm *vm = cookie;490 491 if (unlikely(vm->root_page_table == data)) {492 free_pages((unsigned long)data, get_order(size));493 vm->root_page_table = NULL;494 return;495 }496 497 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))498 return;499 500 /* Return the page to the pt_cache. */501 kmem_cache_free(pt_cache, data);502}503 504static int wait_ready(struct panthor_device *ptdev, u32 as_nr)505{506 int ret;507 u32 val;508 509 /* Wait for the MMU status to indicate there is no active command, in510 * case one is pending.511 */512 ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),513 val, !(val & AS_STATUS_AS_ACTIVE),514 10, 100000);515 516 if (ret) {517 panthor_device_schedule_reset(ptdev);518 drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");519 }520 521 return ret;522}523 524static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)525{526 int status;527 528 /* write AS_COMMAND when MMU is ready to accept another command */529 status = wait_ready(ptdev, as_nr);530 if (!status)531 gpu_write(ptdev, AS_COMMAND(as_nr), cmd);532 533 return status;534}535 536static void lock_region(struct panthor_device *ptdev, u32 as_nr,537 u64 region_start, u64 size)538{539 u8 region_width;540 u64 region;541 u64 region_end = region_start + size;542 543 if (!size)544 return;545 546 /*547 * The locked region is a naturally aligned power of 2 block encoded as548 * log2 minus(1).549 * Calculate the desired start/end and look for the highest bit which550 * differs. The smallest naturally aligned block must include this bit551 * change, the desired region starts with this bit (and subsequent bits)552 * zeroed and ends with the bit (and subsequent bits) set to one.553 */554 region_width = max(fls64(region_start ^ (region_end - 1)),555 const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;556 557 /*558 * Mask off the low bits of region_start (which would be ignored by559 * the hardware anyway)560 */561 region_start &= GENMASK_ULL(63, region_width);562 563 region = region_width | region_start;564 565 /* Lock the region that needs to be updated */566 gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));567 gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));568 write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);569}570 571static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,572 u64 iova, u64 size, u32 op)573{574 lockdep_assert_held(&ptdev->mmu->as.slots_lock);575 576 if (as_nr < 0)577 return 0;578 579 /*580 * If the AS number is greater than zero, then we can be sure581 * the device is up and running, so we don't need to explicitly582 * power it up583 */584 585 if (op != AS_COMMAND_UNLOCK)586 lock_region(ptdev, as_nr, iova, size);587 588 /* Run the MMU operation */589 write_cmd(ptdev, as_nr, op);590 591 /* Wait for the flush to complete */592 return wait_ready(ptdev, as_nr);593}594 595static int mmu_hw_do_operation(struct panthor_vm *vm,596 u64 iova, u64 size, u32 op)597{598 struct panthor_device *ptdev = vm->ptdev;599 int ret;600 601 mutex_lock(&ptdev->mmu->as.slots_lock);602 ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);603 mutex_unlock(&ptdev->mmu->as.slots_lock);604 605 return ret;606}607 608static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,609 u64 transtab, u64 transcfg, u64 memattr)610{611 int ret;612 613 ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);614 if (ret)615 return ret;616 617 gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));618 gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));619 620 gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));621 gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));622 623 gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));624 gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));625 626 return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);627}628 629static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)630{631 int ret;632 633 ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);634 if (ret)635 return ret;636 637 gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);638 gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);639 640 gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);641 gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);642 643 gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);644 gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);645 646 return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);647}648 649static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)650{651 /* Bits 16 to 31 mean REQ_COMPLETE. */652 return value & GENMASK(15, 0);653}654 655static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)656{657 return BIT(as);658}659 660/**661 * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults662 * @vm: VM to check.663 *664 * Return: true if the VM has unhandled faults, false otherwise.665 */666bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm)667{668 return vm->unhandled_fault;669}670 671/**672 * panthor_vm_is_unusable() - Check if the VM is still usable673 * @vm: VM to check.674 *675 * Return: true if the VM is unusable, false otherwise.676 */677bool panthor_vm_is_unusable(struct panthor_vm *vm)678{679 return vm->unusable;680}681 682static void panthor_vm_release_as_locked(struct panthor_vm *vm)683{684 struct panthor_device *ptdev = vm->ptdev;685 686 lockdep_assert_held(&ptdev->mmu->as.slots_lock);687 688 if (drm_WARN_ON(&ptdev->base, vm->as.id < 0))689 return;690 691 ptdev->mmu->as.slots[vm->as.id].vm = NULL;692 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);693 refcount_set(&vm->as.active_cnt, 0);694 list_del_init(&vm->as.lru_node);695 vm->as.id = -1;696}697 698/**699 * panthor_vm_active() - Flag a VM as active700 * @VM: VM to flag as active.701 *702 * Assigns an address space to a VM so it can be used by the GPU/MCU.703 *704 * Return: 0 on success, a negative error code otherwise.705 */706int panthor_vm_active(struct panthor_vm *vm)707{708 struct panthor_device *ptdev = vm->ptdev;709 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);710 struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;711 int ret = 0, as, cookie;712 u64 transtab, transcfg;713 714 if (!drm_dev_enter(&ptdev->base, &cookie))715 return -ENODEV;716 717 if (refcount_inc_not_zero(&vm->as.active_cnt))718 goto out_dev_exit;719 720 mutex_lock(&ptdev->mmu->as.slots_lock);721 722 if (refcount_inc_not_zero(&vm->as.active_cnt))723 goto out_unlock;724 725 as = vm->as.id;726 if (as >= 0) {727 /* Unhandled pagefault on this AS, the MMU was disabled. We need to728 * re-enable the MMU after clearing+unmasking the AS interrupts.729 */730 if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as))731 goto out_enable_as;732 733 goto out_make_active;734 }735 736 /* Check for a free AS */737 if (vm->for_mcu) {738 drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));739 as = 0;740 } else {741 as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));742 }743 744 if (!(BIT(as) & ptdev->gpu_info.as_present)) {745 struct panthor_vm *lru_vm;746 747 lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,748 struct panthor_vm,749 as.lru_node);750 if (drm_WARN_ON(&ptdev->base, !lru_vm)) {751 ret = -EBUSY;752 goto out_unlock;753 }754 755 drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt));756 as = lru_vm->as.id;757 panthor_vm_release_as_locked(lru_vm);758 }759 760 /* Assign the free or reclaimed AS to the FD */761 vm->as.id = as;762 set_bit(as, &ptdev->mmu->as.alloc_mask);763 ptdev->mmu->as.slots[as].vm = vm;764 765out_enable_as:766 transtab = cfg->arm_lpae_s1_cfg.ttbr;767 transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |768 AS_TRANSCFG_PTW_RA |769 AS_TRANSCFG_ADRMODE_AARCH64_4K |770 AS_TRANSCFG_INA_BITS(55 - va_bits);771 if (ptdev->coherent)772 transcfg |= AS_TRANSCFG_PTW_SH_OS;773 774 /* If the VM is re-activated, we clear the fault. */775 vm->unhandled_fault = false;776 777 /* Unhandled pagefault on this AS, clear the fault and re-enable interrupts778 * before enabling the AS.779 */780 if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {781 gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as));782 ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);783 gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);784 }785 786 ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);787 788out_make_active:789 if (!ret) {790 refcount_set(&vm->as.active_cnt, 1);791 list_del_init(&vm->as.lru_node);792 }793 794out_unlock:795 mutex_unlock(&ptdev->mmu->as.slots_lock);796 797out_dev_exit:798 drm_dev_exit(cookie);799 return ret;800}801 802/**803 * panthor_vm_idle() - Flag a VM idle804 * @VM: VM to flag as idle.805 *806 * When we know the GPU is done with the VM (no more jobs to process),807 * we can relinquish the AS slot attached to this VM, if any.808 *809 * We don't release the slot immediately, but instead place the VM in810 * the LRU list, so it can be evicted if another VM needs an AS slot.811 * This way, VMs keep attached to the AS they were given until we run812 * out of free slot, limiting the number of MMU operations (TLB flush813 * and other AS updates).814 */815void panthor_vm_idle(struct panthor_vm *vm)816{817 struct panthor_device *ptdev = vm->ptdev;818 819 if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))820 return;821 822 if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node)))823 list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);824 825 refcount_set(&vm->as.active_cnt, 0);826 mutex_unlock(&ptdev->mmu->as.slots_lock);827}828 829u32 panthor_vm_page_size(struct panthor_vm *vm)830{831 const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);832 u32 pg_shift = ffs(pgt->cfg.pgsize_bitmap) - 1;833 834 return 1u << pg_shift;835}836 837static void panthor_vm_stop(struct panthor_vm *vm)838{839 drm_sched_stop(&vm->sched, NULL);840}841 842static void panthor_vm_start(struct panthor_vm *vm)843{844 drm_sched_start(&vm->sched);845}846 847/**848 * panthor_vm_as() - Get the AS slot attached to a VM849 * @vm: VM to get the AS slot of.850 *851 * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.852 */853int panthor_vm_as(struct panthor_vm *vm)854{855 return vm->as.id;856}857 858static size_t get_pgsize(u64 addr, size_t size, size_t *count)859{860 /*861 * io-pgtable only operates on multiple pages within a single table862 * entry, so we need to split at boundaries of the table size, i.e.863 * the next block size up. The distance from address A to the next864 * boundary of block size B is logically B - A % B, but in unsigned865 * two's complement where B is a power of two we get the equivalence866 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)867 */868 size_t blk_offset = -addr % SZ_2M;869 870 if (blk_offset || size < SZ_2M) {871 *count = min_not_zero(blk_offset, size) / SZ_4K;872 return SZ_4K;873 }874 blk_offset = -addr % SZ_1G ?: SZ_1G;875 *count = min(blk_offset, size) / SZ_2M;876 return SZ_2M;877}878 879static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)880{881 struct panthor_device *ptdev = vm->ptdev;882 int ret = 0, cookie;883 884 if (vm->as.id < 0)885 return 0;886 887 /* If the device is unplugged, we just silently skip the flush. */888 if (!drm_dev_enter(&ptdev->base, &cookie))889 return 0;890 891 ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);892 893 drm_dev_exit(cookie);894 return ret;895}896 897/**898 * panthor_vm_flush_all() - Flush L2 caches for the entirety of a VM's AS899 * @vm: VM whose cache to flush900 *901 * Return: 0 on success, a negative error code if flush failed.902 */903int panthor_vm_flush_all(struct panthor_vm *vm)904{905 return panthor_vm_flush_range(vm, vm->base.mm_start, vm->base.mm_range);906}907 908static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)909{910 struct panthor_device *ptdev = vm->ptdev;911 struct io_pgtable_ops *ops = vm->pgtbl_ops;912 u64 offset = 0;913 914 drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%llx", vm->as.id, iova, size);915 916 while (offset < size) {917 size_t unmapped_sz = 0, pgcount;918 size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);919 920 unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);921 922 if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {923 drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",924 iova + offset + unmapped_sz,925 iova + offset + pgsize * pgcount,926 iova, iova + size);927 panthor_vm_flush_range(vm, iova, offset + unmapped_sz);928 return -EINVAL;929 }930 offset += unmapped_sz;931 }932 933 return panthor_vm_flush_range(vm, iova, size);934}935 936static int937panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,938 struct sg_table *sgt, u64 offset, u64 size)939{940 struct panthor_device *ptdev = vm->ptdev;941 unsigned int count;942 struct scatterlist *sgl;943 struct io_pgtable_ops *ops = vm->pgtbl_ops;944 u64 start_iova = iova;945 int ret;946 947 if (!size)948 return 0;949 950 for_each_sgtable_dma_sg(sgt, sgl, count) {951 dma_addr_t paddr = sg_dma_address(sgl);952 size_t len = sg_dma_len(sgl);953 954 if (len <= offset) {955 offset -= len;956 continue;957 }958 959 paddr += offset;960 len -= offset;961 len = min_t(size_t, len, size);962 size -= len;963 964 drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%pad, len=%zx",965 vm->as.id, iova, &paddr, len);966 967 while (len) {968 size_t pgcount, mapped = 0;969 size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);970 971 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,972 GFP_KERNEL, &mapped);973 iova += mapped;974 paddr += mapped;975 len -= mapped;976 977 if (drm_WARN_ON(&ptdev->base, !ret && !mapped))978 ret = -ENOMEM;979 980 if (ret) {981 /* If something failed, unmap what we've already mapped before982 * returning. The unmap call is not supposed to fail.983 */984 drm_WARN_ON(&ptdev->base,985 panthor_vm_unmap_pages(vm, start_iova,986 iova - start_iova));987 return ret;988 }989 }990 991 if (!size)992 break;993 994 offset = 0;995 }996 997 return panthor_vm_flush_range(vm, start_iova, iova - start_iova);998}999 1000static int flags_to_prot(u32 flags)1001{1002 int prot = 0;1003 1004 if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)1005 prot |= IOMMU_NOEXEC;1006 1007 if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))1008 prot |= IOMMU_CACHE;1009 1010 if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)1011 prot |= IOMMU_READ;1012 else1013 prot |= IOMMU_READ | IOMMU_WRITE;1014 1015 return prot;1016}1017 1018/**1019 * panthor_vm_alloc_va() - Allocate a region in the auto-va space1020 * @VM: VM to allocate a region on.1021 * @va: start of the VA range. Can be PANTHOR_VM_KERNEL_AUTO_VA if the user1022 * wants the VA to be automatically allocated from the auto-VA range.1023 * @size: size of the VA range.1024 * @va_node: drm_mm_node to initialize. Must be zero-initialized.1025 *1026 * Some GPU objects, like heap chunks, are fully managed by the kernel and1027 * need to be mapped to the userspace VM, in the region reserved for kernel1028 * objects.1029 *1030 * This function takes care of allocating a region in the kernel auto-VA space.1031 *1032 * Return: 0 on success, an error code otherwise.1033 */1034int1035panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,1036 struct drm_mm_node *va_node)1037{1038 ssize_t vm_pgsz = panthor_vm_page_size(vm);1039 int ret;1040 1041 if (!size || !IS_ALIGNED(size, vm_pgsz))1042 return -EINVAL;1043 1044 if (va != PANTHOR_VM_KERNEL_AUTO_VA && !IS_ALIGNED(va, vm_pgsz))1045 return -EINVAL;1046 1047 mutex_lock(&vm->mm_lock);1048 if (va != PANTHOR_VM_KERNEL_AUTO_VA) {1049 va_node->start = va;1050 va_node->size = size;1051 ret = drm_mm_reserve_node(&vm->mm, va_node);1052 } else {1053 ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size,1054 size >= SZ_2M ? SZ_2M : SZ_4K,1055 0, vm->kernel_auto_va.start,1056 vm->kernel_auto_va.end,1057 DRM_MM_INSERT_BEST);1058 }1059 mutex_unlock(&vm->mm_lock);1060 1061 return ret;1062}1063 1064/**1065 * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()1066 * @VM: VM to free the region on.1067 * @va_node: Memory node representing the region to free.1068 */1069void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node)1070{1071 mutex_lock(&vm->mm_lock);1072 drm_mm_remove_node(va_node);1073 mutex_unlock(&vm->mm_lock);1074}1075 1076static void panthor_vm_bo_put(struct drm_gpuvm_bo *vm_bo)1077{1078 struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);1079 struct drm_gpuvm *vm = vm_bo->vm;1080 bool unpin;1081 1082 /* We must retain the GEM before calling drm_gpuvm_bo_put(),1083 * otherwise the mutex might be destroyed while we hold it.1084 * Same goes for the VM, since we take the VM resv lock.1085 */1086 drm_gem_object_get(&bo->base.base);1087 drm_gpuvm_get(vm);1088 1089 /* We take the resv lock to protect against concurrent accesses to the1090 * gpuvm evicted/extobj lists that are modified in1091 * drm_gpuvm_bo_destroy(), which is called if drm_gpuvm_bo_put()1092 * releases sthe last vm_bo reference.1093 * We take the BO GPUVA list lock to protect the vm_bo removal from the1094 * GEM vm_bo list.1095 */1096 dma_resv_lock(drm_gpuvm_resv(vm), NULL);1097 mutex_lock(&bo->gpuva_list_lock);1098 unpin = drm_gpuvm_bo_put(vm_bo);1099 mutex_unlock(&bo->gpuva_list_lock);1100 dma_resv_unlock(drm_gpuvm_resv(vm));1101 1102 /* If the vm_bo object was destroyed, release the pin reference that1103 * was hold by this object.1104 */1105 if (unpin && !bo->base.base.import_attach)1106 drm_gem_shmem_unpin(&bo->base);1107 1108 drm_gpuvm_put(vm);1109 drm_gem_object_put(&bo->base.base);1110}1111 1112static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,1113 struct panthor_vm *vm)1114{1115 struct panthor_vma *vma, *tmp_vma;1116 1117 u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -1118 op_ctx->rsvd_page_tables.ptr;1119 1120 if (remaining_pt_count) {1121 kmem_cache_free_bulk(pt_cache, remaining_pt_count,1122 op_ctx->rsvd_page_tables.pages +1123 op_ctx->rsvd_page_tables.ptr);1124 }1125 1126 kfree(op_ctx->rsvd_page_tables.pages);1127 1128 if (op_ctx->map.vm_bo)1129 panthor_vm_bo_put(op_ctx->map.vm_bo);1130 1131 for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)1132 kfree(op_ctx->preallocated_vmas[i]);1133 1134 list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {1135 list_del(&vma->node);1136 panthor_vm_bo_put(vma->base.vm_bo);1137 kfree(vma);1138 }1139}1140 1141static struct panthor_vma *1142panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx)1143{1144 for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) {1145 struct panthor_vma *vma = op_ctx->preallocated_vmas[i];1146 1147 if (vma) {1148 op_ctx->preallocated_vmas[i] = NULL;1149 return vma;1150 }1151 }1152 1153 return NULL;1154}1155 1156static int1157panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx)1158{1159 u32 vma_count;1160 1161 switch (op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {1162 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:1163 /* One VMA for the new mapping, and two more VMAs for the remap case1164 * which might contain both a prev and next VA.1165 */1166 vma_count = 3;1167 break;1168 1169 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:1170 /* Partial unmaps might trigger a remap with either a prev or a next VA,1171 * but not both.1172 */1173 vma_count = 1;1174 break;1175 1176 default:1177 return 0;1178 }1179 1180 for (u32 i = 0; i < vma_count; i++) {1181 struct panthor_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);1182 1183 if (!vma)1184 return -ENOMEM;1185 1186 op_ctx->preallocated_vmas[i] = vma;1187 }1188 1189 return 0;1190}1191 1192#define PANTHOR_VM_BIND_OP_MAP_FLAGS \1193 (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \1194 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \1195 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \1196 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)1197 1198static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,1199 struct panthor_vm *vm,1200 struct panthor_gem_object *bo,1201 u64 offset,1202 u64 size, u64 va,1203 u32 flags)1204{1205 struct drm_gpuvm_bo *preallocated_vm_bo;1206 struct sg_table *sgt = NULL;1207 u64 pt_count;1208 int ret;1209 1210 if (!bo)1211 return -EINVAL;1212 1213 if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||1214 (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)1215 return -EINVAL;1216 1217 /* Make sure the VA and size are aligned and in-bounds. */1218 if (size > bo->base.base.size || offset > bo->base.base.size - size)1219 return -EINVAL;1220 1221 /* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */1222 if (bo->exclusive_vm_root_gem &&1223 bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))1224 return -EINVAL;1225 1226 memset(op_ctx, 0, sizeof(*op_ctx));1227 INIT_LIST_HEAD(&op_ctx->returned_vmas);1228 op_ctx->flags = flags;1229 op_ctx->va.range = size;1230 op_ctx->va.addr = va;1231 1232 ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);1233 if (ret)1234 goto err_cleanup;1235 1236 if (!bo->base.base.import_attach) {1237 /* Pre-reserve the BO pages, so the map operation doesn't have to1238 * allocate.1239 */1240 ret = drm_gem_shmem_pin(&bo->base);1241 if (ret)1242 goto err_cleanup;1243 }1244 1245 sgt = drm_gem_shmem_get_pages_sgt(&bo->base);1246 if (IS_ERR(sgt)) {1247 if (!bo->base.base.import_attach)1248 drm_gem_shmem_unpin(&bo->base);1249 1250 ret = PTR_ERR(sgt);1251 goto err_cleanup;1252 }1253 1254 op_ctx->map.sgt = sgt;1255 1256 preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);1257 if (!preallocated_vm_bo) {1258 if (!bo->base.base.import_attach)1259 drm_gem_shmem_unpin(&bo->base);1260 1261 ret = -ENOMEM;1262 goto err_cleanup;1263 }1264 1265 /* drm_gpuvm_bo_obtain_prealloc() will call drm_gpuvm_bo_put() on our1266 * pre-allocated BO if the <BO,VM> association exists. Given we1267 * only have one ref on preallocated_vm_bo, drm_gpuvm_bo_destroy() will1268 * be called immediately, and we have to hold the VM resv lock when1269 * calling this function.1270 */1271 dma_resv_lock(panthor_vm_resv(vm), NULL);1272 mutex_lock(&bo->gpuva_list_lock);1273 op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);1274 mutex_unlock(&bo->gpuva_list_lock);1275 dma_resv_unlock(panthor_vm_resv(vm));1276 1277 /* If the a vm_bo for this <VM,BO> combination exists, it already1278 * retains a pin ref, and we can release the one we took earlier.1279 *1280 * If our pre-allocated vm_bo is picked, it now retains the pin ref,1281 * which will be released in panthor_vm_bo_put().1282 */1283 if (preallocated_vm_bo != op_ctx->map.vm_bo &&1284 !bo->base.base.import_attach)1285 drm_gem_shmem_unpin(&bo->base);1286 1287 op_ctx->map.bo_offset = offset;1288 1289 /* L1, L2 and L3 page tables.1290 * We could optimize L3 allocation by iterating over the sgt and merging1291 * 2M contiguous blocks, but it's simpler to over-provision and return1292 * the pages if they're not used.1293 */1294 pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +1295 ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +1296 ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);1297 1298 op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,1299 sizeof(*op_ctx->rsvd_page_tables.pages),1300 GFP_KERNEL);1301 if (!op_ctx->rsvd_page_tables.pages) {1302 ret = -ENOMEM;1303 goto err_cleanup;1304 }1305 1306 ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,1307 op_ctx->rsvd_page_tables.pages);1308 op_ctx->rsvd_page_tables.count = ret;1309 if (ret != pt_count) {1310 ret = -ENOMEM;1311 goto err_cleanup;1312 }1313 1314 /* Insert BO into the extobj list last, when we know nothing can fail. */1315 dma_resv_lock(panthor_vm_resv(vm), NULL);1316 drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);1317 dma_resv_unlock(panthor_vm_resv(vm));1318 1319 return 0;1320 1321err_cleanup:1322 panthor_vm_cleanup_op_ctx(op_ctx, vm);1323 return ret;1324}1325 1326static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,1327 struct panthor_vm *vm,1328 u64 va, u64 size)1329{1330 u32 pt_count = 0;1331 int ret;1332 1333 memset(op_ctx, 0, sizeof(*op_ctx));1334 INIT_LIST_HEAD(&op_ctx->returned_vmas);1335 op_ctx->va.range = size;1336 op_ctx->va.addr = va;1337 op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;1338 1339 /* Pre-allocate L3 page tables to account for the split-2M-block1340 * situation on unmap.1341 */1342 if (va != ALIGN(va, SZ_2M))1343 pt_count++;1344 1345 if (va + size != ALIGN(va + size, SZ_2M) &&1346 ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))1347 pt_count++;1348 1349 ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);1350 if (ret)1351 goto err_cleanup;1352 1353 if (pt_count) {1354 op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,1355 sizeof(*op_ctx->rsvd_page_tables.pages),1356 GFP_KERNEL);1357 if (!op_ctx->rsvd_page_tables.pages) {1358 ret = -ENOMEM;1359 goto err_cleanup;1360 }1361 1362 ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,1363 op_ctx->rsvd_page_tables.pages);1364 if (ret != pt_count) {1365 ret = -ENOMEM;1366 goto err_cleanup;1367 }1368 op_ctx->rsvd_page_tables.count = pt_count;1369 }1370 1371 return 0;1372 1373err_cleanup:1374 panthor_vm_cleanup_op_ctx(op_ctx, vm);1375 return ret;1376}1377 1378static void panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx *op_ctx,1379 struct panthor_vm *vm)1380{1381 memset(op_ctx, 0, sizeof(*op_ctx));1382 INIT_LIST_HEAD(&op_ctx->returned_vmas);1383 op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY;1384}1385 1386/**1387 * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address1388 * @vm: VM to look into.1389 * @va: Virtual address to search for.1390 * @bo_offset: Offset of the GEM object mapped at this virtual address.1391 * Only valid on success.1392 *1393 * The object returned by this function might no longer be mapped when the1394 * function returns. It's the caller responsibility to ensure there's no1395 * concurrent map/unmap operations making the returned value invalid, or1396 * make sure it doesn't matter if the object is no longer mapped.1397 *1398 * Return: A valid pointer on success, an ERR_PTR() otherwise.1399 */1400struct panthor_gem_object *1401panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)1402{1403 struct panthor_gem_object *bo = ERR_PTR(-ENOENT);1404 struct drm_gpuva *gpuva;1405 struct panthor_vma *vma;1406 1407 /* Take the VM lock to prevent concurrent map/unmap operations. */1408 mutex_lock(&vm->op_lock);1409 gpuva = drm_gpuva_find_first(&vm->base, va, 1);1410 vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;1411 if (vma && vma->base.gem.obj) {1412 drm_gem_object_get(vma->base.gem.obj);1413 bo = to_panthor_bo(vma->base.gem.obj);1414 *bo_offset = vma->base.gem.offset + (va - vma->base.va.addr);1415 }1416 mutex_unlock(&vm->op_lock);1417 1418 return bo;1419}1420 1421#define PANTHOR_VM_MIN_KERNEL_VA_SIZE SZ_256M1422 1423static u641424panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create *args,1425 u64 full_va_range)1426{1427 u64 user_va_range;1428 1429 /* Make sure we have a minimum amount of VA space for kernel objects. */1430 if (full_va_range < PANTHOR_VM_MIN_KERNEL_VA_SIZE)1431 return 0;1432 1433 if (args->user_va_range) {1434 /* Use the user provided value if != 0. */1435 user_va_range = args->user_va_range;1436 } else if (TASK_SIZE_OF(current) < full_va_range) {1437 /* If the task VM size is smaller than the GPU VA range, pick this1438 * as our default user VA range, so userspace can CPU/GPU map buffers1439 * at the same address.1440 */1441 user_va_range = TASK_SIZE_OF(current);1442 } else {1443 /* If the GPU VA range is smaller than the task VM size, we1444 * just have to live with the fact we won't be able to map1445 * all buffers at the same GPU/CPU address.1446 *1447 * If the GPU VA range is bigger than 4G (more than 32-bit of1448 * VA), we split the range in two, and assign half of it to1449 * the user and the other half to the kernel, if it's not, we1450 * keep the kernel VA space as small as possible.1451 */1452 user_va_range = full_va_range > SZ_4G ?1453 full_va_range / 2 :1454 full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;1455 }1456 1457 if (full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE < user_va_range)1458 user_va_range = full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;1459 1460 return user_va_range;1461}1462 1463#define PANTHOR_VM_CREATE_FLAGS 01464 1465static int1466panthor_vm_create_check_args(const struct panthor_device *ptdev,1467 const struct drm_panthor_vm_create *args,1468 u64 *kernel_va_start, u64 *kernel_va_range)1469{1470 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);1471 u64 full_va_range = 1ull << va_bits;1472 u64 user_va_range;1473 1474 if (args->flags & ~PANTHOR_VM_CREATE_FLAGS)1475 return -EINVAL;1476 1477 user_va_range = panthor_vm_create_get_user_va_range(args, full_va_range);1478 if (!user_va_range || (args->user_va_range && args->user_va_range > user_va_range))1479 return -EINVAL;1480 1481 /* Pick a kernel VA range that's a power of two, to have a clear split. */1482 *kernel_va_range = rounddown_pow_of_two(full_va_range - user_va_range);1483 *kernel_va_start = full_va_range - *kernel_va_range;1484 return 0;1485}1486 1487/*1488 * Only 32 VMs per open file. If that becomes a limiting factor, we can1489 * increase this number.1490 */1491#define PANTHOR_MAX_VMS_PER_FILE 321492 1493/**1494 * panthor_vm_pool_create_vm() - Create a VM1495 * @pool: The VM to create this VM on.1496 * @kernel_va_start: Start of the region reserved for kernel objects.1497 * @kernel_va_range: Size of the region reserved for kernel objects.1498 *1499 * Return: a positive VM ID on success, a negative error code otherwise.1500 */1501int panthor_vm_pool_create_vm(struct panthor_device *ptdev,1502 struct panthor_vm_pool *pool,1503 struct drm_panthor_vm_create *args)1504{1505 u64 kernel_va_start, kernel_va_range;1506 struct panthor_vm *vm;1507 int ret;1508 u32 id;1509 1510 ret = panthor_vm_create_check_args(ptdev, args, &kernel_va_start, &kernel_va_range);1511 if (ret)1512 return ret;1513 1514 vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range,1515 kernel_va_start, kernel_va_range);1516 if (IS_ERR(vm))1517 return PTR_ERR(vm);1518 1519 ret = xa_alloc(&pool->xa, &id, vm,1520 XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);1521 1522 if (ret) {1523 panthor_vm_put(vm);1524 return ret;1525 }1526 1527 args->user_va_range = kernel_va_start;1528 return id;1529}1530 1531static void panthor_vm_destroy(struct panthor_vm *vm)1532{1533 if (!vm)1534 return;1535 1536 vm->destroyed = true;1537 1538 mutex_lock(&vm->heaps.lock);1539 panthor_heap_pool_destroy(vm->heaps.pool);1540 vm->heaps.pool = NULL;1541 mutex_unlock(&vm->heaps.lock);1542 1543 drm_WARN_ON(&vm->ptdev->base,1544 panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range));1545 panthor_vm_put(vm);1546}1547 1548/**1549 * panthor_vm_pool_destroy_vm() - Destroy a VM.1550 * @pool: VM pool.1551 * @handle: VM handle.1552 *1553 * This function doesn't free the VM object or its resources, it just kills1554 * all mappings, and makes sure nothing can be mapped after that point.1555 *1556 * If there was any active jobs at the time this function is called, these1557 * jobs should experience page faults and be killed as a result.1558 *1559 * The VM resources are freed when the last reference on the VM object is1560 * dropped.1561 */1562int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)1563{1564 struct panthor_vm *vm;1565 1566 vm = xa_erase(&pool->xa, handle);1567 1568 panthor_vm_destroy(vm);1569 1570 return vm ? 0 : -EINVAL;1571}1572 1573/**1574 * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle1575 * @pool: VM pool to check.1576 * @handle: Handle of the VM to retrieve.1577 *1578 * Return: A valid pointer if the VM exists, NULL otherwise.1579 */1580struct panthor_vm *1581panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)1582{1583 struct panthor_vm *vm;1584 1585 xa_lock(&pool->xa);1586 vm = panthor_vm_get(xa_load(&pool->xa, handle));1587 xa_unlock(&pool->xa);1588 1589 return vm;1590}1591 1592/**1593 * panthor_vm_pool_destroy() - Destroy a VM pool.1594 * @pfile: File.1595 *1596 * Destroy all VMs in the pool, and release the pool resources.1597 *1598 * Note that VMs can outlive the pool they were created from if other1599 * objects hold a reference to there VMs.1600 */1601void panthor_vm_pool_destroy(struct panthor_file *pfile)1602{1603 struct panthor_vm *vm;1604 unsigned long i;1605 1606 if (!pfile->vms)1607 return;1608 1609 xa_for_each(&pfile->vms->xa, i, vm)1610 panthor_vm_destroy(vm);1611 1612 xa_destroy(&pfile->vms->xa);1613 kfree(pfile->vms);1614}1615 1616/**1617 * panthor_vm_pool_create() - Create a VM pool1618 * @pfile: File.1619 *1620 * Return: 0 on success, a negative error code otherwise.1621 */1622int panthor_vm_pool_create(struct panthor_file *pfile)1623{1624 pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);1625 if (!pfile->vms)1626 return -ENOMEM;1627 1628 xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);1629 return 0;1630}1631 1632/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */1633static void mmu_tlb_flush_all(void *cookie)1634{1635}1636 1637static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)1638{1639}1640 1641static const struct iommu_flush_ops mmu_tlb_ops = {1642 .tlb_flush_all = mmu_tlb_flush_all,1643 .tlb_flush_walk = mmu_tlb_flush_walk,1644};1645 1646static const char *access_type_name(struct panthor_device *ptdev,1647 u32 fault_status)1648{1649 switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {1650 case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:1651 return "ATOMIC";1652 case AS_FAULTSTATUS_ACCESS_TYPE_READ:1653 return "READ";1654 case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:1655 return "WRITE";1656 case AS_FAULTSTATUS_ACCESS_TYPE_EX:1657 return "EXECUTE";1658 default:1659 drm_WARN_ON(&ptdev->base, 1);1660 return NULL;1661 }1662}1663 1664static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)1665{1666 bool has_unhandled_faults = false;1667 1668 status = panthor_mmu_fault_mask(ptdev, status);1669 while (status) {1670 u32 as = ffs(status | (status >> 16)) - 1;1671 u32 mask = panthor_mmu_as_fault_mask(ptdev, as);1672 u32 new_int_mask;1673 u64 addr;1674 u32 fault_status;1675 u32 exception_type;1676 u32 access_type;1677 u32 source_id;1678 1679 fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));1680 addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));1681 addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;1682 1683 /* decode the fault status */1684 exception_type = fault_status & 0xFF;1685 access_type = (fault_status >> 8) & 0x3;1686 source_id = (fault_status >> 16);1687 1688 mutex_lock(&ptdev->mmu->as.slots_lock);1689 1690 ptdev->mmu->as.faulty_mask |= mask;1691 new_int_mask =1692 panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);1693 1694 /* terminal fault, print info about the fault */1695 drm_err(&ptdev->base,1696 "Unhandled Page fault in AS%d at VA 0x%016llX\n"1697 "raw fault status: 0x%X\n"1698 "decoded fault status: %s\n"1699 "exception type 0x%X: %s\n"1700 "access type 0x%X: %s\n"1701 "source id 0x%X\n",1702 as, addr,1703 fault_status,1704 (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),1705 exception_type, panthor_exception_name(ptdev, exception_type),1706 access_type, access_type_name(ptdev, fault_status),1707 source_id);1708 1709 /* Ignore MMU interrupts on this AS until it's been1710 * re-enabled.1711 */1712 ptdev->mmu->irq.mask = new_int_mask;1713 gpu_write(ptdev, MMU_INT_MASK, new_int_mask);1714 1715 if (ptdev->mmu->as.slots[as].vm)1716 ptdev->mmu->as.slots[as].vm->unhandled_fault = true;1717 1718 /* Disable the MMU to kill jobs on this AS. */1719 panthor_mmu_as_disable(ptdev, as);1720 mutex_unlock(&ptdev->mmu->as.slots_lock);1721 1722 status &= ~mask;1723 has_unhandled_faults = true;1724 }1725 1726 if (has_unhandled_faults)1727 panthor_sched_report_mmu_fault(ptdev);1728}1729PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);1730 1731/**1732 * panthor_mmu_suspend() - Suspend the MMU logic1733 * @ptdev: Device.1734 *1735 * All we do here is de-assign the AS slots on all active VMs, so things1736 * get flushed to the main memory, and no further access to these VMs are1737 * possible.1738 *1739 * We also suspend the MMU IRQ.1740 */1741void panthor_mmu_suspend(struct panthor_device *ptdev)1742{1743 mutex_lock(&ptdev->mmu->as.slots_lock);1744 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {1745 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;1746 1747 if (vm) {1748 drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));1749 panthor_vm_release_as_locked(vm);1750 }1751 }1752 mutex_unlock(&ptdev->mmu->as.slots_lock);1753 1754 panthor_mmu_irq_suspend(&ptdev->mmu->irq);1755}1756 1757/**1758 * panthor_mmu_resume() - Resume the MMU logic1759 * @ptdev: Device.1760 *1761 * Resume the IRQ.1762 *1763 * We don't re-enable previously active VMs. We assume other parts of the1764 * driver will call panthor_vm_active() on the VMs they intend to use.1765 */1766void panthor_mmu_resume(struct panthor_device *ptdev)1767{1768 mutex_lock(&ptdev->mmu->as.slots_lock);1769 ptdev->mmu->as.alloc_mask = 0;1770 ptdev->mmu->as.faulty_mask = 0;1771 mutex_unlock(&ptdev->mmu->as.slots_lock);1772 1773 panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));1774}1775 1776/**1777 * panthor_mmu_pre_reset() - Prepare for a reset1778 * @ptdev: Device.1779 *1780 * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we1781 * don't get asked to do a VM operation while the GPU is down.1782 *1783 * We don't cleanly shutdown the AS slots here, because the reset might1784 * come from an AS_ACTIVE_BIT stuck situation.1785 */1786void panthor_mmu_pre_reset(struct panthor_device *ptdev)1787{1788 struct panthor_vm *vm;1789 1790 panthor_mmu_irq_suspend(&ptdev->mmu->irq);1791 1792 mutex_lock(&ptdev->mmu->vm.lock);1793 ptdev->mmu->vm.reset_in_progress = true;1794 list_for_each_entry(vm, &ptdev->mmu->vm.list, node)1795 panthor_vm_stop(vm);1796 mutex_unlock(&ptdev->mmu->vm.lock);1797}1798 1799/**1800 * panthor_mmu_post_reset() - Restore things after a reset1801 * @ptdev: Device.1802 *1803 * Put the MMU logic back in action after a reset. That implies resuming the1804 * IRQ and re-enabling the VM_BIND queues.1805 */1806void panthor_mmu_post_reset(struct panthor_device *ptdev)1807{1808 struct panthor_vm *vm;1809 1810 mutex_lock(&ptdev->mmu->as.slots_lock);1811 1812 /* Now that the reset is effective, we can assume that none of the1813 * AS slots are setup, and clear the faulty flags too.1814 */1815 ptdev->mmu->as.alloc_mask = 0;1816 ptdev->mmu->as.faulty_mask = 0;1817 1818 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {1819 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;1820 1821 if (vm)1822 panthor_vm_release_as_locked(vm);1823 }1824 1825 mutex_unlock(&ptdev->mmu->as.slots_lock);1826 1827 panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));1828 1829 /* Restart the VM_BIND queues. */1830 mutex_lock(&ptdev->mmu->vm.lock);1831 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {1832 panthor_vm_start(vm);1833 }1834 ptdev->mmu->vm.reset_in_progress = false;1835 mutex_unlock(&ptdev->mmu->vm.lock);1836}1837 1838static void panthor_vm_free(struct drm_gpuvm *gpuvm)1839{1840 struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);1841 struct panthor_device *ptdev = vm->ptdev;1842 1843 mutex_lock(&vm->heaps.lock);1844 if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))1845 panthor_heap_pool_destroy(vm->heaps.pool);1846 mutex_unlock(&vm->heaps.lock);1847 mutex_destroy(&vm->heaps.lock);1848 1849 mutex_lock(&ptdev->mmu->vm.lock);1850 list_del(&vm->node);1851 /* Restore the scheduler state so we can call drm_sched_entity_destroy()1852 * and drm_sched_fini(). If get there, that means we have no job left1853 * and no new jobs can be queued, so we can start the scheduler without1854 * risking interfering with the reset.1855 */1856 if (ptdev->mmu->vm.reset_in_progress)1857 panthor_vm_start(vm);1858 mutex_unlock(&ptdev->mmu->vm.lock);1859 1860 drm_sched_entity_destroy(&vm->entity);1861 drm_sched_fini(&vm->sched);1862 1863 mutex_lock(&ptdev->mmu->as.slots_lock);1864 if (vm->as.id >= 0) {1865 int cookie;1866 1867 if (drm_dev_enter(&ptdev->base, &cookie)) {1868 panthor_mmu_as_disable(ptdev, vm->as.id);1869 drm_dev_exit(cookie);1870 }1871 1872 ptdev->mmu->as.slots[vm->as.id].vm = NULL;1873 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);1874 list_del(&vm->as.lru_node);1875 }1876 mutex_unlock(&ptdev->mmu->as.slots_lock);1877 1878 free_io_pgtable_ops(vm->pgtbl_ops);1879 1880 drm_mm_takedown(&vm->mm);1881 kfree(vm);1882}1883 1884/**1885 * panthor_vm_put() - Release a reference on a VM1886 * @vm: VM to release the reference on. Can be NULL.1887 */1888void panthor_vm_put(struct panthor_vm *vm)1889{1890 drm_gpuvm_put(vm ? &vm->base : NULL);1891}1892 1893/**1894 * panthor_vm_get() - Get a VM reference1895 * @vm: VM to get the reference on. Can be NULL.1896 *1897 * Return: @vm value.1898 */1899struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)1900{1901 if (vm)1902 drm_gpuvm_get(&vm->base);1903 1904 return vm;1905}1906 1907/**1908 * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM1909 * @vm: VM to query the heap pool on.1910 * @create: True if the heap pool should be created when it doesn't exist.1911 *1912 * Heap pools are per-VM. This function allows one to retrieve the heap pool1913 * attached to a VM.1914 *1915 * If no heap pool exists yet, and @create is true, we create one.1916 *1917 * The returned panthor_heap_pool should be released with panthor_heap_pool_put().1918 *1919 * Return: A valid pointer on success, an ERR_PTR() otherwise.1920 */1921struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)1922{1923 struct panthor_heap_pool *pool;1924 1925 mutex_lock(&vm->heaps.lock);1926 if (!vm->heaps.pool && create) {1927 if (vm->destroyed)1928 pool = ERR_PTR(-EINVAL);1929 else1930 pool = panthor_heap_pool_create(vm->ptdev, vm);1931 1932 if (!IS_ERR(pool))1933 vm->heaps.pool = panthor_heap_pool_get(pool);1934 } else {1935 pool = panthor_heap_pool_get(vm->heaps.pool);1936 if (!pool)1937 pool = ERR_PTR(-ENOENT);1938 }1939 mutex_unlock(&vm->heaps.lock);1940 1941 return pool;1942}1943 1944static u64 mair_to_memattr(u64 mair)1945{1946 u64 memattr = 0;1947 u32 i;1948 1949 for (i = 0; i < 8; i++) {1950 u8 in_attr = mair >> (8 * i), out_attr;1951 u8 outer = in_attr >> 4, inner = in_attr & 0xf;1952 1953 /* For caching to be enabled, inner and outer caching policy1954 * have to be both write-back, if one of them is write-through1955 * or non-cacheable, we just choose non-cacheable. Device1956 * memory is also translated to non-cacheable.1957 */1958 if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {1959 out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |1960 AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |1961 AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);1962 } else {1963 /* Use SH_CPU_INNER mode so SH_IS, which is used when1964 * IOMMU_CACHE is set, actually maps to the standard1965 * definition of inner-shareable and not Mali's1966 * internal-shareable mode.1967 */1968 out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |1969 AS_MEMATTR_AARCH64_SH_CPU_INNER |1970 AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);1971 }1972 1973 memattr |= (u64)out_attr << (8 * i);1974 }1975 1976 return memattr;1977}1978 1979static void panthor_vma_link(struct panthor_vm *vm,1980 struct panthor_vma *vma,1981 struct drm_gpuvm_bo *vm_bo)1982{1983 struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);1984 1985 mutex_lock(&bo->gpuva_list_lock);1986 drm_gpuva_link(&vma->base, vm_bo);1987 drm_WARN_ON(&vm->ptdev->base, drm_gpuvm_bo_put(vm_bo));1988 mutex_unlock(&bo->gpuva_list_lock);1989}1990 1991static void panthor_vma_unlink(struct panthor_vm *vm,1992 struct panthor_vma *vma)1993{1994 struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);1995 struct drm_gpuvm_bo *vm_bo = drm_gpuvm_bo_get(vma->base.vm_bo);1996 1997 mutex_lock(&bo->gpuva_list_lock);1998 drm_gpuva_unlink(&vma->base);1999 mutex_unlock(&bo->gpuva_list_lock);2000 2001 /* drm_gpuva_unlink() release the vm_bo, but we manually retained it2002 * when entering this function, so we can implement deferred VMA2003 * destruction. Re-assign it here.2004 */2005 vma->base.vm_bo = vm_bo;2006 list_add_tail(&vma->node, &vm->op_ctx->returned_vmas);2007}2008 2009static void panthor_vma_init(struct panthor_vma *vma, u32 flags)2010{2011 INIT_LIST_HEAD(&vma->node);2012 vma->flags = flags;2013}2014 2015#define PANTHOR_VM_MAP_FLAGS \2016 (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \2017 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \2018 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)2019 2020static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)2021{2022 struct panthor_vm *vm = priv;2023 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;2024 struct panthor_vma *vma = panthor_vm_op_ctx_get_vma(op_ctx);2025 int ret;2026 2027 if (!vma)2028 return -EINVAL;2029 2030 panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);2031 2032 ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),2033 op_ctx->map.sgt, op->map.gem.offset,2034 op->map.va.range);2035 if (ret)2036 return ret;2037 2038 /* Ref owned by the mapping now, clear the obj field so we don't release the2039 * pinning/obj ref behind GPUVA's back.2040 */2041 drm_gpuva_map(&vm->base, &vma->base, &op->map);2042 panthor_vma_link(vm, vma, op_ctx->map.vm_bo);2043 op_ctx->map.vm_bo = NULL;2044 return 0;2045}2046 2047static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,2048 void *priv)2049{2050 struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);2051 struct panthor_vm *vm = priv;2052 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;2053 struct panthor_vma *prev_vma = NULL, *next_vma = NULL;2054 u64 unmap_start, unmap_range;2055 int ret;2056 2057 drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);2058 ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);2059 if (ret)2060 return ret;2061 2062 if (op->remap.prev) {2063 prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);2064 panthor_vma_init(prev_vma, unmap_vma->flags);2065 }2066 2067 if (op->remap.next) {2068 next_vma = panthor_vm_op_ctx_get_vma(op_ctx);2069 panthor_vma_init(next_vma, unmap_vma->flags);2070 }2071 2072 drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,2073 next_vma ? &next_vma->base : NULL,2074 &op->remap);2075 2076 if (prev_vma) {2077 /* panthor_vma_link() transfers the vm_bo ownership to2078 * the VMA object. Since the vm_bo we're passing is still2079 * owned by the old mapping which will be released when this2080 * mapping is destroyed, we need to grab a ref here.2081 */2082 panthor_vma_link(vm, prev_vma,2083 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));2084 }2085 2086 if (next_vma) {2087 panthor_vma_link(vm, next_vma,2088 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));2089 }2090 2091 panthor_vma_unlink(vm, unmap_vma);2092 return 0;2093}2094 2095static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,2096 void *priv)2097{2098 struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);2099 struct panthor_vm *vm = priv;2100 int ret;2101 2102 ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,2103 unmap_vma->base.va.range);2104 if (drm_WARN_ON(&vm->ptdev->base, ret))2105 return ret;2106 2107 drm_gpuva_unmap(&op->unmap);2108 panthor_vma_unlink(vm, unmap_vma);2109 return 0;2110}2111 2112static const struct drm_gpuvm_ops panthor_gpuvm_ops = {2113 .vm_free = panthor_vm_free,2114 .sm_step_map = panthor_gpuva_sm_step_map,2115 .sm_step_remap = panthor_gpuva_sm_step_remap,2116 .sm_step_unmap = panthor_gpuva_sm_step_unmap,2117};2118 2119/**2120 * panthor_vm_resv() - Get the dma_resv object attached to a VM.2121 * @vm: VM to get the dma_resv of.2122 *2123 * Return: A dma_resv object.2124 */2125struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)2126{2127 return drm_gpuvm_resv(&vm->base);2128}2129 2130struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm)2131{2132 if (!vm)2133 return NULL;2134 2135 return vm->base.r_obj;2136}2137 2138static int2139panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,2140 bool flag_vm_unusable_on_failure)2141{2142 u32 op_type = op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;2143 int ret;2144 2145 if (op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY)2146 return 0;2147 2148 mutex_lock(&vm->op_lock);2149 vm->op_ctx = op;2150 switch (op_type) {2151 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:2152 if (vm->unusable) {2153 ret = -EINVAL;2154 break;2155 }2156 2157 ret = drm_gpuvm_sm_map(&vm->base, vm, op->va.addr, op->va.range,2158 op->map.vm_bo->obj, op->map.bo_offset);2159 break;2160 2161 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:2162 ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range);2163 break;2164 2165 default:2166 ret = -EINVAL;2167 break;2168 }2169 2170 if (ret && flag_vm_unusable_on_failure)2171 vm->unusable = true;2172 2173 vm->op_ctx = NULL;2174 mutex_unlock(&vm->op_lock);2175 2176 return ret;2177}2178 2179static struct dma_fence *2180panthor_vm_bind_run_job(struct drm_sched_job *sched_job)2181{2182 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);2183 bool cookie;2184 int ret;2185 2186 /* Not only we report an error whose result is propagated to the2187 * drm_sched finished fence, but we also flag the VM as unusable, because2188 * a failure in the async VM_BIND results in an inconsistent state. VM needs2189 * to be destroyed and recreated.2190 */2191 cookie = dma_fence_begin_signalling();2192 ret = panthor_vm_exec_op(job->vm, &job->ctx, true);2193 dma_fence_end_signalling(cookie);2194 2195 return ret ? ERR_PTR(ret) : NULL;2196}2197 2198static void panthor_vm_bind_job_release(struct kref *kref)2199{2200 struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);2201 2202 if (job->base.s_fence)2203 drm_sched_job_cleanup(&job->base);2204 2205 panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);2206 panthor_vm_put(job->vm);2207 kfree(job);2208}2209 2210/**2211 * panthor_vm_bind_job_put() - Release a VM_BIND job reference2212 * @sched_job: Job to release the reference on.2213 */2214void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)2215{2216 struct panthor_vm_bind_job *job =2217 container_of(sched_job, struct panthor_vm_bind_job, base);2218 2219 if (sched_job)2220 kref_put(&job->refcount, panthor_vm_bind_job_release);2221}2222 2223static void2224panthor_vm_bind_free_job(struct drm_sched_job *sched_job)2225{2226 struct panthor_vm_bind_job *job =2227 container_of(sched_job, struct panthor_vm_bind_job, base);2228 2229 drm_sched_job_cleanup(sched_job);2230 2231 /* Do the heavy cleanups asynchronously, so we're out of the2232 * dma-signaling path and can acquire dma-resv locks safely.2233 */2234 queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);2235}2236 2237static enum drm_gpu_sched_stat2238panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)2239{2240 WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");2241 return DRM_GPU_SCHED_STAT_NOMINAL;2242}2243 2244static const struct drm_sched_backend_ops panthor_vm_bind_ops = {2245 .run_job = panthor_vm_bind_run_job,2246 .free_job = panthor_vm_bind_free_job,2247 .timedout_job = panthor_vm_bind_timedout_job,2248};2249 2250/**2251 * panthor_vm_create() - Create a VM2252 * @ptdev: Device.2253 * @for_mcu: True if this is the FW MCU VM.2254 * @kernel_va_start: Start of the range reserved for kernel BO mapping.2255 * @kernel_va_size: Size of the range reserved for kernel BO mapping.2256 * @auto_kernel_va_start: Start of the auto-VA kernel range.2257 * @auto_kernel_va_size: Size of the auto-VA kernel range.2258 *2259 * Return: A valid pointer on success, an ERR_PTR() otherwise.2260 */2261struct panthor_vm *2262panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,2263 u64 kernel_va_start, u64 kernel_va_size,2264 u64 auto_kernel_va_start, u64 auto_kernel_va_size)2265{2266 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);2267 u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);2268 u64 full_va_range = 1ull << va_bits;2269 struct drm_gem_object *dummy_gem;2270 struct drm_gpu_scheduler *sched;2271 struct io_pgtable_cfg pgtbl_cfg;2272 u64 mair, min_va, va_range;2273 struct panthor_vm *vm;2274 int ret;2275 2276 vm = kzalloc(sizeof(*vm), GFP_KERNEL);2277 if (!vm)2278 return ERR_PTR(-ENOMEM);2279 2280 /* We allocate a dummy GEM for the VM. */2281 dummy_gem = drm_gpuvm_resv_object_alloc(&ptdev->base);2282 if (!dummy_gem) {2283 ret = -ENOMEM;2284 goto err_free_vm;2285 }2286 2287 mutex_init(&vm->heaps.lock);2288 vm->for_mcu = for_mcu;2289 vm->ptdev = ptdev;2290 mutex_init(&vm->op_lock);2291 2292 if (for_mcu) {2293 /* CSF MCU is a cortex M7, and can only address 4G */2294 min_va = 0;2295 va_range = SZ_4G;2296 } else {2297 min_va = 0;2298 va_range = full_va_range;2299 }2300 2301 mutex_init(&vm->mm_lock);2302 drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size);2303 vm->kernel_auto_va.start = auto_kernel_va_start;2304 vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;2305 2306 INIT_LIST_HEAD(&vm->node);2307 INIT_LIST_HEAD(&vm->as.lru_node);2308 vm->as.id = -1;2309 refcount_set(&vm->as.active_cnt, 0);2310 2311 pgtbl_cfg = (struct io_pgtable_cfg) {2312 .pgsize_bitmap = SZ_4K | SZ_2M,2313 .ias = va_bits,2314 .oas = pa_bits,2315 .coherent_walk = ptdev->coherent,2316 .tlb = &mmu_tlb_ops,2317 .iommu_dev = ptdev->base.dev,2318 .alloc = alloc_pt,2319 .free = free_pt,2320 };2321 2322 vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);2323 if (!vm->pgtbl_ops) {2324 ret = -EINVAL;2325 goto err_mm_takedown;2326 }2327 2328 /* Bind operations are synchronous for now, no timeout needed. */2329 ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq,2330 1, 1, 0,2331 MAX_SCHEDULE_TIMEOUT, NULL, NULL,2332 "panthor-vm-bind", ptdev->base.dev);2333 if (ret)2334 goto err_free_io_pgtable;2335 2336 sched = &vm->sched;2337 ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL);2338 if (ret)2339 goto err_sched_fini;2340 2341 mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;2342 vm->memattr = mair_to_memattr(mair);2343 2344 mutex_lock(&ptdev->mmu->vm.lock);2345 list_add_tail(&vm->node, &ptdev->mmu->vm.list);2346 2347 /* If a reset is in progress, stop the scheduler. */2348 if (ptdev->mmu->vm.reset_in_progress)2349 panthor_vm_stop(vm);2350 mutex_unlock(&ptdev->mmu->vm.lock);2351 2352 /* We intentionally leave the reserved range to zero, because we want kernel VMAs2353 * to be handled the same way user VMAs are.2354 */2355 drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM",2356 DRM_GPUVM_RESV_PROTECTED, &ptdev->base, dummy_gem,2357 min_va, va_range, 0, 0, &panthor_gpuvm_ops);2358 drm_gem_object_put(dummy_gem);2359 return vm;2360 2361err_sched_fini:2362 drm_sched_fini(&vm->sched);2363 2364err_free_io_pgtable:2365 free_io_pgtable_ops(vm->pgtbl_ops);2366 2367err_mm_takedown:2368 drm_mm_takedown(&vm->mm);2369 drm_gem_object_put(dummy_gem);2370 2371err_free_vm:2372 kfree(vm);2373 return ERR_PTR(ret);2374}2375 2376static int2377panthor_vm_bind_prepare_op_ctx(struct drm_file *file,2378 struct panthor_vm *vm,2379 const struct drm_panthor_vm_bind_op *op,2380 struct panthor_vm_op_ctx *op_ctx)2381{2382 ssize_t vm_pgsz = panthor_vm_page_size(vm);2383 struct drm_gem_object *gem;2384 int ret;2385 2386 /* Aligned on page size. */2387 if (!IS_ALIGNED(op->va | op->size, vm_pgsz))2388 return -EINVAL;2389 2390 switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {2391 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:2392 gem = drm_gem_object_lookup(file, op->bo_handle);2393 ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,2394 gem ? to_panthor_bo(gem) : NULL,2395 op->bo_offset,2396 op->size,2397 op->va,2398 op->flags);2399 drm_gem_object_put(gem);2400 return ret;2401 2402 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:2403 if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)2404 return -EINVAL;2405 2406 if (op->bo_handle || op->bo_offset)2407 return -EINVAL;2408 2409 return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);2410 2411 case DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY:2412 if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)2413 return -EINVAL;2414 2415 if (op->bo_handle || op->bo_offset)2416 return -EINVAL;2417 2418 if (op->va || op->size)2419 return -EINVAL;2420 2421 if (!op->syncs.count)2422 return -EINVAL;2423 2424 panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm);2425 return 0;2426 2427 default:2428 return -EINVAL;2429 }2430}2431 2432static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)2433{2434 struct panthor_vm_bind_job *job =2435 container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);2436 2437 panthor_vm_bind_job_put(&job->base);2438}2439 2440/**2441 * panthor_vm_bind_job_create() - Create a VM_BIND job2442 * @file: File.2443 * @vm: VM targeted by the VM_BIND job.2444 * @op: VM operation data.2445 *2446 * Return: A valid pointer on success, an ERR_PTR() otherwise.2447 */2448struct drm_sched_job *2449panthor_vm_bind_job_create(struct drm_file *file,2450 struct panthor_vm *vm,2451 const struct drm_panthor_vm_bind_op *op)2452{2453 struct panthor_vm_bind_job *job;2454 int ret;2455 2456 if (!vm)2457 return ERR_PTR(-EINVAL);2458 2459 if (vm->destroyed || vm->unusable)2460 return ERR_PTR(-EINVAL);2461 2462 job = kzalloc(sizeof(*job), GFP_KERNEL);2463 if (!job)2464 return ERR_PTR(-ENOMEM);2465 2466 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);2467 if (ret) {2468 kfree(job);2469 return ERR_PTR(ret);2470 }2471 2472 INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);2473 kref_init(&job->refcount);2474 job->vm = panthor_vm_get(vm);2475 2476 ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm);2477 if (ret)2478 goto err_put_job;2479 2480 return &job->base;2481 2482err_put_job:2483 panthor_vm_bind_job_put(&job->base);2484 return ERR_PTR(ret);2485}2486 2487/**2488 * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs2489 * @exec: The locking/preparation context.2490 * @sched_job: The job to prepare resvs on.2491 *2492 * Locks and prepare the VM resv.2493 *2494 * If this is a map operation, locks and prepares the GEM resv.2495 *2496 * Return: 0 on success, a negative error code otherwise.2497 */2498int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,2499 struct drm_sched_job *sched_job)2500{2501 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);2502 int ret;2503 2504 /* Acquire the VM lock an reserve a slot for this VM bind job. */2505 ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1);2506 if (ret)2507 return ret;2508 2509 if (job->ctx.map.vm_bo) {2510 /* Lock/prepare the GEM being mapped. */2511 ret = drm_exec_prepare_obj(exec, job->ctx.map.vm_bo->obj, 1);2512 if (ret)2513 return ret;2514 }2515 2516 return 0;2517}2518 2519/**2520 * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job2521 * @exec: drm_exec context.2522 * @sched_job: Job to update the resvs on.2523 */2524void panthor_vm_bind_job_update_resvs(struct drm_exec *exec,2525 struct drm_sched_job *sched_job)2526{2527 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);2528 2529 /* Explicit sync => we just register our job finished fence as bookkeep. */2530 drm_gpuvm_resv_add_fence(&job->vm->base, exec,2531 &sched_job->s_fence->finished,2532 DMA_RESV_USAGE_BOOKKEEP,2533 DMA_RESV_USAGE_BOOKKEEP);2534}2535 2536void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,2537 struct dma_fence *fence,2538 enum dma_resv_usage private_usage,2539 enum dma_resv_usage extobj_usage)2540{2541 drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage);2542}2543 2544/**2545 * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.2546 * @file: File.2547 * @vm: VM targeted by the VM operation.2548 * @op: Data describing the VM operation.2549 *2550 * Return: 0 on success, a negative error code otherwise.2551 */2552int panthor_vm_bind_exec_sync_op(struct drm_file *file,2553 struct panthor_vm *vm,2554 struct drm_panthor_vm_bind_op *op)2555{2556 struct panthor_vm_op_ctx op_ctx;2557 int ret;2558 2559 /* No sync objects allowed on synchronous operations. */2560 if (op->syncs.count)2561 return -EINVAL;2562 2563 if (!op->size)2564 return 0;2565 2566 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);2567 if (ret)2568 return ret;2569 2570 ret = panthor_vm_exec_op(vm, &op_ctx, false);2571 panthor_vm_cleanup_op_ctx(&op_ctx, vm);2572 2573 return ret;2574}2575 2576/**2577 * panthor_vm_map_bo_range() - Map a GEM object range to a VM2578 * @vm: VM to map the GEM to.2579 * @bo: GEM object to map.2580 * @offset: Offset in the GEM object.2581 * @size: Size to map.2582 * @va: Virtual address to map the object to.2583 * @flags: Combination of drm_panthor_vm_bind_op_flags flags.2584 * Only map-related flags are valid.2585 *2586 * Internal use only. For userspace requests, use2587 * panthor_vm_bind_exec_sync_op() instead.2588 *2589 * Return: 0 on success, a negative error code otherwise.2590 */2591int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,2592 u64 offset, u64 size, u64 va, u32 flags)2593{2594 struct panthor_vm_op_ctx op_ctx;2595 int ret;2596 2597 ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);2598 if (ret)2599 return ret;2600 2601 ret = panthor_vm_exec_op(vm, &op_ctx, false);2602 panthor_vm_cleanup_op_ctx(&op_ctx, vm);2603 2604 return ret;2605}2606 2607/**2608 * panthor_vm_unmap_range() - Unmap a portion of the VA space2609 * @vm: VM to unmap the region from.2610 * @va: Virtual address to unmap. Must be 4k aligned.2611 * @size: Size of the region to unmap. Must be 4k aligned.2612 *2613 * Internal use only. For userspace requests, use2614 * panthor_vm_bind_exec_sync_op() instead.2615 *2616 * Return: 0 on success, a negative error code otherwise.2617 */2618int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size)2619{2620 struct panthor_vm_op_ctx op_ctx;2621 int ret;2622 2623 ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);2624 if (ret)2625 return ret;2626 2627 ret = panthor_vm_exec_op(vm, &op_ctx, false);2628 panthor_vm_cleanup_op_ctx(&op_ctx, vm);2629 2630 return ret;2631}2632 2633/**2634 * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.2635 * @exec: Locking/preparation context.2636 * @vm: VM targeted by the GPU job.2637 * @slot_count: Number of slots to reserve.2638 *2639 * GPU jobs assume all BOs bound to the VM at the time the job is submitted2640 * are available when the job is executed. In order to guarantee that, we2641 * need to reserve a slot on all BOs mapped to a VM and update this slot with2642 * the job fence after its submission.2643 *2644 * Return: 0 on success, a negative error code otherwise.2645 */2646int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm,2647 u32 slot_count)2648{2649 int ret;2650 2651 /* Acquire the VM lock and reserve a slot for this GPU job. */2652 ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count);2653 if (ret)2654 return ret;2655 2656 return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);2657}2658 2659/**2660 * panthor_mmu_unplug() - Unplug the MMU logic2661 * @ptdev: Device.2662 *2663 * No access to the MMU regs should be done after this function is called.2664 * We suspend the IRQ and disable all VMs to guarantee that.2665 */2666void panthor_mmu_unplug(struct panthor_device *ptdev)2667{2668 panthor_mmu_irq_suspend(&ptdev->mmu->irq);2669 2670 mutex_lock(&ptdev->mmu->as.slots_lock);2671 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {2672 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;2673 2674 if (vm) {2675 drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));2676 panthor_vm_release_as_locked(vm);2677 }2678 }2679 mutex_unlock(&ptdev->mmu->as.slots_lock);2680}2681 2682static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)2683{2684 destroy_workqueue(res);2685}2686 2687/**2688 * panthor_mmu_init() - Initialize the MMU logic.2689 * @ptdev: Device.2690 *2691 * Return: 0 on success, a negative error code otherwise.2692 */2693int panthor_mmu_init(struct panthor_device *ptdev)2694{2695 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);2696 struct panthor_mmu *mmu;2697 int ret, irq;2698 2699 mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);2700 if (!mmu)2701 return -ENOMEM;2702 2703 INIT_LIST_HEAD(&mmu->as.lru_list);2704 2705 ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);2706 if (ret)2707 return ret;2708 2709 INIT_LIST_HEAD(&mmu->vm.list);2710 ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);2711 if (ret)2712 return ret;2713 2714 ptdev->mmu = mmu;2715 2716 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");2717 if (irq <= 0)2718 return -ENODEV;2719 2720 ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,2721 panthor_mmu_fault_mask(ptdev, ~0));2722 if (ret)2723 return ret;2724 2725 mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);2726 if (!mmu->vm.wq)2727 return -ENOMEM;2728 2729 /* On 32-bit kernels, the VA space is limited by the io_pgtable_ops abstraction,2730 * which passes iova as an unsigned long. Patch the mmu_features to reflect this2731 * limitation.2732 */2733 if (sizeof(unsigned long) * 8 < va_bits) {2734 ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0);2735 ptdev->gpu_info.mmu_features |= sizeof(unsigned long) * 8;2736 }2737 2738 return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);2739}2740 2741#ifdef CONFIG_DEBUG_FS2742static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)2743{2744 int ret;2745 2746 mutex_lock(&vm->op_lock);2747 ret = drm_debugfs_gpuva_info(m, &vm->base);2748 mutex_unlock(&vm->op_lock);2749 2750 return ret;2751}2752 2753static int show_each_vm(struct seq_file *m, void *arg)2754{2755 struct drm_info_node *node = (struct drm_info_node *)m->private;2756 struct drm_device *ddev = node->minor->dev;2757 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);2758 int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;2759 struct panthor_vm *vm;2760 int ret = 0;2761 2762 mutex_lock(&ptdev->mmu->vm.lock);2763 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {2764 ret = show(vm, m);2765 if (ret < 0)2766 break;2767 2768 seq_puts(m, "\n");2769 }2770 mutex_unlock(&ptdev->mmu->vm.lock);2771 2772 return ret;2773}2774 2775static struct drm_info_list panthor_mmu_debugfs_list[] = {2776 DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),2777};2778 2779/**2780 * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries2781 * @minor: Minor.2782 */2783void panthor_mmu_debugfs_init(struct drm_minor *minor)2784{2785 drm_debugfs_create_files(panthor_mmu_debugfs_list,2786 ARRAY_SIZE(panthor_mmu_debugfs_list),2787 minor->debugfs_root, minor);2788}2789#endif /* CONFIG_DEBUG_FS */2790 2791/**2792 * panthor_mmu_pt_cache_init() - Initialize the page table cache.2793 *2794 * Return: 0 on success, a negative error code otherwise.2795 */2796int panthor_mmu_pt_cache_init(void)2797{2798 pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);2799 if (!pt_cache)2800 return -ENOMEM;2801 2802 return 0;2803}2804 2805/**2806 * panthor_mmu_pt_cache_fini() - Destroy the page table cache.2807 */2808void panthor_mmu_pt_cache_fini(void)2809{2810 kmem_cache_destroy(pt_cache);2811}2812