3600 lines · c
1// SPDX-License-Identifier: GPL-2.0 or MIT2/* Copyright 2023 Collabora ltd. */3 4#include <drm/drm_drv.h>5#include <drm/drm_exec.h>6#include <drm/drm_gem_shmem_helper.h>7#include <drm/drm_managed.h>8#include <drm/gpu_scheduler.h>9#include <drm/panthor_drm.h>10 11#include <linux/build_bug.h>12#include <linux/clk.h>13#include <linux/delay.h>14#include <linux/dma-mapping.h>15#include <linux/dma-resv.h>16#include <linux/firmware.h>17#include <linux/interrupt.h>18#include <linux/io.h>19#include <linux/iopoll.h>20#include <linux/iosys-map.h>21#include <linux/module.h>22#include <linux/platform_device.h>23#include <linux/pm_runtime.h>24 25#include "panthor_devfreq.h"26#include "panthor_device.h"27#include "panthor_fw.h"28#include "panthor_gem.h"29#include "panthor_gpu.h"30#include "panthor_heap.h"31#include "panthor_mmu.h"32#include "panthor_regs.h"33#include "panthor_sched.h"34 35/**36 * DOC: Scheduler37 *38 * Mali CSF hardware adopts a firmware-assisted scheduling model, where39 * the firmware takes care of scheduling aspects, to some extent.40 *41 * The scheduling happens at the scheduling group level, each group42 * contains 1 to N queues (N is FW/hardware dependent, and exposed43 * through the firmware interface). Each queue is assigned a command44 * stream ring buffer, which serves as a way to get jobs submitted to45 * the GPU, among other things.46 *47 * The firmware can schedule a maximum of M groups (M is FW/hardware48 * dependent, and exposed through the firmware interface). Passed49 * this maximum number of groups, the kernel must take care of50 * rotating the groups passed to the firmware so every group gets51 * a chance to have his queues scheduled for execution.52 *53 * The current implementation only supports with kernel-mode queues.54 * In other terms, userspace doesn't have access to the ring-buffer.55 * Instead, userspace passes indirect command stream buffers that are56 * called from the queue ring-buffer by the kernel using a pre-defined57 * sequence of command stream instructions to ensure the userspace driver58 * always gets consistent results (cache maintenance,59 * synchronization, ...).60 *61 * We rely on the drm_gpu_scheduler framework to deal with job62 * dependencies and submission. As any other driver dealing with a63 * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each64 * entity has its own job scheduler. When a job is ready to be executed65 * (all its dependencies are met), it is pushed to the appropriate66 * queue ring-buffer, and the group is scheduled for execution if it67 * wasn't already active.68 *69 * Kernel-side group scheduling is timeslice-based. When we have less70 * groups than there are slots, the periodic tick is disabled and we71 * just let the FW schedule the active groups. When there are more72 * groups than slots, we let each group a chance to execute stuff for73 * a given amount of time, and then re-evaluate and pick new groups74 * to schedule. The group selection algorithm is based on75 * priority+round-robin.76 *77 * Even though user-mode queues is out of the scope right now, the78 * current design takes them into account by avoiding any guess on the79 * group/queue state that would be based on information we wouldn't have80 * if userspace was in charge of the ring-buffer. That's also one of the81 * reason we don't do 'cooperative' scheduling (encoding FW group slot82 * reservation as dma_fence that would be returned from the83 * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as84 * a queue of waiters, ordered by job submission order). This approach85 * would work for kernel-mode queues, but would make user-mode queues a86 * lot more complicated to retrofit.87 */88 89#define JOB_TIMEOUT_MS 500090 91#define MIN_CS_PER_CSG 892 93#define MIN_CSGS 394#define MAX_CSG_PRIO 0xf95 96struct panthor_group;97 98/**99 * struct panthor_csg_slot - Command stream group slot100 *101 * This represents a FW slot for a scheduling group.102 */103struct panthor_csg_slot {104 /** @group: Scheduling group bound to this slot. */105 struct panthor_group *group;106 107 /** @priority: Group priority. */108 u8 priority;109 110 /**111 * @idle: True if the group bound to this slot is idle.112 *113 * A group is idle when it has nothing waiting for execution on114 * all its queues, or when queues are blocked waiting for something115 * to happen (synchronization object).116 */117 bool idle;118};119 120/**121 * enum panthor_csg_priority - Group priority122 */123enum panthor_csg_priority {124 /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */125 PANTHOR_CSG_PRIORITY_LOW = 0,126 127 /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */128 PANTHOR_CSG_PRIORITY_MEDIUM,129 130 /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */131 PANTHOR_CSG_PRIORITY_HIGH,132 133 /**134 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.135 *136 * Real-time priority allows one to preempt scheduling of other137 * non-real-time groups. When such a group becomes executable,138 * it will evict the group with the lowest non-rt priority if139 * there's no free group slot available.140 *141 * Currently not exposed to userspace.142 */143 PANTHOR_CSG_PRIORITY_RT,144 145 /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */146 PANTHOR_CSG_PRIORITY_COUNT,147};148 149/**150 * struct panthor_scheduler - Object used to manage the scheduler151 */152struct panthor_scheduler {153 /** @ptdev: Device. */154 struct panthor_device *ptdev;155 156 /**157 * @wq: Workqueue used by our internal scheduler logic and158 * drm_gpu_scheduler.159 *160 * Used for the scheduler tick, group update or other kind of FW161 * event processing that can't be handled in the threaded interrupt162 * path. Also passed to the drm_gpu_scheduler instances embedded163 * in panthor_queue.164 */165 struct workqueue_struct *wq;166 167 /**168 * @heap_alloc_wq: Workqueue used to schedule tiler_oom works.169 *170 * We have a queue dedicated to heap chunk allocation works to avoid171 * blocking the rest of the scheduler if the allocation tries to172 * reclaim memory.173 */174 struct workqueue_struct *heap_alloc_wq;175 176 /** @tick_work: Work executed on a scheduling tick. */177 struct delayed_work tick_work;178 179 /**180 * @sync_upd_work: Work used to process synchronization object updates.181 *182 * We use this work to unblock queues/groups that were waiting on a183 * synchronization object.184 */185 struct work_struct sync_upd_work;186 187 /**188 * @fw_events_work: Work used to process FW events outside the interrupt path.189 *190 * Even if the interrupt is threaded, we need any event processing191 * that require taking the panthor_scheduler::lock to be processed192 * outside the interrupt path so we don't block the tick logic when193 * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the194 * event processing requires taking this lock, we just delegate all195 * FW event processing to the scheduler workqueue.196 */197 struct work_struct fw_events_work;198 199 /**200 * @fw_events: Bitmask encoding pending FW events.201 */202 atomic_t fw_events;203 204 /**205 * @resched_target: When the next tick should occur.206 *207 * Expressed in jiffies.208 */209 u64 resched_target;210 211 /**212 * @last_tick: When the last tick occurred.213 *214 * Expressed in jiffies.215 */216 u64 last_tick;217 218 /** @tick_period: Tick period in jiffies. */219 u64 tick_period;220 221 /**222 * @lock: Lock protecting access to all the scheduler fields.223 *224 * Should be taken in the tick work, the irq handler, and anywhere the @groups225 * fields are touched.226 */227 struct mutex lock;228 229 /** @groups: Various lists used to classify groups. */230 struct {231 /**232 * @runnable: Runnable group lists.233 *234 * When a group has queues that want to execute something,235 * its panthor_group::run_node should be inserted here.236 *237 * One list per-priority.238 */239 struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];240 241 /**242 * @idle: Idle group lists.243 *244 * When all queues of a group are idle (either because they245 * have nothing to execute, or because they are blocked), the246 * panthor_group::run_node field should be inserted here.247 *248 * One list per-priority.249 */250 struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];251 252 /**253 * @waiting: List of groups whose queues are blocked on a254 * synchronization object.255 *256 * Insert panthor_group::wait_node here when a group is waiting257 * for synchronization objects to be signaled.258 *259 * This list is evaluated in the @sync_upd_work work.260 */261 struct list_head waiting;262 } groups;263 264 /**265 * @csg_slots: FW command stream group slots.266 */267 struct panthor_csg_slot csg_slots[MAX_CSGS];268 269 /** @csg_slot_count: Number of command stream group slots exposed by the FW. */270 u32 csg_slot_count;271 272 /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */273 u32 cs_slot_count;274 275 /** @as_slot_count: Number of address space slots supported by the MMU. */276 u32 as_slot_count;277 278 /** @used_csg_slot_count: Number of command stream group slot currently used. */279 u32 used_csg_slot_count;280 281 /** @sb_slot_count: Number of scoreboard slots. */282 u32 sb_slot_count;283 284 /**285 * @might_have_idle_groups: True if an active group might have become idle.286 *287 * This will force a tick, so other runnable groups can be scheduled if one288 * or more active groups became idle.289 */290 bool might_have_idle_groups;291 292 /** @pm: Power management related fields. */293 struct {294 /** @has_ref: True if the scheduler owns a runtime PM reference. */295 bool has_ref;296 } pm;297 298 /** @reset: Reset related fields. */299 struct {300 /** @lock: Lock protecting the other reset fields. */301 struct mutex lock;302 303 /**304 * @in_progress: True if a reset is in progress.305 *306 * Set to true in panthor_sched_pre_reset() and back to false in307 * panthor_sched_post_reset().308 */309 atomic_t in_progress;310 311 /**312 * @stopped_groups: List containing all groups that were stopped313 * before a reset.314 *315 * Insert panthor_group::run_node in the pre_reset path.316 */317 struct list_head stopped_groups;318 } reset;319};320 321/**322 * struct panthor_syncobj_32b - 32-bit FW synchronization object323 */324struct panthor_syncobj_32b {325 /** @seqno: Sequence number. */326 u32 seqno;327 328 /**329 * @status: Status.330 *331 * Not zero on failure.332 */333 u32 status;334};335 336/**337 * struct panthor_syncobj_64b - 64-bit FW synchronization object338 */339struct panthor_syncobj_64b {340 /** @seqno: Sequence number. */341 u64 seqno;342 343 /**344 * @status: Status.345 *346 * Not zero on failure.347 */348 u32 status;349 350 /** @pad: MBZ. */351 u32 pad;352};353 354/**355 * struct panthor_queue - Execution queue356 */357struct panthor_queue {358 /** @scheduler: DRM scheduler used for this queue. */359 struct drm_gpu_scheduler scheduler;360 361 /** @entity: DRM scheduling entity used for this queue. */362 struct drm_sched_entity entity;363 364 /**365 * @remaining_time: Time remaining before the job timeout expires.366 *367 * The job timeout is suspended when the queue is not scheduled by the368 * FW. Every time we suspend the timer, we need to save the remaining369 * time so we can restore it later on.370 */371 unsigned long remaining_time;372 373 /** @timeout_suspended: True if the job timeout was suspended. */374 bool timeout_suspended;375 376 /**377 * @doorbell_id: Doorbell assigned to this queue.378 *379 * Right now, all groups share the same doorbell, and the doorbell ID380 * is assigned to group_slot + 1 when the group is assigned a slot. But381 * we might decide to provide fine grained doorbell assignment at some382 * point, so don't have to wake up all queues in a group every time one383 * of them is updated.384 */385 u8 doorbell_id;386 387 /**388 * @priority: Priority of the queue inside the group.389 *390 * Must be less than 16 (Only 4 bits available).391 */392 u8 priority;393#define CSF_MAX_QUEUE_PRIO GENMASK(3, 0)394 395 /** @ringbuf: Command stream ring-buffer. */396 struct panthor_kernel_bo *ringbuf;397 398 /** @iface: Firmware interface. */399 struct {400 /** @mem: FW memory allocated for this interface. */401 struct panthor_kernel_bo *mem;402 403 /** @input: Input interface. */404 struct panthor_fw_ringbuf_input_iface *input;405 406 /** @output: Output interface. */407 const struct panthor_fw_ringbuf_output_iface *output;408 409 /** @input_fw_va: FW virtual address of the input interface buffer. */410 u32 input_fw_va;411 412 /** @output_fw_va: FW virtual address of the output interface buffer. */413 u32 output_fw_va;414 } iface;415 416 /**417 * @syncwait: Stores information about the synchronization object this418 * queue is waiting on.419 */420 struct {421 /** @gpu_va: GPU address of the synchronization object. */422 u64 gpu_va;423 424 /** @ref: Reference value to compare against. */425 u64 ref;426 427 /** @gt: True if this is a greater-than test. */428 bool gt;429 430 /** @sync64: True if this is a 64-bit sync object. */431 bool sync64;432 433 /** @bo: Buffer object holding the synchronization object. */434 struct drm_gem_object *obj;435 436 /** @offset: Offset of the synchronization object inside @bo. */437 u64 offset;438 439 /**440 * @kmap: Kernel mapping of the buffer object holding the441 * synchronization object.442 */443 void *kmap;444 } syncwait;445 446 /** @fence_ctx: Fence context fields. */447 struct {448 /** @lock: Used to protect access to all fences allocated by this context. */449 spinlock_t lock;450 451 /**452 * @id: Fence context ID.453 *454 * Allocated with dma_fence_context_alloc().455 */456 u64 id;457 458 /** @seqno: Sequence number of the last initialized fence. */459 atomic64_t seqno;460 461 /**462 * @last_fence: Fence of the last submitted job.463 *464 * We return this fence when we get an empty command stream.465 * This way, we are guaranteed that all earlier jobs have completed466 * when drm_sched_job::s_fence::finished without having to feed467 * the CS ring buffer with a dummy job that only signals the fence.468 */469 struct dma_fence *last_fence;470 471 /**472 * @in_flight_jobs: List containing all in-flight jobs.473 *474 * Used to keep track and signal panthor_job::done_fence when the475 * synchronization object attached to the queue is signaled.476 */477 struct list_head in_flight_jobs;478 } fence_ctx;479};480 481/**482 * enum panthor_group_state - Scheduling group state.483 */484enum panthor_group_state {485 /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */486 PANTHOR_CS_GROUP_CREATED,487 488 /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */489 PANTHOR_CS_GROUP_ACTIVE,490 491 /**492 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is493 * inactive/suspended right now.494 */495 PANTHOR_CS_GROUP_SUSPENDED,496 497 /**498 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.499 *500 * Can no longer be scheduled. The only allowed action is a destruction.501 */502 PANTHOR_CS_GROUP_TERMINATED,503 504 /**505 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.506 *507 * The FW returned an inconsistent state. The group is flagged unusable508 * and can no longer be scheduled. The only allowed action is a509 * destruction.510 *511 * When that happens, we also schedule a FW reset, to start from a fresh512 * state.513 */514 PANTHOR_CS_GROUP_UNKNOWN_STATE,515};516 517/**518 * struct panthor_group - Scheduling group object519 */520struct panthor_group {521 /** @refcount: Reference count */522 struct kref refcount;523 524 /** @ptdev: Device. */525 struct panthor_device *ptdev;526 527 /** @vm: VM bound to the group. */528 struct panthor_vm *vm;529 530 /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */531 u64 compute_core_mask;532 533 /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */534 u64 fragment_core_mask;535 536 /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */537 u64 tiler_core_mask;538 539 /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */540 u8 max_compute_cores;541 542 /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */543 u8 max_fragment_cores;544 545 /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */546 u8 max_tiler_cores;547 548 /** @priority: Group priority (check panthor_csg_priority). */549 u8 priority;550 551 /** @blocked_queues: Bitmask reflecting the blocked queues. */552 u32 blocked_queues;553 554 /** @idle_queues: Bitmask reflecting the idle queues. */555 u32 idle_queues;556 557 /** @fatal_lock: Lock used to protect access to fatal fields. */558 spinlock_t fatal_lock;559 560 /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */561 u32 fatal_queues;562 563 /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */564 atomic_t tiler_oom;565 566 /** @queue_count: Number of queues in this group. */567 u32 queue_count;568 569 /** @queues: Queues owned by this group. */570 struct panthor_queue *queues[MAX_CS_PER_CSG];571 572 /**573 * @csg_id: ID of the FW group slot.574 *575 * -1 when the group is not scheduled/active.576 */577 int csg_id;578 579 /**580 * @destroyed: True when the group has been destroyed.581 *582 * If a group is destroyed it becomes useless: no further jobs can be submitted583 * to its queues. We simply wait for all references to be dropped so we can584 * release the group object.585 */586 bool destroyed;587 588 /**589 * @timedout: True when a timeout occurred on any of the queues owned by590 * this group.591 *592 * Timeouts can be reported by drm_sched or by the FW. If a reset is required,593 * and the group can't be suspended, this also leads to a timeout. In any case,594 * any timeout situation is unrecoverable, and the group becomes useless. We595 * simply wait for all references to be dropped so we can release the group596 * object.597 */598 bool timedout;599 600 /**601 * @syncobjs: Pool of per-queue synchronization objects.602 *603 * One sync object per queue. The position of the sync object is604 * determined by the queue index.605 */606 struct panthor_kernel_bo *syncobjs;607 608 /** @state: Group state. */609 enum panthor_group_state state;610 611 /**612 * @suspend_buf: Suspend buffer.613 *614 * Stores the state of the group and its queues when a group is suspended.615 * Used at resume time to restore the group in its previous state.616 *617 * The size of the suspend buffer is exposed through the FW interface.618 */619 struct panthor_kernel_bo *suspend_buf;620 621 /**622 * @protm_suspend_buf: Protection mode suspend buffer.623 *624 * Stores the state of the group and its queues when a group that's in625 * protection mode is suspended.626 *627 * Used at resume time to restore the group in its previous state.628 *629 * The size of the protection mode suspend buffer is exposed through the630 * FW interface.631 */632 struct panthor_kernel_bo *protm_suspend_buf;633 634 /** @sync_upd_work: Work used to check/signal job fences. */635 struct work_struct sync_upd_work;636 637 /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */638 struct work_struct tiler_oom_work;639 640 /** @term_work: Work used to finish the group termination procedure. */641 struct work_struct term_work;642 643 /**644 * @release_work: Work used to release group resources.645 *646 * We need to postpone the group release to avoid a deadlock when647 * the last ref is released in the tick work.648 */649 struct work_struct release_work;650 651 /**652 * @run_node: Node used to insert the group in the653 * panthor_group::groups::{runnable,idle} and654 * panthor_group::reset.stopped_groups lists.655 */656 struct list_head run_node;657 658 /**659 * @wait_node: Node used to insert the group in the660 * panthor_group::groups::waiting list.661 */662 struct list_head wait_node;663};664 665/**666 * group_queue_work() - Queue a group work667 * @group: Group to queue the work for.668 * @wname: Work name.669 *670 * Grabs a ref and queue a work item to the scheduler workqueue. If671 * the work was already queued, we release the reference we grabbed.672 *673 * Work callbacks must release the reference we grabbed here.674 */675#define group_queue_work(group, wname) \676 do { \677 group_get(group); \678 if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \679 group_put(group); \680 } while (0)681 682/**683 * sched_queue_work() - Queue a scheduler work.684 * @sched: Scheduler object.685 * @wname: Work name.686 *687 * Conditionally queues a scheduler work if no reset is pending/in-progress.688 */689#define sched_queue_work(sched, wname) \690 do { \691 if (!atomic_read(&(sched)->reset.in_progress) && \692 !panthor_device_reset_is_pending((sched)->ptdev)) \693 queue_work((sched)->wq, &(sched)->wname ## _work); \694 } while (0)695 696/**697 * sched_queue_delayed_work() - Queue a scheduler delayed work.698 * @sched: Scheduler object.699 * @wname: Work name.700 * @delay: Work delay in jiffies.701 *702 * Conditionally queues a scheduler delayed work if no reset is703 * pending/in-progress.704 */705#define sched_queue_delayed_work(sched, wname, delay) \706 do { \707 if (!atomic_read(&sched->reset.in_progress) && \708 !panthor_device_reset_is_pending((sched)->ptdev)) \709 mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \710 } while (0)711 712/*713 * We currently set the maximum of groups per file to an arbitrary low value.714 * But this can be updated if we need more.715 */716#define MAX_GROUPS_PER_POOL 128717 718/**719 * struct panthor_group_pool - Group pool720 *721 * Each file get assigned a group pool.722 */723struct panthor_group_pool {724 /** @xa: Xarray used to manage group handles. */725 struct xarray xa;726};727 728/**729 * struct panthor_job - Used to manage GPU job730 */731struct panthor_job {732 /** @base: Inherit from drm_sched_job. */733 struct drm_sched_job base;734 735 /** @refcount: Reference count. */736 struct kref refcount;737 738 /** @group: Group of the queue this job will be pushed to. */739 struct panthor_group *group;740 741 /** @queue_idx: Index of the queue inside @group. */742 u32 queue_idx;743 744 /** @call_info: Information about the userspace command stream call. */745 struct {746 /** @start: GPU address of the userspace command stream. */747 u64 start;748 749 /** @size: Size of the userspace command stream. */750 u32 size;751 752 /**753 * @latest_flush: Flush ID at the time the userspace command754 * stream was built.755 *756 * Needed for the flush reduction mechanism.757 */758 u32 latest_flush;759 } call_info;760 761 /** @ringbuf: Position of this job is in the ring buffer. */762 struct {763 /** @start: Start offset. */764 u64 start;765 766 /** @end: End offset. */767 u64 end;768 } ringbuf;769 770 /**771 * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs772 * list.773 */774 struct list_head node;775 776 /** @done_fence: Fence signaled when the job is finished or cancelled. */777 struct dma_fence *done_fence;778};779 780static void781panthor_queue_put_syncwait_obj(struct panthor_queue *queue)782{783 if (queue->syncwait.kmap) {784 struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);785 786 drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);787 queue->syncwait.kmap = NULL;788 }789 790 drm_gem_object_put(queue->syncwait.obj);791 queue->syncwait.obj = NULL;792}793 794static void *795panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)796{797 struct panthor_device *ptdev = group->ptdev;798 struct panthor_gem_object *bo;799 struct iosys_map map;800 int ret;801 802 if (queue->syncwait.kmap)803 return queue->syncwait.kmap + queue->syncwait.offset;804 805 bo = panthor_vm_get_bo_for_va(group->vm,806 queue->syncwait.gpu_va,807 &queue->syncwait.offset);808 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))809 goto err_put_syncwait_obj;810 811 queue->syncwait.obj = &bo->base.base;812 ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);813 if (drm_WARN_ON(&ptdev->base, ret))814 goto err_put_syncwait_obj;815 816 queue->syncwait.kmap = map.vaddr;817 if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))818 goto err_put_syncwait_obj;819 820 return queue->syncwait.kmap + queue->syncwait.offset;821 822err_put_syncwait_obj:823 panthor_queue_put_syncwait_obj(queue);824 return NULL;825}826 827static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)828{829 if (IS_ERR_OR_NULL(queue))830 return;831 832 if (queue->entity.fence_context)833 drm_sched_entity_destroy(&queue->entity);834 835 if (queue->scheduler.ops)836 drm_sched_fini(&queue->scheduler);837 838 panthor_queue_put_syncwait_obj(queue);839 840 panthor_kernel_bo_destroy(queue->ringbuf);841 panthor_kernel_bo_destroy(queue->iface.mem);842 843 /* Release the last_fence we were holding, if any. */844 dma_fence_put(queue->fence_ctx.last_fence);845 846 kfree(queue);847}848 849static void group_release_work(struct work_struct *work)850{851 struct panthor_group *group = container_of(work,852 struct panthor_group,853 release_work);854 u32 i;855 856 for (i = 0; i < group->queue_count; i++)857 group_free_queue(group, group->queues[i]);858 859 panthor_kernel_bo_destroy(group->suspend_buf);860 panthor_kernel_bo_destroy(group->protm_suspend_buf);861 panthor_kernel_bo_destroy(group->syncobjs);862 863 panthor_vm_put(group->vm);864 kfree(group);865}866 867static void group_release(struct kref *kref)868{869 struct panthor_group *group = container_of(kref,870 struct panthor_group,871 refcount);872 struct panthor_device *ptdev = group->ptdev;873 874 drm_WARN_ON(&ptdev->base, group->csg_id >= 0);875 drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node));876 drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node));877 878 queue_work(panthor_cleanup_wq, &group->release_work);879}880 881static void group_put(struct panthor_group *group)882{883 if (group)884 kref_put(&group->refcount, group_release);885}886 887static struct panthor_group *888group_get(struct panthor_group *group)889{890 if (group)891 kref_get(&group->refcount);892 893 return group;894}895 896/**897 * group_bind_locked() - Bind a group to a group slot898 * @group: Group.899 * @csg_id: Slot.900 *901 * Return: 0 on success, a negative error code otherwise.902 */903static int904group_bind_locked(struct panthor_group *group, u32 csg_id)905{906 struct panthor_device *ptdev = group->ptdev;907 struct panthor_csg_slot *csg_slot;908 int ret;909 910 lockdep_assert_held(&ptdev->scheduler->lock);911 912 if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||913 ptdev->scheduler->csg_slots[csg_id].group))914 return -EINVAL;915 916 ret = panthor_vm_active(group->vm);917 if (ret)918 return ret;919 920 csg_slot = &ptdev->scheduler->csg_slots[csg_id];921 group_get(group);922 group->csg_id = csg_id;923 924 /* Dummy doorbell allocation: doorbell is assigned to the group and925 * all queues use the same doorbell.926 *927 * TODO: Implement LRU-based doorbell assignment, so the most often928 * updated queues get their own doorbell, thus avoiding useless checks929 * on queues belonging to the same group that are rarely updated.930 */931 for (u32 i = 0; i < group->queue_count; i++)932 group->queues[i]->doorbell_id = csg_id + 1;933 934 csg_slot->group = group;935 936 return 0;937}938 939/**940 * group_unbind_locked() - Unbind a group from a slot.941 * @group: Group to unbind.942 *943 * Return: 0 on success, a negative error code otherwise.944 */945static int946group_unbind_locked(struct panthor_group *group)947{948 struct panthor_device *ptdev = group->ptdev;949 struct panthor_csg_slot *slot;950 951 lockdep_assert_held(&ptdev->scheduler->lock);952 953 if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))954 return -EINVAL;955 956 if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))957 return -EINVAL;958 959 slot = &ptdev->scheduler->csg_slots[group->csg_id];960 panthor_vm_idle(group->vm);961 group->csg_id = -1;962 963 /* Tiler OOM events will be re-issued next time the group is scheduled. */964 atomic_set(&group->tiler_oom, 0);965 cancel_work(&group->tiler_oom_work);966 967 for (u32 i = 0; i < group->queue_count; i++)968 group->queues[i]->doorbell_id = -1;969 970 slot->group = NULL;971 972 group_put(group);973 return 0;974}975 976/**977 * cs_slot_prog_locked() - Program a queue slot978 * @ptdev: Device.979 * @csg_id: Group slot ID.980 * @cs_id: Queue slot ID.981 *982 * Program a queue slot with the queue information so things can start being983 * executed on this queue.984 *985 * The group slot must have a group bound to it already (group_bind_locked()).986 */987static void988cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)989{990 struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id];991 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);992 993 lockdep_assert_held(&ptdev->scheduler->lock);994 995 queue->iface.input->extract = queue->iface.output->extract;996 drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract);997 998 cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf);999 cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);1000 cs_iface->input->ringbuf_input = queue->iface.input_fw_va;1001 cs_iface->input->ringbuf_output = queue->iface.output_fw_va;1002 cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) |1003 CS_CONFIG_DOORBELL(queue->doorbell_id);1004 cs_iface->input->ack_irq_mask = ~0;1005 panthor_fw_update_reqs(cs_iface, req,1006 CS_IDLE_SYNC_WAIT |1007 CS_IDLE_EMPTY |1008 CS_STATE_START |1009 CS_EXTRACT_EVENT,1010 CS_IDLE_SYNC_WAIT |1011 CS_IDLE_EMPTY |1012 CS_STATE_MASK |1013 CS_EXTRACT_EVENT);1014 if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) {1015 drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time);1016 queue->timeout_suspended = false;1017 }1018}1019 1020/**1021 * cs_slot_reset_locked() - Reset a queue slot1022 * @ptdev: Device.1023 * @csg_id: Group slot.1024 * @cs_id: Queue slot.1025 *1026 * Change the queue slot state to STOP and suspend the queue timeout if1027 * the queue is not blocked.1028 *1029 * The group slot must have a group bound to it (group_bind_locked()).1030 */1031static int1032cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)1033{1034 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1035 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;1036 struct panthor_queue *queue = group->queues[cs_id];1037 1038 lockdep_assert_held(&ptdev->scheduler->lock);1039 1040 panthor_fw_update_reqs(cs_iface, req,1041 CS_STATE_STOP,1042 CS_STATE_MASK);1043 1044 /* If the queue is blocked, we want to keep the timeout running, so1045 * we can detect unbounded waits and kill the group when that happens.1046 */1047 if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) {1048 queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);1049 queue->timeout_suspended = true;1050 WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS));1051 }1052 1053 return 0;1054}1055 1056/**1057 * csg_slot_sync_priority_locked() - Synchronize the group slot priority1058 * @ptdev: Device.1059 * @csg_id: Group slot ID.1060 *1061 * Group slot priority update happens asynchronously. When we receive a1062 * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can1063 * reflect it to our panthor_csg_slot object.1064 */1065static void1066csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id)1067{1068 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];1069 struct panthor_fw_csg_iface *csg_iface;1070 1071 lockdep_assert_held(&ptdev->scheduler->lock);1072 1073 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1074 csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28;1075}1076 1077/**1078 * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority1079 * @ptdev: Device.1080 * @csg_id: Group slot.1081 * @cs_id: Queue slot.1082 *1083 * Queue state is updated on group suspend or STATUS_UPDATE event.1084 */1085static void1086cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)1087{1088 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;1089 struct panthor_queue *queue = group->queues[cs_id];1090 struct panthor_fw_cs_iface *cs_iface =1091 panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id);1092 1093 u32 status_wait_cond;1094 1095 switch (cs_iface->output->status_blocked_reason) {1096 case CS_STATUS_BLOCKED_REASON_UNBLOCKED:1097 if (queue->iface.input->insert == queue->iface.output->extract &&1098 cs_iface->output->status_scoreboards == 0)1099 group->idle_queues |= BIT(cs_id);1100 break;1101 1102 case CS_STATUS_BLOCKED_REASON_SYNC_WAIT:1103 if (list_empty(&group->wait_node)) {1104 list_move_tail(&group->wait_node,1105 &group->ptdev->scheduler->groups.waiting);1106 }1107 1108 /* The queue is only blocked if there's no deferred operation1109 * pending, which can be checked through the scoreboard status.1110 */1111 if (!cs_iface->output->status_scoreboards)1112 group->blocked_queues |= BIT(cs_id);1113 1114 queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr;1115 queue->syncwait.ref = cs_iface->output->status_wait_sync_value;1116 status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK;1117 queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT;1118 if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) {1119 u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi;1120 1121 queue->syncwait.sync64 = true;1122 queue->syncwait.ref |= sync_val_hi << 32;1123 } else {1124 queue->syncwait.sync64 = false;1125 }1126 break;1127 1128 default:1129 /* Other reasons are not blocking. Consider the queue as runnable1130 * in those cases.1131 */1132 break;1133 }1134}1135 1136static void1137csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id)1138{1139 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];1140 struct panthor_group *group = csg_slot->group;1141 u32 i;1142 1143 lockdep_assert_held(&ptdev->scheduler->lock);1144 1145 group->idle_queues = 0;1146 group->blocked_queues = 0;1147 1148 for (i = 0; i < group->queue_count; i++) {1149 if (group->queues[i])1150 cs_slot_sync_queue_state_locked(ptdev, csg_id, i);1151 }1152}1153 1154static void1155csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id)1156{1157 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];1158 struct panthor_fw_csg_iface *csg_iface;1159 struct panthor_group *group;1160 enum panthor_group_state new_state, old_state;1161 u32 csg_state;1162 1163 lockdep_assert_held(&ptdev->scheduler->lock);1164 1165 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1166 group = csg_slot->group;1167 1168 if (!group)1169 return;1170 1171 old_state = group->state;1172 csg_state = csg_iface->output->ack & CSG_STATE_MASK;1173 switch (csg_state) {1174 case CSG_STATE_START:1175 case CSG_STATE_RESUME:1176 new_state = PANTHOR_CS_GROUP_ACTIVE;1177 break;1178 case CSG_STATE_TERMINATE:1179 new_state = PANTHOR_CS_GROUP_TERMINATED;1180 break;1181 case CSG_STATE_SUSPEND:1182 new_state = PANTHOR_CS_GROUP_SUSPENDED;1183 break;1184 default:1185 /* The unknown state might be caused by a FW state corruption,1186 * which means the group metadata can't be trusted anymore, and1187 * the SUSPEND operation might propagate the corruption to the1188 * suspend buffers. Flag the group state as unknown to make1189 * sure it's unusable after that point.1190 */1191 drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)",1192 csg_id, csg_state);1193 new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE;1194 break;1195 }1196 1197 if (old_state == new_state)1198 return;1199 1200 /* The unknown state might be caused by a FW issue, reset the FW to1201 * take a fresh start.1202 */1203 if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE)1204 panthor_device_schedule_reset(ptdev);1205 1206 if (new_state == PANTHOR_CS_GROUP_SUSPENDED)1207 csg_slot_sync_queues_state_locked(ptdev, csg_id);1208 1209 if (old_state == PANTHOR_CS_GROUP_ACTIVE) {1210 u32 i;1211 1212 /* Reset the queue slots so we start from a clean1213 * state when starting/resuming a new group on this1214 * CSG slot. No wait needed here, and no ringbell1215 * either, since the CS slot will only be re-used1216 * on the next CSG start operation.1217 */1218 for (i = 0; i < group->queue_count; i++) {1219 if (group->queues[i])1220 cs_slot_reset_locked(ptdev, csg_id, i);1221 }1222 }1223 1224 group->state = new_state;1225}1226 1227static int1228csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority)1229{1230 struct panthor_fw_csg_iface *csg_iface;1231 struct panthor_csg_slot *csg_slot;1232 struct panthor_group *group;1233 u32 queue_mask = 0, i;1234 1235 lockdep_assert_held(&ptdev->scheduler->lock);1236 1237 if (priority > MAX_CSG_PRIO)1238 return -EINVAL;1239 1240 if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS))1241 return -EINVAL;1242 1243 csg_slot = &ptdev->scheduler->csg_slots[csg_id];1244 group = csg_slot->group;1245 if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE)1246 return 0;1247 1248 csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id);1249 1250 for (i = 0; i < group->queue_count; i++) {1251 if (group->queues[i]) {1252 cs_slot_prog_locked(ptdev, csg_id, i);1253 queue_mask |= BIT(i);1254 }1255 }1256 1257 csg_iface->input->allow_compute = group->compute_core_mask;1258 csg_iface->input->allow_fragment = group->fragment_core_mask;1259 csg_iface->input->allow_other = group->tiler_core_mask;1260 csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) |1261 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) |1262 CSG_EP_REQ_TILER(group->max_tiler_cores) |1263 CSG_EP_REQ_PRIORITY(priority);1264 csg_iface->input->config = panthor_vm_as(group->vm);1265 1266 if (group->suspend_buf)1267 csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf);1268 else1269 csg_iface->input->suspend_buf = 0;1270 1271 if (group->protm_suspend_buf) {1272 csg_iface->input->protm_suspend_buf =1273 panthor_kernel_bo_gpuva(group->protm_suspend_buf);1274 } else {1275 csg_iface->input->protm_suspend_buf = 0;1276 }1277 1278 csg_iface->input->ack_irq_mask = ~0;1279 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask);1280 return 0;1281}1282 1283static void1284cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,1285 u32 csg_id, u32 cs_id)1286{1287 struct panthor_scheduler *sched = ptdev->scheduler;1288 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];1289 struct panthor_group *group = csg_slot->group;1290 struct panthor_fw_cs_iface *cs_iface;1291 u32 fatal;1292 u64 info;1293 1294 lockdep_assert_held(&sched->lock);1295 1296 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1297 fatal = cs_iface->output->fatal;1298 info = cs_iface->output->fatal_info;1299 1300 if (group)1301 group->fatal_queues |= BIT(cs_id);1302 1303 if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {1304 /* If this exception is unrecoverable, queue a reset, and make1305 * sure we stop scheduling groups until the reset has happened.1306 */1307 panthor_device_schedule_reset(ptdev);1308 cancel_delayed_work(&sched->tick_work);1309 } else {1310 sched_queue_delayed_work(sched, tick, 0);1311 }1312 1313 drm_warn(&ptdev->base,1314 "CSG slot %d CS slot: %d\n"1315 "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"1316 "CS_FATAL.EXCEPTION_DATA: 0x%x\n"1317 "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",1318 csg_id, cs_id,1319 (unsigned int)CS_EXCEPTION_TYPE(fatal),1320 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)),1321 (unsigned int)CS_EXCEPTION_DATA(fatal),1322 info);1323}1324 1325static void1326cs_slot_process_fault_event_locked(struct panthor_device *ptdev,1327 u32 csg_id, u32 cs_id)1328{1329 struct panthor_scheduler *sched = ptdev->scheduler;1330 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];1331 struct panthor_group *group = csg_slot->group;1332 struct panthor_queue *queue = group && cs_id < group->queue_count ?1333 group->queues[cs_id] : NULL;1334 struct panthor_fw_cs_iface *cs_iface;1335 u32 fault;1336 u64 info;1337 1338 lockdep_assert_held(&sched->lock);1339 1340 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1341 fault = cs_iface->output->fault;1342 info = cs_iface->output->fault_info;1343 1344 if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) {1345 u64 cs_extract = queue->iface.output->extract;1346 struct panthor_job *job;1347 1348 spin_lock(&queue->fence_ctx.lock);1349 list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) {1350 if (cs_extract >= job->ringbuf.end)1351 continue;1352 1353 if (cs_extract < job->ringbuf.start)1354 break;1355 1356 dma_fence_set_error(job->done_fence, -EINVAL);1357 }1358 spin_unlock(&queue->fence_ctx.lock);1359 }1360 1361 drm_warn(&ptdev->base,1362 "CSG slot %d CS slot: %d\n"1363 "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"1364 "CS_FAULT.EXCEPTION_DATA: 0x%x\n"1365 "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",1366 csg_id, cs_id,1367 (unsigned int)CS_EXCEPTION_TYPE(fault),1368 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)),1369 (unsigned int)CS_EXCEPTION_DATA(fault),1370 info);1371}1372 1373static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)1374{1375 struct panthor_device *ptdev = group->ptdev;1376 struct panthor_scheduler *sched = ptdev->scheduler;1377 u32 renderpasses_in_flight, pending_frag_count;1378 struct panthor_heap_pool *heaps = NULL;1379 u64 heap_address, new_chunk_va = 0;1380 u32 vt_start, vt_end, frag_end;1381 int ret, csg_id;1382 1383 mutex_lock(&sched->lock);1384 csg_id = group->csg_id;1385 if (csg_id >= 0) {1386 struct panthor_fw_cs_iface *cs_iface;1387 1388 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1389 heaps = panthor_vm_get_heap_pool(group->vm, false);1390 heap_address = cs_iface->output->heap_address;1391 vt_start = cs_iface->output->heap_vt_start;1392 vt_end = cs_iface->output->heap_vt_end;1393 frag_end = cs_iface->output->heap_frag_end;1394 renderpasses_in_flight = vt_start - frag_end;1395 pending_frag_count = vt_end - frag_end;1396 }1397 mutex_unlock(&sched->lock);1398 1399 /* The group got scheduled out, we stop here. We will get a new tiler OOM event1400 * when it's scheduled again.1401 */1402 if (unlikely(csg_id < 0))1403 return 0;1404 1405 if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) {1406 ret = -EINVAL;1407 } else {1408 /* We do the allocation without holding the scheduler lock to avoid1409 * blocking the scheduling.1410 */1411 ret = panthor_heap_grow(heaps, heap_address,1412 renderpasses_in_flight,1413 pending_frag_count, &new_chunk_va);1414 }1415 1416 /* If the heap context doesn't have memory for us, we want to let the1417 * FW try to reclaim memory by waiting for fragment jobs to land or by1418 * executing the tiler OOM exception handler, which is supposed to1419 * implement incremental rendering.1420 */1421 if (ret && ret != -ENOMEM) {1422 drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");1423 group->fatal_queues |= BIT(cs_id);1424 sched_queue_delayed_work(sched, tick, 0);1425 goto out_put_heap_pool;1426 }1427 1428 mutex_lock(&sched->lock);1429 csg_id = group->csg_id;1430 if (csg_id >= 0) {1431 struct panthor_fw_csg_iface *csg_iface;1432 struct panthor_fw_cs_iface *cs_iface;1433 1434 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1435 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1436 1437 cs_iface->input->heap_start = new_chunk_va;1438 cs_iface->input->heap_end = new_chunk_va;1439 panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM);1440 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id));1441 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));1442 }1443 mutex_unlock(&sched->lock);1444 1445 /* We allocated a chunck, but couldn't link it to the heap1446 * context because the group was scheduled out while we were1447 * allocating memory. We need to return this chunk to the heap.1448 */1449 if (unlikely(csg_id < 0 && new_chunk_va))1450 panthor_heap_return_chunk(heaps, heap_address, new_chunk_va);1451 1452 ret = 0;1453 1454out_put_heap_pool:1455 panthor_heap_pool_put(heaps);1456 return ret;1457}1458 1459static void group_tiler_oom_work(struct work_struct *work)1460{1461 struct panthor_group *group =1462 container_of(work, struct panthor_group, tiler_oom_work);1463 u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0);1464 1465 while (tiler_oom) {1466 u32 cs_id = ffs(tiler_oom) - 1;1467 1468 group_process_tiler_oom(group, cs_id);1469 tiler_oom &= ~BIT(cs_id);1470 }1471 1472 group_put(group);1473}1474 1475static void1476cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,1477 u32 csg_id, u32 cs_id)1478{1479 struct panthor_scheduler *sched = ptdev->scheduler;1480 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];1481 struct panthor_group *group = csg_slot->group;1482 1483 lockdep_assert_held(&sched->lock);1484 1485 if (drm_WARN_ON(&ptdev->base, !group))1486 return;1487 1488 atomic_or(BIT(cs_id), &group->tiler_oom);1489 1490 /* We don't use group_queue_work() here because we want to queue the1491 * work item to the heap_alloc_wq.1492 */1493 group_get(group);1494 if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work))1495 group_put(group);1496}1497 1498static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,1499 u32 csg_id, u32 cs_id)1500{1501 struct panthor_fw_cs_iface *cs_iface;1502 u32 req, ack, events;1503 1504 lockdep_assert_held(&ptdev->scheduler->lock);1505 1506 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);1507 req = cs_iface->input->req;1508 ack = cs_iface->output->ack;1509 events = (req ^ ack) & CS_EVT_MASK;1510 1511 if (events & CS_FATAL)1512 cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id);1513 1514 if (events & CS_FAULT)1515 cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id);1516 1517 if (events & CS_TILER_OOM)1518 cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id);1519 1520 /* We don't acknowledge the TILER_OOM event since its handling is1521 * deferred to a separate work.1522 */1523 panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT);1524 1525 return (events & (CS_FAULT | CS_TILER_OOM)) != 0;1526}1527 1528static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)1529{1530 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];1531 struct panthor_fw_csg_iface *csg_iface;1532 1533 lockdep_assert_held(&ptdev->scheduler->lock);1534 1535 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1536 csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;1537}1538 1539static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)1540{1541 struct panthor_scheduler *sched = ptdev->scheduler;1542 1543 lockdep_assert_held(&sched->lock);1544 1545 sched->might_have_idle_groups = true;1546 1547 /* Schedule a tick so we can evict idle groups and schedule non-idle1548 * ones. This will also update runtime PM and devfreq busy/idle states,1549 * so the device can lower its frequency or get suspended.1550 */1551 sched_queue_delayed_work(sched, tick, 0);1552}1553 1554static void csg_slot_sync_update_locked(struct panthor_device *ptdev,1555 u32 csg_id)1556{1557 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];1558 struct panthor_group *group = csg_slot->group;1559 1560 lockdep_assert_held(&ptdev->scheduler->lock);1561 1562 if (group)1563 group_queue_work(group, sync_upd);1564 1565 sched_queue_work(ptdev->scheduler, sync_upd);1566}1567 1568static void1569csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id)1570{1571 struct panthor_scheduler *sched = ptdev->scheduler;1572 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];1573 struct panthor_group *group = csg_slot->group;1574 1575 lockdep_assert_held(&sched->lock);1576 1577 drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);1578 1579 group = csg_slot->group;1580 if (!drm_WARN_ON(&ptdev->base, !group))1581 group->timedout = true;1582 1583 sched_queue_delayed_work(sched, tick, 0);1584}1585 1586static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id)1587{1588 u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events;1589 struct panthor_fw_csg_iface *csg_iface;1590 u32 ring_cs_db_mask = 0;1591 1592 lockdep_assert_held(&ptdev->scheduler->lock);1593 1594 if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))1595 return;1596 1597 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1598 req = READ_ONCE(csg_iface->input->req);1599 ack = READ_ONCE(csg_iface->output->ack);1600 cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req);1601 cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack);1602 csg_events = (req ^ ack) & CSG_EVT_MASK;1603 1604 /* There may not be any pending CSG/CS interrupts to process */1605 if (req == ack && cs_irq_req == cs_irq_ack)1606 return;1607 1608 /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before1609 * examining the CS_ACK & CS_REQ bits. This would ensure that Host1610 * doesn't miss an interrupt for the CS in the race scenario where1611 * whilst Host is servicing an interrupt for the CS, firmware sends1612 * another interrupt for that CS.1613 */1614 csg_iface->input->cs_irq_ack = cs_irq_req;1615 1616 panthor_fw_update_reqs(csg_iface, req, ack,1617 CSG_SYNC_UPDATE |1618 CSG_IDLE |1619 CSG_PROGRESS_TIMER_EVENT);1620 1621 if (csg_events & CSG_IDLE)1622 csg_slot_process_idle_event_locked(ptdev, csg_id);1623 1624 if (csg_events & CSG_PROGRESS_TIMER_EVENT)1625 csg_slot_process_progress_timer_event_locked(ptdev, csg_id);1626 1627 cs_irqs = cs_irq_req ^ cs_irq_ack;1628 while (cs_irqs) {1629 u32 cs_id = ffs(cs_irqs) - 1;1630 1631 if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id))1632 ring_cs_db_mask |= BIT(cs_id);1633 1634 cs_irqs &= ~BIT(cs_id);1635 }1636 1637 if (csg_events & CSG_SYNC_UPDATE)1638 csg_slot_sync_update_locked(ptdev, csg_id);1639 1640 if (ring_cs_db_mask)1641 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask);1642 1643 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));1644}1645 1646static void sched_process_idle_event_locked(struct panthor_device *ptdev)1647{1648 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);1649 1650 lockdep_assert_held(&ptdev->scheduler->lock);1651 1652 /* Acknowledge the idle event and schedule a tick. */1653 panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);1654 sched_queue_delayed_work(ptdev->scheduler, tick, 0);1655}1656 1657/**1658 * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ1659 * @ptdev: Device.1660 */1661static void sched_process_global_irq_locked(struct panthor_device *ptdev)1662{1663 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);1664 u32 req, ack, evts;1665 1666 lockdep_assert_held(&ptdev->scheduler->lock);1667 1668 req = READ_ONCE(glb_iface->input->req);1669 ack = READ_ONCE(glb_iface->output->ack);1670 evts = (req ^ ack) & GLB_EVT_MASK;1671 1672 if (evts & GLB_IDLE)1673 sched_process_idle_event_locked(ptdev);1674}1675 1676static void process_fw_events_work(struct work_struct *work)1677{1678 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,1679 fw_events_work);1680 u32 events = atomic_xchg(&sched->fw_events, 0);1681 struct panthor_device *ptdev = sched->ptdev;1682 1683 mutex_lock(&sched->lock);1684 1685 if (events & JOB_INT_GLOBAL_IF) {1686 sched_process_global_irq_locked(ptdev);1687 events &= ~JOB_INT_GLOBAL_IF;1688 }1689 1690 while (events) {1691 u32 csg_id = ffs(events) - 1;1692 1693 sched_process_csg_irq_locked(ptdev, csg_id);1694 events &= ~BIT(csg_id);1695 }1696 1697 mutex_unlock(&sched->lock);1698}1699 1700/**1701 * panthor_sched_report_fw_events() - Report FW events to the scheduler.1702 */1703void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)1704{1705 if (!ptdev->scheduler)1706 return;1707 1708 atomic_or(events, &ptdev->scheduler->fw_events);1709 sched_queue_work(ptdev->scheduler, fw_events);1710}1711 1712static const char *fence_get_driver_name(struct dma_fence *fence)1713{1714 return "panthor";1715}1716 1717static const char *queue_fence_get_timeline_name(struct dma_fence *fence)1718{1719 return "queue-fence";1720}1721 1722static const struct dma_fence_ops panthor_queue_fence_ops = {1723 .get_driver_name = fence_get_driver_name,1724 .get_timeline_name = queue_fence_get_timeline_name,1725};1726 1727struct panthor_csg_slots_upd_ctx {1728 u32 update_mask;1729 u32 timedout_mask;1730 struct {1731 u32 value;1732 u32 mask;1733 } requests[MAX_CSGS];1734};1735 1736static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx)1737{1738 memset(ctx, 0, sizeof(*ctx));1739}1740 1741static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev,1742 struct panthor_csg_slots_upd_ctx *ctx,1743 u32 csg_id, u32 value, u32 mask)1744{1745 if (drm_WARN_ON(&ptdev->base, !mask) ||1746 drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))1747 return;1748 1749 ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask);1750 ctx->requests[csg_id].mask |= mask;1751 ctx->update_mask |= BIT(csg_id);1752}1753 1754static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,1755 struct panthor_csg_slots_upd_ctx *ctx)1756{1757 struct panthor_scheduler *sched = ptdev->scheduler;1758 u32 update_slots = ctx->update_mask;1759 1760 lockdep_assert_held(&sched->lock);1761 1762 if (!ctx->update_mask)1763 return 0;1764 1765 while (update_slots) {1766 struct panthor_fw_csg_iface *csg_iface;1767 u32 csg_id = ffs(update_slots) - 1;1768 1769 update_slots &= ~BIT(csg_id);1770 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1771 panthor_fw_update_reqs(csg_iface, req,1772 ctx->requests[csg_id].value,1773 ctx->requests[csg_id].mask);1774 }1775 1776 panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask);1777 1778 update_slots = ctx->update_mask;1779 while (update_slots) {1780 struct panthor_fw_csg_iface *csg_iface;1781 u32 csg_id = ffs(update_slots) - 1;1782 u32 req_mask = ctx->requests[csg_id].mask, acked;1783 int ret;1784 1785 update_slots &= ~BIT(csg_id);1786 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);1787 1788 ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100);1789 1790 if (acked & CSG_ENDPOINT_CONFIG)1791 csg_slot_sync_priority_locked(ptdev, csg_id);1792 1793 if (acked & CSG_STATE_MASK)1794 csg_slot_sync_state_locked(ptdev, csg_id);1795 1796 if (acked & CSG_STATUS_UPDATE) {1797 csg_slot_sync_queues_state_locked(ptdev, csg_id);1798 csg_slot_sync_idle_state_locked(ptdev, csg_id);1799 }1800 1801 if (ret && acked != req_mask &&1802 ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {1803 drm_err(&ptdev->base, "CSG %d update request timedout", csg_id);1804 ctx->timedout_mask |= BIT(csg_id);1805 }1806 }1807 1808 if (ctx->timedout_mask)1809 return -ETIMEDOUT;1810 1811 return 0;1812}1813 1814struct panthor_sched_tick_ctx {1815 struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT];1816 struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];1817 u32 idle_group_count;1818 u32 group_count;1819 enum panthor_csg_priority min_priority;1820 struct panthor_vm *vms[MAX_CS_PER_CSG];1821 u32 as_count;1822 bool immediate_tick;1823 u32 csg_upd_failed_mask;1824};1825 1826static bool1827tick_ctx_is_full(const struct panthor_scheduler *sched,1828 const struct panthor_sched_tick_ctx *ctx)1829{1830 return ctx->group_count == sched->csg_slot_count;1831}1832 1833static bool1834group_is_idle(struct panthor_group *group)1835{1836 struct panthor_device *ptdev = group->ptdev;1837 u32 inactive_queues;1838 1839 if (group->csg_id >= 0)1840 return ptdev->scheduler->csg_slots[group->csg_id].idle;1841 1842 inactive_queues = group->idle_queues | group->blocked_queues;1843 return hweight32(inactive_queues) == group->queue_count;1844}1845 1846static bool1847group_can_run(struct panthor_group *group)1848{1849 return group->state != PANTHOR_CS_GROUP_TERMINATED &&1850 group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&1851 !group->destroyed && group->fatal_queues == 0 &&1852 !group->timedout;1853}1854 1855static void1856tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,1857 struct panthor_sched_tick_ctx *ctx,1858 struct list_head *queue,1859 bool skip_idle_groups,1860 bool owned_by_tick_ctx)1861{1862 struct panthor_group *group, *tmp;1863 1864 if (tick_ctx_is_full(sched, ctx))1865 return;1866 1867 list_for_each_entry_safe(group, tmp, queue, run_node) {1868 u32 i;1869 1870 if (!group_can_run(group))1871 continue;1872 1873 if (skip_idle_groups && group_is_idle(group))1874 continue;1875 1876 for (i = 0; i < ctx->as_count; i++) {1877 if (ctx->vms[i] == group->vm)1878 break;1879 }1880 1881 if (i == ctx->as_count && ctx->as_count == sched->as_slot_count)1882 continue;1883 1884 if (!owned_by_tick_ctx)1885 group_get(group);1886 1887 list_move_tail(&group->run_node, &ctx->groups[group->priority]);1888 ctx->group_count++;1889 if (group_is_idle(group))1890 ctx->idle_group_count++;1891 1892 if (i == ctx->as_count)1893 ctx->vms[ctx->as_count++] = group->vm;1894 1895 if (ctx->min_priority > group->priority)1896 ctx->min_priority = group->priority;1897 1898 if (tick_ctx_is_full(sched, ctx))1899 return;1900 }1901}1902 1903static void1904tick_ctx_insert_old_group(struct panthor_scheduler *sched,1905 struct panthor_sched_tick_ctx *ctx,1906 struct panthor_group *group,1907 bool full_tick)1908{1909 struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];1910 struct panthor_group *other_group;1911 1912 if (!full_tick) {1913 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);1914 return;1915 }1916 1917 /* Rotate to make sure groups with lower CSG slot1918 * priorities have a chance to get a higher CSG slot1919 * priority next time they get picked. This priority1920 * has an impact on resource request ordering, so it's1921 * important to make sure we don't let one group starve1922 * all other groups with the same group priority.1923 */1924 list_for_each_entry(other_group,1925 &ctx->old_groups[csg_slot->group->priority],1926 run_node) {1927 struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];1928 1929 if (other_csg_slot->priority > csg_slot->priority) {1930 list_add_tail(&csg_slot->group->run_node, &other_group->run_node);1931 return;1932 }1933 }1934 1935 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);1936}1937 1938static void1939tick_ctx_init(struct panthor_scheduler *sched,1940 struct panthor_sched_tick_ctx *ctx,1941 bool full_tick)1942{1943 struct panthor_device *ptdev = sched->ptdev;1944 struct panthor_csg_slots_upd_ctx upd_ctx;1945 int ret;1946 u32 i;1947 1948 memset(ctx, 0, sizeof(*ctx));1949 csgs_upd_ctx_init(&upd_ctx);1950 1951 ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;1952 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {1953 INIT_LIST_HEAD(&ctx->groups[i]);1954 INIT_LIST_HEAD(&ctx->old_groups[i]);1955 }1956 1957 for (i = 0; i < sched->csg_slot_count; i++) {1958 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];1959 struct panthor_group *group = csg_slot->group;1960 struct panthor_fw_csg_iface *csg_iface;1961 1962 if (!group)1963 continue;1964 1965 csg_iface = panthor_fw_get_csg_iface(ptdev, i);1966 group_get(group);1967 1968 /* If there was unhandled faults on the VM, force processing of1969 * CSG IRQs, so we can flag the faulty queue.1970 */1971 if (panthor_vm_has_unhandled_faults(group->vm)) {1972 sched_process_csg_irq_locked(ptdev, i);1973 1974 /* No fatal fault reported, flag all queues as faulty. */1975 if (!group->fatal_queues)1976 group->fatal_queues |= GENMASK(group->queue_count - 1, 0);1977 }1978 1979 tick_ctx_insert_old_group(sched, ctx, group, full_tick);1980 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,1981 csg_iface->output->ack ^ CSG_STATUS_UPDATE,1982 CSG_STATUS_UPDATE);1983 }1984 1985 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);1986 if (ret) {1987 panthor_device_schedule_reset(ptdev);1988 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;1989 }1990}1991 1992#define NUM_INSTRS_PER_SLOT 161993 1994static void1995group_term_post_processing(struct panthor_group *group)1996{1997 struct panthor_job *job, *tmp;1998 LIST_HEAD(faulty_jobs);1999 bool cookie;2000 u32 i = 0;2001 2002 if (drm_WARN_ON(&group->ptdev->base, group_can_run(group)))2003 return;2004 2005 cookie = dma_fence_begin_signalling();2006 for (i = 0; i < group->queue_count; i++) {2007 struct panthor_queue *queue = group->queues[i];2008 struct panthor_syncobj_64b *syncobj;2009 int err;2010 2011 if (group->fatal_queues & BIT(i))2012 err = -EINVAL;2013 else if (group->timedout)2014 err = -ETIMEDOUT;2015 else2016 err = -ECANCELED;2017 2018 if (!queue)2019 continue;2020 2021 spin_lock(&queue->fence_ctx.lock);2022 list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) {2023 list_move_tail(&job->node, &faulty_jobs);2024 dma_fence_set_error(job->done_fence, err);2025 dma_fence_signal_locked(job->done_fence);2026 }2027 spin_unlock(&queue->fence_ctx.lock);2028 2029 /* Manually update the syncobj seqno to unblock waiters. */2030 syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));2031 syncobj->status = ~0;2032 syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);2033 sched_queue_work(group->ptdev->scheduler, sync_upd);2034 }2035 dma_fence_end_signalling(cookie);2036 2037 list_for_each_entry_safe(job, tmp, &faulty_jobs, node) {2038 list_del_init(&job->node);2039 panthor_job_put(&job->base);2040 }2041}2042 2043static void group_term_work(struct work_struct *work)2044{2045 struct panthor_group *group =2046 container_of(work, struct panthor_group, term_work);2047 2048 group_term_post_processing(group);2049 group_put(group);2050}2051 2052static void2053tick_ctx_cleanup(struct panthor_scheduler *sched,2054 struct panthor_sched_tick_ctx *ctx)2055{2056 struct panthor_device *ptdev = sched->ptdev;2057 struct panthor_group *group, *tmp;2058 u32 i;2059 2060 for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) {2061 list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) {2062 /* If everything went fine, we should only have groups2063 * to be terminated in the old_groups lists.2064 */2065 drm_WARN_ON(&ptdev->base, !ctx->csg_upd_failed_mask &&2066 group_can_run(group));2067 2068 if (!group_can_run(group)) {2069 list_del_init(&group->run_node);2070 list_del_init(&group->wait_node);2071 group_queue_work(group, term);2072 } else if (group->csg_id >= 0) {2073 list_del_init(&group->run_node);2074 } else {2075 list_move(&group->run_node,2076 group_is_idle(group) ?2077 &sched->groups.idle[group->priority] :2078 &sched->groups.runnable[group->priority]);2079 }2080 group_put(group);2081 }2082 }2083 2084 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {2085 /* If everything went fine, the groups to schedule lists should2086 * be empty.2087 */2088 drm_WARN_ON(&ptdev->base,2089 !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i]));2090 2091 list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) {2092 if (group->csg_id >= 0) {2093 list_del_init(&group->run_node);2094 } else {2095 list_move(&group->run_node,2096 group_is_idle(group) ?2097 &sched->groups.idle[group->priority] :2098 &sched->groups.runnable[group->priority]);2099 }2100 group_put(group);2101 }2102 }2103}2104 2105static void2106tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx)2107{2108 struct panthor_group *group, *tmp;2109 struct panthor_device *ptdev = sched->ptdev;2110 struct panthor_csg_slot *csg_slot;2111 int prio, new_csg_prio = MAX_CSG_PRIO, i;2112 u32 free_csg_slots = 0;2113 struct panthor_csg_slots_upd_ctx upd_ctx;2114 int ret;2115 2116 csgs_upd_ctx_init(&upd_ctx);2117 2118 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {2119 /* Suspend or terminate evicted groups. */2120 list_for_each_entry(group, &ctx->old_groups[prio], run_node) {2121 bool term = !group_can_run(group);2122 int csg_id = group->csg_id;2123 2124 if (drm_WARN_ON(&ptdev->base, csg_id < 0))2125 continue;2126 2127 csg_slot = &sched->csg_slots[csg_id];2128 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,2129 term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND,2130 CSG_STATE_MASK);2131 }2132 2133 /* Update priorities on already running groups. */2134 list_for_each_entry(group, &ctx->groups[prio], run_node) {2135 struct panthor_fw_csg_iface *csg_iface;2136 int csg_id = group->csg_id;2137 2138 if (csg_id < 0) {2139 new_csg_prio--;2140 continue;2141 }2142 2143 csg_slot = &sched->csg_slots[csg_id];2144 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);2145 if (csg_slot->priority == new_csg_prio) {2146 new_csg_prio--;2147 continue;2148 }2149 2150 panthor_fw_update_reqs(csg_iface, endpoint_req,2151 CSG_EP_REQ_PRIORITY(new_csg_prio),2152 CSG_EP_REQ_PRIORITY_MASK);2153 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,2154 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,2155 CSG_ENDPOINT_CONFIG);2156 new_csg_prio--;2157 }2158 }2159 2160 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);2161 if (ret) {2162 panthor_device_schedule_reset(ptdev);2163 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;2164 return;2165 }2166 2167 /* Unbind evicted groups. */2168 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {2169 list_for_each_entry(group, &ctx->old_groups[prio], run_node) {2170 /* This group is gone. Process interrupts to clear2171 * any pending interrupts before we start the new2172 * group.2173 */2174 if (group->csg_id >= 0)2175 sched_process_csg_irq_locked(ptdev, group->csg_id);2176 2177 group_unbind_locked(group);2178 }2179 }2180 2181 for (i = 0; i < sched->csg_slot_count; i++) {2182 if (!sched->csg_slots[i].group)2183 free_csg_slots |= BIT(i);2184 }2185 2186 csgs_upd_ctx_init(&upd_ctx);2187 new_csg_prio = MAX_CSG_PRIO;2188 2189 /* Start new groups. */2190 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {2191 list_for_each_entry(group, &ctx->groups[prio], run_node) {2192 int csg_id = group->csg_id;2193 struct panthor_fw_csg_iface *csg_iface;2194 2195 if (csg_id >= 0) {2196 new_csg_prio--;2197 continue;2198 }2199 2200 csg_id = ffs(free_csg_slots) - 1;2201 if (drm_WARN_ON(&ptdev->base, csg_id < 0))2202 break;2203 2204 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);2205 csg_slot = &sched->csg_slots[csg_id];2206 group_bind_locked(group, csg_id);2207 csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--);2208 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,2209 group->state == PANTHOR_CS_GROUP_SUSPENDED ?2210 CSG_STATE_RESUME : CSG_STATE_START,2211 CSG_STATE_MASK);2212 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,2213 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,2214 CSG_ENDPOINT_CONFIG);2215 free_csg_slots &= ~BIT(csg_id);2216 }2217 }2218 2219 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);2220 if (ret) {2221 panthor_device_schedule_reset(ptdev);2222 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;2223 return;2224 }2225 2226 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {2227 list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) {2228 list_del_init(&group->run_node);2229 2230 /* If the group has been destroyed while we were2231 * scheduling, ask for an immediate tick to2232 * re-evaluate as soon as possible and get rid of2233 * this dangling group.2234 */2235 if (group->destroyed)2236 ctx->immediate_tick = true;2237 group_put(group);2238 }2239 2240 /* Return evicted groups to the idle or run queues. Groups2241 * that can no longer be run (because they've been destroyed2242 * or experienced an unrecoverable error) will be scheduled2243 * for destruction in tick_ctx_cleanup().2244 */2245 list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) {2246 if (!group_can_run(group))2247 continue;2248 2249 if (group_is_idle(group))2250 list_move_tail(&group->run_node, &sched->groups.idle[prio]);2251 else2252 list_move_tail(&group->run_node, &sched->groups.runnable[prio]);2253 group_put(group);2254 }2255 }2256 2257 sched->used_csg_slot_count = ctx->group_count;2258 sched->might_have_idle_groups = ctx->idle_group_count > 0;2259}2260 2261static u642262tick_ctx_update_resched_target(struct panthor_scheduler *sched,2263 const struct panthor_sched_tick_ctx *ctx)2264{2265 /* We had space left, no need to reschedule until some external event happens. */2266 if (!tick_ctx_is_full(sched, ctx))2267 goto no_tick;2268 2269 /* If idle groups were scheduled, no need to wake up until some external2270 * event happens (group unblocked, new job submitted, ...).2271 */2272 if (ctx->idle_group_count)2273 goto no_tick;2274 2275 if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))2276 goto no_tick;2277 2278 /* If there are groups of the same priority waiting, we need to2279 * keep the scheduler ticking, otherwise, we'll just wait for2280 * new groups with higher priority to be queued.2281 */2282 if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {2283 u64 resched_target = sched->last_tick + sched->tick_period;2284 2285 if (time_before64(sched->resched_target, sched->last_tick) ||2286 time_before64(resched_target, sched->resched_target))2287 sched->resched_target = resched_target;2288 2289 return sched->resched_target - sched->last_tick;2290 }2291 2292no_tick:2293 sched->resched_target = U64_MAX;2294 return U64_MAX;2295}2296 2297static void tick_work(struct work_struct *work)2298{2299 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,2300 tick_work.work);2301 struct panthor_device *ptdev = sched->ptdev;2302 struct panthor_sched_tick_ctx ctx;2303 u64 remaining_jiffies = 0, resched_delay;2304 u64 now = get_jiffies_64();2305 int prio, ret, cookie;2306 2307 if (!drm_dev_enter(&ptdev->base, &cookie))2308 return;2309 2310 ret = pm_runtime_resume_and_get(ptdev->base.dev);2311 if (drm_WARN_ON(&ptdev->base, ret))2312 goto out_dev_exit;2313 2314 if (time_before64(now, sched->resched_target))2315 remaining_jiffies = sched->resched_target - now;2316 2317 mutex_lock(&sched->lock);2318 if (panthor_device_reset_is_pending(sched->ptdev))2319 goto out_unlock;2320 2321 tick_ctx_init(sched, &ctx, remaining_jiffies != 0);2322 if (ctx.csg_upd_failed_mask)2323 goto out_cleanup_ctx;2324 2325 if (remaining_jiffies) {2326 /* Scheduling forced in the middle of a tick. Only RT groups2327 * can preempt non-RT ones. Currently running RT groups can't be2328 * preempted.2329 */2330 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;2331 prio >= 0 && !tick_ctx_is_full(sched, &ctx);2332 prio--) {2333 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],2334 true, true);2335 if (prio == PANTHOR_CSG_PRIORITY_RT) {2336 tick_ctx_pick_groups_from_list(sched, &ctx,2337 &sched->groups.runnable[prio],2338 true, false);2339 }2340 }2341 }2342 2343 /* First pick non-idle groups */2344 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;2345 prio >= 0 && !tick_ctx_is_full(sched, &ctx);2346 prio--) {2347 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],2348 true, false);2349 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);2350 }2351 2352 /* If we have free CSG slots left, pick idle groups */2353 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;2354 prio >= 0 && !tick_ctx_is_full(sched, &ctx);2355 prio--) {2356 /* Check the old_group queue first to avoid reprogramming the slots */2357 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true);2358 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio],2359 false, false);2360 }2361 2362 tick_ctx_apply(sched, &ctx);2363 if (ctx.csg_upd_failed_mask)2364 goto out_cleanup_ctx;2365 2366 if (ctx.idle_group_count == ctx.group_count) {2367 panthor_devfreq_record_idle(sched->ptdev);2368 if (sched->pm.has_ref) {2369 pm_runtime_put_autosuspend(ptdev->base.dev);2370 sched->pm.has_ref = false;2371 }2372 } else {2373 panthor_devfreq_record_busy(sched->ptdev);2374 if (!sched->pm.has_ref) {2375 pm_runtime_get(ptdev->base.dev);2376 sched->pm.has_ref = true;2377 }2378 }2379 2380 sched->last_tick = now;2381 resched_delay = tick_ctx_update_resched_target(sched, &ctx);2382 if (ctx.immediate_tick)2383 resched_delay = 0;2384 2385 if (resched_delay != U64_MAX)2386 sched_queue_delayed_work(sched, tick, resched_delay);2387 2388out_cleanup_ctx:2389 tick_ctx_cleanup(sched, &ctx);2390 2391out_unlock:2392 mutex_unlock(&sched->lock);2393 pm_runtime_mark_last_busy(ptdev->base.dev);2394 pm_runtime_put_autosuspend(ptdev->base.dev);2395 2396out_dev_exit:2397 drm_dev_exit(cookie);2398}2399 2400static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)2401{2402 struct panthor_queue *queue = group->queues[queue_idx];2403 union {2404 struct panthor_syncobj_64b sync64;2405 struct panthor_syncobj_32b sync32;2406 } *syncobj;2407 bool result;2408 u64 value;2409 2410 syncobj = panthor_queue_get_syncwait_obj(group, queue);2411 if (!syncobj)2412 return -EINVAL;2413 2414 value = queue->syncwait.sync64 ?2415 syncobj->sync64.seqno :2416 syncobj->sync32.seqno;2417 2418 if (queue->syncwait.gt)2419 result = value > queue->syncwait.ref;2420 else2421 result = value <= queue->syncwait.ref;2422 2423 if (result)2424 panthor_queue_put_syncwait_obj(queue);2425 2426 return result;2427}2428 2429static void sync_upd_work(struct work_struct *work)2430{2431 struct panthor_scheduler *sched = container_of(work,2432 struct panthor_scheduler,2433 sync_upd_work);2434 struct panthor_group *group, *tmp;2435 bool immediate_tick = false;2436 2437 mutex_lock(&sched->lock);2438 list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) {2439 u32 tested_queues = group->blocked_queues;2440 u32 unblocked_queues = 0;2441 2442 while (tested_queues) {2443 u32 cs_id = ffs(tested_queues) - 1;2444 int ret;2445 2446 ret = panthor_queue_eval_syncwait(group, cs_id);2447 drm_WARN_ON(&group->ptdev->base, ret < 0);2448 if (ret)2449 unblocked_queues |= BIT(cs_id);2450 2451 tested_queues &= ~BIT(cs_id);2452 }2453 2454 if (unblocked_queues) {2455 group->blocked_queues &= ~unblocked_queues;2456 2457 if (group->csg_id < 0) {2458 list_move(&group->run_node,2459 &sched->groups.runnable[group->priority]);2460 if (group->priority == PANTHOR_CSG_PRIORITY_RT)2461 immediate_tick = true;2462 }2463 }2464 2465 if (!group->blocked_queues)2466 list_del_init(&group->wait_node);2467 }2468 mutex_unlock(&sched->lock);2469 2470 if (immediate_tick)2471 sched_queue_delayed_work(sched, tick, 0);2472}2473 2474static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)2475{2476 struct panthor_device *ptdev = group->ptdev;2477 struct panthor_scheduler *sched = ptdev->scheduler;2478 struct list_head *queue = &sched->groups.runnable[group->priority];2479 u64 delay_jiffies = 0;2480 bool was_idle;2481 u64 now;2482 2483 if (!group_can_run(group))2484 return;2485 2486 /* All updated queues are blocked, no need to wake up the scheduler. */2487 if ((queue_mask & group->blocked_queues) == queue_mask)2488 return;2489 2490 was_idle = group_is_idle(group);2491 group->idle_queues &= ~queue_mask;2492 2493 /* Don't mess up with the lists if we're in a middle of a reset. */2494 if (atomic_read(&sched->reset.in_progress))2495 return;2496 2497 if (was_idle && !group_is_idle(group))2498 list_move_tail(&group->run_node, queue);2499 2500 /* RT groups are preemptive. */2501 if (group->priority == PANTHOR_CSG_PRIORITY_RT) {2502 sched_queue_delayed_work(sched, tick, 0);2503 return;2504 }2505 2506 /* Some groups might be idle, force an immediate tick to2507 * re-evaluate.2508 */2509 if (sched->might_have_idle_groups) {2510 sched_queue_delayed_work(sched, tick, 0);2511 return;2512 }2513 2514 /* Scheduler is ticking, nothing to do. */2515 if (sched->resched_target != U64_MAX) {2516 /* If there are free slots, force immediating ticking. */2517 if (sched->used_csg_slot_count < sched->csg_slot_count)2518 sched_queue_delayed_work(sched, tick, 0);2519 2520 return;2521 }2522 2523 /* Scheduler tick was off, recalculate the resched_target based on the2524 * last tick event, and queue the scheduler work.2525 */2526 now = get_jiffies_64();2527 sched->resched_target = sched->last_tick + sched->tick_period;2528 if (sched->used_csg_slot_count == sched->csg_slot_count &&2529 time_before64(now, sched->resched_target))2530 delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);2531 2532 sched_queue_delayed_work(sched, tick, delay_jiffies);2533}2534 2535static void queue_stop(struct panthor_queue *queue,2536 struct panthor_job *bad_job)2537{2538 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);2539}2540 2541static void queue_start(struct panthor_queue *queue)2542{2543 struct panthor_job *job;2544 2545 /* Re-assign the parent fences. */2546 list_for_each_entry(job, &queue->scheduler.pending_list, base.list)2547 job->base.s_fence->parent = dma_fence_get(job->done_fence);2548 2549 drm_sched_start(&queue->scheduler);2550}2551 2552static void panthor_group_stop(struct panthor_group *group)2553{2554 struct panthor_scheduler *sched = group->ptdev->scheduler;2555 2556 lockdep_assert_held(&sched->reset.lock);2557 2558 for (u32 i = 0; i < group->queue_count; i++)2559 queue_stop(group->queues[i], NULL);2560 2561 group_get(group);2562 list_move_tail(&group->run_node, &sched->reset.stopped_groups);2563}2564 2565static void panthor_group_start(struct panthor_group *group)2566{2567 struct panthor_scheduler *sched = group->ptdev->scheduler;2568 2569 lockdep_assert_held(&group->ptdev->scheduler->reset.lock);2570 2571 for (u32 i = 0; i < group->queue_count; i++)2572 queue_start(group->queues[i]);2573 2574 if (group_can_run(group)) {2575 list_move_tail(&group->run_node,2576 group_is_idle(group) ?2577 &sched->groups.idle[group->priority] :2578 &sched->groups.runnable[group->priority]);2579 } else {2580 list_del_init(&group->run_node);2581 list_del_init(&group->wait_node);2582 group_queue_work(group, term);2583 }2584 2585 group_put(group);2586}2587 2588static void panthor_sched_immediate_tick(struct panthor_device *ptdev)2589{2590 struct panthor_scheduler *sched = ptdev->scheduler;2591 2592 sched_queue_delayed_work(sched, tick, 0);2593}2594 2595/**2596 * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.2597 */2598void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)2599{2600 /* Force a tick to immediately kill faulty groups. */2601 if (ptdev->scheduler)2602 panthor_sched_immediate_tick(ptdev);2603}2604 2605void panthor_sched_resume(struct panthor_device *ptdev)2606{2607 /* Force a tick to re-evaluate after a resume. */2608 panthor_sched_immediate_tick(ptdev);2609}2610 2611void panthor_sched_suspend(struct panthor_device *ptdev)2612{2613 struct panthor_scheduler *sched = ptdev->scheduler;2614 struct panthor_csg_slots_upd_ctx upd_ctx;2615 struct panthor_group *group;2616 u32 suspended_slots;2617 u32 i;2618 2619 mutex_lock(&sched->lock);2620 csgs_upd_ctx_init(&upd_ctx);2621 for (i = 0; i < sched->csg_slot_count; i++) {2622 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];2623 2624 if (csg_slot->group) {2625 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,2626 group_can_run(csg_slot->group) ?2627 CSG_STATE_SUSPEND : CSG_STATE_TERMINATE,2628 CSG_STATE_MASK);2629 }2630 }2631 2632 suspended_slots = upd_ctx.update_mask;2633 2634 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);2635 suspended_slots &= ~upd_ctx.timedout_mask;2636 2637 if (upd_ctx.timedout_mask) {2638 u32 slot_mask = upd_ctx.timedout_mask;2639 2640 drm_err(&ptdev->base, "CSG suspend failed, escalating to termination");2641 csgs_upd_ctx_init(&upd_ctx);2642 while (slot_mask) {2643 u32 csg_id = ffs(slot_mask) - 1;2644 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];2645 2646 /* We consider group suspension failures as fatal and flag the2647 * group as unusable by setting timedout=true.2648 */2649 csg_slot->group->timedout = true;2650 2651 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,2652 CSG_STATE_TERMINATE,2653 CSG_STATE_MASK);2654 slot_mask &= ~BIT(csg_id);2655 }2656 2657 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);2658 2659 slot_mask = upd_ctx.timedout_mask;2660 while (slot_mask) {2661 u32 csg_id = ffs(slot_mask) - 1;2662 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];2663 2664 /* Terminate command timedout, but the soft-reset will2665 * automatically terminate all active groups, so let's2666 * force the state to halted here.2667 */2668 if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED)2669 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;2670 slot_mask &= ~BIT(csg_id);2671 }2672 }2673 2674 /* Flush L2 and LSC caches to make sure suspend state is up-to-date.2675 * If the flush fails, flag all queues for termination.2676 */2677 if (suspended_slots) {2678 bool flush_caches_failed = false;2679 u32 slot_mask = suspended_slots;2680 2681 if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0))2682 flush_caches_failed = true;2683 2684 while (slot_mask) {2685 u32 csg_id = ffs(slot_mask) - 1;2686 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];2687 2688 if (flush_caches_failed)2689 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;2690 else2691 csg_slot_sync_update_locked(ptdev, csg_id);2692 2693 slot_mask &= ~BIT(csg_id);2694 }2695 }2696 2697 for (i = 0; i < sched->csg_slot_count; i++) {2698 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];2699 2700 group = csg_slot->group;2701 if (!group)2702 continue;2703 2704 group_get(group);2705 2706 if (group->csg_id >= 0)2707 sched_process_csg_irq_locked(ptdev, group->csg_id);2708 2709 group_unbind_locked(group);2710 2711 drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));2712 2713 if (group_can_run(group)) {2714 list_add(&group->run_node,2715 &sched->groups.idle[group->priority]);2716 } else {2717 /* We don't bother stopping the scheduler if the group is2718 * faulty, the group termination work will finish the job.2719 */2720 list_del_init(&group->wait_node);2721 group_queue_work(group, term);2722 }2723 group_put(group);2724 }2725 mutex_unlock(&sched->lock);2726}2727 2728void panthor_sched_pre_reset(struct panthor_device *ptdev)2729{2730 struct panthor_scheduler *sched = ptdev->scheduler;2731 struct panthor_group *group, *group_tmp;2732 u32 i;2733 2734 mutex_lock(&sched->reset.lock);2735 atomic_set(&sched->reset.in_progress, true);2736 2737 /* Cancel all scheduler works. Once this is done, these works can't be2738 * scheduled again until the reset operation is complete.2739 */2740 cancel_work_sync(&sched->sync_upd_work);2741 cancel_delayed_work_sync(&sched->tick_work);2742 2743 panthor_sched_suspend(ptdev);2744 2745 /* Stop all groups that might still accept jobs, so we don't get passed2746 * new jobs while we're resetting.2747 */2748 for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {2749 /* All groups should be in the idle lists. */2750 drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));2751 list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)2752 panthor_group_stop(group);2753 }2754 2755 for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) {2756 list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node)2757 panthor_group_stop(group);2758 }2759 2760 mutex_unlock(&sched->reset.lock);2761}2762 2763void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)2764{2765 struct panthor_scheduler *sched = ptdev->scheduler;2766 struct panthor_group *group, *group_tmp;2767 2768 mutex_lock(&sched->reset.lock);2769 2770 list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) {2771 /* Consider all previously running group as terminated if the2772 * reset failed.2773 */2774 if (reset_failed)2775 group->state = PANTHOR_CS_GROUP_TERMINATED;2776 2777 panthor_group_start(group);2778 }2779 2780 /* We're done resetting the GPU, clear the reset.in_progress bit so we can2781 * kick the scheduler.2782 */2783 atomic_set(&sched->reset.in_progress, false);2784 mutex_unlock(&sched->reset.lock);2785 2786 /* No need to queue a tick and update syncs if the reset failed. */2787 if (!reset_failed) {2788 sched_queue_delayed_work(sched, tick, 0);2789 sched_queue_work(sched, sync_upd);2790 }2791}2792 2793static void group_sync_upd_work(struct work_struct *work)2794{2795 struct panthor_group *group =2796 container_of(work, struct panthor_group, sync_upd_work);2797 struct panthor_job *job, *job_tmp;2798 LIST_HEAD(done_jobs);2799 u32 queue_idx;2800 bool cookie;2801 2802 cookie = dma_fence_begin_signalling();2803 for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {2804 struct panthor_queue *queue = group->queues[queue_idx];2805 struct panthor_syncobj_64b *syncobj;2806 2807 if (!queue)2808 continue;2809 2810 syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));2811 2812 spin_lock(&queue->fence_ctx.lock);2813 list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {2814 if (syncobj->seqno < job->done_fence->seqno)2815 break;2816 2817 list_move_tail(&job->node, &done_jobs);2818 dma_fence_signal_locked(job->done_fence);2819 }2820 spin_unlock(&queue->fence_ctx.lock);2821 }2822 dma_fence_end_signalling(cookie);2823 2824 list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {2825 list_del_init(&job->node);2826 panthor_job_put(&job->base);2827 }2828 2829 group_put(group);2830}2831 2832static struct dma_fence *2833queue_run_job(struct drm_sched_job *sched_job)2834{2835 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);2836 struct panthor_group *group = job->group;2837 struct panthor_queue *queue = group->queues[job->queue_idx];2838 struct panthor_device *ptdev = group->ptdev;2839 struct panthor_scheduler *sched = ptdev->scheduler;2840 u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);2841 u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1);2842 u64 addr_reg = ptdev->csif_info.cs_reg_count -2843 ptdev->csif_info.unpreserved_cs_reg_count;2844 u64 val_reg = addr_reg + 2;2845 u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +2846 job->queue_idx * sizeof(struct panthor_syncobj_64b);2847 u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);2848 struct dma_fence *done_fence;2849 int ret;2850 2851 u64 call_instrs[NUM_INSTRS_PER_SLOT] = {2852 /* MOV32 rX+2, cs.latest_flush */2853 (2ull << 56) | (val_reg << 48) | job->call_info.latest_flush,2854 2855 /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */2856 (36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233,2857 2858 /* MOV48 rX:rX+1, cs.start */2859 (1ull << 56) | (addr_reg << 48) | job->call_info.start,2860 2861 /* MOV32 rX+2, cs.size */2862 (2ull << 56) | (val_reg << 48) | job->call_info.size,2863 2864 /* WAIT(0) => waits for FLUSH_CACHE2 instruction */2865 (3ull << 56) | (1 << 16),2866 2867 /* CALL rX:rX+1, rX+2 */2868 (32ull << 56) | (addr_reg << 40) | (val_reg << 32),2869 2870 /* MOV48 rX:rX+1, sync_addr */2871 (1ull << 56) | (addr_reg << 48) | sync_addr,2872 2873 /* MOV48 rX+2, #1 */2874 (1ull << 56) | (val_reg << 48) | 1,2875 2876 /* WAIT(all) */2877 (3ull << 56) | (waitall_mask << 16),2878 2879 /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/2880 (51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1,2881 2882 /* ERROR_BARRIER, so we can recover from faults at job2883 * boundaries.2884 */2885 (47ull << 56),2886 };2887 2888 /* Need to be cacheline aligned to please the prefetcher. */2889 static_assert(sizeof(call_instrs) % 64 == 0,2890 "call_instrs is not aligned on a cacheline");2891 2892 /* Stream size is zero, nothing to do except making sure all previously2893 * submitted jobs are done before we signal the2894 * drm_sched_job::s_fence::finished fence.2895 */2896 if (!job->call_info.size) {2897 job->done_fence = dma_fence_get(queue->fence_ctx.last_fence);2898 return dma_fence_get(job->done_fence);2899 }2900 2901 ret = pm_runtime_resume_and_get(ptdev->base.dev);2902 if (drm_WARN_ON(&ptdev->base, ret))2903 return ERR_PTR(ret);2904 2905 mutex_lock(&sched->lock);2906 if (!group_can_run(group)) {2907 done_fence = ERR_PTR(-ECANCELED);2908 goto out_unlock;2909 }2910 2911 dma_fence_init(job->done_fence,2912 &panthor_queue_fence_ops,2913 &queue->fence_ctx.lock,2914 queue->fence_ctx.id,2915 atomic64_inc_return(&queue->fence_ctx.seqno));2916 2917 memcpy(queue->ringbuf->kmap + ringbuf_insert,2918 call_instrs, sizeof(call_instrs));2919 2920 panthor_job_get(&job->base);2921 spin_lock(&queue->fence_ctx.lock);2922 list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs);2923 spin_unlock(&queue->fence_ctx.lock);2924 2925 job->ringbuf.start = queue->iface.input->insert;2926 job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs);2927 2928 /* Make sure the ring buffer is updated before the INSERT2929 * register.2930 */2931 wmb();2932 2933 queue->iface.input->extract = queue->iface.output->extract;2934 queue->iface.input->insert = job->ringbuf.end;2935 2936 if (group->csg_id < 0) {2937 /* If the queue is blocked, we want to keep the timeout running, so we2938 * can detect unbounded waits and kill the group when that happens.2939 * Otherwise, we suspend the timeout so the time we spend waiting for2940 * a CSG slot is not counted.2941 */2942 if (!(group->blocked_queues & BIT(job->queue_idx)) &&2943 !queue->timeout_suspended) {2944 queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);2945 queue->timeout_suspended = true;2946 }2947 2948 group_schedule_locked(group, BIT(job->queue_idx));2949 } else {2950 gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);2951 if (!sched->pm.has_ref &&2952 !(group->blocked_queues & BIT(job->queue_idx))) {2953 pm_runtime_get(ptdev->base.dev);2954 sched->pm.has_ref = true;2955 }2956 panthor_devfreq_record_busy(sched->ptdev);2957 }2958 2959 /* Update the last fence. */2960 dma_fence_put(queue->fence_ctx.last_fence);2961 queue->fence_ctx.last_fence = dma_fence_get(job->done_fence);2962 2963 done_fence = dma_fence_get(job->done_fence);2964 2965out_unlock:2966 mutex_unlock(&sched->lock);2967 pm_runtime_mark_last_busy(ptdev->base.dev);2968 pm_runtime_put_autosuspend(ptdev->base.dev);2969 2970 return done_fence;2971}2972 2973static enum drm_gpu_sched_stat2974queue_timedout_job(struct drm_sched_job *sched_job)2975{2976 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);2977 struct panthor_group *group = job->group;2978 struct panthor_device *ptdev = group->ptdev;2979 struct panthor_scheduler *sched = ptdev->scheduler;2980 struct panthor_queue *queue = group->queues[job->queue_idx];2981 2982 drm_warn(&ptdev->base, "job timeout\n");2983 2984 drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress));2985 2986 queue_stop(queue, job);2987 2988 mutex_lock(&sched->lock);2989 group->timedout = true;2990 if (group->csg_id >= 0) {2991 sched_queue_delayed_work(ptdev->scheduler, tick, 0);2992 } else {2993 /* Remove from the run queues, so the scheduler can't2994 * pick the group on the next tick.2995 */2996 list_del_init(&group->run_node);2997 list_del_init(&group->wait_node);2998 2999 group_queue_work(group, term);3000 }3001 mutex_unlock(&sched->lock);3002 3003 queue_start(queue);3004 3005 return DRM_GPU_SCHED_STAT_NOMINAL;3006}3007 3008static void queue_free_job(struct drm_sched_job *sched_job)3009{3010 drm_sched_job_cleanup(sched_job);3011 panthor_job_put(sched_job);3012}3013 3014static const struct drm_sched_backend_ops panthor_queue_sched_ops = {3015 .run_job = queue_run_job,3016 .timedout_job = queue_timedout_job,3017 .free_job = queue_free_job,3018};3019 3020static struct panthor_queue *3021group_create_queue(struct panthor_group *group,3022 const struct drm_panthor_queue_create *args)3023{3024 struct drm_gpu_scheduler *drm_sched;3025 struct panthor_queue *queue;3026 int ret;3027 3028 if (args->pad[0] || args->pad[1] || args->pad[2])3029 return ERR_PTR(-EINVAL);3030 3031 if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K ||3032 !is_power_of_2(args->ringbuf_size))3033 return ERR_PTR(-EINVAL);3034 3035 if (args->priority > CSF_MAX_QUEUE_PRIO)3036 return ERR_PTR(-EINVAL);3037 3038 queue = kzalloc(sizeof(*queue), GFP_KERNEL);3039 if (!queue)3040 return ERR_PTR(-ENOMEM);3041 3042 queue->fence_ctx.id = dma_fence_context_alloc(1);3043 spin_lock_init(&queue->fence_ctx.lock);3044 INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);3045 3046 queue->priority = args->priority;3047 3048 queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm,3049 args->ringbuf_size,3050 DRM_PANTHOR_BO_NO_MMAP,3051 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |3052 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,3053 PANTHOR_VM_KERNEL_AUTO_VA);3054 if (IS_ERR(queue->ringbuf)) {3055 ret = PTR_ERR(queue->ringbuf);3056 goto err_free_queue;3057 }3058 3059 ret = panthor_kernel_bo_vmap(queue->ringbuf);3060 if (ret)3061 goto err_free_queue;3062 3063 queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev,3064 &queue->iface.input,3065 &queue->iface.output,3066 &queue->iface.input_fw_va,3067 &queue->iface.output_fw_va);3068 if (IS_ERR(queue->iface.mem)) {3069 ret = PTR_ERR(queue->iface.mem);3070 goto err_free_queue;3071 }3072 3073 ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops,3074 group->ptdev->scheduler->wq, 1,3075 args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)),3076 0, msecs_to_jiffies(JOB_TIMEOUT_MS),3077 group->ptdev->reset.wq,3078 NULL, "panthor-queue", group->ptdev->base.dev);3079 if (ret)3080 goto err_free_queue;3081 3082 drm_sched = &queue->scheduler;3083 ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL);3084 3085 return queue;3086 3087err_free_queue:3088 group_free_queue(group, queue);3089 return ERR_PTR(ret);3090}3091 3092#define MAX_GROUPS_PER_POOL 1283093 3094int panthor_group_create(struct panthor_file *pfile,3095 const struct drm_panthor_group_create *group_args,3096 const struct drm_panthor_queue_create *queue_args)3097{3098 struct panthor_device *ptdev = pfile->ptdev;3099 struct panthor_group_pool *gpool = pfile->groups;3100 struct panthor_scheduler *sched = ptdev->scheduler;3101 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);3102 struct panthor_group *group = NULL;3103 u32 gid, i, suspend_size;3104 int ret;3105 3106 if (group_args->pad)3107 return -EINVAL;3108 3109 if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT)3110 return -EINVAL;3111 3112 if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) ||3113 (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) ||3114 (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present))3115 return -EINVAL;3116 3117 if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores ||3118 hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores ||3119 hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores)3120 return -EINVAL;3121 3122 group = kzalloc(sizeof(*group), GFP_KERNEL);3123 if (!group)3124 return -ENOMEM;3125 3126 spin_lock_init(&group->fatal_lock);3127 kref_init(&group->refcount);3128 group->state = PANTHOR_CS_GROUP_CREATED;3129 group->csg_id = -1;3130 3131 group->ptdev = ptdev;3132 group->max_compute_cores = group_args->max_compute_cores;3133 group->compute_core_mask = group_args->compute_core_mask;3134 group->max_fragment_cores = group_args->max_fragment_cores;3135 group->fragment_core_mask = group_args->fragment_core_mask;3136 group->max_tiler_cores = group_args->max_tiler_cores;3137 group->tiler_core_mask = group_args->tiler_core_mask;3138 group->priority = group_args->priority;3139 3140 INIT_LIST_HEAD(&group->wait_node);3141 INIT_LIST_HEAD(&group->run_node);3142 INIT_WORK(&group->term_work, group_term_work);3143 INIT_WORK(&group->sync_upd_work, group_sync_upd_work);3144 INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);3145 INIT_WORK(&group->release_work, group_release_work);3146 3147 group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id);3148 if (!group->vm) {3149 ret = -EINVAL;3150 goto err_put_group;3151 }3152 3153 suspend_size = csg_iface->control->suspend_size;3154 group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);3155 if (IS_ERR(group->suspend_buf)) {3156 ret = PTR_ERR(group->suspend_buf);3157 group->suspend_buf = NULL;3158 goto err_put_group;3159 }3160 3161 suspend_size = csg_iface->control->protm_suspend_size;3162 group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);3163 if (IS_ERR(group->protm_suspend_buf)) {3164 ret = PTR_ERR(group->protm_suspend_buf);3165 group->protm_suspend_buf = NULL;3166 goto err_put_group;3167 }3168 3169 group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,3170 group_args->queues.count *3171 sizeof(struct panthor_syncobj_64b),3172 DRM_PANTHOR_BO_NO_MMAP,3173 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |3174 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,3175 PANTHOR_VM_KERNEL_AUTO_VA);3176 if (IS_ERR(group->syncobjs)) {3177 ret = PTR_ERR(group->syncobjs);3178 goto err_put_group;3179 }3180 3181 ret = panthor_kernel_bo_vmap(group->syncobjs);3182 if (ret)3183 goto err_put_group;3184 3185 memset(group->syncobjs->kmap, 0,3186 group_args->queues.count * sizeof(struct panthor_syncobj_64b));3187 3188 for (i = 0; i < group_args->queues.count; i++) {3189 group->queues[i] = group_create_queue(group, &queue_args[i]);3190 if (IS_ERR(group->queues[i])) {3191 ret = PTR_ERR(group->queues[i]);3192 group->queues[i] = NULL;3193 goto err_put_group;3194 }3195 3196 group->queue_count++;3197 }3198 3199 group->idle_queues = GENMASK(group->queue_count - 1, 0);3200 3201 ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL);3202 if (ret)3203 goto err_put_group;3204 3205 mutex_lock(&sched->reset.lock);3206 if (atomic_read(&sched->reset.in_progress)) {3207 panthor_group_stop(group);3208 } else {3209 mutex_lock(&sched->lock);3210 list_add_tail(&group->run_node,3211 &sched->groups.idle[group->priority]);3212 mutex_unlock(&sched->lock);3213 }3214 mutex_unlock(&sched->reset.lock);3215 3216 return gid;3217 3218err_put_group:3219 group_put(group);3220 return ret;3221}3222 3223int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle)3224{3225 struct panthor_group_pool *gpool = pfile->groups;3226 struct panthor_device *ptdev = pfile->ptdev;3227 struct panthor_scheduler *sched = ptdev->scheduler;3228 struct panthor_group *group;3229 3230 group = xa_erase(&gpool->xa, group_handle);3231 if (!group)3232 return -EINVAL;3233 3234 for (u32 i = 0; i < group->queue_count; i++) {3235 if (group->queues[i])3236 drm_sched_entity_destroy(&group->queues[i]->entity);3237 }3238 3239 mutex_lock(&sched->reset.lock);3240 mutex_lock(&sched->lock);3241 group->destroyed = true;3242 if (group->csg_id >= 0) {3243 sched_queue_delayed_work(sched, tick, 0);3244 } else if (!atomic_read(&sched->reset.in_progress)) {3245 /* Remove from the run queues, so the scheduler can't3246 * pick the group on the next tick.3247 */3248 list_del_init(&group->run_node);3249 list_del_init(&group->wait_node);3250 group_queue_work(group, term);3251 }3252 mutex_unlock(&sched->lock);3253 mutex_unlock(&sched->reset.lock);3254 3255 group_put(group);3256 return 0;3257}3258 3259static struct panthor_group *group_from_handle(struct panthor_group_pool *pool,3260 u32 group_handle)3261{3262 struct panthor_group *group;3263 3264 xa_lock(&pool->xa);3265 group = group_get(xa_load(&pool->xa, group_handle));3266 xa_unlock(&pool->xa);3267 3268 return group;3269}3270 3271int panthor_group_get_state(struct panthor_file *pfile,3272 struct drm_panthor_group_get_state *get_state)3273{3274 struct panthor_group_pool *gpool = pfile->groups;3275 struct panthor_device *ptdev = pfile->ptdev;3276 struct panthor_scheduler *sched = ptdev->scheduler;3277 struct panthor_group *group;3278 3279 if (get_state->pad)3280 return -EINVAL;3281 3282 group = group_from_handle(gpool, get_state->group_handle);3283 if (!group)3284 return -EINVAL;3285 3286 memset(get_state, 0, sizeof(*get_state));3287 3288 mutex_lock(&sched->lock);3289 if (group->timedout)3290 get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;3291 if (group->fatal_queues) {3292 get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;3293 get_state->fatal_queues = group->fatal_queues;3294 }3295 mutex_unlock(&sched->lock);3296 3297 group_put(group);3298 return 0;3299}3300 3301int panthor_group_pool_create(struct panthor_file *pfile)3302{3303 struct panthor_group_pool *gpool;3304 3305 gpool = kzalloc(sizeof(*gpool), GFP_KERNEL);3306 if (!gpool)3307 return -ENOMEM;3308 3309 xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1);3310 pfile->groups = gpool;3311 return 0;3312}3313 3314void panthor_group_pool_destroy(struct panthor_file *pfile)3315{3316 struct panthor_group_pool *gpool = pfile->groups;3317 struct panthor_group *group;3318 unsigned long i;3319 3320 if (IS_ERR_OR_NULL(gpool))3321 return;3322 3323 xa_for_each(&gpool->xa, i, group)3324 panthor_group_destroy(pfile, i);3325 3326 xa_destroy(&gpool->xa);3327 kfree(gpool);3328 pfile->groups = NULL;3329}3330 3331static void job_release(struct kref *ref)3332{3333 struct panthor_job *job = container_of(ref, struct panthor_job, refcount);3334 3335 drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node));3336 3337 if (job->base.s_fence)3338 drm_sched_job_cleanup(&job->base);3339 3340 if (job->done_fence && job->done_fence->ops)3341 dma_fence_put(job->done_fence);3342 else3343 dma_fence_free(job->done_fence);3344 3345 group_put(job->group);3346 3347 kfree(job);3348}3349 3350struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job)3351{3352 if (sched_job) {3353 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);3354 3355 kref_get(&job->refcount);3356 }3357 3358 return sched_job;3359}3360 3361void panthor_job_put(struct drm_sched_job *sched_job)3362{3363 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);3364 3365 if (sched_job)3366 kref_put(&job->refcount, job_release);3367}3368 3369struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job)3370{3371 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);3372 3373 return job->group->vm;3374}3375 3376struct drm_sched_job *3377panthor_job_create(struct panthor_file *pfile,3378 u16 group_handle,3379 const struct drm_panthor_queue_submit *qsubmit)3380{3381 struct panthor_group_pool *gpool = pfile->groups;3382 struct panthor_job *job;3383 int ret;3384 3385 if (qsubmit->pad)3386 return ERR_PTR(-EINVAL);3387 3388 /* If stream_addr is zero, so stream_size should be. */3389 if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0))3390 return ERR_PTR(-EINVAL);3391 3392 /* Make sure the address is aligned on 64-byte (cacheline) and the size is3393 * aligned on 8-byte (instruction size).3394 */3395 if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7))3396 return ERR_PTR(-EINVAL);3397 3398 /* bits 24:30 must be zero. */3399 if (qsubmit->latest_flush & GENMASK(30, 24))3400 return ERR_PTR(-EINVAL);3401 3402 job = kzalloc(sizeof(*job), GFP_KERNEL);3403 if (!job)3404 return ERR_PTR(-ENOMEM);3405 3406 kref_init(&job->refcount);3407 job->queue_idx = qsubmit->queue_index;3408 job->call_info.size = qsubmit->stream_size;3409 job->call_info.start = qsubmit->stream_addr;3410 job->call_info.latest_flush = qsubmit->latest_flush;3411 INIT_LIST_HEAD(&job->node);3412 3413 job->group = group_from_handle(gpool, group_handle);3414 if (!job->group) {3415 ret = -EINVAL;3416 goto err_put_job;3417 }3418 3419 if (!group_can_run(job->group)) {3420 ret = -EINVAL;3421 goto err_put_job;3422 }3423 3424 if (job->queue_idx >= job->group->queue_count ||3425 !job->group->queues[job->queue_idx]) {3426 ret = -EINVAL;3427 goto err_put_job;3428 }3429 3430 /* Empty command streams don't need a fence, they'll pick the one from3431 * the previously submitted job.3432 */3433 if (job->call_info.size) {3434 job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL);3435 if (!job->done_fence) {3436 ret = -ENOMEM;3437 goto err_put_job;3438 }3439 }3440 3441 ret = drm_sched_job_init(&job->base,3442 &job->group->queues[job->queue_idx]->entity,3443 1, job->group);3444 if (ret)3445 goto err_put_job;3446 3447 return &job->base;3448 3449err_put_job:3450 panthor_job_put(&job->base);3451 return ERR_PTR(ret);3452}3453 3454void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job)3455{3456 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);3457 3458 panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished,3459 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);3460}3461 3462void panthor_sched_unplug(struct panthor_device *ptdev)3463{3464 struct panthor_scheduler *sched = ptdev->scheduler;3465 3466 cancel_delayed_work_sync(&sched->tick_work);3467 3468 mutex_lock(&sched->lock);3469 if (sched->pm.has_ref) {3470 pm_runtime_put(ptdev->base.dev);3471 sched->pm.has_ref = false;3472 }3473 mutex_unlock(&sched->lock);3474}3475 3476static void panthor_sched_fini(struct drm_device *ddev, void *res)3477{3478 struct panthor_scheduler *sched = res;3479 int prio;3480 3481 if (!sched || !sched->csg_slot_count)3482 return;3483 3484 cancel_delayed_work_sync(&sched->tick_work);3485 3486 if (sched->wq)3487 destroy_workqueue(sched->wq);3488 3489 if (sched->heap_alloc_wq)3490 destroy_workqueue(sched->heap_alloc_wq);3491 3492 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {3493 drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio]));3494 drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio]));3495 }3496 3497 drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting));3498}3499 3500int panthor_sched_init(struct panthor_device *ptdev)3501{3502 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);3503 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);3504 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0);3505 struct panthor_scheduler *sched;3506 u32 gpu_as_count, num_groups;3507 int prio, ret;3508 3509 sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL);3510 if (!sched)3511 return -ENOMEM;3512 3513 /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That3514 * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here.3515 */3516 num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num);3517 3518 /* The FW-side scheduler might deadlock if two groups with the same3519 * priority try to access a set of resources that overlaps, with part3520 * of the resources being allocated to one group and the other part to3521 * the other group, both groups waiting for the remaining resources to3522 * be allocated. To avoid that, it is recommended to assign each CSG a3523 * different priority. In theory we could allow several groups to have3524 * the same CSG priority if they don't request the same resources, but3525 * that makes the scheduling logic more complicated, so let's clamp3526 * the number of CSG slots to MAX_CSG_PRIO + 1 for now.3527 */3528 num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups);3529 3530 /* We need at least one AS for the MCU and one for the GPU contexts. */3531 gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1));3532 if (!gpu_as_count) {3533 drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)",3534 gpu_as_count + 1);3535 return -EINVAL;3536 }3537 3538 sched->ptdev = ptdev;3539 sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features);3540 sched->csg_slot_count = num_groups;3541 sched->cs_slot_count = csg_iface->control->stream_num;3542 sched->as_slot_count = gpu_as_count;3543 ptdev->csif_info.csg_slot_count = sched->csg_slot_count;3544 ptdev->csif_info.cs_slot_count = sched->cs_slot_count;3545 ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count;3546 3547 sched->last_tick = 0;3548 sched->resched_target = U64_MAX;3549 sched->tick_period = msecs_to_jiffies(10);3550 INIT_DELAYED_WORK(&sched->tick_work, tick_work);3551 INIT_WORK(&sched->sync_upd_work, sync_upd_work);3552 INIT_WORK(&sched->fw_events_work, process_fw_events_work);3553 3554 ret = drmm_mutex_init(&ptdev->base, &sched->lock);3555 if (ret)3556 return ret;3557 3558 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {3559 INIT_LIST_HEAD(&sched->groups.runnable[prio]);3560 INIT_LIST_HEAD(&sched->groups.idle[prio]);3561 }3562 INIT_LIST_HEAD(&sched->groups.waiting);3563 3564 ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock);3565 if (ret)3566 return ret;3567 3568 INIT_LIST_HEAD(&sched->reset.stopped_groups);3569 3570 /* sched->heap_alloc_wq will be used for heap chunk allocation on3571 * tiler OOM events, which means we can't use the same workqueue for3572 * the scheduler because works queued by the scheduler are in3573 * the dma-signalling path. Allocate a dedicated heap_alloc_wq to3574 * work around this limitation.3575 *3576 * FIXME: Ultimately, what we need is a failable/non-blocking GEM3577 * allocation path that we can call when a heap OOM is reported. The3578 * FW is smart enough to fall back on other methods if the kernel can't3579 * allocate memory, and fail the tiling job if none of these3580 * countermeasures worked.3581 *3582 * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the3583 * system is running out of memory.3584 */3585 sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0);3586 sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);3587 if (!sched->wq || !sched->heap_alloc_wq) {3588 panthor_sched_fini(&ptdev->base, sched);3589 drm_err(&ptdev->base, "Failed to allocate the workqueues");3590 return -ENOMEM;3591 }3592 3593 ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched);3594 if (ret)3595 return ret;3596 3597 ptdev->scheduler = sched;3598 return 0;3599}3600