355 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_exec.h"7 8#include <drm/drm_device.h>9#include <drm/drm_exec.h>10#include <drm/drm_file.h>11#include <uapi/drm/xe_drm.h>12#include <linux/delay.h>13 14#include "xe_bo.h"15#include "xe_device.h"16#include "xe_exec_queue.h"17#include "xe_hw_engine_group.h"18#include "xe_macros.h"19#include "xe_ring_ops_types.h"20#include "xe_sched_job.h"21#include "xe_sync.h"22#include "xe_vm.h"23 24/**25 * DOC: Execbuf (User GPU command submission)26 *27 * Execs have historically been rather complicated in DRM drivers (at least in28 * the i915) because a few things:29 *30 * - Passing in a list BO which are read / written to creating implicit syncs31 * - Binding at exec time32 * - Flow controlling the ring at exec time33 *34 * In XE we avoid all of this complication by not allowing a BO list to be35 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as36 * seperate operations, and using the DRM scheduler to flow control the ring.37 * Let's deep dive on each of these.38 *39 * We can get away from a BO list by forcing the user to use in / out fences on40 * every exec rather than the kernel tracking dependencies of BO (e.g. if the41 * user knows an exec writes to a BO and reads from the BO in the next exec, it42 * is the user's responsibility to pass in / out fence between the two execs).43 *44 * We do not allow a user to trigger a bind at exec time rather we have a VM45 * bind IOCTL which uses the same in / out fence interface as exec. In that46 * sense, a VM bind is basically the same operation as an exec from the user47 * perspective. e.g. If an exec depends on a VM bind use the in / out fence48 * interface (struct drm_xe_sync) to synchronize like syncing between two49 * dependent execs.50 *51 * Although a user cannot trigger a bind, we still have to rebind userptrs in52 * the VM that have been invalidated since the last exec, likewise we also have53 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds54 * behind any pending kernel operations on any external BOs in VM or any BOs55 * private to the VM. This is accomplished by the rebinds waiting on BOs56 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs57 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and58 * for external BOs).59 *60 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute61 * mode VMs we use preempt fences and a rebind worker (TODO: add link).62 *63 * There is no need to flow control the ring in the exec as we write the ring at64 * submission time and set the DRM scheduler max job limit SIZE_OF_RING /65 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the66 * ring is available.67 *68 * All of this results in a rather simple exec implementation.69 *70 * Flow71 * ~~~~72 *73 * .. code-block::74 *75 * Parse input arguments76 * Wait for any async VM bind passed as in-fences to start77 * <----------------------------------------------------------------------|78 * Lock global VM lock in read mode |79 * Pin userptrs (also finds userptr invalidated since last exec) |80 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) |81 * Validate BOs that have been evicted |82 * Create job |83 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) |84 * Add rebind fence dependency to job |85 * Add job VM dma-resv bookkeeping slot (non-compute mode) |86 * Add job to external BOs dma-resv write slots (non-compute mode) |87 * Check if any userptrs invalidated since pin ------ Drop locks ---------|88 * Install in / out fences for job89 * Submit job90 * Unlock all91 */92 93/*94 * Add validation and rebinding to the drm_exec locking loop, since both can95 * trigger eviction which may require sleeping dma_resv locks.96 */97static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)98{99 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm);100 101 /* The fence slot added here is intended for the exec sched job. */102 return xe_vm_validate_rebind(vm, &vm_exec->exec, 1);103}104 105int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)106{107 struct xe_device *xe = to_xe_device(dev);108 struct xe_file *xef = to_xe_file(file);109 struct drm_xe_exec *args = data;110 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs);111 u64 __user *addresses_user = u64_to_user_ptr(args->address);112 struct xe_exec_queue *q;113 struct xe_sync_entry *syncs = NULL;114 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];115 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};116 struct drm_exec *exec = &vm_exec.exec;117 u32 i, num_syncs, num_ufence = 0;118 struct xe_sched_job *job;119 struct xe_vm *vm;120 bool write_locked, skip_retry = false;121 ktime_t end = 0;122 int err = 0;123 struct xe_hw_engine_group *group;124 enum xe_hw_engine_group_execution_mode mode, previous_mode;125 126 if (XE_IOCTL_DBG(xe, args->extensions) ||127 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||128 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))129 return -EINVAL;130 131 q = xe_exec_queue_lookup(xef, args->exec_queue_id);132 if (XE_IOCTL_DBG(xe, !q))133 return -ENOENT;134 135 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) {136 err = -EINVAL;137 goto err_exec_queue;138 }139 140 if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&141 q->width != args->num_batch_buffer)) {142 err = -EINVAL;143 goto err_exec_queue;144 }145 146 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) {147 err = -ECANCELED;148 goto err_exec_queue;149 }150 151 if (args->num_syncs) {152 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);153 if (!syncs) {154 err = -ENOMEM;155 goto err_exec_queue;156 }157 }158 159 vm = q->vm;160 161 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {162 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],163 &syncs_user[num_syncs], SYNC_PARSE_FLAG_EXEC |164 (xe_vm_in_lr_mode(vm) ?165 SYNC_PARSE_FLAG_LR_MODE : 0));166 if (err)167 goto err_syncs;168 169 if (xe_sync_is_ufence(&syncs[num_syncs]))170 num_ufence++;171 }172 173 if (XE_IOCTL_DBG(xe, num_ufence > 1)) {174 err = -EINVAL;175 goto err_syncs;176 }177 178 if (xe_exec_queue_is_parallel(q)) {179 err = __copy_from_user(addresses, addresses_user, sizeof(u64) *180 q->width);181 if (err) {182 err = -EFAULT;183 goto err_syncs;184 }185 }186 187 group = q->hwe->hw_engine_group;188 mode = xe_hw_engine_group_find_exec_mode(q);189 190 if (mode == EXEC_MODE_DMA_FENCE) {191 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);192 if (err)193 goto err_syncs;194 }195 196retry:197 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) {198 err = down_write_killable(&vm->lock);199 write_locked = true;200 } else {201 /* We don't allow execs while the VM is in error state */202 err = down_read_interruptible(&vm->lock);203 write_locked = false;204 }205 if (err)206 goto err_hw_exec_mode;207 208 if (write_locked) {209 err = xe_vm_userptr_pin(vm);210 downgrade_write(&vm->lock);211 write_locked = false;212 if (err)213 goto err_unlock_list;214 }215 216 if (!args->num_batch_buffer) {217 err = xe_vm_lock(vm, true);218 if (err)219 goto err_unlock_list;220 221 if (!xe_vm_in_lr_mode(vm)) {222 struct dma_fence *fence;223 224 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);225 if (IS_ERR(fence)) {226 err = PTR_ERR(fence);227 xe_vm_unlock(vm);228 goto err_unlock_list;229 }230 for (i = 0; i < num_syncs; i++)231 xe_sync_entry_signal(&syncs[i], fence);232 xe_exec_queue_last_fence_set(q, vm, fence);233 dma_fence_put(fence);234 }235 236 xe_vm_unlock(vm);237 goto err_unlock_list;238 }239 240 vm_exec.vm = &vm->gpuvm;241 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;242 if (xe_vm_in_lr_mode(vm)) {243 drm_exec_init(exec, vm_exec.flags, 0);244 } else {245 err = drm_gpuvm_exec_lock(&vm_exec);246 if (err) {247 if (xe_vm_validate_should_retry(exec, err, &end))248 err = -EAGAIN;249 goto err_unlock_list;250 }251 }252 253 if (xe_vm_is_closed_or_banned(q->vm)) {254 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n");255 err = -ECANCELED;256 goto err_exec;257 }258 259 if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) {260 err = -EWOULDBLOCK; /* Aliased to -EAGAIN */261 skip_retry = true;262 goto err_exec;263 }264 265 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ?266 addresses : &args->address);267 if (IS_ERR(job)) {268 err = PTR_ERR(job);269 goto err_exec;270 }271 272 /* Wait behind rebinds */273 if (!xe_vm_in_lr_mode(vm)) {274 err = xe_sched_job_add_deps(job,275 xe_vm_resv(vm),276 DMA_RESV_USAGE_KERNEL);277 if (err)278 goto err_put_job;279 }280 281 for (i = 0; i < num_syncs && !err; i++)282 err = xe_sync_entry_add_deps(&syncs[i], job);283 if (err)284 goto err_put_job;285 286 if (!xe_vm_in_lr_mode(vm)) {287 err = xe_sched_job_last_fence_add_dep(job, vm);288 if (err)289 goto err_put_job;290 291 err = down_read_interruptible(&vm->userptr.notifier_lock);292 if (err)293 goto err_put_job;294 295 err = __xe_vm_userptr_needs_repin(vm);296 if (err)297 goto err_repin;298 }299 300 /*301 * Point of no return, if we error after this point just set an error on302 * the job and let the DRM scheduler / backend clean up the job.303 */304 xe_sched_job_arm(job);305 if (!xe_vm_in_lr_mode(vm))306 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,307 DMA_RESV_USAGE_BOOKKEEP,308 DMA_RESV_USAGE_BOOKKEEP);309 310 for (i = 0; i < num_syncs; i++) {311 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);312 xe_sched_job_init_user_fence(job, &syncs[i]);313 }314 315 if (xe_exec_queue_is_lr(q))316 q->ring_ops->emit_job(job);317 if (!xe_vm_in_lr_mode(vm))318 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);319 xe_sched_job_push(job);320 xe_vm_reactivate_rebind(vm);321 322 if (!err && !xe_vm_in_lr_mode(vm)) {323 spin_lock(&xe->ttm.lru_lock);324 ttm_lru_bulk_move_tail(&vm->lru_bulk_move);325 spin_unlock(&xe->ttm.lru_lock);326 }327 328 if (mode == EXEC_MODE_LR)329 xe_hw_engine_group_resume_faulting_lr_jobs(group);330 331err_repin:332 if (!xe_vm_in_lr_mode(vm))333 up_read(&vm->userptr.notifier_lock);334err_put_job:335 if (err)336 xe_sched_job_put(job);337err_exec:338 drm_exec_fini(exec);339err_unlock_list:340 up_read(&vm->lock);341 if (err == -EAGAIN && !skip_retry)342 goto retry;343err_hw_exec_mode:344 if (mode == EXEC_MODE_DMA_FENCE)345 xe_hw_engine_group_put(group);346err_syncs:347 while (num_syncs--)348 xe_sync_entry_cleanup(&syncs[num_syncs]);349 kfree(syncs);350err_exec_queue:351 xe_exec_queue_put(q);352 353 return err;354}355