784 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020-2024 Intel Corporation4 */5 6#include <drm/drm_file.h>7 8#include <linux/bitfield.h>9#include <linux/highmem.h>10#include <linux/pci.h>11#include <linux/module.h>12#include <uapi/drm/ivpu_accel.h>13 14#include "ivpu_drv.h"15#include "ivpu_fw.h"16#include "ivpu_hw.h"17#include "ivpu_ipc.h"18#include "ivpu_job.h"19#include "ivpu_jsm_msg.h"20#include "ivpu_pm.h"21#include "vpu_boot_api.h"22 23#define CMD_BUF_IDX 024#define JOB_ID_JOB_MASK GENMASK(7, 0)25#define JOB_ID_CONTEXT_MASK GENMASK(31, 8)26#define JOB_MAX_BUFFER_COUNT 6553527 28static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)29{30 ivpu_hw_db_set(vdev, cmdq->db_id);31}32 33static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,34 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)35{36 u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE);37 u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE);38 struct ivpu_addr_range range;39 40 if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)41 return 0;42 43 range.start = vdev->hw->ranges.user.end - (primary_size * IVPU_NUM_CMDQS_PER_CTX);44 range.end = vdev->hw->ranges.user.end;45 cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, primary_size,46 DRM_IVPU_BO_WC);47 if (!cmdq->primary_preempt_buf) {48 ivpu_err(vdev, "Failed to create primary preemption buffer\n");49 return -ENOMEM;50 }51 52 range.start = vdev->hw->ranges.shave.end - (secondary_size * IVPU_NUM_CMDQS_PER_CTX);53 range.end = vdev->hw->ranges.shave.end;54 cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &range, secondary_size,55 DRM_IVPU_BO_WC);56 if (!cmdq->secondary_preempt_buf) {57 ivpu_err(vdev, "Failed to create secondary preemption buffer\n");58 goto err_free_primary;59 }60 61 return 0;62 63err_free_primary:64 ivpu_bo_free(cmdq->primary_preempt_buf);65 return -ENOMEM;66}67 68static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,69 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)70{71 if (vdev->hw->sched_mode != VPU_SCHEDULING_MODE_HW)72 return;73 74 drm_WARN_ON(&vdev->drm, !cmdq->primary_preempt_buf);75 drm_WARN_ON(&vdev->drm, !cmdq->secondary_preempt_buf);76 ivpu_bo_free(cmdq->primary_preempt_buf);77 ivpu_bo_free(cmdq->secondary_preempt_buf);78}79 80static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv)81{82 struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};83 struct ivpu_device *vdev = file_priv->vdev;84 struct ivpu_cmdq *cmdq;85 int ret;86 87 cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);88 if (!cmdq)89 return NULL;90 91 ret = xa_alloc(&vdev->db_xa, &cmdq->db_id, NULL, db_xa_limit, GFP_KERNEL);92 if (ret) {93 ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret);94 goto err_free_cmdq;95 }96 97 cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);98 if (!cmdq->mem)99 goto err_erase_xa;100 101 ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);102 if (ret)103 goto err_free_cmdq_mem;104 105 return cmdq;106 107err_free_cmdq_mem:108 ivpu_bo_free(cmdq->mem);109err_erase_xa:110 xa_erase(&vdev->db_xa, cmdq->db_id);111err_free_cmdq:112 kfree(cmdq);113 return NULL;114}115 116static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)117{118 if (!cmdq)119 return;120 121 ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);122 ivpu_bo_free(cmdq->mem);123 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);124 kfree(cmdq);125}126 127static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,128 u8 priority)129{130 struct ivpu_device *vdev = file_priv->vdev;131 int ret;132 133 ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->db_id,134 task_pid_nr(current), engine,135 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));136 if (ret)137 return ret;138 139 ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->db_id,140 priority);141 if (ret)142 return ret;143 144 return 0;145}146 147static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)148{149 struct ivpu_device *vdev = file_priv->vdev;150 int ret;151 152 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW)153 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->db_id, cmdq->db_id,154 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));155 else156 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,157 cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));158 159 if (!ret)160 ivpu_dbg(vdev, JOB, "DB %d registered to ctx %d\n", cmdq->db_id, file_priv->ctx.id);161 162 return ret;163}164 165static int166ivpu_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine, u8 priority)167{168 struct ivpu_device *vdev = file_priv->vdev;169 struct vpu_job_queue_header *jobq_header;170 int ret;171 172 lockdep_assert_held(&file_priv->lock);173 174 if (cmdq->db_registered)175 return 0;176 177 cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /178 sizeof(struct vpu_job_queue_entry));179 180 cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);181 jobq_header = &cmdq->jobq->header;182 jobq_header->engine_idx = engine;183 jobq_header->head = 0;184 jobq_header->tail = 0;185 wmb(); /* Flush WC buffer for jobq->header */186 187 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {188 ret = ivpu_hws_cmdq_init(file_priv, cmdq, engine, priority);189 if (ret)190 return ret;191 }192 193 ret = ivpu_register_db(file_priv, cmdq);194 if (ret)195 return ret;196 197 cmdq->db_registered = true;198 199 return 0;200}201 202static int ivpu_cmdq_fini(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)203{204 struct ivpu_device *vdev = file_priv->vdev;205 int ret;206 207 lockdep_assert_held(&file_priv->lock);208 209 if (!cmdq->db_registered)210 return 0;211 212 cmdq->db_registered = false;213 214 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW) {215 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->db_id);216 if (!ret)217 ivpu_dbg(vdev, JOB, "Command queue %d destroyed\n", cmdq->db_id);218 }219 220 ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);221 if (!ret)222 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);223 224 return 0;225}226 227static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine,228 u8 priority)229{230 int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);231 struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];232 int ret;233 234 lockdep_assert_held(&file_priv->lock);235 236 if (!cmdq) {237 cmdq = ivpu_cmdq_alloc(file_priv);238 if (!cmdq)239 return NULL;240 file_priv->cmdq[cmdq_idx] = cmdq;241 }242 243 ret = ivpu_cmdq_init(file_priv, cmdq, engine, priority);244 if (ret)245 return NULL;246 247 return cmdq;248}249 250static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine, u8 priority)251{252 int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);253 struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];254 255 lockdep_assert_held(&file_priv->lock);256 257 if (cmdq) {258 file_priv->cmdq[cmdq_idx] = NULL;259 ivpu_cmdq_fini(file_priv, cmdq);260 ivpu_cmdq_free(file_priv, cmdq);261 }262}263 264void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)265{266 u16 engine;267 u8 priority;268 269 lockdep_assert_held(&file_priv->lock);270 271 for (engine = 0; engine < IVPU_NUM_ENGINES; engine++)272 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++)273 ivpu_cmdq_release_locked(file_priv, engine, priority);274}275 276/*277 * Mark the doorbell as unregistered278 * This function needs to be called when the VPU hardware is restarted279 * and FW loses job queue state. The next time job queue is used it280 * will be registered again.281 */282static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)283{284 u16 engine;285 u8 priority;286 287 mutex_lock(&file_priv->lock);288 289 for (engine = 0; engine < IVPU_NUM_ENGINES; engine++) {290 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++) {291 int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);292 struct ivpu_cmdq *cmdq = file_priv->cmdq[cmdq_idx];293 294 if (cmdq)295 cmdq->db_registered = false;296 }297 }298 299 mutex_unlock(&file_priv->lock);300}301 302void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)303{304 struct ivpu_file_priv *file_priv;305 unsigned long ctx_id;306 307 mutex_lock(&vdev->context_list_lock);308 309 xa_for_each(&vdev->context_xa, ctx_id, file_priv)310 ivpu_cmdq_reset(file_priv);311 312 mutex_unlock(&vdev->context_list_lock);313}314 315static void ivpu_cmdq_fini_all(struct ivpu_file_priv *file_priv)316{317 u16 engine;318 u8 priority;319 320 for (engine = 0; engine < IVPU_NUM_ENGINES; engine++) {321 for (priority = 0; priority < IVPU_NUM_PRIORITIES; priority++) {322 int cmdq_idx = IVPU_CMDQ_INDEX(engine, priority);323 324 if (file_priv->cmdq[cmdq_idx])325 ivpu_cmdq_fini(file_priv, file_priv->cmdq[cmdq_idx]);326 }327 }328}329 330void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv)331{332 struct ivpu_device *vdev = file_priv->vdev;333 334 lockdep_assert_held(&file_priv->lock);335 336 ivpu_cmdq_fini_all(file_priv);337 338 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_OS)339 ivpu_jsm_context_release(vdev, file_priv->ctx.id);340}341 342static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)343{344 struct ivpu_device *vdev = job->vdev;345 struct vpu_job_queue_header *header = &cmdq->jobq->header;346 struct vpu_job_queue_entry *entry;347 u32 tail = READ_ONCE(header->tail);348 u32 next_entry = (tail + 1) % cmdq->entry_count;349 350 /* Check if there is space left in job queue */351 if (next_entry == header->head) {352 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",353 job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);354 return -EBUSY;355 }356 357 entry = &cmdq->jobq->job[tail];358 entry->batch_buf_addr = job->cmd_buf_vpu_addr;359 entry->job_id = job->job_id;360 entry->flags = 0;361 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))362 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;363 364 if (vdev->hw->sched_mode == VPU_SCHEDULING_MODE_HW &&365 (unlikely(!(ivpu_test_mode & IVPU_TEST_MODE_PREEMPTION_DISABLE)))) {366 entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;367 entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);368 entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;369 entry->secondary_preempt_buf_size = ivpu_bo_size(cmdq->secondary_preempt_buf);370 }371 372 wmb(); /* Ensure that tail is updated after filling entry */373 header->tail = next_entry;374 wmb(); /* Flush WC buffer for jobq header */375 376 return 0;377}378 379struct ivpu_fence {380 struct dma_fence base;381 spinlock_t lock; /* protects base */382 struct ivpu_device *vdev;383};384 385static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)386{387 return container_of(fence, struct ivpu_fence, base);388}389 390static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)391{392 return DRIVER_NAME;393}394 395static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)396{397 struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);398 399 return dev_name(ivpu_fence->vdev->drm.dev);400}401 402static const struct dma_fence_ops ivpu_fence_ops = {403 .get_driver_name = ivpu_fence_get_driver_name,404 .get_timeline_name = ivpu_fence_get_timeline_name,405};406 407static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)408{409 struct ivpu_fence *fence;410 411 fence = kzalloc(sizeof(*fence), GFP_KERNEL);412 if (!fence)413 return NULL;414 415 fence->vdev = vdev;416 spin_lock_init(&fence->lock);417 dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);418 419 return &fence->base;420}421 422static void ivpu_job_destroy(struct ivpu_job *job)423{424 struct ivpu_device *vdev = job->vdev;425 u32 i;426 427 ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",428 job->job_id, job->file_priv->ctx.id, job->engine_idx);429 430 for (i = 0; i < job->bo_count; i++)431 if (job->bos[i])432 drm_gem_object_put(&job->bos[i]->base.base);433 434 dma_fence_put(job->done_fence);435 ivpu_file_priv_put(&job->file_priv);436 kfree(job);437}438 439static struct ivpu_job *440ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)441{442 struct ivpu_device *vdev = file_priv->vdev;443 struct ivpu_job *job;444 445 job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);446 if (!job)447 return NULL;448 449 job->vdev = vdev;450 job->engine_idx = engine_idx;451 job->bo_count = bo_count;452 job->done_fence = ivpu_fence_create(vdev);453 if (!job->done_fence) {454 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");455 goto err_free_job;456 }457 458 job->file_priv = ivpu_file_priv_get(file_priv);459 460 ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);461 return job;462 463err_free_job:464 kfree(job);465 return NULL;466}467 468static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id)469{470 struct ivpu_job *job;471 472 xa_lock(&vdev->submitted_jobs_xa);473 job = __xa_erase(&vdev->submitted_jobs_xa, job_id);474 475 if (xa_empty(&vdev->submitted_jobs_xa) && job) {476 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts),477 vdev->busy_time);478 }479 480 xa_unlock(&vdev->submitted_jobs_xa);481 482 return job;483}484 485static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)486{487 struct ivpu_job *job;488 489 job = ivpu_job_remove_from_submitted_jobs(vdev, job_id);490 if (!job)491 return -ENOENT;492 493 if (job->file_priv->has_mmu_faults)494 job_status = DRM_IVPU_JOB_STATUS_ABORTED;495 496 job->bos[CMD_BUF_IDX]->job_status = job_status;497 dma_fence_signal(job->done_fence);498 499 ivpu_dbg(vdev, JOB, "Job complete: id %3u ctx %2d engine %d status 0x%x\n",500 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);501 502 ivpu_job_destroy(job);503 ivpu_stop_job_timeout_detection(vdev);504 505 ivpu_rpm_put(vdev);506 return 0;507}508 509void ivpu_jobs_abort_all(struct ivpu_device *vdev)510{511 struct ivpu_job *job;512 unsigned long id;513 514 xa_for_each(&vdev->submitted_jobs_xa, id, job)515 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);516}517 518static int ivpu_job_submit(struct ivpu_job *job, u8 priority)519{520 struct ivpu_file_priv *file_priv = job->file_priv;521 struct ivpu_device *vdev = job->vdev;522 struct xa_limit job_id_range;523 struct ivpu_cmdq *cmdq;524 bool is_first_job;525 int ret;526 527 ret = ivpu_rpm_get(vdev);528 if (ret < 0)529 return ret;530 531 mutex_lock(&file_priv->lock);532 533 cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx, priority);534 if (!cmdq) {535 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d engine %d prio %d\n",536 file_priv->ctx.id, job->engine_idx, priority);537 ret = -EINVAL;538 goto err_unlock_file_priv;539 }540 541 job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));542 job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;543 544 xa_lock(&vdev->submitted_jobs_xa);545 is_first_job = xa_empty(&vdev->submitted_jobs_xa);546 ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);547 if (ret) {548 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",549 file_priv->ctx.id);550 ret = -EBUSY;551 goto err_unlock_submitted_jobs_xa;552 }553 554 ret = ivpu_cmdq_push_job(cmdq, job);555 if (ret)556 goto err_erase_xa;557 558 ivpu_start_job_timeout_detection(vdev);559 560 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {561 cmdq->jobq->header.head = cmdq->jobq->header.tail;562 wmb(); /* Flush WC buffer for jobq header */563 } else {564 ivpu_cmdq_ring_db(vdev, cmdq);565 if (is_first_job)566 vdev->busy_start_ts = ktime_get();567 }568 569 ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n",570 job->job_id, file_priv->ctx.id, job->engine_idx, priority,571 job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);572 573 xa_unlock(&vdev->submitted_jobs_xa);574 575 mutex_unlock(&file_priv->lock);576 577 if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))578 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);579 580 return 0;581 582err_erase_xa:583 __xa_erase(&vdev->submitted_jobs_xa, job->job_id);584err_unlock_submitted_jobs_xa:585 xa_unlock(&vdev->submitted_jobs_xa);586err_unlock_file_priv:587 mutex_unlock(&file_priv->lock);588 ivpu_rpm_put(vdev);589 return ret;590}591 592static int593ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,594 u32 buf_count, u32 commands_offset)595{596 struct ivpu_file_priv *file_priv = file->driver_priv;597 struct ivpu_device *vdev = file_priv->vdev;598 struct ww_acquire_ctx acquire_ctx;599 enum dma_resv_usage usage;600 struct ivpu_bo *bo;601 int ret;602 u32 i;603 604 for (i = 0; i < buf_count; i++) {605 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);606 607 if (!obj)608 return -ENOENT;609 610 job->bos[i] = to_ivpu_bo(obj);611 612 ret = ivpu_bo_pin(job->bos[i]);613 if (ret)614 return ret;615 }616 617 bo = job->bos[CMD_BUF_IDX];618 if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {619 ivpu_warn(vdev, "Buffer is already in use\n");620 return -EBUSY;621 }622 623 if (commands_offset >= ivpu_bo_size(bo)) {624 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);625 return -EINVAL;626 }627 628 job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;629 630 ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,631 &acquire_ctx);632 if (ret) {633 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);634 return ret;635 }636 637 for (i = 0; i < buf_count; i++) {638 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);639 if (ret) {640 ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);641 goto unlock_reservations;642 }643 }644 645 for (i = 0; i < buf_count; i++) {646 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;647 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);648 }649 650unlock_reservations:651 drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);652 653 wmb(); /* Flush write combining buffers */654 655 return ret;656}657 658static inline u8 ivpu_job_to_hws_priority(struct ivpu_file_priv *file_priv, u8 priority)659{660 if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT)661 return DRM_IVPU_JOB_PRIORITY_NORMAL;662 663 return priority - 1;664}665 666int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)667{668 struct ivpu_file_priv *file_priv = file->driver_priv;669 struct ivpu_device *vdev = file_priv->vdev;670 struct drm_ivpu_submit *params = data;671 struct ivpu_job *job;672 u32 *buf_handles;673 int idx, ret;674 u8 priority;675 676 if (params->engine > DRM_IVPU_ENGINE_COPY)677 return -EINVAL;678 679 if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)680 return -EINVAL;681 682 if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)683 return -EINVAL;684 685 if (!IS_ALIGNED(params->commands_offset, 8))686 return -EINVAL;687 688 if (!file_priv->ctx.id)689 return -EINVAL;690 691 if (file_priv->has_mmu_faults)692 return -EBADFD;693 694 buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);695 if (!buf_handles)696 return -ENOMEM;697 698 ret = copy_from_user(buf_handles,699 (void __user *)params->buffers_ptr,700 params->buffer_count * sizeof(u32));701 if (ret) {702 ret = -EFAULT;703 goto err_free_handles;704 }705 706 if (!drm_dev_enter(&vdev->drm, &idx)) {707 ret = -ENODEV;708 goto err_free_handles;709 }710 711 ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",712 file_priv->ctx.id, params->buffer_count);713 714 job = ivpu_job_create(file_priv, params->engine, params->buffer_count);715 if (!job) {716 ivpu_err(vdev, "Failed to create job\n");717 ret = -ENOMEM;718 goto err_exit_dev;719 }720 721 ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,722 params->commands_offset);723 if (ret) {724 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);725 goto err_destroy_job;726 }727 728 priority = ivpu_job_to_hws_priority(file_priv, params->priority);729 730 down_read(&vdev->pm->reset_lock);731 ret = ivpu_job_submit(job, priority);732 up_read(&vdev->pm->reset_lock);733 if (ret)734 goto err_signal_fence;735 736 drm_dev_exit(idx);737 kfree(buf_handles);738 return ret;739 740err_signal_fence:741 dma_fence_signal(job->done_fence);742err_destroy_job:743 ivpu_job_destroy(job);744err_exit_dev:745 drm_dev_exit(idx);746err_free_handles:747 kfree(buf_handles);748 return ret;749}750 751static void752ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,753 struct vpu_jsm_msg *jsm_msg)754{755 struct vpu_ipc_msg_payload_job_done *payload;756 int ret;757 758 if (!jsm_msg) {759 ivpu_err(vdev, "IPC message has no JSM payload\n");760 return;761 }762 763 if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {764 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);765 return;766 }767 768 payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;769 ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);770 if (!ret && !xa_empty(&vdev->submitted_jobs_xa))771 ivpu_start_job_timeout_detection(vdev);772}773 774void ivpu_job_done_consumer_init(struct ivpu_device *vdev)775{776 ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,777 VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);778}779 780void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)781{782 ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);783}784