685 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (c) 2020 NVIDIA Corporation */3 4#include <linux/dma-fence-array.h>5#include <linux/dma-mapping.h>6#include <linux/file.h>7#include <linux/host1x.h>8#include <linux/iommu.h>9#include <linux/kref.h>10#include <linux/list.h>11#include <linux/nospec.h>12#include <linux/pm_runtime.h>13#include <linux/scatterlist.h>14#include <linux/slab.h>15#include <linux/sync_file.h>16 17#include <drm/drm_drv.h>18#include <drm/drm_file.h>19#include <drm/drm_syncobj.h>20 21#include "drm.h"22#include "gem.h"23#include "submit.h"24#include "uapi.h"25 26#define SUBMIT_ERR(context, fmt, ...) \27 dev_err_ratelimited(context->client->base.dev, \28 "%s: job submission failed: " fmt "\n", \29 current->comm, ##__VA_ARGS__)30 31struct gather_bo {32 struct host1x_bo base;33 34 struct kref ref;35 36 struct device *dev;37 u32 *gather_data;38 dma_addr_t gather_data_dma;39 size_t gather_data_words;40};41 42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)43{44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);45 46 kref_get(&bo->ref);47 48 return host_bo;49}50 51static void gather_bo_release(struct kref *ref)52{53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref);54 55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma,56 0);57 kfree(bo);58}59 60static void gather_bo_put(struct host1x_bo *host_bo)61{62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);63 64 kref_put(&bo->ref, gather_bo_release);65}66 67static struct host1x_bo_mapping *68gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)69{70 struct gather_bo *gather = container_of(bo, struct gather_bo, base);71 struct host1x_bo_mapping *map;72 int err;73 74 map = kzalloc(sizeof(*map), GFP_KERNEL);75 if (!map)76 return ERR_PTR(-ENOMEM);77 78 kref_init(&map->ref);79 map->bo = host1x_bo_get(bo);80 map->direction = direction;81 map->dev = dev;82 83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL);84 if (!map->sgt) {85 err = -ENOMEM;86 goto free;87 }88 89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,90 gather->gather_data_words * 4);91 if (err)92 goto free_sgt;93 94 err = dma_map_sgtable(dev, map->sgt, direction, 0);95 if (err)96 goto free_sgt;97 98 map->phys = sg_dma_address(map->sgt->sgl);99 map->size = gather->gather_data_words * 4;100 map->chunks = err;101 102 return map;103 104free_sgt:105 sg_free_table(map->sgt);106 kfree(map->sgt);107free:108 kfree(map);109 return ERR_PTR(err);110}111 112static void gather_bo_unpin(struct host1x_bo_mapping *map)113{114 if (!map)115 return;116 117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);118 sg_free_table(map->sgt);119 kfree(map->sgt);120 host1x_bo_put(map->bo);121 122 kfree(map);123}124 125static void *gather_bo_mmap(struct host1x_bo *host_bo)126{127 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);128 129 return bo->gather_data;130}131 132static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)133{134}135 136static const struct host1x_bo_ops gather_bo_ops = {137 .get = gather_bo_get,138 .put = gather_bo_put,139 .pin = gather_bo_pin,140 .unpin = gather_bo_unpin,141 .mmap = gather_bo_mmap,142 .munmap = gather_bo_munmap,143};144 145static struct tegra_drm_mapping *146tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)147{148 struct tegra_drm_mapping *mapping;149 150 xa_lock(&context->mappings);151 152 mapping = xa_load(&context->mappings, id);153 if (mapping)154 kref_get(&mapping->ref);155 156 xa_unlock(&context->mappings);157 158 return mapping;159}160 161static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)162{163 size_t copy_len;164 void *data;165 166 if (check_mul_overflow(count, size, ©_len))167 return ERR_PTR(-EINVAL);168 169 if (copy_len > 0x4000)170 return ERR_PTR(-E2BIG);171 172 data = vmemdup_user(from, copy_len);173 if (IS_ERR(data))174 return ERR_CAST(data);175 176 return data;177}178 179static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,180 struct tegra_drm_context *context,181 struct drm_tegra_channel_submit *args)182{183 struct gather_bo *bo;184 size_t copy_len;185 186 if (args->gather_data_words == 0) {187 SUBMIT_ERR(context, "gather_data_words cannot be zero");188 return -EINVAL;189 }190 191 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) {192 SUBMIT_ERR(context, "gather_data_words is too large");193 return -EINVAL;194 }195 196 bo = kzalloc(sizeof(*bo), GFP_KERNEL);197 if (!bo) {198 SUBMIT_ERR(context, "failed to allocate memory for bo info");199 return -ENOMEM;200 }201 202 host1x_bo_init(&bo->base, &gather_bo_ops);203 kref_init(&bo->ref);204 bo->dev = dev;205 206 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma,207 GFP_KERNEL | __GFP_NOWARN, 0);208 if (!bo->gather_data) {209 SUBMIT_ERR(context, "failed to allocate memory for gather data");210 kfree(bo);211 return -ENOMEM;212 }213 214 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) {215 SUBMIT_ERR(context, "failed to copy gather data from userspace");216 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0);217 kfree(bo);218 return -EFAULT;219 }220 221 bo->gather_data_words = args->gather_data_words;222 223 *pbo = bo;224 225 return 0;226}227 228static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,229 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)230{231 /* TODO check that target_offset is within bounds */232 dma_addr_t iova = mapping->iova + buf->reloc.target_offset;233 u32 written_ptr;234 235#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT236 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)237 iova |= BIT_ULL(39);238#endif239 240 written_ptr = iova >> buf->reloc.shift;241 242 if (buf->reloc.gather_offset_words >= bo->gather_data_words) {243 SUBMIT_ERR(context,244 "relocation has too large gather offset (%u vs gather length %zu)",245 buf->reloc.gather_offset_words, bo->gather_data_words);246 return -EINVAL;247 }248 249 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,250 bo->gather_data_words);251 252 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;253 254 return 0;255}256 257static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,258 struct drm_tegra_channel_submit *args,259 struct tegra_drm_submit_data *job_data)260{261 struct tegra_drm_used_mapping *mappings;262 struct drm_tegra_submit_buf *bufs;263 int err;264 u32 i;265 266 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs,267 sizeof(*bufs));268 if (IS_ERR(bufs)) {269 SUBMIT_ERR(context, "failed to copy bufs array from userspace");270 return PTR_ERR(bufs);271 }272 273 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL);274 if (!mappings) {275 SUBMIT_ERR(context, "failed to allocate memory for mapping info");276 err = -ENOMEM;277 goto done;278 }279 280 for (i = 0; i < args->num_bufs; i++) {281 struct drm_tegra_submit_buf *buf = &bufs[i];282 struct tegra_drm_mapping *mapping;283 284 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {285 SUBMIT_ERR(context, "invalid flag specified for buffer");286 err = -EINVAL;287 goto drop_refs;288 }289 290 mapping = tegra_drm_mapping_get(context, buf->mapping);291 if (!mapping) {292 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);293 err = -EINVAL;294 goto drop_refs;295 }296 297 err = submit_write_reloc(context, bo, buf, mapping);298 if (err) {299 tegra_drm_mapping_put(mapping);300 goto drop_refs;301 }302 303 mappings[i].mapping = mapping;304 mappings[i].flags = buf->flags;305 }306 307 job_data->used_mappings = mappings;308 job_data->num_used_mappings = i;309 310 err = 0;311 312 goto done;313 314drop_refs:315 while (i--)316 tegra_drm_mapping_put(mappings[i].mapping);317 318 kfree(mappings);319 job_data->used_mappings = NULL;320 321done:322 kvfree(bufs);323 324 return err;325}326 327static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,328 struct xarray *syncpoints, struct drm_tegra_channel_submit *args)329{330 struct host1x_syncpt *sp;331 332 if (args->syncpt.flags) {333 SUBMIT_ERR(context, "invalid flag specified for syncpt");334 return -EINVAL;335 }336 337 /* Syncpt ref will be dropped on job release */338 sp = xa_load(syncpoints, args->syncpt.id);339 if (!sp) {340 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");341 return -EINVAL;342 }343 344 job->syncpt = host1x_syncpt_get(sp);345 job->syncpt_incrs = args->syncpt.increments;346 347 return 0;348}349 350static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,351 struct drm_tegra_submit_cmd_gather_uptr *cmd,352 struct gather_bo *bo, u32 *offset,353 struct tegra_drm_submit_data *job_data,354 u32 *class)355{356 u32 next_offset;357 358 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {359 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");360 return -EINVAL;361 }362 363 /* Check for maximum gather size */364 if (cmd->words > 16383) {365 SUBMIT_ERR(context, "too many words in GATHER_UPTR command");366 return -EINVAL;367 }368 369 if (check_add_overflow(*offset, cmd->words, &next_offset)) {370 SUBMIT_ERR(context, "too many total words in job");371 return -EINVAL;372 }373 374 if (next_offset > bo->gather_data_words) {375 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");376 return -EINVAL;377 }378 379 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset,380 cmd->words, job_data, class)) {381 SUBMIT_ERR(context, "job was rejected by firewall");382 return -EINVAL;383 }384 385 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);386 387 *offset = next_offset;388 389 return 0;390}391 392static struct host1x_job *393submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,394 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,395 struct xarray *syncpoints)396{397 struct drm_tegra_submit_cmd *cmds;398 u32 i, gather_offset = 0, class;399 struct host1x_job *job;400 int err;401 402 /* Set initial class for firewall. */403 class = context->client->base.class;404 405 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds,406 sizeof(*cmds));407 if (IS_ERR(cmds)) {408 SUBMIT_ERR(context, "failed to copy cmds array from userspace");409 return ERR_CAST(cmds);410 }411 412 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true);413 if (!job) {414 SUBMIT_ERR(context, "failed to allocate memory for job");415 job = ERR_PTR(-ENOMEM);416 goto done;417 }418 419 err = submit_get_syncpt(context, job, syncpoints, args);420 if (err < 0)421 goto free_job;422 423 job->client = &context->client->base;424 job->class = context->client->base.class;425 job->serialize = true;426 427 for (i = 0; i < args->num_cmds; i++) {428 struct drm_tegra_submit_cmd *cmd = &cmds[i];429 430 if (cmd->flags) {431 SUBMIT_ERR(context, "unknown flags given for cmd");432 err = -EINVAL;433 goto free_job;434 }435 436 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {437 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo,438 &gather_offset, job_data, &class);439 if (err)440 goto free_job;441 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {442 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {443 SUBMIT_ERR(context, "non-zero reserved value");444 err = -EINVAL;445 goto free_job;446 }447 448 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,449 false, class);450 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {451 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {452 SUBMIT_ERR(context, "non-zero reserved value");453 err = -EINVAL;454 goto free_job;455 }456 457 if (cmd->wait_syncpt.id != args->syncpt.id) {458 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");459 err = -EINVAL;460 goto free_job;461 }462 463 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,464 true, class);465 } else {466 SUBMIT_ERR(context, "unknown cmd type");467 err = -EINVAL;468 goto free_job;469 }470 }471 472 if (gather_offset == 0) {473 SUBMIT_ERR(context, "job must have at least one gather");474 err = -EINVAL;475 goto free_job;476 }477 478 goto done;479 480free_job:481 host1x_job_put(job);482 job = ERR_PTR(err);483 484done:485 kvfree(cmds);486 487 return job;488}489 490static void release_job(struct host1x_job *job)491{492 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);493 struct tegra_drm_submit_data *job_data = job->user_data;494 u32 i;495 496 if (job->memory_context)497 host1x_memory_context_put(job->memory_context);498 499 for (i = 0; i < job_data->num_used_mappings; i++)500 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);501 502 kfree(job_data->used_mappings);503 kfree(job_data);504 505 pm_runtime_mark_last_busy(client->base.dev);506 pm_runtime_put_autosuspend(client->base.dev);507}508 509int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,510 struct drm_file *file)511{512 struct tegra_drm_file *fpriv = file->driver_priv;513 struct drm_tegra_channel_submit *args = data;514 struct tegra_drm_submit_data *job_data;515 struct drm_syncobj *syncobj = NULL;516 struct tegra_drm_context *context;517 struct host1x_job *job;518 struct gather_bo *bo;519 u32 i;520 int err;521 522 mutex_lock(&fpriv->lock);523 524 context = xa_load(&fpriv->contexts, args->context);525 if (!context) {526 mutex_unlock(&fpriv->lock);527 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,528 current->comm, args->context);529 return -EINVAL;530 }531 532 if (args->syncobj_in) {533 struct dma_fence *fence;534 535 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence);536 if (err) {537 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);538 goto unlock;539 }540 541 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000));542 dma_fence_put(fence);543 if (err) {544 SUBMIT_ERR(context, "wait for syncobj_in timed out");545 goto unlock;546 }547 }548 549 if (args->syncobj_out) {550 syncobj = drm_syncobj_find(file, args->syncobj_out);551 if (!syncobj) {552 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);553 err = -ENOENT;554 goto unlock;555 }556 }557 558 /* Allocate gather BO and copy gather words in. */559 err = submit_copy_gather_data(&bo, drm->dev, context, args);560 if (err)561 goto unlock;562 563 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL);564 if (!job_data) {565 SUBMIT_ERR(context, "failed to allocate memory for job data");566 err = -ENOMEM;567 goto put_bo;568 }569 570 /* Get data buffer mappings and do relocation patching. */571 err = submit_process_bufs(context, bo, args, job_data);572 if (err)573 goto free_job_data;574 575 /* Allocate host1x_job and add gathers and waits to it. */576 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints);577 if (IS_ERR(job)) {578 err = PTR_ERR(job);579 goto free_job_data;580 }581 582 /* Map gather data for Host1x. */583 err = host1x_job_pin(job, context->client->base.dev);584 if (err) {585 SUBMIT_ERR(context, "failed to pin job: %d", err);586 goto put_job;587 }588 589 if (context->client->ops->get_streamid_offset) {590 err = context->client->ops->get_streamid_offset(591 context->client, &job->engine_streamid_offset);592 if (err) {593 SUBMIT_ERR(context, "failed to get streamid offset: %d", err);594 goto unpin_job;595 }596 }597 598 if (context->memory_context && context->client->ops->can_use_memory_ctx) {599 bool supported;600 601 err = context->client->ops->can_use_memory_ctx(context->client, &supported);602 if (err) {603 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err);604 goto unpin_job;605 }606 607 if (supported) {608 job->memory_context = context->memory_context;609 host1x_memory_context_get(job->memory_context);610 }611 } else if (context->client->ops->get_streamid_offset) {612 /*613 * Job submission will need to temporarily change stream ID,614 * so need to tell it what to change it back to.615 */616 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev,617 &job->engine_fallback_streamid))618 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS;619 }620 621 /* Boot engine. */622 err = pm_runtime_resume_and_get(context->client->base.dev);623 if (err < 0) {624 SUBMIT_ERR(context, "could not power up engine: %d", err);625 goto put_memory_context;626 }627 628 job->user_data = job_data;629 job->release = release_job;630 job->timeout = 10000;631 632 /*633 * job_data is now part of job reference counting, so don't release634 * it from here.635 */636 job_data = NULL;637 638 /* Submit job to hardware. */639 err = host1x_job_submit(job);640 if (err) {641 SUBMIT_ERR(context, "host1x job submission failed: %d", err);642 goto unpin_job;643 }644 645 /* Return postfences to userspace and add fences to DMA reservations. */646 args->syncpt.value = job->syncpt_end;647 648 if (syncobj) {649 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true);650 if (IS_ERR(fence)) {651 err = PTR_ERR(fence);652 SUBMIT_ERR(context, "failed to create postfence: %d", err);653 }654 655 drm_syncobj_replace_fence(syncobj, fence);656 }657 658 goto put_job;659 660put_memory_context:661 if (job->memory_context)662 host1x_memory_context_put(job->memory_context);663unpin_job:664 host1x_job_unpin(job);665put_job:666 host1x_job_put(job);667free_job_data:668 if (job_data && job_data->used_mappings) {669 for (i = 0; i < job_data->num_used_mappings; i++)670 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);671 672 kfree(job_data->used_mappings);673 }674 675 kfree(job_data);676put_bo:677 gather_bo_put(&bo->base);678unlock:679 if (syncobj)680 drm_syncobj_put(syncobj);681 682 mutex_unlock(&fpriv->lock);683 return err;684}685