1458 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Avionic Design GmbH4 * Copyright (C) 2012-2016 NVIDIA CORPORATION. All rights reserved.5 */6 7#include <linux/bitops.h>8#include <linux/host1x.h>9#include <linux/idr.h>10#include <linux/iommu.h>11#include <linux/module.h>12#include <linux/platform_device.h>13#include <linux/pm_runtime.h>14 15#include <drm/drm_aperture.h>16#include <drm/drm_atomic.h>17#include <drm/drm_atomic_helper.h>18#include <drm/drm_debugfs.h>19#include <drm/drm_drv.h>20#include <drm/drm_fourcc.h>21#include <drm/drm_framebuffer.h>22#include <drm/drm_ioctl.h>23#include <drm/drm_prime.h>24#include <drm/drm_vblank.h>25 26#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)27#include <asm/dma-iommu.h>28#endif29 30#include "dc.h"31#include "drm.h"32#include "gem.h"33#include "uapi.h"34 35#define DRIVER_NAME "tegra"36#define DRIVER_DESC "NVIDIA Tegra graphics"37#define DRIVER_DATE "20120330"38#define DRIVER_MAJOR 139#define DRIVER_MINOR 040#define DRIVER_PATCHLEVEL 041 42#define CARVEOUT_SZ SZ_64M43#define CDMA_GATHER_FETCHES_MAX_NB 1638344 45static int tegra_atomic_check(struct drm_device *drm,46 struct drm_atomic_state *state)47{48 int err;49 50 err = drm_atomic_helper_check(drm, state);51 if (err < 0)52 return err;53 54 return tegra_display_hub_atomic_check(drm, state);55}56 57static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {58 .fb_create = tegra_fb_create,59 .atomic_check = tegra_atomic_check,60 .atomic_commit = drm_atomic_helper_commit,61};62 63static void tegra_atomic_post_commit(struct drm_device *drm,64 struct drm_atomic_state *old_state)65{66 struct drm_crtc_state *old_crtc_state __maybe_unused;67 struct drm_crtc *crtc;68 unsigned int i;69 70 for_each_old_crtc_in_state(old_state, crtc, old_crtc_state, i)71 tegra_crtc_atomic_post_commit(crtc, old_state);72}73 74static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state)75{76 struct drm_device *drm = old_state->dev;77 struct tegra_drm *tegra = drm->dev_private;78 79 if (tegra->hub) {80 bool fence_cookie = dma_fence_begin_signalling();81 82 drm_atomic_helper_commit_modeset_disables(drm, old_state);83 tegra_display_hub_atomic_commit(drm, old_state);84 drm_atomic_helper_commit_planes(drm, old_state, 0);85 drm_atomic_helper_commit_modeset_enables(drm, old_state);86 drm_atomic_helper_commit_hw_done(old_state);87 dma_fence_end_signalling(fence_cookie);88 drm_atomic_helper_wait_for_vblanks(drm, old_state);89 drm_atomic_helper_cleanup_planes(drm, old_state);90 } else {91 drm_atomic_helper_commit_tail_rpm(old_state);92 }93 94 tegra_atomic_post_commit(drm, old_state);95}96 97static const struct drm_mode_config_helper_funcs98tegra_drm_mode_config_helpers = {99 .atomic_commit_tail = tegra_atomic_commit_tail,100};101 102static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp)103{104 struct tegra_drm_file *fpriv;105 106 fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);107 if (!fpriv)108 return -ENOMEM;109 110 idr_init_base(&fpriv->legacy_contexts, 1);111 xa_init_flags(&fpriv->contexts, XA_FLAGS_ALLOC1);112 xa_init(&fpriv->syncpoints);113 mutex_init(&fpriv->lock);114 filp->driver_priv = fpriv;115 116 return 0;117}118 119static void tegra_drm_context_free(struct tegra_drm_context *context)120{121 context->client->ops->close_channel(context);122 pm_runtime_put(context->client->base.dev);123 kfree(context);124}125 126static int host1x_reloc_copy_from_user(struct host1x_reloc *dest,127 struct drm_tegra_reloc __user *src,128 struct drm_device *drm,129 struct drm_file *file)130{131 u32 cmdbuf, target;132 int err;133 134 err = get_user(cmdbuf, &src->cmdbuf.handle);135 if (err < 0)136 return err;137 138 err = get_user(dest->cmdbuf.offset, &src->cmdbuf.offset);139 if (err < 0)140 return err;141 142 err = get_user(target, &src->target.handle);143 if (err < 0)144 return err;145 146 err = get_user(dest->target.offset, &src->target.offset);147 if (err < 0)148 return err;149 150 err = get_user(dest->shift, &src->shift);151 if (err < 0)152 return err;153 154 dest->flags = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE;155 156 dest->cmdbuf.bo = tegra_gem_lookup(file, cmdbuf);157 if (!dest->cmdbuf.bo)158 return -ENOENT;159 160 dest->target.bo = tegra_gem_lookup(file, target);161 if (!dest->target.bo)162 return -ENOENT;163 164 return 0;165}166 167int tegra_drm_submit(struct tegra_drm_context *context,168 struct drm_tegra_submit *args, struct drm_device *drm,169 struct drm_file *file)170{171 struct host1x_client *client = &context->client->base;172 unsigned int num_cmdbufs = args->num_cmdbufs;173 unsigned int num_relocs = args->num_relocs;174 struct drm_tegra_cmdbuf __user *user_cmdbufs;175 struct drm_tegra_reloc __user *user_relocs;176 struct drm_tegra_syncpt __user *user_syncpt;177 struct drm_tegra_syncpt syncpt;178 struct host1x *host1x = dev_get_drvdata(drm->dev->parent);179 struct drm_gem_object **refs;180 struct host1x_syncpt *sp = NULL;181 struct host1x_job *job;182 unsigned int num_refs;183 int err;184 185 user_cmdbufs = u64_to_user_ptr(args->cmdbufs);186 user_relocs = u64_to_user_ptr(args->relocs);187 user_syncpt = u64_to_user_ptr(args->syncpts);188 189 /* We don't yet support other than one syncpt_incr struct per submit */190 if (args->num_syncpts != 1)191 return -EINVAL;192 193 /* We don't yet support waitchks */194 if (args->num_waitchks != 0)195 return -EINVAL;196 197 job = host1x_job_alloc(context->channel, args->num_cmdbufs,198 args->num_relocs, false);199 if (!job)200 return -ENOMEM;201 202 job->num_relocs = args->num_relocs;203 job->client = client;204 job->class = client->class;205 job->serialize = true;206 job->syncpt_recovery = true;207 208 /*209 * Track referenced BOs so that they can be unreferenced after the210 * submission is complete.211 */212 num_refs = num_cmdbufs + num_relocs * 2;213 214 refs = kmalloc_array(num_refs, sizeof(*refs), GFP_KERNEL);215 if (!refs) {216 err = -ENOMEM;217 goto put;218 }219 220 /* reuse as an iterator later */221 num_refs = 0;222 223 while (num_cmdbufs) {224 struct drm_tegra_cmdbuf cmdbuf;225 struct host1x_bo *bo;226 struct tegra_bo *obj;227 u64 offset;228 229 if (copy_from_user(&cmdbuf, user_cmdbufs, sizeof(cmdbuf))) {230 err = -EFAULT;231 goto fail;232 }233 234 /*235 * The maximum number of CDMA gather fetches is 16383, a higher236 * value means the words count is malformed.237 */238 if (cmdbuf.words > CDMA_GATHER_FETCHES_MAX_NB) {239 err = -EINVAL;240 goto fail;241 }242 243 bo = tegra_gem_lookup(file, cmdbuf.handle);244 if (!bo) {245 err = -ENOENT;246 goto fail;247 }248 249 offset = (u64)cmdbuf.offset + (u64)cmdbuf.words * sizeof(u32);250 obj = host1x_to_tegra_bo(bo);251 refs[num_refs++] = &obj->gem;252 253 /*254 * Gather buffer base address must be 4-bytes aligned,255 * unaligned offset is malformed and cause commands stream256 * corruption on the buffer address relocation.257 */258 if (offset & 3 || offset > obj->gem.size) {259 err = -EINVAL;260 goto fail;261 }262 263 host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);264 num_cmdbufs--;265 user_cmdbufs++;266 }267 268 /* copy and resolve relocations from submit */269 while (num_relocs--) {270 struct host1x_reloc *reloc;271 struct tegra_bo *obj;272 273 err = host1x_reloc_copy_from_user(&job->relocs[num_relocs],274 &user_relocs[num_relocs], drm,275 file);276 if (err < 0)277 goto fail;278 279 reloc = &job->relocs[num_relocs];280 obj = host1x_to_tegra_bo(reloc->cmdbuf.bo);281 refs[num_refs++] = &obj->gem;282 283 /*284 * The unaligned cmdbuf offset will cause an unaligned write285 * during of the relocations patching, corrupting the commands286 * stream.287 */288 if (reloc->cmdbuf.offset & 3 ||289 reloc->cmdbuf.offset >= obj->gem.size) {290 err = -EINVAL;291 goto fail;292 }293 294 obj = host1x_to_tegra_bo(reloc->target.bo);295 refs[num_refs++] = &obj->gem;296 297 if (reloc->target.offset >= obj->gem.size) {298 err = -EINVAL;299 goto fail;300 }301 }302 303 if (copy_from_user(&syncpt, user_syncpt, sizeof(syncpt))) {304 err = -EFAULT;305 goto fail;306 }307 308 /* Syncpoint ref will be dropped on job release. */309 sp = host1x_syncpt_get_by_id(host1x, syncpt.id);310 if (!sp) {311 err = -ENOENT;312 goto fail;313 }314 315 job->is_addr_reg = context->client->ops->is_addr_reg;316 job->is_valid_class = context->client->ops->is_valid_class;317 job->syncpt_incrs = syncpt.incrs;318 job->syncpt = sp;319 job->timeout = 10000;320 321 if (args->timeout && args->timeout < 10000)322 job->timeout = args->timeout;323 324 err = host1x_job_pin(job, context->client->base.dev);325 if (err)326 goto fail;327 328 err = host1x_job_submit(job);329 if (err) {330 host1x_job_unpin(job);331 goto fail;332 }333 334 args->fence = job->syncpt_end;335 336fail:337 while (num_refs--)338 drm_gem_object_put(refs[num_refs]);339 340 kfree(refs);341 342put:343 host1x_job_put(job);344 return err;345}346 347 348#ifdef CONFIG_DRM_TEGRA_STAGING349static int tegra_gem_create(struct drm_device *drm, void *data,350 struct drm_file *file)351{352 struct drm_tegra_gem_create *args = data;353 struct tegra_bo *bo;354 355 bo = tegra_bo_create_with_handle(file, drm, args->size, args->flags,356 &args->handle);357 if (IS_ERR(bo))358 return PTR_ERR(bo);359 360 return 0;361}362 363static int tegra_gem_mmap(struct drm_device *drm, void *data,364 struct drm_file *file)365{366 struct drm_tegra_gem_mmap *args = data;367 struct drm_gem_object *gem;368 struct tegra_bo *bo;369 370 gem = drm_gem_object_lookup(file, args->handle);371 if (!gem)372 return -EINVAL;373 374 bo = to_tegra_bo(gem);375 376 args->offset = drm_vma_node_offset_addr(&bo->gem.vma_node);377 378 drm_gem_object_put(gem);379 380 return 0;381}382 383static int tegra_syncpt_read(struct drm_device *drm, void *data,384 struct drm_file *file)385{386 struct host1x *host = dev_get_drvdata(drm->dev->parent);387 struct drm_tegra_syncpt_read *args = data;388 struct host1x_syncpt *sp;389 390 sp = host1x_syncpt_get_by_id_noref(host, args->id);391 if (!sp)392 return -EINVAL;393 394 args->value = host1x_syncpt_read_min(sp);395 return 0;396}397 398static int tegra_syncpt_incr(struct drm_device *drm, void *data,399 struct drm_file *file)400{401 struct host1x *host1x = dev_get_drvdata(drm->dev->parent);402 struct drm_tegra_syncpt_incr *args = data;403 struct host1x_syncpt *sp;404 405 sp = host1x_syncpt_get_by_id_noref(host1x, args->id);406 if (!sp)407 return -EINVAL;408 409 return host1x_syncpt_incr(sp);410}411 412static int tegra_syncpt_wait(struct drm_device *drm, void *data,413 struct drm_file *file)414{415 struct host1x *host1x = dev_get_drvdata(drm->dev->parent);416 struct drm_tegra_syncpt_wait *args = data;417 struct host1x_syncpt *sp;418 419 sp = host1x_syncpt_get_by_id_noref(host1x, args->id);420 if (!sp)421 return -EINVAL;422 423 return host1x_syncpt_wait(sp, args->thresh,424 msecs_to_jiffies(args->timeout),425 &args->value);426}427 428static int tegra_client_open(struct tegra_drm_file *fpriv,429 struct tegra_drm_client *client,430 struct tegra_drm_context *context)431{432 int err;433 434 err = pm_runtime_resume_and_get(client->base.dev);435 if (err)436 return err;437 438 err = client->ops->open_channel(client, context);439 if (err < 0) {440 pm_runtime_put(client->base.dev);441 return err;442 }443 444 err = idr_alloc(&fpriv->legacy_contexts, context, 1, 0, GFP_KERNEL);445 if (err < 0) {446 client->ops->close_channel(context);447 pm_runtime_put(client->base.dev);448 return err;449 }450 451 context->client = client;452 context->id = err;453 454 return 0;455}456 457static int tegra_open_channel(struct drm_device *drm, void *data,458 struct drm_file *file)459{460 struct tegra_drm_file *fpriv = file->driver_priv;461 struct tegra_drm *tegra = drm->dev_private;462 struct drm_tegra_open_channel *args = data;463 struct tegra_drm_context *context;464 struct tegra_drm_client *client;465 int err = -ENODEV;466 467 context = kzalloc(sizeof(*context), GFP_KERNEL);468 if (!context)469 return -ENOMEM;470 471 mutex_lock(&fpriv->lock);472 473 list_for_each_entry(client, &tegra->clients, list)474 if (client->base.class == args->client) {475 err = tegra_client_open(fpriv, client, context);476 if (err < 0)477 break;478 479 args->context = context->id;480 break;481 }482 483 if (err < 0)484 kfree(context);485 486 mutex_unlock(&fpriv->lock);487 return err;488}489 490static int tegra_close_channel(struct drm_device *drm, void *data,491 struct drm_file *file)492{493 struct tegra_drm_file *fpriv = file->driver_priv;494 struct drm_tegra_close_channel *args = data;495 struct tegra_drm_context *context;496 int err = 0;497 498 mutex_lock(&fpriv->lock);499 500 context = idr_find(&fpriv->legacy_contexts, args->context);501 if (!context) {502 err = -EINVAL;503 goto unlock;504 }505 506 idr_remove(&fpriv->legacy_contexts, context->id);507 tegra_drm_context_free(context);508 509unlock:510 mutex_unlock(&fpriv->lock);511 return err;512}513 514static int tegra_get_syncpt(struct drm_device *drm, void *data,515 struct drm_file *file)516{517 struct tegra_drm_file *fpriv = file->driver_priv;518 struct drm_tegra_get_syncpt *args = data;519 struct tegra_drm_context *context;520 struct host1x_syncpt *syncpt;521 int err = 0;522 523 mutex_lock(&fpriv->lock);524 525 context = idr_find(&fpriv->legacy_contexts, args->context);526 if (!context) {527 err = -ENODEV;528 goto unlock;529 }530 531 if (args->index >= context->client->base.num_syncpts) {532 err = -EINVAL;533 goto unlock;534 }535 536 syncpt = context->client->base.syncpts[args->index];537 args->id = host1x_syncpt_id(syncpt);538 539unlock:540 mutex_unlock(&fpriv->lock);541 return err;542}543 544static int tegra_submit(struct drm_device *drm, void *data,545 struct drm_file *file)546{547 struct tegra_drm_file *fpriv = file->driver_priv;548 struct drm_tegra_submit *args = data;549 struct tegra_drm_context *context;550 int err;551 552 mutex_lock(&fpriv->lock);553 554 context = idr_find(&fpriv->legacy_contexts, args->context);555 if (!context) {556 err = -ENODEV;557 goto unlock;558 }559 560 err = context->client->ops->submit(context, args, drm, file);561 562unlock:563 mutex_unlock(&fpriv->lock);564 return err;565}566 567static int tegra_get_syncpt_base(struct drm_device *drm, void *data,568 struct drm_file *file)569{570 struct tegra_drm_file *fpriv = file->driver_priv;571 struct drm_tegra_get_syncpt_base *args = data;572 struct tegra_drm_context *context;573 struct host1x_syncpt_base *base;574 struct host1x_syncpt *syncpt;575 int err = 0;576 577 mutex_lock(&fpriv->lock);578 579 context = idr_find(&fpriv->legacy_contexts, args->context);580 if (!context) {581 err = -ENODEV;582 goto unlock;583 }584 585 if (args->syncpt >= context->client->base.num_syncpts) {586 err = -EINVAL;587 goto unlock;588 }589 590 syncpt = context->client->base.syncpts[args->syncpt];591 592 base = host1x_syncpt_get_base(syncpt);593 if (!base) {594 err = -ENXIO;595 goto unlock;596 }597 598 args->id = host1x_syncpt_base_id(base);599 600unlock:601 mutex_unlock(&fpriv->lock);602 return err;603}604 605static int tegra_gem_set_tiling(struct drm_device *drm, void *data,606 struct drm_file *file)607{608 struct drm_tegra_gem_set_tiling *args = data;609 enum tegra_bo_tiling_mode mode;610 struct drm_gem_object *gem;611 unsigned long value = 0;612 struct tegra_bo *bo;613 614 switch (args->mode) {615 case DRM_TEGRA_GEM_TILING_MODE_PITCH:616 mode = TEGRA_BO_TILING_MODE_PITCH;617 618 if (args->value != 0)619 return -EINVAL;620 621 break;622 623 case DRM_TEGRA_GEM_TILING_MODE_TILED:624 mode = TEGRA_BO_TILING_MODE_TILED;625 626 if (args->value != 0)627 return -EINVAL;628 629 break;630 631 case DRM_TEGRA_GEM_TILING_MODE_BLOCK:632 mode = TEGRA_BO_TILING_MODE_BLOCK;633 634 if (args->value > 5)635 return -EINVAL;636 637 value = args->value;638 break;639 640 default:641 return -EINVAL;642 }643 644 gem = drm_gem_object_lookup(file, args->handle);645 if (!gem)646 return -ENOENT;647 648 bo = to_tegra_bo(gem);649 650 bo->tiling.mode = mode;651 bo->tiling.value = value;652 653 drm_gem_object_put(gem);654 655 return 0;656}657 658static int tegra_gem_get_tiling(struct drm_device *drm, void *data,659 struct drm_file *file)660{661 struct drm_tegra_gem_get_tiling *args = data;662 struct drm_gem_object *gem;663 struct tegra_bo *bo;664 int err = 0;665 666 gem = drm_gem_object_lookup(file, args->handle);667 if (!gem)668 return -ENOENT;669 670 bo = to_tegra_bo(gem);671 672 switch (bo->tiling.mode) {673 case TEGRA_BO_TILING_MODE_PITCH:674 args->mode = DRM_TEGRA_GEM_TILING_MODE_PITCH;675 args->value = 0;676 break;677 678 case TEGRA_BO_TILING_MODE_TILED:679 args->mode = DRM_TEGRA_GEM_TILING_MODE_TILED;680 args->value = 0;681 break;682 683 case TEGRA_BO_TILING_MODE_BLOCK:684 args->mode = DRM_TEGRA_GEM_TILING_MODE_BLOCK;685 args->value = bo->tiling.value;686 break;687 688 default:689 err = -EINVAL;690 break;691 }692 693 drm_gem_object_put(gem);694 695 return err;696}697 698static int tegra_gem_set_flags(struct drm_device *drm, void *data,699 struct drm_file *file)700{701 struct drm_tegra_gem_set_flags *args = data;702 struct drm_gem_object *gem;703 struct tegra_bo *bo;704 705 if (args->flags & ~DRM_TEGRA_GEM_FLAGS)706 return -EINVAL;707 708 gem = drm_gem_object_lookup(file, args->handle);709 if (!gem)710 return -ENOENT;711 712 bo = to_tegra_bo(gem);713 bo->flags = 0;714 715 if (args->flags & DRM_TEGRA_GEM_BOTTOM_UP)716 bo->flags |= TEGRA_BO_BOTTOM_UP;717 718 drm_gem_object_put(gem);719 720 return 0;721}722 723static int tegra_gem_get_flags(struct drm_device *drm, void *data,724 struct drm_file *file)725{726 struct drm_tegra_gem_get_flags *args = data;727 struct drm_gem_object *gem;728 struct tegra_bo *bo;729 730 gem = drm_gem_object_lookup(file, args->handle);731 if (!gem)732 return -ENOENT;733 734 bo = to_tegra_bo(gem);735 args->flags = 0;736 737 if (bo->flags & TEGRA_BO_BOTTOM_UP)738 args->flags |= DRM_TEGRA_GEM_BOTTOM_UP;739 740 drm_gem_object_put(gem);741 742 return 0;743}744#endif745 746static const struct drm_ioctl_desc tegra_drm_ioctls[] = {747#ifdef CONFIG_DRM_TEGRA_STAGING748 DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_OPEN, tegra_drm_ioctl_channel_open,749 DRM_RENDER_ALLOW),750 DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_CLOSE, tegra_drm_ioctl_channel_close,751 DRM_RENDER_ALLOW),752 DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_MAP, tegra_drm_ioctl_channel_map,753 DRM_RENDER_ALLOW),754 DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_UNMAP, tegra_drm_ioctl_channel_unmap,755 DRM_RENDER_ALLOW),756 DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_SUBMIT, tegra_drm_ioctl_channel_submit,757 DRM_RENDER_ALLOW),758 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_ALLOCATE, tegra_drm_ioctl_syncpoint_allocate,759 DRM_RENDER_ALLOW),760 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_FREE, tegra_drm_ioctl_syncpoint_free,761 DRM_RENDER_ALLOW),762 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPOINT_WAIT, tegra_drm_ioctl_syncpoint_wait,763 DRM_RENDER_ALLOW),764 765 DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, DRM_RENDER_ALLOW),766 DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, DRM_RENDER_ALLOW),767 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read,768 DRM_RENDER_ALLOW),769 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr,770 DRM_RENDER_ALLOW),771 DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait,772 DRM_RENDER_ALLOW),773 DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel,774 DRM_RENDER_ALLOW),775 DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel,776 DRM_RENDER_ALLOW),777 DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt,778 DRM_RENDER_ALLOW),779 DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit,780 DRM_RENDER_ALLOW),781 DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base,782 DRM_RENDER_ALLOW),783 DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling,784 DRM_RENDER_ALLOW),785 DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling,786 DRM_RENDER_ALLOW),787 DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags,788 DRM_RENDER_ALLOW),789 DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags,790 DRM_RENDER_ALLOW),791#endif792};793 794static const struct file_operations tegra_drm_fops = {795 .owner = THIS_MODULE,796 .open = drm_open,797 .release = drm_release,798 .unlocked_ioctl = drm_ioctl,799 .mmap = tegra_drm_mmap,800 .poll = drm_poll,801 .read = drm_read,802 .compat_ioctl = drm_compat_ioctl,803 .llseek = noop_llseek,804 .fop_flags = FOP_UNSIGNED_OFFSET,805};806 807static int tegra_drm_context_cleanup(int id, void *p, void *data)808{809 struct tegra_drm_context *context = p;810 811 tegra_drm_context_free(context);812 813 return 0;814}815 816static void tegra_drm_postclose(struct drm_device *drm, struct drm_file *file)817{818 struct tegra_drm_file *fpriv = file->driver_priv;819 820 mutex_lock(&fpriv->lock);821 idr_for_each(&fpriv->legacy_contexts, tegra_drm_context_cleanup, NULL);822 tegra_drm_uapi_close_file(fpriv);823 mutex_unlock(&fpriv->lock);824 825 idr_destroy(&fpriv->legacy_contexts);826 mutex_destroy(&fpriv->lock);827 kfree(fpriv);828}829 830#ifdef CONFIG_DEBUG_FS831static int tegra_debugfs_framebuffers(struct seq_file *s, void *data)832{833 struct drm_info_node *node = (struct drm_info_node *)s->private;834 struct drm_device *drm = node->minor->dev;835 struct drm_framebuffer *fb;836 837 mutex_lock(&drm->mode_config.fb_lock);838 839 list_for_each_entry(fb, &drm->mode_config.fb_list, head) {840 seq_printf(s, "%3d: user size: %d x %d, depth %d, %d bpp, refcount %d\n",841 fb->base.id, fb->width, fb->height,842 fb->format->depth,843 fb->format->cpp[0] * 8,844 drm_framebuffer_read_refcount(fb));845 }846 847 mutex_unlock(&drm->mode_config.fb_lock);848 849 return 0;850}851 852static int tegra_debugfs_iova(struct seq_file *s, void *data)853{854 struct drm_info_node *node = (struct drm_info_node *)s->private;855 struct drm_device *drm = node->minor->dev;856 struct tegra_drm *tegra = drm->dev_private;857 struct drm_printer p = drm_seq_file_printer(s);858 859 if (tegra->domain) {860 mutex_lock(&tegra->mm_lock);861 drm_mm_print(&tegra->mm, &p);862 mutex_unlock(&tegra->mm_lock);863 }864 865 return 0;866}867 868static struct drm_info_list tegra_debugfs_list[] = {869 { "framebuffers", tegra_debugfs_framebuffers, 0 },870 { "iova", tegra_debugfs_iova, 0 },871};872 873static void tegra_debugfs_init(struct drm_minor *minor)874{875 drm_debugfs_create_files(tegra_debugfs_list,876 ARRAY_SIZE(tegra_debugfs_list),877 minor->debugfs_root, minor);878}879#endif880 881static const struct drm_driver tegra_drm_driver = {882 .driver_features = DRIVER_MODESET | DRIVER_GEM |883 DRIVER_ATOMIC | DRIVER_RENDER | DRIVER_SYNCOBJ,884 .open = tegra_drm_open,885 .postclose = tegra_drm_postclose,886 887#if defined(CONFIG_DEBUG_FS)888 .debugfs_init = tegra_debugfs_init,889#endif890 891 .gem_prime_import = tegra_gem_prime_import,892 893 .dumb_create = tegra_bo_dumb_create,894 895 .ioctls = tegra_drm_ioctls,896 .num_ioctls = ARRAY_SIZE(tegra_drm_ioctls),897 .fops = &tegra_drm_fops,898 899 .name = DRIVER_NAME,900 .desc = DRIVER_DESC,901 .date = DRIVER_DATE,902 .major = DRIVER_MAJOR,903 .minor = DRIVER_MINOR,904 .patchlevel = DRIVER_PATCHLEVEL,905};906 907int tegra_drm_register_client(struct tegra_drm *tegra,908 struct tegra_drm_client *client)909{910 /*911 * When MLOCKs are implemented, change to allocate a shared channel912 * only when MLOCKs are disabled.913 */914 client->shared_channel = host1x_channel_request(&client->base);915 if (!client->shared_channel)916 return -EBUSY;917 918 mutex_lock(&tegra->clients_lock);919 list_add_tail(&client->list, &tegra->clients);920 client->drm = tegra;921 mutex_unlock(&tegra->clients_lock);922 923 return 0;924}925 926int tegra_drm_unregister_client(struct tegra_drm *tegra,927 struct tegra_drm_client *client)928{929 mutex_lock(&tegra->clients_lock);930 list_del_init(&client->list);931 client->drm = NULL;932 mutex_unlock(&tegra->clients_lock);933 934 if (client->shared_channel)935 host1x_channel_put(client->shared_channel);936 937 return 0;938}939 940int host1x_client_iommu_attach(struct host1x_client *client)941{942 struct iommu_domain *domain = iommu_get_domain_for_dev(client->dev);943 struct drm_device *drm = dev_get_drvdata(client->host);944 struct tegra_drm *tegra = drm->dev_private;945 struct iommu_group *group = NULL;946 int err;947 948#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU)949 if (client->dev->archdata.mapping) {950 struct dma_iommu_mapping *mapping =951 to_dma_iommu_mapping(client->dev);952 arm_iommu_detach_device(client->dev);953 arm_iommu_release_mapping(mapping);954 955 domain = iommu_get_domain_for_dev(client->dev);956 }957#endif958 959 /*960 * If the host1x client is already attached to an IOMMU domain that is961 * not the shared IOMMU domain, don't try to attach it to a different962 * domain. This allows using the IOMMU-backed DMA API.963 */964 if (domain && domain->type != IOMMU_DOMAIN_IDENTITY &&965 domain != tegra->domain)966 return 0;967 968 if (tegra->domain) {969 group = iommu_group_get(client->dev);970 if (!group)971 return -ENODEV;972 973 if (domain != tegra->domain) {974 err = iommu_attach_group(tegra->domain, group);975 if (err < 0) {976 iommu_group_put(group);977 return err;978 }979 }980 981 tegra->use_explicit_iommu = true;982 }983 984 client->group = group;985 986 return 0;987}988 989void host1x_client_iommu_detach(struct host1x_client *client)990{991 struct drm_device *drm = dev_get_drvdata(client->host);992 struct tegra_drm *tegra = drm->dev_private;993 struct iommu_domain *domain;994 995 if (client->group) {996 /*997 * Devices that are part of the same group may no longer be998 * attached to a domain at this point because their group may999 * have been detached by an earlier client.1000 */1001 domain = iommu_get_domain_for_dev(client->dev);1002 if (domain)1003 iommu_detach_group(tegra->domain, client->group);1004 1005 iommu_group_put(client->group);1006 client->group = NULL;1007 }1008}1009 1010void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)1011{1012 struct iova *alloc;1013 void *virt;1014 gfp_t gfp;1015 int err;1016 1017 if (tegra->domain)1018 size = iova_align(&tegra->carveout.domain, size);1019 else1020 size = PAGE_ALIGN(size);1021 1022 gfp = GFP_KERNEL | __GFP_ZERO;1023 if (!tegra->domain) {1024 /*1025 * Many units only support 32-bit addresses, even on 64-bit1026 * SoCs. If there is no IOMMU to translate into a 32-bit IO1027 * virtual address space, force allocations to be in the1028 * lower 32-bit range.1029 */1030 gfp |= GFP_DMA;1031 }1032 1033 virt = (void *)__get_free_pages(gfp, get_order(size));1034 if (!virt)1035 return ERR_PTR(-ENOMEM);1036 1037 if (!tegra->domain) {1038 /*1039 * If IOMMU is disabled, devices address physical memory1040 * directly.1041 */1042 *dma = virt_to_phys(virt);1043 return virt;1044 }1045 1046 alloc = alloc_iova(&tegra->carveout.domain,1047 size >> tegra->carveout.shift,1048 tegra->carveout.limit, true);1049 if (!alloc) {1050 err = -EBUSY;1051 goto free_pages;1052 }1053 1054 *dma = iova_dma_addr(&tegra->carveout.domain, alloc);1055 err = iommu_map(tegra->domain, *dma, virt_to_phys(virt),1056 size, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL);1057 if (err < 0)1058 goto free_iova;1059 1060 return virt;1061 1062free_iova:1063 __free_iova(&tegra->carveout.domain, alloc);1064free_pages:1065 free_pages((unsigned long)virt, get_order(size));1066 1067 return ERR_PTR(err);1068}1069 1070void tegra_drm_free(struct tegra_drm *tegra, size_t size, void *virt,1071 dma_addr_t dma)1072{1073 if (tegra->domain)1074 size = iova_align(&tegra->carveout.domain, size);1075 else1076 size = PAGE_ALIGN(size);1077 1078 if (tegra->domain) {1079 iommu_unmap(tegra->domain, dma, size);1080 free_iova(&tegra->carveout.domain,1081 iova_pfn(&tegra->carveout.domain, dma));1082 }1083 1084 free_pages((unsigned long)virt, get_order(size));1085}1086 1087static bool host1x_drm_wants_iommu(struct host1x_device *dev)1088{1089 struct host1x *host1x = dev_get_drvdata(dev->dev.parent);1090 struct iommu_domain *domain;1091 1092 /* Our IOMMU usage policy doesn't currently play well with GART */1093 if (of_machine_is_compatible("nvidia,tegra20"))1094 return false;1095 1096 /*1097 * If the Tegra DRM clients are backed by an IOMMU, push buffers are1098 * likely to be allocated beyond the 32-bit boundary if sufficient1099 * system memory is available. This is problematic on earlier Tegra1100 * generations where host1x supports a maximum of 32 address bits in1101 * the GATHER opcode. In this case, unless host1x is behind an IOMMU1102 * as well it won't be able to process buffers allocated beyond the1103 * 32-bit boundary.1104 *1105 * The DMA API will use bounce buffers in this case, so that could1106 * perhaps still be made to work, even if less efficient, but there1107 * is another catch: in order to perform cache maintenance on pages1108 * allocated for discontiguous buffers we need to map and unmap the1109 * SG table representing these buffers. This is fine for something1110 * small like a push buffer, but it exhausts the bounce buffer pool1111 * (typically on the order of a few MiB) for framebuffers (many MiB1112 * for any modern resolution).1113 *1114 * Work around this by making sure that Tegra DRM clients only use1115 * an IOMMU if the parent host1x also uses an IOMMU.1116 *1117 * Note that there's still a small gap here that we don't cover: if1118 * the DMA API is backed by an IOMMU there's no way to control which1119 * device is attached to an IOMMU and which isn't, except via wiring1120 * up the device tree appropriately. This is considered an problem1121 * of integration, so care must be taken for the DT to be consistent.1122 */1123 domain = iommu_get_domain_for_dev(dev->dev.parent);1124 1125 /*1126 * Tegra20 and Tegra30 don't support addressing memory beyond the1127 * 32-bit boundary, so the regular GATHER opcodes will always be1128 * sufficient and whether or not the host1x is attached to an IOMMU1129 * doesn't matter.1130 */1131 if (!domain && host1x_get_dma_mask(host1x) <= DMA_BIT_MASK(32))1132 return true;1133 1134 return domain != NULL;1135}1136 1137static int host1x_drm_probe(struct host1x_device *dev)1138{1139 struct device *dma_dev = dev->dev.parent;1140 struct tegra_drm *tegra;1141 struct drm_device *drm;1142 int err;1143 1144 drm = drm_dev_alloc(&tegra_drm_driver, &dev->dev);1145 if (IS_ERR(drm))1146 return PTR_ERR(drm);1147 1148 tegra = kzalloc(sizeof(*tegra), GFP_KERNEL);1149 if (!tegra) {1150 err = -ENOMEM;1151 goto put;1152 }1153 1154 if (host1x_drm_wants_iommu(dev) && device_iommu_mapped(dma_dev)) {1155 tegra->domain = iommu_paging_domain_alloc(dma_dev);1156 if (IS_ERR(tegra->domain)) {1157 err = PTR_ERR(tegra->domain);1158 goto free;1159 }1160 1161 err = iova_cache_get();1162 if (err < 0)1163 goto domain;1164 }1165 1166 mutex_init(&tegra->clients_lock);1167 INIT_LIST_HEAD(&tegra->clients);1168 1169 dev_set_drvdata(&dev->dev, drm);1170 drm->dev_private = tegra;1171 tegra->drm = drm;1172 1173 drm_mode_config_init(drm);1174 1175 drm->mode_config.min_width = 0;1176 drm->mode_config.min_height = 0;1177 drm->mode_config.max_width = 0;1178 drm->mode_config.max_height = 0;1179 1180 drm->mode_config.normalize_zpos = true;1181 1182 drm->mode_config.funcs = &tegra_drm_mode_config_funcs;1183 drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;1184 1185 drm_kms_helper_poll_init(drm);1186 1187 err = host1x_device_init(dev);1188 if (err < 0)1189 goto poll;1190 1191 /*1192 * Now that all display controller have been initialized, the maximum1193 * supported resolution is known and the bitmask for horizontal and1194 * vertical bitfields can be computed.1195 */1196 tegra->hmask = drm->mode_config.max_width - 1;1197 tegra->vmask = drm->mode_config.max_height - 1;1198 1199 if (tegra->use_explicit_iommu) {1200 u64 carveout_start, carveout_end, gem_start, gem_end;1201 u64 dma_mask = dma_get_mask(&dev->dev);1202 dma_addr_t start, end;1203 unsigned long order;1204 1205 start = tegra->domain->geometry.aperture_start & dma_mask;1206 end = tegra->domain->geometry.aperture_end & dma_mask;1207 1208 gem_start = start;1209 gem_end = end - CARVEOUT_SZ;1210 carveout_start = gem_end + 1;1211 carveout_end = end;1212 1213 order = __ffs(tegra->domain->pgsize_bitmap);1214 init_iova_domain(&tegra->carveout.domain, 1UL << order,1215 carveout_start >> order);1216 1217 tegra->carveout.shift = iova_shift(&tegra->carveout.domain);1218 tegra->carveout.limit = carveout_end >> tegra->carveout.shift;1219 1220 drm_mm_init(&tegra->mm, gem_start, gem_end - gem_start + 1);1221 mutex_init(&tegra->mm_lock);1222 1223 DRM_DEBUG_DRIVER("IOMMU apertures:\n");1224 DRM_DEBUG_DRIVER(" GEM: %#llx-%#llx\n", gem_start, gem_end);1225 DRM_DEBUG_DRIVER(" Carveout: %#llx-%#llx\n", carveout_start,1226 carveout_end);1227 } else if (tegra->domain) {1228 iommu_domain_free(tegra->domain);1229 tegra->domain = NULL;1230 iova_cache_put();1231 }1232 1233 if (tegra->hub) {1234 err = tegra_display_hub_prepare(tegra->hub);1235 if (err < 0)1236 goto device;1237 }1238 1239 /* syncpoints are used for full 32-bit hardware VBLANK counters */1240 drm->max_vblank_count = 0xffffffff;1241 1242 err = drm_vblank_init(drm, drm->mode_config.num_crtc);1243 if (err < 0)1244 goto hub;1245 1246 drm_mode_config_reset(drm);1247 1248 /*1249 * Only take over from a potential firmware framebuffer if any CRTCs1250 * have been registered. This must not be a fatal error because there1251 * are other accelerators that are exposed via this driver.1252 *1253 * Another case where this happens is on Tegra234 where the display1254 * hardware is no longer part of the host1x complex, so this driver1255 * will not expose any modesetting features.1256 */1257 if (drm->mode_config.num_crtc > 0) {1258 err = drm_aperture_remove_framebuffers(&tegra_drm_driver);1259 if (err < 0)1260 goto hub;1261 } else {1262 /*1263 * Indicate to userspace that this doesn't expose any display1264 * capabilities.1265 */1266 drm->driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC);1267 }1268 1269 err = drm_dev_register(drm, 0);1270 if (err < 0)1271 goto hub;1272 1273 tegra_fbdev_setup(drm);1274 1275 return 0;1276 1277hub:1278 if (tegra->hub)1279 tegra_display_hub_cleanup(tegra->hub);1280device:1281 if (tegra->domain) {1282 mutex_destroy(&tegra->mm_lock);1283 drm_mm_takedown(&tegra->mm);1284 put_iova_domain(&tegra->carveout.domain);1285 iova_cache_put();1286 }1287 1288 host1x_device_exit(dev);1289poll:1290 drm_kms_helper_poll_fini(drm);1291 drm_mode_config_cleanup(drm);1292domain:1293 if (tegra->domain)1294 iommu_domain_free(tegra->domain);1295free:1296 kfree(tegra);1297put:1298 drm_dev_put(drm);1299 return err;1300}1301 1302static int host1x_drm_remove(struct host1x_device *dev)1303{1304 struct drm_device *drm = dev_get_drvdata(&dev->dev);1305 struct tegra_drm *tegra = drm->dev_private;1306 int err;1307 1308 drm_dev_unregister(drm);1309 1310 drm_kms_helper_poll_fini(drm);1311 drm_atomic_helper_shutdown(drm);1312 drm_mode_config_cleanup(drm);1313 1314 if (tegra->hub)1315 tegra_display_hub_cleanup(tegra->hub);1316 1317 err = host1x_device_exit(dev);1318 if (err < 0)1319 dev_err(&dev->dev, "host1x device cleanup failed: %d\n", err);1320 1321 if (tegra->domain) {1322 mutex_destroy(&tegra->mm_lock);1323 drm_mm_takedown(&tegra->mm);1324 put_iova_domain(&tegra->carveout.domain);1325 iova_cache_put();1326 iommu_domain_free(tegra->domain);1327 }1328 1329 kfree(tegra);1330 drm_dev_put(drm);1331 1332 return 0;1333}1334 1335static void host1x_drm_shutdown(struct host1x_device *dev)1336{1337 drm_atomic_helper_shutdown(dev_get_drvdata(&dev->dev));1338}1339 1340#ifdef CONFIG_PM_SLEEP1341static int host1x_drm_suspend(struct device *dev)1342{1343 struct drm_device *drm = dev_get_drvdata(dev);1344 1345 return drm_mode_config_helper_suspend(drm);1346}1347 1348static int host1x_drm_resume(struct device *dev)1349{1350 struct drm_device *drm = dev_get_drvdata(dev);1351 1352 return drm_mode_config_helper_resume(drm);1353}1354#endif1355 1356static SIMPLE_DEV_PM_OPS(host1x_drm_pm_ops, host1x_drm_suspend,1357 host1x_drm_resume);1358 1359static const struct of_device_id host1x_drm_subdevs[] = {1360 { .compatible = "nvidia,tegra20-dc", },1361 { .compatible = "nvidia,tegra20-hdmi", },1362 { .compatible = "nvidia,tegra20-gr2d", },1363 { .compatible = "nvidia,tegra20-gr3d", },1364 { .compatible = "nvidia,tegra30-dc", },1365 { .compatible = "nvidia,tegra30-hdmi", },1366 { .compatible = "nvidia,tegra30-gr2d", },1367 { .compatible = "nvidia,tegra30-gr3d", },1368 { .compatible = "nvidia,tegra114-dc", },1369 { .compatible = "nvidia,tegra114-dsi", },1370 { .compatible = "nvidia,tegra114-hdmi", },1371 { .compatible = "nvidia,tegra114-gr2d", },1372 { .compatible = "nvidia,tegra114-gr3d", },1373 { .compatible = "nvidia,tegra124-dc", },1374 { .compatible = "nvidia,tegra124-sor", },1375 { .compatible = "nvidia,tegra124-hdmi", },1376 { .compatible = "nvidia,tegra124-dsi", },1377 { .compatible = "nvidia,tegra124-vic", },1378 { .compatible = "nvidia,tegra132-dsi", },1379 { .compatible = "nvidia,tegra210-dc", },1380 { .compatible = "nvidia,tegra210-dsi", },1381 { .compatible = "nvidia,tegra210-sor", },1382 { .compatible = "nvidia,tegra210-sor1", },1383 { .compatible = "nvidia,tegra210-vic", },1384 { .compatible = "nvidia,tegra210-nvdec", },1385 { .compatible = "nvidia,tegra186-display", },1386 { .compatible = "nvidia,tegra186-dc", },1387 { .compatible = "nvidia,tegra186-sor", },1388 { .compatible = "nvidia,tegra186-sor1", },1389 { .compatible = "nvidia,tegra186-vic", },1390 { .compatible = "nvidia,tegra186-nvdec", },1391 { .compatible = "nvidia,tegra194-display", },1392 { .compatible = "nvidia,tegra194-dc", },1393 { .compatible = "nvidia,tegra194-sor", },1394 { .compatible = "nvidia,tegra194-vic", },1395 { .compatible = "nvidia,tegra194-nvdec", },1396 { .compatible = "nvidia,tegra234-vic", },1397 { .compatible = "nvidia,tegra234-nvdec", },1398 { /* sentinel */ }1399};1400 1401static struct host1x_driver host1x_drm_driver = {1402 .driver = {1403 .name = "drm",1404 .pm = &host1x_drm_pm_ops,1405 },1406 .probe = host1x_drm_probe,1407 .remove = host1x_drm_remove,1408 .shutdown = host1x_drm_shutdown,1409 .subdevs = host1x_drm_subdevs,1410};1411 1412static struct platform_driver * const drivers[] = {1413 &tegra_display_hub_driver,1414 &tegra_dc_driver,1415 &tegra_hdmi_driver,1416 &tegra_dsi_driver,1417 &tegra_dpaux_driver,1418 &tegra_sor_driver,1419 &tegra_gr2d_driver,1420 &tegra_gr3d_driver,1421 &tegra_vic_driver,1422 &tegra_nvdec_driver,1423};1424 1425static int __init host1x_drm_init(void)1426{1427 int err;1428 1429 if (drm_firmware_drivers_only())1430 return -ENODEV;1431 1432 err = host1x_driver_register(&host1x_drm_driver);1433 if (err < 0)1434 return err;1435 1436 err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));1437 if (err < 0)1438 goto unregister_host1x;1439 1440 return 0;1441 1442unregister_host1x:1443 host1x_driver_unregister(&host1x_drm_driver);1444 return err;1445}1446module_init(host1x_drm_init);1447 1448static void __exit host1x_drm_exit(void)1449{1450 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));1451 host1x_driver_unregister(&host1x_drm_driver);1452}1453module_exit(host1x_drm_exit);1454 1455MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>");1456MODULE_DESCRIPTION("NVIDIA Tegra DRM driver");1457MODULE_LICENSE("GPL v2");1458