505 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */3 4#include <linux/module.h>5#include <linux/of.h>6#include <linux/platform_device.h>7#include <linux/uaccess.h>8#include <linux/slab.h>9#include <linux/pm_runtime.h>10#include <drm/drm_ioctl.h>11#include <drm/drm_drv.h>12#include <drm/drm_prime.h>13#include <drm/lima_drm.h>14 15#include "lima_device.h"16#include "lima_drv.h"17#include "lima_gem.h"18#include "lima_vm.h"19 20int lima_sched_timeout_ms;21uint lima_heap_init_nr_pages = 8;22uint lima_max_error_tasks;23uint lima_job_hang_limit;24 25MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms");26module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444);27 28MODULE_PARM_DESC(heap_init_nr_pages, "heap buffer init number of pages");29module_param_named(heap_init_nr_pages, lima_heap_init_nr_pages, uint, 0444);30 31MODULE_PARM_DESC(max_error_tasks, "max number of error tasks to save");32module_param_named(max_error_tasks, lima_max_error_tasks, uint, 0644);33 34MODULE_PARM_DESC(job_hang_limit, "number of times to allow a job to hang before dropping it (default 0)");35module_param_named(job_hang_limit, lima_job_hang_limit, uint, 0444);36 37static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file)38{39 struct drm_lima_get_param *args = data;40 struct lima_device *ldev = to_lima_dev(dev);41 42 if (args->pad)43 return -EINVAL;44 45 switch (args->param) {46 case DRM_LIMA_PARAM_GPU_ID:47 switch (ldev->id) {48 case lima_gpu_mali400:49 args->value = DRM_LIMA_PARAM_GPU_ID_MALI400;50 break;51 case lima_gpu_mali450:52 args->value = DRM_LIMA_PARAM_GPU_ID_MALI450;53 break;54 default:55 args->value = DRM_LIMA_PARAM_GPU_ID_UNKNOWN;56 break;57 }58 break;59 60 case DRM_LIMA_PARAM_NUM_PP:61 args->value = ldev->pipe[lima_pipe_pp].num_processor;62 break;63 64 case DRM_LIMA_PARAM_GP_VERSION:65 args->value = ldev->gp_version;66 break;67 68 case DRM_LIMA_PARAM_PP_VERSION:69 args->value = ldev->pp_version;70 break;71 72 default:73 return -EINVAL;74 }75 76 return 0;77}78 79static int lima_ioctl_gem_create(struct drm_device *dev, void *data, struct drm_file *file)80{81 struct drm_lima_gem_create *args = data;82 83 if (args->pad)84 return -EINVAL;85 86 if (args->flags & ~(LIMA_BO_FLAG_HEAP))87 return -EINVAL;88 89 if (args->size == 0)90 return -EINVAL;91 92 return lima_gem_create_handle(dev, file, args->size, args->flags, &args->handle);93}94 95static int lima_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file)96{97 struct drm_lima_gem_info *args = data;98 99 return lima_gem_get_info(file, args->handle, &args->va, &args->offset);100}101 102static int lima_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file)103{104 struct drm_lima_gem_submit *args = data;105 struct lima_device *ldev = to_lima_dev(dev);106 struct lima_drm_priv *priv = file->driver_priv;107 struct drm_lima_gem_submit_bo *bos;108 struct lima_sched_pipe *pipe;109 struct lima_sched_task *task;110 struct lima_ctx *ctx;111 struct lima_submit submit = {0};112 size_t size;113 int err = 0;114 115 if (args->pipe >= lima_pipe_num || args->nr_bos == 0)116 return -EINVAL;117 118 if (args->flags & ~(LIMA_SUBMIT_FLAG_EXPLICIT_FENCE))119 return -EINVAL;120 121 pipe = ldev->pipe + args->pipe;122 if (args->frame_size != pipe->frame_size)123 return -EINVAL;124 125 bos = kvcalloc(args->nr_bos, sizeof(*submit.bos) + sizeof(*submit.lbos), GFP_KERNEL);126 if (!bos)127 return -ENOMEM;128 129 size = args->nr_bos * sizeof(*submit.bos);130 if (copy_from_user(bos, u64_to_user_ptr(args->bos), size)) {131 err = -EFAULT;132 goto out0;133 }134 135 task = kmem_cache_zalloc(pipe->task_slab, GFP_KERNEL);136 if (!task) {137 err = -ENOMEM;138 goto out0;139 }140 141 task->frame = task + 1;142 if (copy_from_user(task->frame, u64_to_user_ptr(args->frame), args->frame_size)) {143 err = -EFAULT;144 goto out1;145 }146 147 err = pipe->task_validate(pipe, task);148 if (err)149 goto out1;150 151 ctx = lima_ctx_get(&priv->ctx_mgr, args->ctx);152 if (!ctx) {153 err = -ENOENT;154 goto out1;155 }156 157 submit.pipe = args->pipe;158 submit.bos = bos;159 submit.lbos = (void *)bos + size;160 submit.nr_bos = args->nr_bos;161 submit.task = task;162 submit.ctx = ctx;163 submit.flags = args->flags;164 submit.in_sync[0] = args->in_sync[0];165 submit.in_sync[1] = args->in_sync[1];166 submit.out_sync = args->out_sync;167 168 err = lima_gem_submit(file, &submit);169 170 lima_ctx_put(ctx);171out1:172 if (err)173 kmem_cache_free(pipe->task_slab, task);174out0:175 kvfree(bos);176 return err;177}178 179static int lima_ioctl_gem_wait(struct drm_device *dev, void *data, struct drm_file *file)180{181 struct drm_lima_gem_wait *args = data;182 183 if (args->op & ~(LIMA_GEM_WAIT_READ|LIMA_GEM_WAIT_WRITE))184 return -EINVAL;185 186 return lima_gem_wait(file, args->handle, args->op, args->timeout_ns);187}188 189static int lima_ioctl_ctx_create(struct drm_device *dev, void *data, struct drm_file *file)190{191 struct drm_lima_ctx_create *args = data;192 struct lima_drm_priv *priv = file->driver_priv;193 struct lima_device *ldev = to_lima_dev(dev);194 195 if (args->_pad)196 return -EINVAL;197 198 return lima_ctx_create(ldev, &priv->ctx_mgr, &args->id);199}200 201static int lima_ioctl_ctx_free(struct drm_device *dev, void *data, struct drm_file *file)202{203 struct drm_lima_ctx_create *args = data;204 struct lima_drm_priv *priv = file->driver_priv;205 206 if (args->_pad)207 return -EINVAL;208 209 return lima_ctx_free(&priv->ctx_mgr, args->id);210}211 212static int lima_drm_driver_open(struct drm_device *dev, struct drm_file *file)213{214 int err;215 struct lima_drm_priv *priv;216 struct lima_device *ldev = to_lima_dev(dev);217 218 priv = kzalloc(sizeof(*priv), GFP_KERNEL);219 if (!priv)220 return -ENOMEM;221 222 priv->vm = lima_vm_create(ldev);223 if (!priv->vm) {224 err = -ENOMEM;225 goto err_out0;226 }227 228 lima_ctx_mgr_init(&priv->ctx_mgr);229 230 file->driver_priv = priv;231 return 0;232 233err_out0:234 kfree(priv);235 return err;236}237 238static void lima_drm_driver_postclose(struct drm_device *dev, struct drm_file *file)239{240 struct lima_drm_priv *priv = file->driver_priv;241 242 lima_ctx_mgr_fini(&priv->ctx_mgr);243 lima_vm_put(priv->vm);244 kfree(priv);245}246 247static const struct drm_ioctl_desc lima_drm_driver_ioctls[] = {248 DRM_IOCTL_DEF_DRV(LIMA_GET_PARAM, lima_ioctl_get_param, DRM_RENDER_ALLOW),249 DRM_IOCTL_DEF_DRV(LIMA_GEM_CREATE, lima_ioctl_gem_create, DRM_RENDER_ALLOW),250 DRM_IOCTL_DEF_DRV(LIMA_GEM_INFO, lima_ioctl_gem_info, DRM_RENDER_ALLOW),251 DRM_IOCTL_DEF_DRV(LIMA_GEM_SUBMIT, lima_ioctl_gem_submit, DRM_RENDER_ALLOW),252 DRM_IOCTL_DEF_DRV(LIMA_GEM_WAIT, lima_ioctl_gem_wait, DRM_RENDER_ALLOW),253 DRM_IOCTL_DEF_DRV(LIMA_CTX_CREATE, lima_ioctl_ctx_create, DRM_RENDER_ALLOW),254 DRM_IOCTL_DEF_DRV(LIMA_CTX_FREE, lima_ioctl_ctx_free, DRM_RENDER_ALLOW),255};256 257DEFINE_DRM_GEM_FOPS(lima_drm_driver_fops);258 259/*260 * Changelog:261 *262 * - 1.1.0 - add heap buffer support263 */264 265static const struct drm_driver lima_drm_driver = {266 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,267 .open = lima_drm_driver_open,268 .postclose = lima_drm_driver_postclose,269 .ioctls = lima_drm_driver_ioctls,270 .num_ioctls = ARRAY_SIZE(lima_drm_driver_ioctls),271 .fops = &lima_drm_driver_fops,272 .name = "lima",273 .desc = "lima DRM",274 .date = "20191231",275 .major = 1,276 .minor = 1,277 .patchlevel = 0,278 279 .gem_create_object = lima_gem_create_object,280 .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,281};282 283struct lima_block_reader {284 void *dst;285 size_t base;286 size_t count;287 size_t off;288 ssize_t read;289};290 291static bool lima_read_block(struct lima_block_reader *reader,292 void *src, size_t src_size)293{294 size_t max_off = reader->base + src_size;295 296 if (reader->off < max_off) {297 size_t size = min_t(size_t, max_off - reader->off,298 reader->count);299 300 memcpy(reader->dst, src + (reader->off - reader->base), size);301 302 reader->dst += size;303 reader->off += size;304 reader->read += size;305 reader->count -= size;306 }307 308 reader->base = max_off;309 310 return !!reader->count;311}312 313static ssize_t lima_error_state_read(struct file *filp, struct kobject *kobj,314 struct bin_attribute *attr, char *buf,315 loff_t off, size_t count)316{317 struct device *dev = kobj_to_dev(kobj);318 struct lima_device *ldev = dev_get_drvdata(dev);319 struct lima_sched_error_task *et;320 struct lima_block_reader reader = {321 .dst = buf,322 .count = count,323 .off = off,324 };325 326 mutex_lock(&ldev->error_task_list_lock);327 328 if (lima_read_block(&reader, &ldev->dump, sizeof(ldev->dump))) {329 list_for_each_entry(et, &ldev->error_task_list, list) {330 if (!lima_read_block(&reader, et->data, et->size))331 break;332 }333 }334 335 mutex_unlock(&ldev->error_task_list_lock);336 return reader.read;337}338 339static ssize_t lima_error_state_write(struct file *file, struct kobject *kobj,340 struct bin_attribute *attr, char *buf,341 loff_t off, size_t count)342{343 struct device *dev = kobj_to_dev(kobj);344 struct lima_device *ldev = dev_get_drvdata(dev);345 struct lima_sched_error_task *et, *tmp;346 347 mutex_lock(&ldev->error_task_list_lock);348 349 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {350 list_del(&et->list);351 kvfree(et);352 }353 354 ldev->dump.size = 0;355 ldev->dump.num_tasks = 0;356 357 mutex_unlock(&ldev->error_task_list_lock);358 359 return count;360}361 362static const struct bin_attribute lima_error_state_attr = {363 .attr.name = "error",364 .attr.mode = 0600,365 .size = 0,366 .read = lima_error_state_read,367 .write = lima_error_state_write,368};369 370static int lima_pdev_probe(struct platform_device *pdev)371{372 struct lima_device *ldev;373 struct drm_device *ddev;374 const struct lima_compatible *comp;375 int err;376 377 err = lima_sched_slab_init();378 if (err)379 return err;380 381 ldev = devm_kzalloc(&pdev->dev, sizeof(*ldev), GFP_KERNEL);382 if (!ldev) {383 err = -ENOMEM;384 goto err_out0;385 }386 387 ldev->dev = &pdev->dev;388 comp = of_device_get_match_data(&pdev->dev);389 if (!comp) {390 err = -ENODEV;391 goto err_out0;392 }393 394 ldev->id = comp->id;395 396 platform_set_drvdata(pdev, ldev);397 398 /* Allocate and initialize the DRM device. */399 ddev = drm_dev_alloc(&lima_drm_driver, &pdev->dev);400 if (IS_ERR(ddev)) {401 err = PTR_ERR(ddev);402 goto err_out0;403 }404 405 ddev->dev_private = ldev;406 ldev->ddev = ddev;407 408 err = lima_device_init(ldev);409 if (err)410 goto err_out1;411 412 err = lima_devfreq_init(ldev);413 if (err) {414 dev_err(&pdev->dev, "Fatal error during devfreq init\n");415 goto err_out2;416 }417 418 pm_runtime_set_active(ldev->dev);419 pm_runtime_mark_last_busy(ldev->dev);420 pm_runtime_set_autosuspend_delay(ldev->dev, 200);421 pm_runtime_use_autosuspend(ldev->dev);422 pm_runtime_enable(ldev->dev);423 424 /*425 * Register the DRM device with the core and the connectors with426 * sysfs.427 */428 err = drm_dev_register(ddev, 0);429 if (err < 0)430 goto err_out3;431 432 if (sysfs_create_bin_file(&ldev->dev->kobj, &lima_error_state_attr))433 dev_warn(ldev->dev, "fail to create error state sysfs\n");434 435 return 0;436 437err_out3:438 pm_runtime_disable(ldev->dev);439 lima_devfreq_fini(ldev);440err_out2:441 lima_device_fini(ldev);442err_out1:443 drm_dev_put(ddev);444err_out0:445 lima_sched_slab_fini();446 return err;447}448 449static void lima_pdev_remove(struct platform_device *pdev)450{451 struct lima_device *ldev = platform_get_drvdata(pdev);452 struct drm_device *ddev = ldev->ddev;453 454 sysfs_remove_bin_file(&ldev->dev->kobj, &lima_error_state_attr);455 456 drm_dev_unregister(ddev);457 458 /* stop autosuspend to make sure device is in active state */459 pm_runtime_set_autosuspend_delay(ldev->dev, -1);460 pm_runtime_disable(ldev->dev);461 462 lima_devfreq_fini(ldev);463 lima_device_fini(ldev);464 465 drm_dev_put(ddev);466 lima_sched_slab_fini();467}468 469static const struct lima_compatible lima_mali400_data = {470 .id = lima_gpu_mali400,471};472 473static const struct lima_compatible lima_mali450_data = {474 .id = lima_gpu_mali450,475};476 477static const struct of_device_id dt_match[] = {478 { .compatible = "arm,mali-400", .data = &lima_mali400_data },479 { .compatible = "arm,mali-450", .data = &lima_mali450_data },480 {}481};482MODULE_DEVICE_TABLE(of, dt_match);483 484static const struct dev_pm_ops lima_pm_ops = {485 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)486 SET_RUNTIME_PM_OPS(lima_device_suspend, lima_device_resume, NULL)487};488 489static struct platform_driver lima_platform_driver = {490 .probe = lima_pdev_probe,491 .remove_new = lima_pdev_remove,492 .driver = {493 .name = "lima",494 .pm = &lima_pm_ops,495 .of_match_table = dt_match,496 },497};498 499module_platform_driver(lima_platform_driver);500 501MODULE_AUTHOR("Lima Project Developers");502MODULE_DESCRIPTION("Lima DRM Driver");503MODULE_LICENSE("GPL v2");504MODULE_SOFTDEP("pre: governor_simpleondemand");505