498 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_cccb.h"5#include "pvr_context.h"6#include "pvr_device.h"7#include "pvr_drv.h"8#include "pvr_gem.h"9#include "pvr_job.h"10#include "pvr_power.h"11#include "pvr_rogue_fwif.h"12#include "pvr_rogue_fwif_common.h"13#include "pvr_rogue_fwif_resetframework.h"14#include "pvr_stream.h"15#include "pvr_stream_defs.h"16#include "pvr_vm.h"17 18#include <drm/drm_auth.h>19#include <drm/drm_managed.h>20 21#include <linux/bug.h>22#include <linux/errno.h>23#include <linux/kernel.h>24#include <linux/list.h>25#include <linux/sched.h>26#include <linux/slab.h>27#include <linux/spinlock.h>28#include <linux/string.h>29#include <linux/types.h>30#include <linux/xarray.h>31 32static int33remap_priority(struct pvr_file *pvr_file, s32 uapi_priority,34 enum pvr_context_priority *priority_out)35{36 switch (uapi_priority) {37 case DRM_PVR_CTX_PRIORITY_LOW:38 *priority_out = PVR_CTX_PRIORITY_LOW;39 break;40 case DRM_PVR_CTX_PRIORITY_NORMAL:41 *priority_out = PVR_CTX_PRIORITY_MEDIUM;42 break;43 case DRM_PVR_CTX_PRIORITY_HIGH:44 if (!capable(CAP_SYS_NICE) && !drm_is_current_master(from_pvr_file(pvr_file)))45 return -EACCES;46 *priority_out = PVR_CTX_PRIORITY_HIGH;47 break;48 default:49 return -EINVAL;50 }51 52 return 0;53}54 55static int get_fw_obj_size(enum drm_pvr_ctx_type type)56{57 switch (type) {58 case DRM_PVR_CTX_TYPE_RENDER:59 return sizeof(struct rogue_fwif_fwrendercontext);60 case DRM_PVR_CTX_TYPE_COMPUTE:61 return sizeof(struct rogue_fwif_fwcomputecontext);62 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:63 return sizeof(struct rogue_fwif_fwtransfercontext);64 }65 66 return -EINVAL;67}68 69static int70process_static_context_state(struct pvr_device *pvr_dev, const struct pvr_stream_cmd_defs *cmd_defs,71 u64 stream_user_ptr, u32 stream_size, void *dest)72{73 void *stream;74 int err;75 76 stream = kzalloc(stream_size, GFP_KERNEL);77 if (!stream)78 return -ENOMEM;79 80 if (copy_from_user(stream, u64_to_user_ptr(stream_user_ptr), stream_size)) {81 err = -EFAULT;82 goto err_free;83 }84 85 err = pvr_stream_process(pvr_dev, cmd_defs, stream, stream_size, dest);86 if (err)87 goto err_free;88 89 kfree(stream);90 91 return 0;92 93err_free:94 kfree(stream);95 96 return err;97}98 99static int init_render_fw_objs(struct pvr_context *ctx,100 struct drm_pvr_ioctl_create_context_args *args,101 void *fw_ctx_map)102{103 struct rogue_fwif_static_rendercontext_state *static_rendercontext_state;104 struct rogue_fwif_fwrendercontext *fw_render_context = fw_ctx_map;105 106 if (!args->static_context_state_len)107 return -EINVAL;108 109 static_rendercontext_state = &fw_render_context->static_render_context_state;110 111 /* Copy static render context state from userspace. */112 return process_static_context_state(ctx->pvr_dev,113 &pvr_static_render_context_state_stream,114 args->static_context_state,115 args->static_context_state_len,116 &static_rendercontext_state->ctxswitch_regs[0]);117}118 119static int init_compute_fw_objs(struct pvr_context *ctx,120 struct drm_pvr_ioctl_create_context_args *args,121 void *fw_ctx_map)122{123 struct rogue_fwif_fwcomputecontext *fw_compute_context = fw_ctx_map;124 struct rogue_fwif_cdm_registers_cswitch *ctxswitch_regs;125 126 if (!args->static_context_state_len)127 return -EINVAL;128 129 ctxswitch_regs = &fw_compute_context->static_compute_context_state.ctxswitch_regs;130 131 /* Copy static render context state from userspace. */132 return process_static_context_state(ctx->pvr_dev,133 &pvr_static_compute_context_state_stream,134 args->static_context_state,135 args->static_context_state_len,136 ctxswitch_regs);137}138 139static int init_transfer_fw_objs(struct pvr_context *ctx,140 struct drm_pvr_ioctl_create_context_args *args,141 void *fw_ctx_map)142{143 if (args->static_context_state_len)144 return -EINVAL;145 146 return 0;147}148 149static int init_fw_objs(struct pvr_context *ctx,150 struct drm_pvr_ioctl_create_context_args *args,151 void *fw_ctx_map)152{153 switch (ctx->type) {154 case DRM_PVR_CTX_TYPE_RENDER:155 return init_render_fw_objs(ctx, args, fw_ctx_map);156 case DRM_PVR_CTX_TYPE_COMPUTE:157 return init_compute_fw_objs(ctx, args, fw_ctx_map);158 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:159 return init_transfer_fw_objs(ctx, args, fw_ctx_map);160 }161 162 return -EINVAL;163}164 165static void166ctx_fw_data_init(void *cpu_ptr, void *priv)167{168 struct pvr_context *ctx = priv;169 170 memcpy(cpu_ptr, ctx->data, ctx->data_size);171}172 173/**174 * pvr_context_destroy_queues() - Destroy all queues attached to a context.175 * @ctx: Context to destroy queues on.176 *177 * Should be called when the last reference to a context object is dropped.178 * It releases all resources attached to the queues bound to this context.179 */180static void pvr_context_destroy_queues(struct pvr_context *ctx)181{182 switch (ctx->type) {183 case DRM_PVR_CTX_TYPE_RENDER:184 pvr_queue_destroy(ctx->queues.fragment);185 pvr_queue_destroy(ctx->queues.geometry);186 break;187 case DRM_PVR_CTX_TYPE_COMPUTE:188 pvr_queue_destroy(ctx->queues.compute);189 break;190 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:191 pvr_queue_destroy(ctx->queues.transfer);192 break;193 }194}195 196/**197 * pvr_context_create_queues() - Create all queues attached to a context.198 * @ctx: Context to create queues on.199 * @args: Context creation arguments passed by userspace.200 * @fw_ctx_map: CPU mapping of the FW context object.201 *202 * Return:203 * * 0 on success, or204 * * A negative error code otherwise.205 */206static int pvr_context_create_queues(struct pvr_context *ctx,207 struct drm_pvr_ioctl_create_context_args *args,208 void *fw_ctx_map)209{210 int err;211 212 switch (ctx->type) {213 case DRM_PVR_CTX_TYPE_RENDER:214 ctx->queues.geometry = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_GEOMETRY,215 args, fw_ctx_map);216 if (IS_ERR(ctx->queues.geometry)) {217 err = PTR_ERR(ctx->queues.geometry);218 ctx->queues.geometry = NULL;219 goto err_destroy_queues;220 }221 222 ctx->queues.fragment = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_FRAGMENT,223 args, fw_ctx_map);224 if (IS_ERR(ctx->queues.fragment)) {225 err = PTR_ERR(ctx->queues.fragment);226 ctx->queues.fragment = NULL;227 goto err_destroy_queues;228 }229 return 0;230 231 case DRM_PVR_CTX_TYPE_COMPUTE:232 ctx->queues.compute = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_COMPUTE,233 args, fw_ctx_map);234 if (IS_ERR(ctx->queues.compute)) {235 err = PTR_ERR(ctx->queues.compute);236 ctx->queues.compute = NULL;237 goto err_destroy_queues;238 }239 return 0;240 241 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:242 ctx->queues.transfer = pvr_queue_create(ctx, DRM_PVR_JOB_TYPE_TRANSFER_FRAG,243 args, fw_ctx_map);244 if (IS_ERR(ctx->queues.transfer)) {245 err = PTR_ERR(ctx->queues.transfer);246 ctx->queues.transfer = NULL;247 goto err_destroy_queues;248 }249 return 0;250 }251 252 return -EINVAL;253 254err_destroy_queues:255 pvr_context_destroy_queues(ctx);256 return err;257}258 259/**260 * pvr_context_kill_queues() - Kill queues attached to context.261 * @ctx: Context to kill queues on.262 *263 * Killing the queues implies making them unusable for future jobs, while still264 * letting the currently submitted jobs a chance to finish. Queue resources will265 * stay around until pvr_context_destroy_queues() is called.266 */267static void pvr_context_kill_queues(struct pvr_context *ctx)268{269 switch (ctx->type) {270 case DRM_PVR_CTX_TYPE_RENDER:271 pvr_queue_kill(ctx->queues.fragment);272 pvr_queue_kill(ctx->queues.geometry);273 break;274 case DRM_PVR_CTX_TYPE_COMPUTE:275 pvr_queue_kill(ctx->queues.compute);276 break;277 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:278 pvr_queue_kill(ctx->queues.transfer);279 break;280 }281}282 283/**284 * pvr_context_create() - Create a context.285 * @pvr_file: File to attach the created context to.286 * @args: Context creation arguments.287 *288 * Return:289 * * 0 on success, or290 * * A negative error code on failure.291 */292int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_context_args *args)293{294 struct pvr_device *pvr_dev = pvr_file->pvr_dev;295 struct pvr_context *ctx;296 int ctx_size;297 int err;298 299 /* Context creation flags are currently unused and must be zero. */300 if (args->flags)301 return -EINVAL;302 303 ctx_size = get_fw_obj_size(args->type);304 if (ctx_size < 0)305 return ctx_size;306 307 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);308 if (!ctx)309 return -ENOMEM;310 311 ctx->data_size = ctx_size;312 ctx->type = args->type;313 ctx->flags = args->flags;314 ctx->pvr_dev = pvr_dev;315 kref_init(&ctx->ref_count);316 317 err = remap_priority(pvr_file, args->priority, &ctx->priority);318 if (err)319 goto err_free_ctx;320 321 ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle);322 if (IS_ERR(ctx->vm_ctx)) {323 err = PTR_ERR(ctx->vm_ctx);324 goto err_free_ctx;325 }326 327 ctx->data = kzalloc(ctx_size, GFP_KERNEL);328 if (!ctx->data) {329 err = -ENOMEM;330 goto err_put_vm;331 }332 333 err = pvr_context_create_queues(ctx, args, ctx->data);334 if (err)335 goto err_free_ctx_data;336 337 err = init_fw_objs(ctx, args, ctx->data);338 if (err)339 goto err_destroy_queues;340 341 err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED,342 ctx_fw_data_init, ctx, &ctx->fw_obj);343 if (err)344 goto err_free_ctx_data;345 346 err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL);347 if (err)348 goto err_destroy_fw_obj;349 350 err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL);351 if (err) {352 /*353 * It's possible that another thread could have taken a reference on the context at354 * this point as it is in the ctx_ids xarray. Therefore instead of directly355 * destroying the context, drop a reference instead.356 */357 pvr_context_put(ctx);358 return err;359 }360 361 spin_lock(&pvr_dev->ctx_list_lock);362 list_add_tail(&ctx->file_link, &pvr_file->contexts);363 spin_unlock(&pvr_dev->ctx_list_lock);364 365 return 0;366 367err_destroy_fw_obj:368 pvr_fw_object_destroy(ctx->fw_obj);369 370err_destroy_queues:371 pvr_context_destroy_queues(ctx);372 373err_free_ctx_data:374 kfree(ctx->data);375 376err_put_vm:377 pvr_vm_context_put(ctx->vm_ctx);378 379err_free_ctx:380 kfree(ctx);381 return err;382}383 384static void385pvr_context_release(struct kref *ref_count)386{387 struct pvr_context *ctx =388 container_of(ref_count, struct pvr_context, ref_count);389 struct pvr_device *pvr_dev = ctx->pvr_dev;390 391 WARN_ON(in_interrupt());392 spin_lock(&pvr_dev->ctx_list_lock);393 list_del(&ctx->file_link);394 spin_unlock(&pvr_dev->ctx_list_lock);395 396 xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id);397 pvr_context_destroy_queues(ctx);398 pvr_fw_object_destroy(ctx->fw_obj);399 kfree(ctx->data);400 pvr_vm_context_put(ctx->vm_ctx);401 kfree(ctx);402}403 404/**405 * pvr_context_put() - Release reference on context406 * @ctx: Target context.407 */408void409pvr_context_put(struct pvr_context *ctx)410{411 if (ctx)412 kref_put(&ctx->ref_count, pvr_context_release);413}414 415/**416 * pvr_context_destroy() - Destroy context417 * @pvr_file: Pointer to pvr_file structure.418 * @handle: Userspace context handle.419 *420 * Removes context from context list and drops initial reference. Context will421 * then be destroyed once all outstanding references are dropped.422 *423 * Return:424 * * 0 on success, or425 * * -%EINVAL if context not in context list.426 */427int428pvr_context_destroy(struct pvr_file *pvr_file, u32 handle)429{430 struct pvr_context *ctx = xa_erase(&pvr_file->ctx_handles, handle);431 432 if (!ctx)433 return -EINVAL;434 435 /* Make sure nothing can be queued to the queues after that point. */436 pvr_context_kill_queues(ctx);437 438 /* Release the reference held by the handle set. */439 pvr_context_put(ctx);440 441 return 0;442}443 444/**445 * pvr_destroy_contexts_for_file: Destroy any contexts associated with the given file446 * @pvr_file: Pointer to pvr_file structure.447 *448 * Removes all contexts associated with @pvr_file from the device context list and drops initial449 * references. Contexts will then be destroyed once all outstanding references are dropped.450 */451void pvr_destroy_contexts_for_file(struct pvr_file *pvr_file)452{453 struct pvr_device *pvr_dev = pvr_file->pvr_dev;454 struct pvr_context *ctx;455 unsigned long handle;456 457 xa_for_each(&pvr_file->ctx_handles, handle, ctx)458 pvr_context_destroy(pvr_file, handle);459 460 spin_lock(&pvr_dev->ctx_list_lock);461 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link);462 463 while (!list_entry_is_head(ctx, &pvr_file->contexts, file_link)) {464 list_del_init(&ctx->file_link);465 466 if (pvr_context_get_if_referenced(ctx)) {467 spin_unlock(&pvr_dev->ctx_list_lock);468 469 pvr_vm_unmap_all(ctx->vm_ctx);470 471 pvr_context_put(ctx);472 spin_lock(&pvr_dev->ctx_list_lock);473 }474 ctx = list_first_entry(&pvr_file->contexts, struct pvr_context, file_link);475 }476 spin_unlock(&pvr_dev->ctx_list_lock);477}478 479/**480 * pvr_context_device_init() - Device level initialization for queue related resources.481 * @pvr_dev: The device to initialize.482 */483void pvr_context_device_init(struct pvr_device *pvr_dev)484{485 xa_init_flags(&pvr_dev->ctx_ids, XA_FLAGS_ALLOC1);486 spin_lock_init(&pvr_dev->ctx_list_lock);487}488 489/**490 * pvr_context_device_fini() - Device level cleanup for queue related resources.491 * @pvr_dev: The device to cleanup.492 */493void pvr_context_device_fini(struct pvr_device *pvr_dev)494{495 WARN_ON(!xa_empty(&pvr_dev->ctx_ids));496 xa_destroy(&pvr_dev->ctx_ids);497}498