672 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2015-2021, 2023 Linaro Limited4 */5#include <linux/device.h>6#include <linux/err.h>7#include <linux/errno.h>8#include <linux/mm.h>9#include <linux/slab.h>10#include <linux/tee_core.h>11#include <linux/types.h>12#include "optee_private.h"13 14#define MAX_ARG_PARAM_COUNT 615 16/*17 * How much memory we allocate for each entry. This doesn't have to be a18 * single page, but it makes sense to keep at least keep it as multiples of19 * the page size.20 */21#define SHM_ENTRY_SIZE PAGE_SIZE22 23/*24 * We need to have a compile time constant to be able to determine the25 * maximum needed size of the bit field.26 */27#define MIN_ARG_SIZE OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT)28#define MAX_ARG_COUNT_PER_ENTRY (SHM_ENTRY_SIZE / MIN_ARG_SIZE)29 30/*31 * Shared memory for argument structs are cached here. The number of32 * arguments structs that can fit is determined at runtime depending on the33 * needed RPC parameter count reported by secure world34 * (optee->rpc_param_count).35 */36struct optee_shm_arg_entry {37 struct list_head list_node;38 struct tee_shm *shm;39 DECLARE_BITMAP(map, MAX_ARG_COUNT_PER_ENTRY);40};41 42void optee_cq_init(struct optee_call_queue *cq, int thread_count)43{44 mutex_init(&cq->mutex);45 INIT_LIST_HEAD(&cq->waiters);46 47 /*48 * If cq->total_thread_count is 0 then we're not trying to keep49 * track of how many free threads we have, instead we're relying on50 * the secure world to tell us when we're out of thread and have to51 * wait for another thread to become available.52 */53 cq->total_thread_count = thread_count;54 cq->free_thread_count = thread_count;55}56 57void optee_cq_wait_init(struct optee_call_queue *cq,58 struct optee_call_waiter *w, bool sys_thread)59{60 unsigned int free_thread_threshold;61 bool need_wait = false;62 63 memset(w, 0, sizeof(*w));64 65 /*66 * We're preparing to make a call to secure world. In case we can't67 * allocate a thread in secure world we'll end up waiting in68 * optee_cq_wait_for_completion().69 *70 * Normally if there's no contention in secure world the call will71 * complete and we can cleanup directly with optee_cq_wait_final().72 */73 mutex_lock(&cq->mutex);74 75 /*76 * We add ourselves to the queue, but we don't wait. This77 * guarantees that we don't lose a completion if secure world78 * returns busy and another thread just exited and try to complete79 * someone.80 */81 init_completion(&w->c);82 list_add_tail(&w->list_node, &cq->waiters);83 w->sys_thread = sys_thread;84 85 if (cq->total_thread_count) {86 if (sys_thread || !cq->sys_thread_req_count)87 free_thread_threshold = 0;88 else89 free_thread_threshold = 1;90 91 if (cq->free_thread_count > free_thread_threshold)92 cq->free_thread_count--;93 else94 need_wait = true;95 }96 97 mutex_unlock(&cq->mutex);98 99 while (need_wait) {100 optee_cq_wait_for_completion(cq, w);101 mutex_lock(&cq->mutex);102 103 if (sys_thread || !cq->sys_thread_req_count)104 free_thread_threshold = 0;105 else106 free_thread_threshold = 1;107 108 if (cq->free_thread_count > free_thread_threshold) {109 cq->free_thread_count--;110 need_wait = false;111 }112 113 mutex_unlock(&cq->mutex);114 }115}116 117void optee_cq_wait_for_completion(struct optee_call_queue *cq,118 struct optee_call_waiter *w)119{120 wait_for_completion(&w->c);121 122 mutex_lock(&cq->mutex);123 124 /* Move to end of list to get out of the way for other waiters */125 list_del(&w->list_node);126 reinit_completion(&w->c);127 list_add_tail(&w->list_node, &cq->waiters);128 129 mutex_unlock(&cq->mutex);130}131 132static void optee_cq_complete_one(struct optee_call_queue *cq)133{134 struct optee_call_waiter *w;135 136 /* Wake a waiting system session if any, prior to a normal session */137 list_for_each_entry(w, &cq->waiters, list_node) {138 if (w->sys_thread && !completion_done(&w->c)) {139 complete(&w->c);140 return;141 }142 }143 144 list_for_each_entry(w, &cq->waiters, list_node) {145 if (!completion_done(&w->c)) {146 complete(&w->c);147 break;148 }149 }150}151 152void optee_cq_wait_final(struct optee_call_queue *cq,153 struct optee_call_waiter *w)154{155 /*156 * We're done with the call to secure world. The thread in secure157 * world that was used for this call is now available for some158 * other task to use.159 */160 mutex_lock(&cq->mutex);161 162 /* Get out of the list */163 list_del(&w->list_node);164 165 cq->free_thread_count++;166 167 /* Wake up one eventual waiting task */168 optee_cq_complete_one(cq);169 170 /*171 * If we're completed we've got a completion from another task that172 * was just done with its call to secure world. Since yet another173 * thread now is available in secure world wake up another eventual174 * waiting task.175 */176 if (completion_done(&w->c))177 optee_cq_complete_one(cq);178 179 mutex_unlock(&cq->mutex);180}181 182/* Count registered system sessions to reserved a system thread or not */183static bool optee_cq_incr_sys_thread_count(struct optee_call_queue *cq)184{185 if (cq->total_thread_count <= 1)186 return false;187 188 mutex_lock(&cq->mutex);189 cq->sys_thread_req_count++;190 mutex_unlock(&cq->mutex);191 192 return true;193}194 195static void optee_cq_decr_sys_thread_count(struct optee_call_queue *cq)196{197 mutex_lock(&cq->mutex);198 cq->sys_thread_req_count--;199 /* If there's someone waiting, let it resume */200 optee_cq_complete_one(cq);201 mutex_unlock(&cq->mutex);202}203 204/* Requires the filpstate mutex to be held */205static struct optee_session *find_session(struct optee_context_data *ctxdata,206 u32 session_id)207{208 struct optee_session *sess;209 210 list_for_each_entry(sess, &ctxdata->sess_list, list_node)211 if (sess->session_id == session_id)212 return sess;213 214 return NULL;215}216 217void optee_shm_arg_cache_init(struct optee *optee, u32 flags)218{219 INIT_LIST_HEAD(&optee->shm_arg_cache.shm_args);220 mutex_init(&optee->shm_arg_cache.mutex);221 optee->shm_arg_cache.flags = flags;222}223 224void optee_shm_arg_cache_uninit(struct optee *optee)225{226 struct list_head *head = &optee->shm_arg_cache.shm_args;227 struct optee_shm_arg_entry *entry;228 229 mutex_destroy(&optee->shm_arg_cache.mutex);230 while (!list_empty(head)) {231 entry = list_first_entry(head, struct optee_shm_arg_entry,232 list_node);233 list_del(&entry->list_node);234 if (find_first_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY) !=235 MAX_ARG_COUNT_PER_ENTRY) {236 pr_err("Freeing non-free entry\n");237 }238 tee_shm_free(entry->shm);239 kfree(entry);240 }241}242 243size_t optee_msg_arg_size(size_t rpc_param_count)244{245 size_t sz = OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT);246 247 if (rpc_param_count)248 sz += OPTEE_MSG_GET_ARG_SIZE(rpc_param_count);249 250 return sz;251}252 253/**254 * optee_get_msg_arg() - Provide shared memory for argument struct255 * @ctx: Caller TEE context256 * @num_params: Number of parameter to store257 * @entry_ret: Entry pointer, needed when freeing the buffer258 * @shm_ret: Shared memory buffer259 * @offs_ret: Offset of argument strut in shared memory buffer260 *261 * @returns a pointer to the argument struct in memory, else an ERR_PTR262 */263struct optee_msg_arg *optee_get_msg_arg(struct tee_context *ctx,264 size_t num_params,265 struct optee_shm_arg_entry **entry_ret,266 struct tee_shm **shm_ret,267 u_int *offs_ret)268{269 struct optee *optee = tee_get_drvdata(ctx->teedev);270 size_t sz = optee_msg_arg_size(optee->rpc_param_count);271 struct optee_shm_arg_entry *entry;272 struct optee_msg_arg *ma;273 size_t args_per_entry;274 u_long bit;275 u_int offs;276 void *res;277 278 if (num_params > MAX_ARG_PARAM_COUNT)279 return ERR_PTR(-EINVAL);280 281 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_SHARED)282 args_per_entry = SHM_ENTRY_SIZE / sz;283 else284 args_per_entry = 1;285 286 mutex_lock(&optee->shm_arg_cache.mutex);287 list_for_each_entry(entry, &optee->shm_arg_cache.shm_args, list_node) {288 bit = find_first_zero_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY);289 if (bit < args_per_entry)290 goto have_entry;291 }292 293 /*294 * No entry was found, let's allocate a new.295 */296 entry = kzalloc(sizeof(*entry), GFP_KERNEL);297 if (!entry) {298 res = ERR_PTR(-ENOMEM);299 goto out;300 }301 302 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_ALLOC_PRIV)303 res = tee_shm_alloc_priv_buf(ctx, SHM_ENTRY_SIZE);304 else305 res = tee_shm_alloc_kernel_buf(ctx, SHM_ENTRY_SIZE);306 307 if (IS_ERR(res)) {308 kfree(entry);309 goto out;310 }311 entry->shm = res;312 list_add(&entry->list_node, &optee->shm_arg_cache.shm_args);313 bit = 0;314 315have_entry:316 offs = bit * sz;317 res = tee_shm_get_va(entry->shm, offs);318 if (IS_ERR(res))319 goto out;320 ma = res;321 set_bit(bit, entry->map);322 memset(ma, 0, sz);323 ma->num_params = num_params;324 *entry_ret = entry;325 *shm_ret = entry->shm;326 *offs_ret = offs;327out:328 mutex_unlock(&optee->shm_arg_cache.mutex);329 return res;330}331 332/**333 * optee_free_msg_arg() - Free previsouly obtained shared memory334 * @ctx: Caller TEE context335 * @entry: Pointer returned when the shared memory was obtained336 * @offs: Offset of shared memory buffer to free337 *338 * This function frees the shared memory obtained with optee_get_msg_arg().339 */340void optee_free_msg_arg(struct tee_context *ctx,341 struct optee_shm_arg_entry *entry, u_int offs)342{343 struct optee *optee = tee_get_drvdata(ctx->teedev);344 size_t sz = optee_msg_arg_size(optee->rpc_param_count);345 u_long bit;346 347 if (offs > SHM_ENTRY_SIZE || offs % sz) {348 pr_err("Invalid offs %u\n", offs);349 return;350 }351 bit = offs / sz;352 353 mutex_lock(&optee->shm_arg_cache.mutex);354 355 if (!test_bit(bit, entry->map))356 pr_err("Bit pos %lu is already free\n", bit);357 clear_bit(bit, entry->map);358 359 mutex_unlock(&optee->shm_arg_cache.mutex);360}361 362int optee_open_session(struct tee_context *ctx,363 struct tee_ioctl_open_session_arg *arg,364 struct tee_param *param)365{366 struct optee *optee = tee_get_drvdata(ctx->teedev);367 struct optee_context_data *ctxdata = ctx->data;368 struct optee_shm_arg_entry *entry;369 struct tee_shm *shm;370 struct optee_msg_arg *msg_arg;371 struct optee_session *sess = NULL;372 uuid_t client_uuid;373 u_int offs;374 int rc;375 376 /* +2 for the meta parameters added below */377 msg_arg = optee_get_msg_arg(ctx, arg->num_params + 2,378 &entry, &shm, &offs);379 if (IS_ERR(msg_arg))380 return PTR_ERR(msg_arg);381 382 msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION;383 msg_arg->cancel_id = arg->cancel_id;384 385 /*386 * Initialize and add the meta parameters needed when opening a387 * session.388 */389 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |390 OPTEE_MSG_ATTR_META;391 msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |392 OPTEE_MSG_ATTR_META;393 memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));394 msg_arg->params[1].u.value.c = arg->clnt_login;395 396 rc = tee_session_calc_client_uuid(&client_uuid, arg->clnt_login,397 arg->clnt_uuid);398 if (rc)399 goto out;400 export_uuid(msg_arg->params[1].u.octets, &client_uuid);401 402 rc = optee->ops->to_msg_param(optee, msg_arg->params + 2,403 arg->num_params, param);404 if (rc)405 goto out;406 407 sess = kzalloc(sizeof(*sess), GFP_KERNEL);408 if (!sess) {409 rc = -ENOMEM;410 goto out;411 }412 413 if (optee->ops->do_call_with_arg(ctx, shm, offs,414 sess->use_sys_thread)) {415 msg_arg->ret = TEEC_ERROR_COMMUNICATION;416 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;417 }418 419 if (msg_arg->ret == TEEC_SUCCESS) {420 /* A new session has been created, add it to the list. */421 sess->session_id = msg_arg->session;422 mutex_lock(&ctxdata->mutex);423 list_add(&sess->list_node, &ctxdata->sess_list);424 mutex_unlock(&ctxdata->mutex);425 } else {426 kfree(sess);427 }428 429 if (optee->ops->from_msg_param(optee, param, arg->num_params,430 msg_arg->params + 2)) {431 arg->ret = TEEC_ERROR_COMMUNICATION;432 arg->ret_origin = TEEC_ORIGIN_COMMS;433 /* Close session again to avoid leakage */434 optee_close_session(ctx, msg_arg->session);435 } else {436 arg->session = msg_arg->session;437 arg->ret = msg_arg->ret;438 arg->ret_origin = msg_arg->ret_origin;439 }440out:441 optee_free_msg_arg(ctx, entry, offs);442 443 return rc;444}445 446int optee_system_session(struct tee_context *ctx, u32 session)447{448 struct optee *optee = tee_get_drvdata(ctx->teedev);449 struct optee_context_data *ctxdata = ctx->data;450 struct optee_session *sess;451 int rc = -EINVAL;452 453 mutex_lock(&ctxdata->mutex);454 455 sess = find_session(ctxdata, session);456 if (sess && (sess->use_sys_thread ||457 optee_cq_incr_sys_thread_count(&optee->call_queue))) {458 sess->use_sys_thread = true;459 rc = 0;460 }461 462 mutex_unlock(&ctxdata->mutex);463 464 return rc;465}466 467int optee_close_session_helper(struct tee_context *ctx, u32 session,468 bool system_thread)469{470 struct optee *optee = tee_get_drvdata(ctx->teedev);471 struct optee_shm_arg_entry *entry;472 struct optee_msg_arg *msg_arg;473 struct tee_shm *shm;474 u_int offs;475 476 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);477 if (IS_ERR(msg_arg))478 return PTR_ERR(msg_arg);479 480 msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;481 msg_arg->session = session;482 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);483 484 optee_free_msg_arg(ctx, entry, offs);485 486 if (system_thread)487 optee_cq_decr_sys_thread_count(&optee->call_queue);488 489 return 0;490}491 492int optee_close_session(struct tee_context *ctx, u32 session)493{494 struct optee_context_data *ctxdata = ctx->data;495 struct optee_session *sess;496 bool system_thread;497 498 /* Check that the session is valid and remove it from the list */499 mutex_lock(&ctxdata->mutex);500 sess = find_session(ctxdata, session);501 if (sess)502 list_del(&sess->list_node);503 mutex_unlock(&ctxdata->mutex);504 if (!sess)505 return -EINVAL;506 system_thread = sess->use_sys_thread;507 kfree(sess);508 509 return optee_close_session_helper(ctx, session, system_thread);510}511 512int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,513 struct tee_param *param)514{515 struct optee *optee = tee_get_drvdata(ctx->teedev);516 struct optee_context_data *ctxdata = ctx->data;517 struct optee_shm_arg_entry *entry;518 struct optee_msg_arg *msg_arg;519 struct optee_session *sess;520 struct tee_shm *shm;521 bool system_thread;522 u_int offs;523 int rc;524 525 /* Check that the session is valid */526 mutex_lock(&ctxdata->mutex);527 sess = find_session(ctxdata, arg->session);528 if (sess)529 system_thread = sess->use_sys_thread;530 mutex_unlock(&ctxdata->mutex);531 if (!sess)532 return -EINVAL;533 534 msg_arg = optee_get_msg_arg(ctx, arg->num_params,535 &entry, &shm, &offs);536 if (IS_ERR(msg_arg))537 return PTR_ERR(msg_arg);538 msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND;539 msg_arg->func = arg->func;540 msg_arg->session = arg->session;541 msg_arg->cancel_id = arg->cancel_id;542 543 rc = optee->ops->to_msg_param(optee, msg_arg->params, arg->num_params,544 param);545 if (rc)546 goto out;547 548 if (optee->ops->do_call_with_arg(ctx, shm, offs, system_thread)) {549 msg_arg->ret = TEEC_ERROR_COMMUNICATION;550 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;551 }552 553 if (optee->ops->from_msg_param(optee, param, arg->num_params,554 msg_arg->params)) {555 msg_arg->ret = TEEC_ERROR_COMMUNICATION;556 msg_arg->ret_origin = TEEC_ORIGIN_COMMS;557 }558 559 arg->ret = msg_arg->ret;560 arg->ret_origin = msg_arg->ret_origin;561out:562 optee_free_msg_arg(ctx, entry, offs);563 return rc;564}565 566int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)567{568 struct optee *optee = tee_get_drvdata(ctx->teedev);569 struct optee_context_data *ctxdata = ctx->data;570 struct optee_shm_arg_entry *entry;571 struct optee_msg_arg *msg_arg;572 struct optee_session *sess;573 bool system_thread;574 struct tee_shm *shm;575 u_int offs;576 577 /* Check that the session is valid */578 mutex_lock(&ctxdata->mutex);579 sess = find_session(ctxdata, session);580 if (sess)581 system_thread = sess->use_sys_thread;582 mutex_unlock(&ctxdata->mutex);583 if (!sess)584 return -EINVAL;585 586 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);587 if (IS_ERR(msg_arg))588 return PTR_ERR(msg_arg);589 590 msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;591 msg_arg->session = session;592 msg_arg->cancel_id = cancel_id;593 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);594 595 optee_free_msg_arg(ctx, entry, offs);596 return 0;597}598 599static bool is_normal_memory(pgprot_t p)600{601#if defined(CONFIG_ARM)602 return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) ||603 ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK));604#elif defined(CONFIG_ARM64)605 return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);606#else607#error "Unsupported architecture"608#endif609}610 611static int __check_mem_type(struct mm_struct *mm, unsigned long start,612 unsigned long end)613{614 struct vm_area_struct *vma;615 VMA_ITERATOR(vmi, mm, start);616 617 for_each_vma_range(vmi, vma, end) {618 if (!is_normal_memory(vma->vm_page_prot))619 return -EINVAL;620 }621 622 return 0;623}624 625int optee_check_mem_type(unsigned long start, size_t num_pages)626{627 struct mm_struct *mm = current->mm;628 int rc;629 630 /*631 * Allow kernel address to register with OP-TEE as kernel632 * pages are configured as normal memory only.633 */634 if (virt_addr_valid((void *)start) || is_vmalloc_addr((void *)start))635 return 0;636 637 mmap_read_lock(mm);638 rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE);639 mmap_read_unlock(mm);640 641 return rc;642}643 644static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)645{646 struct optee *optee = tee_get_drvdata(ctx->teedev);647 struct optee_shm_arg_entry *entry;648 struct optee_msg_arg *msg_arg;649 struct tee_shm *shm;650 u_int offs;651 652 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);653 if (IS_ERR(msg_arg))654 return PTR_ERR(msg_arg);655 656 msg_arg->cmd = cmd;657 optee->ops->do_call_with_arg(ctx, shm, offs, false);658 659 optee_free_msg_arg(ctx, entry, offs);660 return 0;661}662 663int optee_do_bottom_half(struct tee_context *ctx)664{665 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);666}667 668int optee_stop_async_notif(struct tee_context *ctx)669{670 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);671}672