1837 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2015-2021, 2023 Linaro Limited4 * Copyright (c) 2016, EPAM Systems5 */6 7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt8 9#include <linux/arm-smccc.h>10#include <linux/cpuhotplug.h>11#include <linux/errno.h>12#include <linux/firmware.h>13#include <linux/interrupt.h>14#include <linux/io.h>15#include <linux/irqdomain.h>16#include <linux/kernel.h>17#include <linux/mm.h>18#include <linux/module.h>19#include <linux/of.h>20#include <linux/of_irq.h>21#include <linux/of_platform.h>22#include <linux/platform_device.h>23#include <linux/rpmb.h>24#include <linux/sched.h>25#include <linux/slab.h>26#include <linux/string.h>27#include <linux/tee_core.h>28#include <linux/types.h>29#include <linux/workqueue.h>30#include "optee_private.h"31#include "optee_smc.h"32#include "optee_rpc_cmd.h"33#include <linux/kmemleak.h>34#define CREATE_TRACE_POINTS35#include "optee_trace.h"36 37/*38 * This file implement the SMC ABI used when communicating with secure world39 * OP-TEE OS via raw SMCs.40 * This file is divided into the following sections:41 * 1. Convert between struct tee_param and struct optee_msg_param42 * 2. Low level support functions to register shared memory in secure world43 * 3. Dynamic shared memory pool based on alloc_pages()44 * 4. Do a normal scheduled call into secure world45 * 5. Asynchronous notification46 * 6. Driver initialization.47 */48 49/*50 * A typical OP-TEE private shm allocation is 224 bytes (argument struct51 * with 6 parameters, needed for open session). So with an alignment of 51252 * we'll waste a bit more than 50%. However, it's only expected that we'll53 * have a handful of these structs allocated at a time. Most memory will54 * be allocated aligned to the page size, So all in all this should scale55 * up and down quite well.56 */57#define OPTEE_MIN_STATIC_POOL_ALIGN 9 /* 512 bytes aligned */58 59/* SMC ABI considers at most a single TEE firmware */60static unsigned int pcpu_irq_num;61 62static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)63{64 enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);65 66 return 0;67}68 69static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)70{71 disable_percpu_irq(pcpu_irq_num);72 73 return 0;74}75 76/*77 * 1. Convert between struct tee_param and struct optee_msg_param78 *79 * optee_from_msg_param() and optee_to_msg_param() are the main80 * functions.81 */82 83static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,84 const struct optee_msg_param *mp)85{86 struct tee_shm *shm;87 phys_addr_t pa;88 int rc;89 90 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +91 attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;92 p->u.memref.size = mp->u.tmem.size;93 shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;94 if (!shm) {95 p->u.memref.shm_offs = 0;96 p->u.memref.shm = NULL;97 return 0;98 }99 100 rc = tee_shm_get_pa(shm, 0, &pa);101 if (rc)102 return rc;103 104 p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;105 p->u.memref.shm = shm;106 107 return 0;108}109 110static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,111 const struct optee_msg_param *mp)112{113 struct tee_shm *shm;114 115 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +116 attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;117 p->u.memref.size = mp->u.rmem.size;118 shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;119 120 if (shm) {121 p->u.memref.shm_offs = mp->u.rmem.offs;122 p->u.memref.shm = shm;123 } else {124 p->u.memref.shm_offs = 0;125 p->u.memref.shm = NULL;126 }127}128 129/**130 * optee_from_msg_param() - convert from OPTEE_MSG parameters to131 * struct tee_param132 * @optee: main service struct133 * @params: subsystem internal parameter representation134 * @num_params: number of elements in the parameter arrays135 * @msg_params: OPTEE_MSG parameters136 * Returns 0 on success or <0 on failure137 */138static int optee_from_msg_param(struct optee *optee, struct tee_param *params,139 size_t num_params,140 const struct optee_msg_param *msg_params)141{142 int rc;143 size_t n;144 145 for (n = 0; n < num_params; n++) {146 struct tee_param *p = params + n;147 const struct optee_msg_param *mp = msg_params + n;148 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;149 150 switch (attr) {151 case OPTEE_MSG_ATTR_TYPE_NONE:152 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;153 memset(&p->u, 0, sizeof(p->u));154 break;155 case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:156 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:157 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:158 optee_from_msg_param_value(p, attr, mp);159 break;160 case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:161 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:162 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:163 rc = from_msg_param_tmp_mem(p, attr, mp);164 if (rc)165 return rc;166 break;167 case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:168 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:169 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:170 from_msg_param_reg_mem(p, attr, mp);171 break;172 173 default:174 return -EINVAL;175 }176 }177 return 0;178}179 180static int to_msg_param_tmp_mem(struct optee_msg_param *mp,181 const struct tee_param *p)182{183 int rc;184 phys_addr_t pa;185 186 mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -187 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;188 189 mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;190 mp->u.tmem.size = p->u.memref.size;191 192 if (!p->u.memref.shm) {193 mp->u.tmem.buf_ptr = 0;194 return 0;195 }196 197 rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);198 if (rc)199 return rc;200 201 mp->u.tmem.buf_ptr = pa;202 mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<203 OPTEE_MSG_ATTR_CACHE_SHIFT;204 205 return 0;206}207 208static int to_msg_param_reg_mem(struct optee_msg_param *mp,209 const struct tee_param *p)210{211 mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -212 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;213 214 mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;215 mp->u.rmem.size = p->u.memref.size;216 mp->u.rmem.offs = p->u.memref.shm_offs;217 return 0;218}219 220/**221 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters222 * @optee: main service struct223 * @msg_params: OPTEE_MSG parameters224 * @num_params: number of elements in the parameter arrays225 * @params: subsystem itnernal parameter representation226 * Returns 0 on success or <0 on failure227 */228static int optee_to_msg_param(struct optee *optee,229 struct optee_msg_param *msg_params,230 size_t num_params, const struct tee_param *params)231{232 int rc;233 size_t n;234 235 for (n = 0; n < num_params; n++) {236 const struct tee_param *p = params + n;237 struct optee_msg_param *mp = msg_params + n;238 239 switch (p->attr) {240 case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:241 mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;242 memset(&mp->u, 0, sizeof(mp->u));243 break;244 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:245 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:246 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:247 optee_to_msg_param_value(mp, p);248 break;249 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:250 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:251 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:252 if (tee_shm_is_dynamic(p->u.memref.shm))253 rc = to_msg_param_reg_mem(mp, p);254 else255 rc = to_msg_param_tmp_mem(mp, p);256 if (rc)257 return rc;258 break;259 default:260 return -EINVAL;261 }262 }263 return 0;264}265 266/*267 * 2. Low level support functions to register shared memory in secure world268 *269 * Functions to enable/disable shared memory caching in secure world, that270 * is, lazy freeing of previously allocated shared memory. Freeing is271 * performed when a request has been compled.272 *273 * Functions to register and unregister shared memory both for normal274 * clients and for tee-supplicant.275 */276 277/**278 * optee_enable_shm_cache() - Enables caching of some shared memory allocation279 * in OP-TEE280 * @optee: main service struct281 */282static void optee_enable_shm_cache(struct optee *optee)283{284 struct optee_call_waiter w;285 286 /* We need to retry until secure world isn't busy. */287 optee_cq_wait_init(&optee->call_queue, &w, false);288 while (true) {289 struct arm_smccc_res res;290 291 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,292 0, 0, 0, 0, 0, 0, 0, &res);293 if (res.a0 == OPTEE_SMC_RETURN_OK)294 break;295 optee_cq_wait_for_completion(&optee->call_queue, &w);296 }297 optee_cq_wait_final(&optee->call_queue, &w);298}299 300/**301 * __optee_disable_shm_cache() - Disables caching of some shared memory302 * allocation in OP-TEE303 * @optee: main service struct304 * @is_mapped: true if the cached shared memory addresses were mapped by this305 * kernel, are safe to dereference, and should be freed306 */307static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)308{309 struct optee_call_waiter w;310 311 /* We need to retry until secure world isn't busy. */312 optee_cq_wait_init(&optee->call_queue, &w, false);313 while (true) {314 union {315 struct arm_smccc_res smccc;316 struct optee_smc_disable_shm_cache_result result;317 } res;318 319 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,320 0, 0, 0, 0, 0, 0, 0, &res.smccc);321 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)322 break; /* All shm's freed */323 if (res.result.status == OPTEE_SMC_RETURN_OK) {324 struct tee_shm *shm;325 326 /*327 * Shared memory references that were not mapped by328 * this kernel must be ignored to prevent a crash.329 */330 if (!is_mapped)331 continue;332 333 shm = reg_pair_to_ptr(res.result.shm_upper32,334 res.result.shm_lower32);335 tee_shm_free(shm);336 } else {337 optee_cq_wait_for_completion(&optee->call_queue, &w);338 }339 }340 optee_cq_wait_final(&optee->call_queue, &w);341}342 343/**344 * optee_disable_shm_cache() - Disables caching of mapped shared memory345 * allocations in OP-TEE346 * @optee: main service struct347 */348static void optee_disable_shm_cache(struct optee *optee)349{350 return __optee_disable_shm_cache(optee, true);351}352 353/**354 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory355 * allocations in OP-TEE which are not356 * currently mapped357 * @optee: main service struct358 */359static void optee_disable_unmapped_shm_cache(struct optee *optee)360{361 return __optee_disable_shm_cache(optee, false);362}363 364#define PAGELIST_ENTRIES_PER_PAGE \365 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)366 367/*368 * The final entry in each pagelist page is a pointer to the next369 * pagelist page.370 */371static size_t get_pages_list_size(size_t num_entries)372{373 int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);374 375 return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;376}377 378static u64 *optee_allocate_pages_list(size_t num_entries)379{380 return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);381}382 383static void optee_free_pages_list(void *list, size_t num_entries)384{385 free_pages_exact(list, get_pages_list_size(num_entries));386}387 388/**389 * optee_fill_pages_list() - write list of user pages to given shared390 * buffer.391 *392 * @dst: page-aligned buffer where list of pages will be stored393 * @pages: array of pages that represents shared buffer394 * @num_pages: number of entries in @pages395 * @page_offset: offset of user buffer from page start396 *397 * @dst should be big enough to hold list of user page addresses and398 * links to the next pages of buffer399 */400static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,401 size_t page_offset)402{403 int n = 0;404 phys_addr_t optee_page;405 /*406 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h407 * for details.408 */409 struct {410 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];411 u64 next_page_data;412 } *pages_data;413 414 /*415 * Currently OP-TEE uses 4k page size and it does not looks416 * like this will change in the future. On other hand, there are417 * no know ARM architectures with page size < 4k.418 * Thus the next built assert looks redundant. But the following419 * code heavily relies on this assumption, so it is better be420 * safe than sorry.421 */422 BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);423 424 pages_data = (void *)dst;425 /*426 * If linux page is bigger than 4k, and user buffer offset is427 * larger than 4k/8k/12k/etc this will skip first 4k pages,428 * because they bear no value data for OP-TEE.429 */430 optee_page = page_to_phys(*pages) +431 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);432 433 while (true) {434 pages_data->pages_list[n++] = optee_page;435 436 if (n == PAGELIST_ENTRIES_PER_PAGE) {437 pages_data->next_page_data =438 virt_to_phys(pages_data + 1);439 pages_data++;440 n = 0;441 }442 443 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;444 if (!(optee_page & ~PAGE_MASK)) {445 if (!--num_pages)446 break;447 pages++;448 optee_page = page_to_phys(*pages);449 }450 }451}452 453static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,454 struct page **pages, size_t num_pages,455 unsigned long start)456{457 struct optee *optee = tee_get_drvdata(ctx->teedev);458 struct optee_msg_arg *msg_arg;459 struct tee_shm *shm_arg;460 u64 *pages_list;461 size_t sz;462 int rc;463 464 if (!num_pages)465 return -EINVAL;466 467 rc = optee_check_mem_type(start, num_pages);468 if (rc)469 return rc;470 471 pages_list = optee_allocate_pages_list(num_pages);472 if (!pages_list)473 return -ENOMEM;474 475 /*476 * We're about to register shared memory we can't register shared477 * memory for this request or there's a catch-22.478 *479 * So in this we'll have to do the good old temporary private480 * allocation instead of using optee_get_msg_arg().481 */482 sz = optee_msg_arg_size(optee->rpc_param_count);483 shm_arg = tee_shm_alloc_priv_buf(ctx, sz);484 if (IS_ERR(shm_arg)) {485 rc = PTR_ERR(shm_arg);486 goto out;487 }488 msg_arg = tee_shm_get_va(shm_arg, 0);489 if (IS_ERR(msg_arg)) {490 rc = PTR_ERR(msg_arg);491 goto out;492 }493 494 optee_fill_pages_list(pages_list, pages, num_pages,495 tee_shm_get_page_offset(shm));496 497 memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));498 msg_arg->num_params = 1;499 msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;500 msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |501 OPTEE_MSG_ATTR_NONCONTIG;502 msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;503 msg_arg->params->u.tmem.size = tee_shm_get_size(shm);504 /*505 * In the least bits of msg_arg->params->u.tmem.buf_ptr we506 * store buffer offset from 4k page, as described in OP-TEE ABI.507 */508 msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |509 (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));510 511 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||512 msg_arg->ret != TEEC_SUCCESS)513 rc = -EINVAL;514 515 tee_shm_free(shm_arg);516out:517 optee_free_pages_list(pages_list, num_pages);518 return rc;519}520 521static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)522{523 struct optee *optee = tee_get_drvdata(ctx->teedev);524 struct optee_msg_arg *msg_arg;525 struct tee_shm *shm_arg;526 int rc = 0;527 size_t sz;528 529 /*530 * We're about to unregister shared memory and we may not be able531 * register shared memory for this request in case we're called532 * from optee_shm_arg_cache_uninit().533 *534 * So in order to keep things simple in this function just as in535 * optee_shm_register() we'll use temporary private allocation536 * instead of using optee_get_msg_arg().537 */538 sz = optee_msg_arg_size(optee->rpc_param_count);539 shm_arg = tee_shm_alloc_priv_buf(ctx, sz);540 if (IS_ERR(shm_arg))541 return PTR_ERR(shm_arg);542 msg_arg = tee_shm_get_va(shm_arg, 0);543 if (IS_ERR(msg_arg)) {544 rc = PTR_ERR(msg_arg);545 goto out;546 }547 548 memset(msg_arg, 0, sz);549 msg_arg->num_params = 1;550 msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;551 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;552 msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;553 554 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||555 msg_arg->ret != TEEC_SUCCESS)556 rc = -EINVAL;557out:558 tee_shm_free(shm_arg);559 return rc;560}561 562static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,563 struct page **pages, size_t num_pages,564 unsigned long start)565{566 /*567 * We don't want to register supplicant memory in OP-TEE.568 * Instead information about it will be passed in RPC code.569 */570 return optee_check_mem_type(start, num_pages);571}572 573static int optee_shm_unregister_supp(struct tee_context *ctx,574 struct tee_shm *shm)575{576 return 0;577}578 579/*580 * 3. Dynamic shared memory pool based on alloc_pages()581 *582 * Implements an OP-TEE specific shared memory pool which is used583 * when dynamic shared memory is supported by secure world.584 *585 * The main function is optee_shm_pool_alloc_pages().586 */587 588static int pool_op_alloc(struct tee_shm_pool *pool,589 struct tee_shm *shm, size_t size, size_t align)590{591 /*592 * Shared memory private to the OP-TEE driver doesn't need593 * to be registered with OP-TEE.594 */595 if (shm->flags & TEE_SHM_PRIV)596 return tee_dyn_shm_alloc_helper(shm, size, align, NULL);597 598 return tee_dyn_shm_alloc_helper(shm, size, align, optee_shm_register);599}600 601static void pool_op_free(struct tee_shm_pool *pool,602 struct tee_shm *shm)603{604 if (!(shm->flags & TEE_SHM_PRIV))605 tee_dyn_shm_free_helper(shm, optee_shm_unregister);606 else607 tee_dyn_shm_free_helper(shm, NULL);608}609 610static void pool_op_destroy_pool(struct tee_shm_pool *pool)611{612 kfree(pool);613}614 615static const struct tee_shm_pool_ops pool_ops = {616 .alloc = pool_op_alloc,617 .free = pool_op_free,618 .destroy_pool = pool_op_destroy_pool,619};620 621/**622 * optee_shm_pool_alloc_pages() - create page-based allocator pool623 *624 * This pool is used when OP-TEE supports dymanic SHM. In this case625 * command buffers and such are allocated from kernel's own memory.626 */627static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)628{629 struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);630 631 if (!pool)632 return ERR_PTR(-ENOMEM);633 634 pool->ops = &pool_ops;635 636 return pool;637}638 639/*640 * 4. Do a normal scheduled call into secure world641 *642 * The function optee_smc_do_call_with_arg() performs a normal scheduled643 * call into secure world. During this call may normal world request help644 * from normal world using RPCs, Remote Procedure Calls. This includes645 * delivery of non-secure interrupts to for instance allow rescheduling of646 * the current task.647 */648 649static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,650 struct optee_msg_arg *arg)651{652 struct tee_shm *shm;653 654 arg->ret_origin = TEEC_ORIGIN_COMMS;655 656 if (arg->num_params != 1 ||657 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {658 arg->ret = TEEC_ERROR_BAD_PARAMETERS;659 return;660 }661 662 shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;663 switch (arg->params[0].u.value.a) {664 case OPTEE_RPC_SHM_TYPE_APPL:665 optee_rpc_cmd_free_suppl(ctx, shm);666 break;667 case OPTEE_RPC_SHM_TYPE_KERNEL:668 tee_shm_free(shm);669 break;670 default:671 arg->ret = TEEC_ERROR_BAD_PARAMETERS;672 }673 arg->ret = TEEC_SUCCESS;674}675 676static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,677 struct optee *optee,678 struct optee_msg_arg *arg,679 struct optee_call_ctx *call_ctx)680{681 struct tee_shm *shm;682 size_t sz;683 size_t n;684 struct page **pages;685 size_t page_count;686 687 arg->ret_origin = TEEC_ORIGIN_COMMS;688 689 if (!arg->num_params ||690 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {691 arg->ret = TEEC_ERROR_BAD_PARAMETERS;692 return;693 }694 695 for (n = 1; n < arg->num_params; n++) {696 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {697 arg->ret = TEEC_ERROR_BAD_PARAMETERS;698 return;699 }700 }701 702 sz = arg->params[0].u.value.b;703 switch (arg->params[0].u.value.a) {704 case OPTEE_RPC_SHM_TYPE_APPL:705 shm = optee_rpc_cmd_alloc_suppl(ctx, sz);706 break;707 case OPTEE_RPC_SHM_TYPE_KERNEL:708 shm = tee_shm_alloc_priv_buf(optee->ctx, sz);709 break;710 default:711 arg->ret = TEEC_ERROR_BAD_PARAMETERS;712 return;713 }714 715 if (IS_ERR(shm)) {716 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;717 return;718 }719 720 /*721 * If there are pages it's dynamically allocated shared memory (not722 * from the reserved shared memory pool) and needs to be723 * registered.724 */725 pages = tee_shm_get_pages(shm, &page_count);726 if (pages) {727 u64 *pages_list;728 729 pages_list = optee_allocate_pages_list(page_count);730 if (!pages_list) {731 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;732 goto bad;733 }734 735 call_ctx->pages_list = pages_list;736 call_ctx->num_entries = page_count;737 738 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |739 OPTEE_MSG_ATTR_NONCONTIG;740 /*741 * In the least bits of u.tmem.buf_ptr we store buffer offset742 * from 4k page, as described in OP-TEE ABI.743 */744 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |745 (tee_shm_get_page_offset(shm) &746 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));747 748 optee_fill_pages_list(pages_list, pages, page_count,749 tee_shm_get_page_offset(shm));750 } else {751 phys_addr_t pa;752 753 if (tee_shm_get_pa(shm, 0, &pa)) {754 arg->ret = TEEC_ERROR_BAD_PARAMETERS;755 goto bad;756 }757 758 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;759 arg->params[0].u.tmem.buf_ptr = pa;760 }761 arg->params[0].u.tmem.size = tee_shm_get_size(shm);762 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;763 764 arg->ret = TEEC_SUCCESS;765 return;766bad:767 tee_shm_free(shm);768}769 770static void free_pages_list(struct optee_call_ctx *call_ctx)771{772 if (call_ctx->pages_list) {773 optee_free_pages_list(call_ctx->pages_list,774 call_ctx->num_entries);775 call_ctx->pages_list = NULL;776 call_ctx->num_entries = 0;777 }778}779 780static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)781{782 free_pages_list(call_ctx);783}784 785static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,786 struct optee_msg_arg *arg,787 struct optee_call_ctx *call_ctx)788{789 790 switch (arg->cmd) {791 case OPTEE_RPC_CMD_SHM_ALLOC:792 free_pages_list(call_ctx);793 handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);794 break;795 case OPTEE_RPC_CMD_SHM_FREE:796 handle_rpc_func_cmd_shm_free(ctx, arg);797 break;798 default:799 optee_rpc_cmd(ctx, optee, arg);800 }801}802 803/**804 * optee_handle_rpc() - handle RPC from secure world805 * @ctx: context doing the RPC806 * @rpc_arg: pointer to RPC arguments if any, or NULL if none807 * @param: value of registers for the RPC808 * @call_ctx: call context. Preserved during one OP-TEE invocation809 *810 * Result of RPC is written back into @param.811 */812static void optee_handle_rpc(struct tee_context *ctx,813 struct optee_msg_arg *rpc_arg,814 struct optee_rpc_param *param,815 struct optee_call_ctx *call_ctx)816{817 struct tee_device *teedev = ctx->teedev;818 struct optee *optee = tee_get_drvdata(teedev);819 struct optee_msg_arg *arg;820 struct tee_shm *shm;821 phys_addr_t pa;822 823 switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {824 case OPTEE_SMC_RPC_FUNC_ALLOC:825 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);826 if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {827 reg_pair_from_64(¶m->a1, ¶m->a2, pa);828 reg_pair_from_64(¶m->a4, ¶m->a5,829 (unsigned long)shm);830 } else {831 param->a1 = 0;832 param->a2 = 0;833 param->a4 = 0;834 param->a5 = 0;835 }836 kmemleak_not_leak(shm);837 break;838 case OPTEE_SMC_RPC_FUNC_FREE:839 shm = reg_pair_to_ptr(param->a1, param->a2);840 tee_shm_free(shm);841 break;842 case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:843 /*844 * A foreign interrupt was raised while secure world was845 * executing, since they are handled in Linux a dummy RPC is846 * performed to let Linux take the interrupt through the normal847 * vector.848 */849 break;850 case OPTEE_SMC_RPC_FUNC_CMD:851 if (rpc_arg) {852 arg = rpc_arg;853 } else {854 shm = reg_pair_to_ptr(param->a1, param->a2);855 arg = tee_shm_get_va(shm, 0);856 if (IS_ERR(arg)) {857 pr_err("%s: tee_shm_get_va %p failed\n",858 __func__, shm);859 break;860 }861 }862 863 handle_rpc_func_cmd(ctx, optee, arg, call_ctx);864 break;865 default:866 pr_warn("Unknown RPC func 0x%x\n",867 (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));868 break;869 }870 871 param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;872}873 874/**875 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world876 * @ctx: calling context877 * @shm: shared memory holding the message to pass to secure world878 * @offs: offset of the message in @shm879 * @system_thread: true if caller requests TEE system thread support880 *881 * Does and SMC to OP-TEE in secure world and handles eventual resulting882 * Remote Procedure Calls (RPC) from OP-TEE.883 *884 * Returns return code from secure world, 0 is OK885 */886static int optee_smc_do_call_with_arg(struct tee_context *ctx,887 struct tee_shm *shm, u_int offs,888 bool system_thread)889{890 struct optee *optee = tee_get_drvdata(ctx->teedev);891 struct optee_call_waiter w;892 struct optee_rpc_param param = { };893 struct optee_call_ctx call_ctx = { };894 struct optee_msg_arg *rpc_arg = NULL;895 int rc;896 897 if (optee->rpc_param_count) {898 struct optee_msg_arg *arg;899 unsigned int rpc_arg_offs;900 901 arg = tee_shm_get_va(shm, offs);902 if (IS_ERR(arg))903 return PTR_ERR(arg);904 905 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);906 rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);907 if (IS_ERR(rpc_arg))908 return PTR_ERR(rpc_arg);909 }910 911 if (rpc_arg && tee_shm_is_dynamic(shm)) {912 param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;913 reg_pair_from_64(¶m.a1, ¶m.a2, (u_long)shm);914 param.a3 = offs;915 } else {916 phys_addr_t parg;917 918 rc = tee_shm_get_pa(shm, offs, &parg);919 if (rc)920 return rc;921 922 if (rpc_arg)923 param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;924 else925 param.a0 = OPTEE_SMC_CALL_WITH_ARG;926 reg_pair_from_64(¶m.a1, ¶m.a2, parg);927 }928 /* Initialize waiter */929 optee_cq_wait_init(&optee->call_queue, &w, system_thread);930 while (true) {931 struct arm_smccc_res res;932 933 trace_optee_invoke_fn_begin(¶m);934 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,935 param.a4, param.a5, param.a6, param.a7,936 &res);937 trace_optee_invoke_fn_end(¶m, &res);938 939 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {940 /*941 * Out of threads in secure world, wait for a thread942 * become available.943 */944 optee_cq_wait_for_completion(&optee->call_queue, &w);945 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {946 cond_resched();947 param.a0 = res.a0;948 param.a1 = res.a1;949 param.a2 = res.a2;950 param.a3 = res.a3;951 optee_handle_rpc(ctx, rpc_arg, ¶m, &call_ctx);952 } else {953 rc = res.a0;954 break;955 }956 }957 958 optee_rpc_finalize_call(&call_ctx);959 /*960 * We're done with our thread in secure world, if there's any961 * thread waiters wake up one.962 */963 optee_cq_wait_final(&optee->call_queue, &w);964 965 return rc;966}967 968/*969 * 5. Asynchronous notification970 */971 972static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,973 bool *value_pending)974{975 struct arm_smccc_res res;976 977 invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);978 979 if (res.a0) {980 *value_valid = false;981 return 0;982 }983 *value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);984 *value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);985 return res.a1;986}987 988static irqreturn_t irq_handler(struct optee *optee)989{990 bool do_bottom_half = false;991 bool value_valid;992 bool value_pending;993 u32 value;994 995 do {996 value = get_async_notif_value(optee->smc.invoke_fn,997 &value_valid, &value_pending);998 if (!value_valid)999 break;1000 1001 if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)1002 do_bottom_half = true;1003 else1004 optee_notif_send(optee, value);1005 } while (value_pending);1006 1007 if (do_bottom_half)1008 return IRQ_WAKE_THREAD;1009 return IRQ_HANDLED;1010}1011 1012static irqreturn_t notif_irq_handler(int irq, void *dev_id)1013{1014 struct optee *optee = dev_id;1015 1016 return irq_handler(optee);1017}1018 1019static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)1020{1021 struct optee *optee = dev_id;1022 1023 optee_do_bottom_half(optee->ctx);1024 1025 return IRQ_HANDLED;1026}1027 1028static int init_irq(struct optee *optee, u_int irq)1029{1030 int rc;1031 1032 rc = request_threaded_irq(irq, notif_irq_handler,1033 notif_irq_thread_fn,1034 0, "optee_notification", optee);1035 if (rc)1036 return rc;1037 1038 optee->smc.notif_irq = irq;1039 1040 return 0;1041}1042 1043static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)1044{1045 struct optee_pcpu *pcpu = dev_id;1046 struct optee *optee = pcpu->optee;1047 1048 if (irq_handler(optee) == IRQ_WAKE_THREAD)1049 queue_work(optee->smc.notif_pcpu_wq,1050 &optee->smc.notif_pcpu_work);1051 1052 return IRQ_HANDLED;1053}1054 1055static void notif_pcpu_irq_work_fn(struct work_struct *work)1056{1057 struct optee_smc *optee_smc = container_of(work, struct optee_smc,1058 notif_pcpu_work);1059 struct optee *optee = container_of(optee_smc, struct optee, smc);1060 1061 optee_do_bottom_half(optee->ctx);1062}1063 1064static int init_pcpu_irq(struct optee *optee, u_int irq)1065{1066 struct optee_pcpu __percpu *optee_pcpu;1067 int cpu, rc;1068 1069 optee_pcpu = alloc_percpu(struct optee_pcpu);1070 if (!optee_pcpu)1071 return -ENOMEM;1072 1073 for_each_present_cpu(cpu)1074 per_cpu_ptr(optee_pcpu, cpu)->optee = optee;1075 1076 rc = request_percpu_irq(irq, notif_pcpu_irq_handler,1077 "optee_pcpu_notification", optee_pcpu);1078 if (rc)1079 goto err_free_pcpu;1080 1081 INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);1082 optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");1083 if (!optee->smc.notif_pcpu_wq) {1084 rc = -EINVAL;1085 goto err_free_pcpu_irq;1086 }1087 1088 optee->smc.optee_pcpu = optee_pcpu;1089 optee->smc.notif_irq = irq;1090 1091 pcpu_irq_num = irq;1092 rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",1093 optee_cpuhp_enable_pcpu_irq,1094 optee_cpuhp_disable_pcpu_irq);1095 if (!rc)1096 rc = -EINVAL;1097 if (rc < 0)1098 goto err_free_pcpu_irq;1099 1100 optee->smc.notif_cpuhp_state = rc;1101 1102 return 0;1103 1104err_free_pcpu_irq:1105 free_percpu_irq(irq, optee_pcpu);1106err_free_pcpu:1107 free_percpu(optee_pcpu);1108 1109 return rc;1110}1111 1112static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)1113{1114 if (irq_is_percpu_devid(irq))1115 return init_pcpu_irq(optee, irq);1116 else1117 return init_irq(optee, irq);1118}1119 1120static void uninit_pcpu_irq(struct optee *optee)1121{1122 cpuhp_remove_state(optee->smc.notif_cpuhp_state);1123 1124 destroy_workqueue(optee->smc.notif_pcpu_wq);1125 1126 free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);1127 free_percpu(optee->smc.optee_pcpu);1128}1129 1130static void optee_smc_notif_uninit_irq(struct optee *optee)1131{1132 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {1133 optee_stop_async_notif(optee->ctx);1134 if (optee->smc.notif_irq) {1135 if (irq_is_percpu_devid(optee->smc.notif_irq))1136 uninit_pcpu_irq(optee);1137 else1138 free_irq(optee->smc.notif_irq, optee);1139 1140 irq_dispose_mapping(optee->smc.notif_irq);1141 }1142 }1143}1144 1145/*1146 * 6. Driver initialization1147 *1148 * During driver initialization is secure world probed to find out which1149 * features it supports so the driver can be initialized with a matching1150 * configuration. This involves for instance support for dynamic shared1151 * memory instead of a static memory carvout.1152 */1153 1154static void optee_get_version(struct tee_device *teedev,1155 struct tee_ioctl_version_data *vers)1156{1157 struct tee_ioctl_version_data v = {1158 .impl_id = TEE_IMPL_ID_OPTEE,1159 .impl_caps = TEE_OPTEE_CAP_TZ,1160 .gen_caps = TEE_GEN_CAP_GP,1161 };1162 struct optee *optee = tee_get_drvdata(teedev);1163 1164 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)1165 v.gen_caps |= TEE_GEN_CAP_REG_MEM;1166 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)1167 v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;1168 *vers = v;1169}1170 1171static int optee_smc_open(struct tee_context *ctx)1172{1173 struct optee *optee = tee_get_drvdata(ctx->teedev);1174 u32 sec_caps = optee->smc.sec_caps;1175 1176 return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);1177}1178 1179static const struct tee_driver_ops optee_clnt_ops = {1180 .get_version = optee_get_version,1181 .open = optee_smc_open,1182 .release = optee_release,1183 .open_session = optee_open_session,1184 .close_session = optee_close_session,1185 .system_session = optee_system_session,1186 .invoke_func = optee_invoke_func,1187 .cancel_req = optee_cancel_req,1188 .shm_register = optee_shm_register,1189 .shm_unregister = optee_shm_unregister,1190};1191 1192static const struct tee_desc optee_clnt_desc = {1193 .name = DRIVER_NAME "-clnt",1194 .ops = &optee_clnt_ops,1195 .owner = THIS_MODULE,1196};1197 1198static const struct tee_driver_ops optee_supp_ops = {1199 .get_version = optee_get_version,1200 .open = optee_smc_open,1201 .release = optee_release_supp,1202 .supp_recv = optee_supp_recv,1203 .supp_send = optee_supp_send,1204 .shm_register = optee_shm_register_supp,1205 .shm_unregister = optee_shm_unregister_supp,1206};1207 1208static const struct tee_desc optee_supp_desc = {1209 .name = DRIVER_NAME "-supp",1210 .ops = &optee_supp_ops,1211 .owner = THIS_MODULE,1212 .flags = TEE_DESC_PRIVILEGED,1213};1214 1215static const struct optee_ops optee_ops = {1216 .do_call_with_arg = optee_smc_do_call_with_arg,1217 .to_msg_param = optee_to_msg_param,1218 .from_msg_param = optee_from_msg_param,1219};1220 1221static int enable_async_notif(optee_invoke_fn *invoke_fn)1222{1223 struct arm_smccc_res res;1224 1225 invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);1226 1227 if (res.a0)1228 return -EINVAL;1229 return 0;1230}1231 1232static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)1233{1234 struct arm_smccc_res res;1235 1236 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);1237 1238 if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&1239 res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)1240 return true;1241 return false;1242}1243 1244#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE1245static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)1246{1247 struct arm_smccc_res res;1248 1249 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);1250 1251 if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&1252 res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&1253 res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&1254 res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)1255 return true;1256 return false;1257}1258#endif1259 1260static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)1261{1262 union {1263 struct arm_smccc_res smccc;1264 struct optee_smc_call_get_os_revision_result result;1265 } res = {1266 .result = {1267 .build_id = 01268 }1269 };1270 1271 invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,1272 &res.smccc);1273 1274 if (res.result.build_id)1275 pr_info("revision %lu.%lu (%08lx)", res.result.major,1276 res.result.minor, res.result.build_id);1277 else1278 pr_info("revision %lu.%lu", res.result.major, res.result.minor);1279}1280 1281static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)1282{1283 union {1284 struct arm_smccc_res smccc;1285 struct optee_smc_calls_revision_result result;1286 } res;1287 1288 invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);1289 1290 if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&1291 (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)1292 return true;1293 return false;1294}1295 1296static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,1297 u32 *sec_caps, u32 *max_notif_value,1298 unsigned int *rpc_param_count)1299{1300 union {1301 struct arm_smccc_res smccc;1302 struct optee_smc_exchange_capabilities_result result;1303 } res;1304 u32 a1 = 0;1305 1306 /*1307 * TODO This isn't enough to tell if it's UP system (from kernel1308 * point of view) or not, is_smp() returns the information1309 * needed, but can't be called directly from here.1310 */1311 if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)1312 a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;1313 1314 invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,1315 &res.smccc);1316 1317 if (res.result.status != OPTEE_SMC_RETURN_OK)1318 return false;1319 1320 *sec_caps = res.result.capabilities;1321 if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)1322 *max_notif_value = res.result.max_notif_value;1323 else1324 *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;1325 if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)1326 *rpc_param_count = (u8)res.result.data;1327 else1328 *rpc_param_count = 0;1329 1330 return true;1331}1332 1333static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn)1334{1335 struct arm_smccc_res res;1336 1337 invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res);1338 if (res.a0)1339 return 0;1340 return res.a1;1341}1342 1343static struct tee_shm_pool *1344optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)1345{1346 union {1347 struct arm_smccc_res smccc;1348 struct optee_smc_get_shm_config_result result;1349 } res;1350 unsigned long vaddr;1351 phys_addr_t paddr;1352 size_t size;1353 phys_addr_t begin;1354 phys_addr_t end;1355 void *va;1356 void *rc;1357 1358 invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);1359 if (res.result.status != OPTEE_SMC_RETURN_OK) {1360 pr_err("static shm service not available\n");1361 return ERR_PTR(-ENOENT);1362 }1363 1364 if (res.result.settings != OPTEE_SMC_SHM_CACHED) {1365 pr_err("only normal cached shared memory supported\n");1366 return ERR_PTR(-EINVAL);1367 }1368 1369 begin = roundup(res.result.start, PAGE_SIZE);1370 end = rounddown(res.result.start + res.result.size, PAGE_SIZE);1371 paddr = begin;1372 size = end - begin;1373 1374 va = memremap(paddr, size, MEMREMAP_WB);1375 if (!va) {1376 pr_err("shared memory ioremap failed\n");1377 return ERR_PTR(-EINVAL);1378 }1379 vaddr = (unsigned long)va;1380 1381 rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,1382 OPTEE_MIN_STATIC_POOL_ALIGN);1383 if (IS_ERR(rc))1384 memunmap(va);1385 else1386 *memremaped_shm = va;1387 1388 return rc;1389}1390 1391/* Simple wrapper functions to be able to use a function pointer */1392static void optee_smccc_smc(unsigned long a0, unsigned long a1,1393 unsigned long a2, unsigned long a3,1394 unsigned long a4, unsigned long a5,1395 unsigned long a6, unsigned long a7,1396 struct arm_smccc_res *res)1397{1398 arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);1399}1400 1401static void optee_smccc_hvc(unsigned long a0, unsigned long a1,1402 unsigned long a2, unsigned long a3,1403 unsigned long a4, unsigned long a5,1404 unsigned long a6, unsigned long a7,1405 struct arm_smccc_res *res)1406{1407 arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);1408}1409 1410static optee_invoke_fn *get_invoke_func(struct device *dev)1411{1412 const char *method;1413 1414 pr_info("probing for conduit method.\n");1415 1416 if (device_property_read_string(dev, "method", &method)) {1417 pr_warn("missing \"method\" property\n");1418 return ERR_PTR(-ENXIO);1419 }1420 1421 if (!strcmp("hvc", method))1422 return optee_smccc_hvc;1423 else if (!strcmp("smc", method))1424 return optee_smccc_smc;1425 1426 pr_warn("invalid \"method\" property: %s\n", method);1427 return ERR_PTR(-EINVAL);1428}1429 1430/* optee_remove - Device Removal Routine1431 * @pdev: platform device information struct1432 *1433 * optee_remove is called by platform subsystem to alert the driver1434 * that it should release the device1435 */1436static void optee_smc_remove(struct platform_device *pdev)1437{1438 struct optee *optee = platform_get_drvdata(pdev);1439 1440 /*1441 * Ask OP-TEE to free all cached shared memory objects to decrease1442 * reference counters and also avoid wild pointers in secure world1443 * into the old shared memory range.1444 */1445 if (!optee->rpc_param_count)1446 optee_disable_shm_cache(optee);1447 1448 optee_smc_notif_uninit_irq(optee);1449 1450 optee_remove_common(optee);1451 1452 if (optee->smc.memremaped_shm)1453 memunmap(optee->smc.memremaped_shm);1454 1455 kfree(optee);1456}1457 1458/* optee_shutdown - Device Removal Routine1459 * @pdev: platform device information struct1460 *1461 * platform_shutdown is called by the platform subsystem to alert1462 * the driver that a shutdown, reboot, or kexec is happening and1463 * device must be disabled.1464 */1465static void optee_shutdown(struct platform_device *pdev)1466{1467 struct optee *optee = platform_get_drvdata(pdev);1468 1469 if (!optee->rpc_param_count)1470 optee_disable_shm_cache(optee);1471}1472 1473#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE1474 1475#define OPTEE_FW_IMAGE "optee/tee.bin"1476 1477static optee_invoke_fn *cpuhp_invoke_fn;1478 1479static int optee_cpuhp_probe(unsigned int cpu)1480{1481 /*1482 * Invoking a call on a CPU will cause OP-TEE to perform the required1483 * setup for that CPU. Just invoke the call to get the UID since that1484 * has no side effects.1485 */1486 if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))1487 return 0;1488 else1489 return -EINVAL;1490}1491 1492static int optee_load_fw(struct platform_device *pdev,1493 optee_invoke_fn *invoke_fn)1494{1495 const struct firmware *fw = NULL;1496 struct arm_smccc_res res;1497 phys_addr_t data_pa;1498 u8 *data_buf = NULL;1499 u64 data_size;1500 u32 data_pa_high, data_pa_low;1501 u32 data_size_high, data_size_low;1502 int rc;1503 int hp_state;1504 1505 if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))1506 return 0;1507 1508 rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);1509 if (rc) {1510 /*1511 * The firmware in the rootfs will not be accessible until we1512 * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until1513 * that point.1514 */1515 if (system_state < SYSTEM_RUNNING)1516 return -EPROBE_DEFER;1517 goto fw_err;1518 }1519 1520 data_size = fw->size;1521 /*1522 * This uses the GFP_DMA flag to ensure we are allocated memory in the1523 * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.1524 */1525 data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA);1526 if (!data_buf) {1527 rc = -ENOMEM;1528 goto fw_err;1529 }1530 data_pa = virt_to_phys(data_buf);1531 reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);1532 reg_pair_from_64(&data_size_high, &data_size_low, data_size);1533 goto fw_load;1534 1535fw_err:1536 pr_warn("image loading failed\n");1537 data_pa_high = 0;1538 data_pa_low = 0;1539 data_size_high = 0;1540 data_size_low = 0;1541 1542fw_load:1543 /*1544 * Always invoke the SMC, even if loading the image fails, to indicate1545 * to EL3 that we have passed the point where it should allow invoking1546 * this SMC.1547 */1548 pr_warn("OP-TEE image loaded from kernel, this can be insecure");1549 invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,1550 data_pa_high, data_pa_low, 0, 0, 0, &res);1551 if (!rc)1552 rc = res.a0;1553 if (fw)1554 release_firmware(fw);1555 kfree(data_buf);1556 1557 if (!rc) {1558 /*1559 * We need to initialize OP-TEE on all other running cores as1560 * well. Any cores that aren't running yet will get initialized1561 * when they are brought up by the power management functions in1562 * TF-A which are registered by the OP-TEE SPD. Due to that we1563 * can un-register the callback right after registering it.1564 */1565 cpuhp_invoke_fn = invoke_fn;1566 hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",1567 optee_cpuhp_probe, NULL);1568 if (hp_state < 0) {1569 pr_warn("Failed with CPU hotplug setup for OP-TEE");1570 return -EINVAL;1571 }1572 cpuhp_remove_state(hp_state);1573 cpuhp_invoke_fn = NULL;1574 }1575 1576 return rc;1577}1578#else1579static inline int optee_load_fw(struct platform_device *pdev,1580 optee_invoke_fn *invoke_fn)1581{1582 return 0;1583}1584#endif1585 1586static int optee_probe(struct platform_device *pdev)1587{1588 optee_invoke_fn *invoke_fn;1589 struct tee_shm_pool *pool = ERR_PTR(-EINVAL);1590 struct optee *optee = NULL;1591 void *memremaped_shm = NULL;1592 unsigned int rpc_param_count;1593 unsigned int thread_count;1594 struct tee_device *teedev;1595 struct tee_context *ctx;1596 u32 max_notif_value;1597 u32 arg_cache_flags;1598 u32 sec_caps;1599 int rc;1600 1601 invoke_fn = get_invoke_func(&pdev->dev);1602 if (IS_ERR(invoke_fn))1603 return PTR_ERR(invoke_fn);1604 1605 rc = optee_load_fw(pdev, invoke_fn);1606 if (rc)1607 return rc;1608 1609 if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {1610 pr_warn("api uid mismatch\n");1611 return -EINVAL;1612 }1613 1614 optee_msg_get_os_revision(invoke_fn);1615 1616 if (!optee_msg_api_revision_is_compatible(invoke_fn)) {1617 pr_warn("api revision mismatch\n");1618 return -EINVAL;1619 }1620 1621 thread_count = optee_msg_get_thread_count(invoke_fn);1622 if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,1623 &max_notif_value,1624 &rpc_param_count)) {1625 pr_warn("capabilities mismatch\n");1626 return -EINVAL;1627 }1628 1629 /*1630 * Try to use dynamic shared memory if possible1631 */1632 if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {1633 /*1634 * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask1635 * optee_get_msg_arg() to pre-register (by having1636 * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass1637 * an argument struct.1638 *1639 * With the page is pre-registered we can use a non-zero1640 * offset for argument struct, this is indicated with1641 * OPTEE_SHM_ARG_SHARED.1642 *1643 * This means that optee_smc_do_call_with_arg() will use1644 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.1645 */1646 if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)1647 arg_cache_flags = OPTEE_SHM_ARG_SHARED;1648 else1649 arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;1650 1651 pool = optee_shm_pool_alloc_pages();1652 }1653 1654 /*1655 * If dynamic shared memory is not available or failed - try static one1656 */1657 if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {1658 /*1659 * The static memory pool can use non-zero page offsets so1660 * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.1661 *1662 * optee_get_msg_arg() should not pre-register the1663 * allocated page used to pass an argument struct, this is1664 * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.1665 *1666 * This means that optee_smc_do_call_with_arg() will use1667 * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else1668 * OPTEE_SMC_CALL_WITH_RPC_ARG.1669 */1670 arg_cache_flags = OPTEE_SHM_ARG_SHARED |1671 OPTEE_SHM_ARG_ALLOC_PRIV;1672 pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);1673 }1674 1675 if (IS_ERR(pool))1676 return PTR_ERR(pool);1677 1678 optee = kzalloc(sizeof(*optee), GFP_KERNEL);1679 if (!optee) {1680 rc = -ENOMEM;1681 goto err_free_pool;1682 }1683 1684 optee->ops = &optee_ops;1685 optee->smc.invoke_fn = invoke_fn;1686 optee->smc.sec_caps = sec_caps;1687 optee->rpc_param_count = rpc_param_count;1688 1689 if (IS_REACHABLE(CONFIG_RPMB) &&1690 (sec_caps & OPTEE_SMC_SEC_CAP_RPMB_PROBE))1691 optee->in_kernel_rpmb_routing = true;1692 1693 teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);1694 if (IS_ERR(teedev)) {1695 rc = PTR_ERR(teedev);1696 goto err_free_optee;1697 }1698 optee->teedev = teedev;1699 1700 teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);1701 if (IS_ERR(teedev)) {1702 rc = PTR_ERR(teedev);1703 goto err_unreg_teedev;1704 }1705 optee->supp_teedev = teedev;1706 1707 optee_set_dev_group(optee);1708 1709 rc = tee_device_register(optee->teedev);1710 if (rc)1711 goto err_unreg_supp_teedev;1712 1713 rc = tee_device_register(optee->supp_teedev);1714 if (rc)1715 goto err_unreg_supp_teedev;1716 1717 optee_cq_init(&optee->call_queue, thread_count);1718 optee_supp_init(&optee->supp);1719 optee->smc.memremaped_shm = memremaped_shm;1720 optee->pool = pool;1721 optee_shm_arg_cache_init(optee, arg_cache_flags);1722 mutex_init(&optee->rpmb_dev_mutex);1723 1724 platform_set_drvdata(pdev, optee);1725 ctx = teedev_open(optee->teedev);1726 if (IS_ERR(ctx)) {1727 rc = PTR_ERR(ctx);1728 goto err_supp_uninit;1729 }1730 optee->ctx = ctx;1731 rc = optee_notif_init(optee, max_notif_value);1732 if (rc)1733 goto err_close_ctx;1734 1735 if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {1736 unsigned int irq;1737 1738 rc = platform_get_irq(pdev, 0);1739 if (rc < 0) {1740 pr_err("platform_get_irq: ret %d\n", rc);1741 goto err_notif_uninit;1742 }1743 irq = rc;1744 1745 rc = optee_smc_notif_init_irq(optee, irq);1746 if (rc) {1747 irq_dispose_mapping(irq);1748 goto err_notif_uninit;1749 }1750 enable_async_notif(optee->smc.invoke_fn);1751 pr_info("Asynchronous notifications enabled\n");1752 }1753 1754 /*1755 * Ensure that there are no pre-existing shm objects before enabling1756 * the shm cache so that there's no chance of receiving an invalid1757 * address during shutdown. This could occur, for example, if we're1758 * kexec booting from an older kernel that did not properly cleanup the1759 * shm cache.1760 */1761 optee_disable_unmapped_shm_cache(optee);1762 1763 /*1764 * Only enable the shm cache in case we're not able to pass the RPC1765 * arg struct right after the normal arg struct.1766 */1767 if (!optee->rpc_param_count)1768 optee_enable_shm_cache(optee);1769 1770 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)1771 pr_info("dynamic shared memory is enabled\n");1772 1773 rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);1774 if (rc)1775 goto err_disable_shm_cache;1776 1777 INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb);1778 optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev;1779 blocking_notifier_chain_register(&optee_rpmb_intf_added,1780 &optee->rpmb_intf);1781 pr_info("initialized driver\n");1782 return 0;1783 1784err_disable_shm_cache:1785 if (!optee->rpc_param_count)1786 optee_disable_shm_cache(optee);1787 optee_smc_notif_uninit_irq(optee);1788 optee_unregister_devices();1789err_notif_uninit:1790 optee_notif_uninit(optee);1791err_close_ctx:1792 teedev_close_context(ctx);1793err_supp_uninit:1794 rpmb_dev_put(optee->rpmb_dev);1795 mutex_destroy(&optee->rpmb_dev_mutex);1796 optee_shm_arg_cache_uninit(optee);1797 optee_supp_uninit(&optee->supp);1798 mutex_destroy(&optee->call_queue.mutex);1799err_unreg_supp_teedev:1800 tee_device_unregister(optee->supp_teedev);1801err_unreg_teedev:1802 tee_device_unregister(optee->teedev);1803err_free_optee:1804 kfree(optee);1805err_free_pool:1806 tee_shm_pool_free(pool);1807 if (memremaped_shm)1808 memunmap(memremaped_shm);1809 return rc;1810}1811 1812static const struct of_device_id optee_dt_match[] = {1813 { .compatible = "linaro,optee-tz" },1814 {},1815};1816MODULE_DEVICE_TABLE(of, optee_dt_match);1817 1818static struct platform_driver optee_driver = {1819 .probe = optee_probe,1820 .remove_new = optee_smc_remove,1821 .shutdown = optee_shutdown,1822 .driver = {1823 .name = "optee",1824 .of_match_table = optee_dt_match,1825 },1826};1827 1828int optee_smc_abi_register(void)1829{1830 return platform_driver_register(&optee_driver);1831}1832 1833void optee_smc_abi_unregister(void)1834{1835 platform_driver_unregister(&optee_driver);1836}1837