481 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Memory allocator for buffers shared with the TrustZone.4 *5 * Copyright (C) 2023-2024 Linaro Ltd.6 */7 8#include <linux/bug.h>9#include <linux/cleanup.h>10#include <linux/dma-mapping.h>11#include <linux/err.h>12#include <linux/firmware/qcom/qcom_tzmem.h>13#include <linux/genalloc.h>14#include <linux/gfp.h>15#include <linux/kernel.h>16#include <linux/list.h>17#include <linux/mm.h>18#include <linux/radix-tree.h>19#include <linux/slab.h>20#include <linux/spinlock.h>21#include <linux/types.h>22 23#include "qcom_tzmem.h"24 25struct qcom_tzmem_area {26 struct list_head list;27 void *vaddr;28 dma_addr_t paddr;29 size_t size;30 void *priv;31};32 33struct qcom_tzmem_pool {34 struct gen_pool *genpool;35 struct list_head areas;36 enum qcom_tzmem_policy policy;37 size_t increment;38 size_t max_size;39 spinlock_t lock;40};41 42struct qcom_tzmem_chunk {43 size_t size;44 struct qcom_tzmem_pool *owner;45};46 47static struct device *qcom_tzmem_dev;48static RADIX_TREE(qcom_tzmem_chunks, GFP_ATOMIC);49static DEFINE_SPINLOCK(qcom_tzmem_chunks_lock);50 51#if IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_GENERIC)52 53static int qcom_tzmem_init(void)54{55 return 0;56}57 58static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)59{60 return 0;61}62 63static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)64{65 66}67 68#elif IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE)69 70#include <linux/firmware/qcom/qcom_scm.h>71#include <linux/of.h>72 73#define QCOM_SHM_BRIDGE_NUM_VM_SHIFT 974 75static bool qcom_tzmem_using_shm_bridge;76 77/* List of machines that are known to not support SHM bridge correctly. */78static const char *const qcom_tzmem_blacklist[] = {79 "qcom,sc8180x",80 "qcom,sdm670", /* failure in GPU firmware loading */81 "qcom,sdm845", /* reset in rmtfs memory assignment */82 "qcom,sm8150", /* reset in rmtfs memory assignment */83 NULL84};85 86static int qcom_tzmem_init(void)87{88 const char *const *platform;89 int ret;90 91 for (platform = qcom_tzmem_blacklist; *platform; platform++) {92 if (of_machine_is_compatible(*platform))93 goto notsupp;94 }95 96 ret = qcom_scm_shm_bridge_enable();97 if (ret == -EOPNOTSUPP)98 goto notsupp;99 100 if (!ret)101 qcom_tzmem_using_shm_bridge = true;102 103 return ret;104 105notsupp:106 dev_info(qcom_tzmem_dev, "SHM Bridge not supported\n");107 return 0;108}109 110static int qcom_tzmem_init_area(struct qcom_tzmem_area *area)111{112 u64 pfn_and_ns_perm, ipfn_and_s_perm, size_and_flags;113 int ret;114 115 if (!qcom_tzmem_using_shm_bridge)116 return 0;117 118 pfn_and_ns_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;119 ipfn_and_s_perm = (u64)area->paddr | QCOM_SCM_PERM_RW;120 size_and_flags = area->size | (1 << QCOM_SHM_BRIDGE_NUM_VM_SHIFT);121 122 u64 *handle __free(kfree) = kzalloc(sizeof(*handle), GFP_KERNEL);123 if (!handle)124 return -ENOMEM;125 126 ret = qcom_scm_shm_bridge_create(qcom_tzmem_dev, pfn_and_ns_perm,127 ipfn_and_s_perm, size_and_flags,128 QCOM_SCM_VMID_HLOS, handle);129 if (ret)130 return ret;131 132 area->priv = no_free_ptr(handle);133 134 return 0;135}136 137static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)138{139 u64 *handle = area->priv;140 141 if (!qcom_tzmem_using_shm_bridge)142 return;143 144 qcom_scm_shm_bridge_delete(qcom_tzmem_dev, *handle);145 kfree(handle);146}147 148#endif /* CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE */149 150static int qcom_tzmem_pool_add_memory(struct qcom_tzmem_pool *pool,151 size_t size, gfp_t gfp)152{153 int ret;154 155 struct qcom_tzmem_area *area __free(kfree) = kzalloc(sizeof(*area),156 gfp);157 if (!area)158 return -ENOMEM;159 160 area->size = PAGE_ALIGN(size);161 162 area->vaddr = dma_alloc_coherent(qcom_tzmem_dev, area->size,163 &area->paddr, gfp);164 if (!area->vaddr)165 return -ENOMEM;166 167 ret = qcom_tzmem_init_area(area);168 if (ret) {169 dma_free_coherent(qcom_tzmem_dev, area->size,170 area->vaddr, area->paddr);171 return ret;172 }173 174 ret = gen_pool_add_virt(pool->genpool, (unsigned long)area->vaddr,175 (phys_addr_t)area->paddr, size, -1);176 if (ret) {177 dma_free_coherent(qcom_tzmem_dev, area->size,178 area->vaddr, area->paddr);179 return ret;180 }181 182 scoped_guard(spinlock_irqsave, &pool->lock)183 list_add_tail(&area->list, &pool->areas);184 185 area = NULL;186 return 0;187}188 189/**190 * qcom_tzmem_pool_new() - Create a new TZ memory pool.191 * @config: Pool configuration.192 *193 * Create a new pool of memory suitable for sharing with the TrustZone.194 *195 * Must not be used in atomic context.196 *197 * Return: New memory pool address or ERR_PTR() on error.198 */199struct qcom_tzmem_pool *200qcom_tzmem_pool_new(const struct qcom_tzmem_pool_config *config)201{202 int ret = -ENOMEM;203 204 might_sleep();205 206 switch (config->policy) {207 case QCOM_TZMEM_POLICY_STATIC:208 if (!config->initial_size)209 return ERR_PTR(-EINVAL);210 break;211 case QCOM_TZMEM_POLICY_MULTIPLIER:212 if (!config->increment)213 return ERR_PTR(-EINVAL);214 break;215 case QCOM_TZMEM_POLICY_ON_DEMAND:216 break;217 default:218 return ERR_PTR(-EINVAL);219 }220 221 struct qcom_tzmem_pool *pool __free(kfree) = kzalloc(sizeof(*pool),222 GFP_KERNEL);223 if (!pool)224 return ERR_PTR(-ENOMEM);225 226 pool->genpool = gen_pool_create(PAGE_SHIFT, -1);227 if (!pool->genpool)228 return ERR_PTR(-ENOMEM);229 230 gen_pool_set_algo(pool->genpool, gen_pool_best_fit, NULL);231 232 pool->policy = config->policy;233 pool->increment = config->increment;234 pool->max_size = config->max_size;235 INIT_LIST_HEAD(&pool->areas);236 spin_lock_init(&pool->lock);237 238 if (config->initial_size) {239 ret = qcom_tzmem_pool_add_memory(pool, config->initial_size,240 GFP_KERNEL);241 if (ret) {242 gen_pool_destroy(pool->genpool);243 return ERR_PTR(ret);244 }245 }246 247 return_ptr(pool);248}249EXPORT_SYMBOL_GPL(qcom_tzmem_pool_new);250 251/**252 * qcom_tzmem_pool_free() - Destroy a TZ memory pool and free all resources.253 * @pool: Memory pool to free.254 *255 * Must not be called if any of the allocated chunks has not been freed.256 * Must not be used in atomic context.257 */258void qcom_tzmem_pool_free(struct qcom_tzmem_pool *pool)259{260 struct qcom_tzmem_area *area, *next;261 struct qcom_tzmem_chunk *chunk;262 struct radix_tree_iter iter;263 bool non_empty = false;264 void __rcu **slot;265 266 might_sleep();267 268 if (!pool)269 return;270 271 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {272 radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {273 chunk = radix_tree_deref_slot_protected(slot,274 &qcom_tzmem_chunks_lock);275 276 if (chunk->owner == pool)277 non_empty = true;278 }279 }280 281 WARN(non_empty, "Freeing TZ memory pool with memory still allocated");282 283 list_for_each_entry_safe(area, next, &pool->areas, list) {284 list_del(&area->list);285 qcom_tzmem_cleanup_area(area);286 dma_free_coherent(qcom_tzmem_dev, area->size,287 area->vaddr, area->paddr);288 kfree(area);289 }290 291 gen_pool_destroy(pool->genpool);292 kfree(pool);293}294EXPORT_SYMBOL_GPL(qcom_tzmem_pool_free);295 296static void devm_qcom_tzmem_pool_free(void *data)297{298 struct qcom_tzmem_pool *pool = data;299 300 qcom_tzmem_pool_free(pool);301}302 303/**304 * devm_qcom_tzmem_pool_new() - Managed variant of qcom_tzmem_pool_new().305 * @dev: Device managing this resource.306 * @config: Pool configuration.307 *308 * Must not be used in atomic context.309 *310 * Return: Address of the managed pool or ERR_PTR() on failure.311 */312struct qcom_tzmem_pool *313devm_qcom_tzmem_pool_new(struct device *dev,314 const struct qcom_tzmem_pool_config *config)315{316 struct qcom_tzmem_pool *pool;317 int ret;318 319 pool = qcom_tzmem_pool_new(config);320 if (IS_ERR(pool))321 return pool;322 323 ret = devm_add_action_or_reset(dev, devm_qcom_tzmem_pool_free, pool);324 if (ret)325 return ERR_PTR(ret);326 327 return pool;328}329EXPORT_SYMBOL_GPL(devm_qcom_tzmem_pool_new);330 331static bool qcom_tzmem_try_grow_pool(struct qcom_tzmem_pool *pool,332 size_t requested, gfp_t gfp)333{334 size_t current_size = gen_pool_size(pool->genpool);335 336 if (pool->max_size && (current_size + requested) > pool->max_size)337 return false;338 339 switch (pool->policy) {340 case QCOM_TZMEM_POLICY_STATIC:341 return false;342 case QCOM_TZMEM_POLICY_MULTIPLIER:343 requested = current_size * pool->increment;344 break;345 case QCOM_TZMEM_POLICY_ON_DEMAND:346 break;347 }348 349 return !qcom_tzmem_pool_add_memory(pool, requested, gfp);350}351 352/**353 * qcom_tzmem_alloc() - Allocate a memory chunk suitable for sharing with TZ.354 * @pool: TZ memory pool from which to allocate memory.355 * @size: Number of bytes to allocate.356 * @gfp: GFP flags.357 *358 * Can be used in any context.359 *360 * Return:361 * Address of the allocated buffer or NULL if no more memory can be allocated.362 * The buffer must be released using qcom_tzmem_free().363 */364void *qcom_tzmem_alloc(struct qcom_tzmem_pool *pool, size_t size, gfp_t gfp)365{366 unsigned long vaddr;367 int ret;368 369 if (!size)370 return NULL;371 372 size = PAGE_ALIGN(size);373 374 struct qcom_tzmem_chunk *chunk __free(kfree) = kzalloc(sizeof(*chunk),375 gfp);376 if (!chunk)377 return NULL;378 379again:380 vaddr = gen_pool_alloc(pool->genpool, size);381 if (!vaddr) {382 if (qcom_tzmem_try_grow_pool(pool, size, gfp))383 goto again;384 385 return NULL;386 }387 388 chunk->size = size;389 chunk->owner = pool;390 391 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock) {392 ret = radix_tree_insert(&qcom_tzmem_chunks, vaddr, chunk);393 if (ret) {394 gen_pool_free(pool->genpool, vaddr, size);395 return NULL;396 }397 398 chunk = NULL;399 }400 401 return (void *)vaddr;402}403EXPORT_SYMBOL_GPL(qcom_tzmem_alloc);404 405/**406 * qcom_tzmem_free() - Release a buffer allocated from a TZ memory pool.407 * @vaddr: Virtual address of the buffer.408 *409 * Can be used in any context.410 */411void qcom_tzmem_free(void *vaddr)412{413 struct qcom_tzmem_chunk *chunk;414 415 scoped_guard(spinlock_irqsave, &qcom_tzmem_chunks_lock)416 chunk = radix_tree_delete_item(&qcom_tzmem_chunks,417 (unsigned long)vaddr, NULL);418 419 if (!chunk) {420 WARN(1, "Virtual address %p not owned by TZ memory allocator",421 vaddr);422 return;423 }424 425 scoped_guard(spinlock_irqsave, &chunk->owner->lock)426 gen_pool_free(chunk->owner->genpool, (unsigned long)vaddr,427 chunk->size);428 kfree(chunk);429}430EXPORT_SYMBOL_GPL(qcom_tzmem_free);431 432/**433 * qcom_tzmem_to_phys() - Map the virtual address of TZ memory to physical.434 * @vaddr: Virtual address of memory allocated from a TZ memory pool.435 *436 * Can be used in any context. The address must point to memory allocated437 * using qcom_tzmem_alloc().438 *439 * Returns:440 * Physical address mapped from the virtual or 0 if the mapping failed.441 */442phys_addr_t qcom_tzmem_to_phys(void *vaddr)443{444 struct qcom_tzmem_chunk *chunk;445 struct radix_tree_iter iter;446 void __rcu **slot;447 phys_addr_t ret;448 449 guard(spinlock_irqsave)(&qcom_tzmem_chunks_lock);450 451 radix_tree_for_each_slot(slot, &qcom_tzmem_chunks, &iter, 0) {452 chunk = radix_tree_deref_slot_protected(slot,453 &qcom_tzmem_chunks_lock);454 455 ret = gen_pool_virt_to_phys(chunk->owner->genpool,456 (unsigned long)vaddr);457 if (ret == -1)458 continue;459 460 return ret;461 }462 463 return 0;464}465EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys);466 467int qcom_tzmem_enable(struct device *dev)468{469 if (qcom_tzmem_dev)470 return -EBUSY;471 472 qcom_tzmem_dev = dev;473 474 return qcom_tzmem_init();475}476EXPORT_SYMBOL_GPL(qcom_tzmem_enable);477 478MODULE_DESCRIPTION("TrustZone memory allocator for Qualcomm firmware drivers");479MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@linaro.org>");480MODULE_LICENSE("GPL");481