683 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.4 * Copyright (C) 2019-2024 Linaro Ltd.5 */6 7#include <linux/dma-mapping.h>8#include <linux/io.h>9#include <linux/iommu.h>10#include <linux/platform_device.h>11#include <linux/types.h>12 13#include <linux/soc/qcom/smem.h>14 15#include "gsi_trans.h"16#include "ipa.h"17#include "ipa_cmd.h"18#include "ipa_data.h"19#include "ipa_mem.h"20#include "ipa_reg.h"21#include "ipa_table.h"22 23/* "Canary" value placed between memory regions to detect overflow */24#define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)25 26/* SMEM host id representing the modem. */27#define QCOM_SMEM_HOST_MODEM 128 29const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)30{31 u32 i;32 33 for (i = 0; i < ipa->mem_count; i++) {34 const struct ipa_mem *mem = &ipa->mem[i];35 36 if (mem->id == mem_id)37 return mem;38 }39 40 return NULL;41}42 43/* Add an immediate command to a transaction that zeroes a memory region */44static void45ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)46{47 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);48 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);49 dma_addr_t addr = ipa->zero_addr;50 51 if (!mem->size)52 return;53 54 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);55}56 57/**58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas59 * @ipa: IPA pointer60 *61 * Set up the shared memory regions in IPA local memory. This involves62 * zero-filling memory regions, and in the case of header memory, telling63 * the IPA where it's located.64 *65 * This function performs the initial setup of this memory. If the modem66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().67 *68 * The AP informs the modem where its portions of memory are located69 * in a QMI exchange that occurs at modem startup.70 *71 * There is no need for a matching ipa_mem_teardown() function.72 *73 * Return: 0 if successful, or a negative error code74 */75int ipa_mem_setup(struct ipa *ipa)76{77 dma_addr_t addr = ipa->zero_addr;78 const struct ipa_mem *mem;79 struct gsi_trans *trans;80 const struct reg *reg;81 u32 offset;82 u16 size;83 u32 val;84 85 /* Get a transaction to define the header memory region and to zero86 * the processing context and modem memory regions.87 */88 trans = ipa_cmd_trans_alloc(ipa, 4);89 if (!trans) {90 dev_err(ipa->dev, "no transaction for memory setup\n");91 return -EBUSY;92 }93 94 /* Initialize IPA-local header memory. The AP header region, if95 * present, is contiguous with and follows the modem header region,96 * and they are initialized together.97 */98 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);99 offset = mem->offset;100 size = mem->size;101 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);102 if (mem)103 size += mem->size;104 105 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);106 107 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);108 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);109 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);110 111 gsi_trans_commit_wait(trans);112 113 /* Tell the hardware where the processing context area is located */114 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);115 offset = ipa->mem_offset + mem->offset;116 117 reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT);118 val = reg_encode(reg, IPA_BASE_ADDR, offset);119 iowrite32(val, ipa->reg_virt + reg_offset(reg));120 121 return 0;122}123 124/* Is the given memory region ID is valid for the current IPA version? */125static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)126{127 enum ipa_version version = ipa->version;128 129 switch (mem_id) {130 case IPA_MEM_UC_SHARED:131 case IPA_MEM_UC_INFO:132 case IPA_MEM_V4_FILTER_HASHED:133 case IPA_MEM_V4_FILTER:134 case IPA_MEM_V6_FILTER_HASHED:135 case IPA_MEM_V6_FILTER:136 case IPA_MEM_V4_ROUTE_HASHED:137 case IPA_MEM_V4_ROUTE:138 case IPA_MEM_V6_ROUTE_HASHED:139 case IPA_MEM_V6_ROUTE:140 case IPA_MEM_MODEM_HEADER:141 case IPA_MEM_AP_HEADER:142 case IPA_MEM_MODEM_PROC_CTX:143 case IPA_MEM_AP_PROC_CTX:144 case IPA_MEM_MODEM:145 case IPA_MEM_UC_EVENT_RING:146 case IPA_MEM_PDN_CONFIG:147 case IPA_MEM_STATS_QUOTA_MODEM:148 case IPA_MEM_STATS_QUOTA_AP:149 case IPA_MEM_END_MARKER: /* pseudo region */150 break;151 152 case IPA_MEM_STATS_TETHERING:153 case IPA_MEM_STATS_DROP:154 if (version < IPA_VERSION_4_0)155 return false;156 break;157 158 case IPA_MEM_STATS_V4_FILTER:159 case IPA_MEM_STATS_V6_FILTER:160 case IPA_MEM_STATS_V4_ROUTE:161 case IPA_MEM_STATS_V6_ROUTE:162 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)163 return false;164 break;165 166 case IPA_MEM_AP_V4_FILTER:167 case IPA_MEM_AP_V6_FILTER:168 if (version < IPA_VERSION_5_0)169 return false;170 break;171 172 case IPA_MEM_NAT_TABLE:173 case IPA_MEM_STATS_FILTER_ROUTE:174 if (version < IPA_VERSION_4_5)175 return false;176 break;177 178 default:179 return false;180 }181 182 return true;183}184 185/* Must the given memory region be present in the configuration? */186static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)187{188 switch (mem_id) {189 case IPA_MEM_UC_SHARED:190 case IPA_MEM_UC_INFO:191 case IPA_MEM_V4_FILTER_HASHED:192 case IPA_MEM_V4_FILTER:193 case IPA_MEM_V6_FILTER_HASHED:194 case IPA_MEM_V6_FILTER:195 case IPA_MEM_V4_ROUTE_HASHED:196 case IPA_MEM_V4_ROUTE:197 case IPA_MEM_V6_ROUTE_HASHED:198 case IPA_MEM_V6_ROUTE:199 case IPA_MEM_MODEM_HEADER:200 case IPA_MEM_MODEM_PROC_CTX:201 case IPA_MEM_AP_PROC_CTX:202 case IPA_MEM_MODEM:203 return true;204 205 case IPA_MEM_PDN_CONFIG:206 case IPA_MEM_STATS_QUOTA_MODEM:207 return ipa->version >= IPA_VERSION_4_0;208 209 case IPA_MEM_STATS_TETHERING:210 return ipa->version >= IPA_VERSION_4_0 &&211 ipa->version != IPA_VERSION_5_0;212 213 default:214 return false; /* Anything else is optional */215 }216}217 218static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)219{220 enum ipa_mem_id mem_id = mem->id;221 struct device *dev = ipa->dev;222 u16 size_multiple;223 224 /* Make sure the memory region is valid for this version of IPA */225 if (!ipa_mem_id_valid(ipa, mem_id)) {226 dev_err(dev, "region id %u not valid\n", mem_id);227 return false;228 }229 230 if (!mem->size && !mem->canary_count) {231 dev_err(dev, "empty memory region %u\n", mem_id);232 return false;233 }234 235 /* Other than modem memory, sizes must be a multiple of 8 */236 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;237 if (mem->size % size_multiple)238 dev_err(dev, "region %u size not a multiple of %u bytes\n",239 mem_id, size_multiple);240 else if (mem->offset % 8)241 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);242 else if (mem->offset < mem->canary_count * sizeof(__le32))243 dev_err(dev, "region %u offset too small for %hu canaries\n",244 mem_id, mem->canary_count);245 else if (mem_id == IPA_MEM_END_MARKER && mem->size)246 dev_err(dev, "non-zero end marker region size\n");247 else248 return true;249 250 return false;251}252 253/* Verify each defined memory region is valid. */254static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)255{256 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };257 struct device *dev = ipa->dev;258 enum ipa_mem_id mem_id;259 u32 i;260 261 if (mem_data->local_count > IPA_MEM_COUNT) {262 dev_err(dev, "too many memory regions (%u > %u)\n",263 mem_data->local_count, IPA_MEM_COUNT);264 return false;265 }266 267 for (i = 0; i < mem_data->local_count; i++) {268 const struct ipa_mem *mem = &mem_data->local[i];269 270 if (__test_and_set_bit(mem->id, regions)) {271 dev_err(dev, "duplicate memory region %u\n", mem->id);272 return false;273 }274 275 /* Defined regions have non-zero size and/or canary count */276 if (!ipa_mem_valid_one(ipa, mem))277 return false;278 }279 280 /* Now see if any required regions are not defined */281 for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) {282 if (ipa_mem_id_required(ipa, mem_id))283 dev_err(dev, "required memory region %u missing\n",284 mem_id);285 }286 287 return true;288}289 290/* Do all memory regions fit within the IPA local memory? */291static bool ipa_mem_size_valid(struct ipa *ipa)292{293 struct device *dev = ipa->dev;294 u32 limit = ipa->mem_size;295 u32 i;296 297 for (i = 0; i < ipa->mem_count; i++) {298 const struct ipa_mem *mem = &ipa->mem[i];299 300 if (mem->offset + mem->size <= limit)301 continue;302 303 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",304 mem->id, limit);305 306 return false;307 }308 309 return true;310}311 312/**313 * ipa_mem_config() - Configure IPA shared memory314 * @ipa: IPA pointer315 *316 * Return: 0 if successful, or a negative error code317 */318int ipa_mem_config(struct ipa *ipa)319{320 struct device *dev = ipa->dev;321 const struct ipa_mem *mem;322 const struct reg *reg;323 dma_addr_t addr;324 u32 mem_size;325 void *virt;326 u32 val;327 u32 i;328 329 /* Check the advertised location and size of the shared memory area */330 reg = ipa_reg(ipa, SHARED_MEM_SIZE);331 val = ioread32(ipa->reg_virt + reg_offset(reg));332 333 /* The fields in the register are in 8 byte units */334 ipa->mem_offset = 8 * reg_decode(reg, MEM_BADDR, val);335 336 /* Make sure the end is within the region's mapped space */337 mem_size = 8 * reg_decode(reg, MEM_SIZE, val);338 339 /* If the sizes don't match, issue a warning */340 if (ipa->mem_offset + mem_size < ipa->mem_size) {341 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",342 mem_size);343 ipa->mem_size = mem_size;344 } else if (ipa->mem_offset + mem_size > ipa->mem_size) {345 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",346 mem_size);347 }348 349 /* We know our memory size; make sure regions are all in range */350 if (!ipa_mem_size_valid(ipa))351 return -EINVAL;352 353 /* Prealloc DMA memory for zeroing regions */354 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);355 if (!virt)356 return -ENOMEM;357 ipa->zero_addr = addr;358 ipa->zero_virt = virt;359 ipa->zero_size = IPA_MEM_MAX;360 361 /* For each defined region, write "canary" values in the362 * space prior to the region's base address if indicated.363 */364 for (i = 0; i < ipa->mem_count; i++) {365 u16 canary_count = ipa->mem[i].canary_count;366 __le32 *canary;367 368 if (!canary_count)369 continue;370 371 /* Write canary values in the space before the region */372 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;373 do374 *--canary = IPA_MEM_CANARY_VAL;375 while (--canary_count);376 }377 378 /* Verify the microcontroller ring alignment (if defined) */379 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);380 if (mem && mem->offset % 1024) {381 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");382 goto err_dma_free;383 }384 385 return 0;386 387err_dma_free:388 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);389 390 return -EINVAL;391}392 393/* Inverse of ipa_mem_config() */394void ipa_mem_deconfig(struct ipa *ipa)395{396 struct device *dev = ipa->dev;397 398 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);399 ipa->zero_size = 0;400 ipa->zero_virt = NULL;401 ipa->zero_addr = 0;402}403 404/**405 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem406 * @ipa: IPA pointer407 *408 * Zero regions of IPA-local memory used by the modem. These are configured409 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and410 * restarts via SSR we need to re-initialize them. A QMI message tells the411 * modem where to find regions of IPA local memory it needs to know about412 * (these included).413 */414int ipa_mem_zero_modem(struct ipa *ipa)415{416 struct gsi_trans *trans;417 418 /* Get a transaction to zero the modem memory, modem header,419 * and modem processing context regions.420 */421 trans = ipa_cmd_trans_alloc(ipa, 3);422 if (!trans) {423 dev_err(ipa->dev, "no transaction to zero modem memory\n");424 return -EBUSY;425 }426 427 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);428 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);429 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);430 431 gsi_trans_commit_wait(trans);432 433 return 0;434}435 436/**437 * ipa_imem_init() - Initialize IMEM memory used by the IPA438 * @ipa: IPA pointer439 * @addr: Physical address of the IPA region in IMEM440 * @size: Size (bytes) of the IPA region in IMEM441 *442 * IMEM is a block of shared memory separate from system DRAM, and443 * a portion of this memory is available for the IPA to use. The444 * modem accesses this memory directly, but the IPA accesses it445 * via the IOMMU, using the AP's credentials.446 *447 * If this region exists (size > 0) we map it for read/write access448 * through the IOMMU using the IPA device.449 *450 * Note: @addr and @size are not guaranteed to be page-aligned.451 */452static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)453{454 struct device *dev = ipa->dev;455 struct iommu_domain *domain;456 unsigned long iova;457 phys_addr_t phys;458 int ret;459 460 if (!size)461 return 0; /* IMEM memory not used */462 463 domain = iommu_get_domain_for_dev(dev);464 if (!domain) {465 dev_err(dev, "no IOMMU domain found for IMEM\n");466 return -EINVAL;467 }468 469 /* Align the address down and the size up to page boundaries */470 phys = addr & PAGE_MASK;471 size = PAGE_ALIGN(size + addr - phys);472 iova = phys; /* We just want a direct mapping */473 474 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,475 GFP_KERNEL);476 if (ret)477 return ret;478 479 ipa->imem_iova = iova;480 ipa->imem_size = size;481 482 return 0;483}484 485static void ipa_imem_exit(struct ipa *ipa)486{487 struct device *dev = ipa->dev;488 struct iommu_domain *domain;489 490 if (!ipa->imem_size)491 return;492 493 domain = iommu_get_domain_for_dev(dev);494 if (domain) {495 size_t size;496 497 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);498 if (size != ipa->imem_size)499 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",500 size, ipa->imem_size);501 } else {502 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");503 }504 505 ipa->imem_size = 0;506 ipa->imem_iova = 0;507}508 509/**510 * ipa_smem_init() - Initialize SMEM memory used by the IPA511 * @ipa: IPA pointer512 * @item: Item ID of SMEM memory513 * @size: Size (bytes) of SMEM memory region514 *515 * SMEM is a managed block of shared DRAM, from which numbered "items"516 * can be allocated. One item is designated for use by the IPA.517 *518 * The modem accesses SMEM memory directly, but the IPA accesses it519 * via the IOMMU, using the AP's credentials.520 *521 * If size provided is non-zero, we allocate it and map it for522 * access through the IOMMU.523 *524 * Note: @size and the item address are is not guaranteed to be page-aligned.525 */526static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size)527{528 struct device *dev = ipa->dev;529 struct iommu_domain *domain;530 unsigned long iova;531 phys_addr_t phys;532 phys_addr_t addr;533 size_t actual;534 void *virt;535 int ret;536 537 if (!size)538 return 0; /* SMEM memory not used */539 540 /* SMEM is memory shared between the AP and another system entity541 * (in this case, the modem). An allocation from SMEM is persistent542 * until the AP reboots; there is no way to free an allocated SMEM543 * region. Allocation only reserves the space; to use it you need544 * to "get" a pointer it (this does not imply reference counting).545 * The item might have already been allocated, in which case we546 * use it unless the size isn't what we expect.547 */548 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size);549 if (ret && ret != -EEXIST) {550 dev_err(dev, "error %d allocating size %zu SMEM item %u\n",551 ret, size, item);552 return ret;553 }554 555 /* Now get the address of the SMEM memory region */556 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual);557 if (IS_ERR(virt)) {558 ret = PTR_ERR(virt);559 dev_err(dev, "error %d getting SMEM item %u\n", ret, item);560 return ret;561 }562 563 /* In case the region was already allocated, verify the size */564 if (ret && actual != size) {565 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n",566 item, actual, size);567 return -EINVAL;568 }569 570 domain = iommu_get_domain_for_dev(dev);571 if (!domain) {572 dev_err(dev, "no IOMMU domain found for SMEM\n");573 return -EINVAL;574 }575 576 /* Align the address down and the size up to a page boundary */577 addr = qcom_smem_virt_to_phys(virt);578 phys = addr & PAGE_MASK;579 size = PAGE_ALIGN(size + addr - phys);580 iova = phys; /* We just want a direct mapping */581 582 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,583 GFP_KERNEL);584 if (ret)585 return ret;586 587 ipa->smem_iova = iova;588 ipa->smem_size = size;589 590 return 0;591}592 593static void ipa_smem_exit(struct ipa *ipa)594{595 struct device *dev = ipa->dev;596 struct iommu_domain *domain;597 598 domain = iommu_get_domain_for_dev(dev);599 if (domain) {600 size_t size;601 602 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);603 if (size != ipa->smem_size)604 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",605 size, ipa->smem_size);606 607 } else {608 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");609 }610 611 ipa->smem_size = 0;612 ipa->smem_iova = 0;613}614 615/* Perform memory region-related initialization */616int ipa_mem_init(struct ipa *ipa, struct platform_device *pdev,617 const struct ipa_mem_data *mem_data)618{619 struct device *dev = &pdev->dev;620 struct resource *res;621 int ret;622 623 /* Make sure the set of defined memory regions is valid */624 if (!ipa_mem_valid(ipa, mem_data))625 return -EINVAL;626 627 ipa->mem_count = mem_data->local_count;628 ipa->mem = mem_data->local;629 630 /* Check the route and filter table memory regions */631 if (!ipa_table_mem_valid(ipa, false))632 return -EINVAL;633 if (!ipa_table_mem_valid(ipa, true))634 return -EINVAL;635 636 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));637 if (ret) {638 dev_err(dev, "error %d setting DMA mask\n", ret);639 return ret;640 }641 642 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ipa-shared");643 if (!res) {644 dev_err(dev,645 "DT error getting \"ipa-shared\" memory property\n");646 return -ENODEV;647 }648 649 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);650 if (!ipa->mem_virt) {651 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");652 return -ENOMEM;653 }654 655 ipa->mem_addr = res->start;656 ipa->mem_size = resource_size(res);657 658 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size);659 if (ret)660 goto err_unmap;661 662 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size);663 if (ret)664 goto err_imem_exit;665 666 return 0;667 668err_imem_exit:669 ipa_imem_exit(ipa);670err_unmap:671 memunmap(ipa->mem_virt);672 673 return ret;674}675 676/* Inverse of ipa_mem_init() */677void ipa_mem_exit(struct ipa *ipa)678{679 ipa_smem_exit(ipa);680 ipa_imem_exit(ipa);681 memunmap(ipa->mem_virt);682}683