847 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2013--2024 Intel Corporation4 */5#include <asm/barrier.h>6 7#include <linux/align.h>8#include <linux/atomic.h>9#include <linux/bitops.h>10#include <linux/bits.h>11#include <linux/bug.h>12#include <linux/cacheflush.h>13#include <linux/dma-mapping.h>14#include <linux/err.h>15#include <linux/gfp.h>16#include <linux/io.h>17#include <linux/iova.h>18#include <linux/math.h>19#include <linux/minmax.h>20#include <linux/mm.h>21#include <linux/pfn.h>22#include <linux/slab.h>23#include <linux/spinlock.h>24#include <linux/types.h>25#include <linux/vmalloc.h>26 27#include "ipu6.h"28#include "ipu6-dma.h"29#include "ipu6-mmu.h"30#include "ipu6-platform-regs.h"31 32#define ISP_PAGE_SHIFT 1233#define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT)34#define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1))35 36#define ISP_L1PT_SHIFT 2237#define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1))38 39#define ISP_L2PT_SHIFT 1240#define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK))))41 42#define ISP_L1PT_PTES 102443#define ISP_L2PT_PTES 102444 45#define ISP_PADDR_SHIFT 1246 47#define REG_TLB_INVALIDATE 0x000048 49#define REG_L1_PHYS 0x0004 /* 27-bit pfn */50#define REG_INFO 0x000851 52#define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)53 54static void tlb_invalidate(struct ipu6_mmu *mmu)55{56 unsigned long flags;57 unsigned int i;58 59 spin_lock_irqsave(&mmu->ready_lock, flags);60 if (!mmu->ready) {61 spin_unlock_irqrestore(&mmu->ready_lock, flags);62 return;63 }64 65 for (i = 0; i < mmu->nr_mmus; i++) {66 /*67 * To avoid the HW bug induced dead lock in some of the IPU668 * MMUs on successive invalidate calls, we need to first do a69 * read to the page table base before writing the invalidate70 * register. MMUs which need to implement this WA, will have71 * the insert_read_before_invalidate flags set as true.72 * Disregard the return value of the read.73 */74 if (mmu->mmu_hw[i].insert_read_before_invalidate)75 readl(mmu->mmu_hw[i].base + REG_L1_PHYS);76 77 writel(0xffffffff, mmu->mmu_hw[i].base +78 REG_TLB_INVALIDATE);79 /*80 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)81 * When the actual MMIO write reaches the IPU6 TLB Invalidate82 * register, wmb() will force the TLB invalidate out if the CPU83 * attempts to update the IOMMU page table (or sooner).84 */85 wmb();86 }87 spin_unlock_irqrestore(&mmu->ready_lock, flags);88}89 90#ifdef DEBUG91static void page_table_dump(struct ipu6_mmu_info *mmu_info)92{93 u32 l1_idx;94 95 dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");96 97 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {98 u32 l2_idx;99 u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;100 101 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)102 continue;103 dev_dbg(mmu_info->dev,104 "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pa\n",105 l1_idx, iova, iova + ISP_PAGE_SIZE,106 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]));107 108 for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {109 u32 *l2_pt = mmu_info->l2_pts[l1_idx];110 u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);111 112 if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)113 continue;114 115 dev_dbg(mmu_info->dev,116 "\tl2 entry %u; iova 0x%8.8x, phys %pa\n",117 l2_idx, iova2,118 TBL_PHYS_ADDR(l2_pt[l2_idx]));119 }120 }121 122 dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");123}124#endif /* DEBUG */125 126static dma_addr_t map_single(struct ipu6_mmu_info *mmu_info, void *ptr)127{128 dma_addr_t dma;129 130 dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL);131 if (dma_mapping_error(mmu_info->dev, dma))132 return 0;133 134 return dma;135}136 137static int get_dummy_page(struct ipu6_mmu_info *mmu_info)138{139 void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);140 dma_addr_t dma;141 142 if (!pt)143 return -ENOMEM;144 145 dev_dbg(mmu_info->dev, "dummy_page: get_zeroed_page() == %p\n", pt);146 147 dma = map_single(mmu_info, pt);148 if (!dma) {149 dev_err(mmu_info->dev, "Failed to map dummy page\n");150 goto err_free_page;151 }152 153 mmu_info->dummy_page = pt;154 mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT;155 156 return 0;157 158err_free_page:159 free_page((unsigned long)pt);160 return -ENOMEM;161}162 163static void free_dummy_page(struct ipu6_mmu_info *mmu_info)164{165 dma_unmap_single(mmu_info->dev,166 TBL_PHYS_ADDR(mmu_info->dummy_page_pteval),167 PAGE_SIZE, DMA_BIDIRECTIONAL);168 free_page((unsigned long)mmu_info->dummy_page);169}170 171static int alloc_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)172{173 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);174 dma_addr_t dma;175 unsigned int i;176 177 if (!pt)178 return -ENOMEM;179 180 dev_dbg(mmu_info->dev, "dummy_l2: get_zeroed_page() = %p\n", pt);181 182 dma = map_single(mmu_info, pt);183 if (!dma) {184 dev_err(mmu_info->dev, "Failed to map l2pt page\n");185 goto err_free_page;186 }187 188 for (i = 0; i < ISP_L2PT_PTES; i++)189 pt[i] = mmu_info->dummy_page_pteval;190 191 mmu_info->dummy_l2_pt = pt;192 mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT;193 194 return 0;195 196err_free_page:197 free_page((unsigned long)pt);198 return -ENOMEM;199}200 201static void free_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)202{203 dma_unmap_single(mmu_info->dev,204 TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval),205 PAGE_SIZE, DMA_BIDIRECTIONAL);206 free_page((unsigned long)mmu_info->dummy_l2_pt);207}208 209static u32 *alloc_l1_pt(struct ipu6_mmu_info *mmu_info)210{211 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);212 dma_addr_t dma;213 unsigned int i;214 215 if (!pt)216 return NULL;217 218 dev_dbg(mmu_info->dev, "alloc_l1: get_zeroed_page() = %p\n", pt);219 220 for (i = 0; i < ISP_L1PT_PTES; i++)221 pt[i] = mmu_info->dummy_l2_pteval;222 223 dma = map_single(mmu_info, pt);224 if (!dma) {225 dev_err(mmu_info->dev, "Failed to map l1pt page\n");226 goto err_free_page;227 }228 229 mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT;230 dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma);231 232 return pt;233 234err_free_page:235 free_page((unsigned long)pt);236 return NULL;237}238 239static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info)240{241 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);242 unsigned int i;243 244 if (!pt)245 return NULL;246 247 dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt);248 249 for (i = 0; i < ISP_L1PT_PTES; i++)250 pt[i] = mmu_info->dummy_page_pteval;251 252 return pt;253}254 255static int l2_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,256 phys_addr_t paddr, size_t size)257{258 u32 l1_idx = iova >> ISP_L1PT_SHIFT;259 u32 iova_start = iova;260 u32 *l2_pt, *l2_virt;261 unsigned int l2_idx;262 unsigned long flags;263 dma_addr_t dma;264 u32 l1_entry;265 266 dev_dbg(mmu_info->dev,267 "mapping l2 page table for l1 index %u (iova %8.8x)\n",268 l1_idx, (u32)iova);269 270 spin_lock_irqsave(&mmu_info->lock, flags);271 l1_entry = mmu_info->l1_pt[l1_idx];272 if (l1_entry == mmu_info->dummy_l2_pteval) {273 l2_virt = mmu_info->l2_pts[l1_idx];274 if (likely(!l2_virt)) {275 l2_virt = alloc_l2_pt(mmu_info);276 if (!l2_virt) {277 spin_unlock_irqrestore(&mmu_info->lock, flags);278 return -ENOMEM;279 }280 }281 282 dma = map_single(mmu_info, l2_virt);283 if (!dma) {284 dev_err(mmu_info->dev, "Failed to map l2pt page\n");285 free_page((unsigned long)l2_virt);286 spin_unlock_irqrestore(&mmu_info->lock, flags);287 return -EINVAL;288 }289 290 l1_entry = dma >> ISP_PADDR_SHIFT;291 292 dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n",293 l1_idx, l2_virt);294 mmu_info->l1_pt[l1_idx] = l1_entry;295 mmu_info->l2_pts[l1_idx] = l2_virt;296 clflush_cache_range((void *)&mmu_info->l1_pt[l1_idx],297 sizeof(mmu_info->l1_pt[l1_idx]));298 }299 300 l2_pt = mmu_info->l2_pts[l1_idx];301 302 dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry);303 304 paddr = ALIGN(paddr, ISP_PAGE_SIZE);305 306 l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;307 308 dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx,309 l2_pt[l2_idx]);310 if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) {311 spin_unlock_irqrestore(&mmu_info->lock, flags);312 return -EINVAL;313 }314 315 l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT;316 317 clflush_cache_range((void *)&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));318 spin_unlock_irqrestore(&mmu_info->lock, flags);319 320 dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx,321 l2_pt[l2_idx]);322 323 return 0;324}325 326static int __ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,327 phys_addr_t paddr, size_t size)328{329 u32 iova_start = round_down(iova, ISP_PAGE_SIZE);330 u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE);331 332 dev_dbg(mmu_info->dev,333 "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",334 iova_start, iova_end, size, paddr);335 336 return l2_map(mmu_info, iova_start, paddr, size);337}338 339static size_t l2_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,340 phys_addr_t dummy, size_t size)341{342 u32 l1_idx = iova >> ISP_L1PT_SHIFT;343 u32 iova_start = iova;344 unsigned int l2_idx;345 size_t unmapped = 0;346 unsigned long flags;347 u32 *l2_pt;348 349 dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",350 l1_idx, iova);351 352 spin_lock_irqsave(&mmu_info->lock, flags);353 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) {354 spin_unlock_irqrestore(&mmu_info->lock, flags);355 dev_err(mmu_info->dev,356 "unmap iova 0x%8.8lx l1 idx %u which was not mapped\n",357 iova, l1_idx);358 return 0;359 }360 361 for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;362 (iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT)363 < iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) {364 l2_pt = mmu_info->l2_pts[l1_idx];365 dev_dbg(mmu_info->dev,366 "unmap l2 index %u with pteval 0x%10.10llx\n",367 l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));368 l2_pt[l2_idx] = mmu_info->dummy_page_pteval;369 370 clflush_cache_range((void *)&l2_pt[l2_idx],371 sizeof(l2_pt[l2_idx]));372 unmapped++;373 }374 spin_unlock_irqrestore(&mmu_info->lock, flags);375 376 return unmapped << ISP_PAGE_SHIFT;377}378 379static size_t __ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info,380 unsigned long iova, size_t size)381{382 return l2_unmap(mmu_info, iova, 0, size);383}384 385static int allocate_trash_buffer(struct ipu6_mmu *mmu)386{387 unsigned int n_pages = PHYS_PFN(PAGE_ALIGN(IPU6_MMUV2_TRASH_RANGE));388 struct iova *iova;389 unsigned int i;390 dma_addr_t dma;391 unsigned long iova_addr;392 int ret;393 394 /* Allocate 8MB in iova range */395 iova = alloc_iova(&mmu->dmap->iovad, n_pages,396 PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);397 if (!iova) {398 dev_err(mmu->dev, "cannot allocate iova range for trash\n");399 return -ENOMEM;400 }401 402 dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,403 PAGE_SIZE, DMA_BIDIRECTIONAL);404 if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {405 dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");406 ret = -ENOMEM;407 goto out_free_iova;408 }409 410 mmu->pci_trash_page = dma;411 412 /*413 * Map the 8MB iova address range to the same physical trash page414 * mmu->trash_page which is already reserved at the probe415 */416 iova_addr = iova->pfn_lo;417 for (i = 0; i < n_pages; i++) {418 ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),419 mmu->pci_trash_page, PAGE_SIZE);420 if (ret) {421 dev_err(mmu->dev,422 "mapping trash buffer range failed\n");423 goto out_unmap;424 }425 426 iova_addr++;427 }428 429 mmu->iova_trash_page = PFN_PHYS(iova->pfn_lo);430 dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",431 mmu->mmid, (unsigned int)mmu->iova_trash_page);432 return 0;433 434out_unmap:435 ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),436 PFN_PHYS(iova_size(iova)));437 dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,438 PAGE_SIZE, DMA_BIDIRECTIONAL);439out_free_iova:440 __free_iova(&mmu->dmap->iovad, iova);441 return ret;442}443 444int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)445{446 struct ipu6_mmu_info *mmu_info;447 unsigned long flags;448 unsigned int i;449 450 mmu_info = mmu->dmap->mmu_info;451 452 /* Initialise the each MMU HW block */453 for (i = 0; i < mmu->nr_mmus; i++) {454 struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];455 unsigned int j;456 u16 block_addr;457 458 /* Write page table address per MMU */459 writel((phys_addr_t)mmu_info->l1_pt_dma,460 mmu->mmu_hw[i].base + REG_L1_PHYS);461 462 /* Set info bits per MMU */463 writel(mmu->mmu_hw[i].info_bits,464 mmu->mmu_hw[i].base + REG_INFO);465 466 /* Configure MMU TLB stream configuration for L1 */467 for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams;468 block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {469 if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {470 dev_err(mmu->dev, "invalid L1 configuration\n");471 return -EINVAL;472 }473 474 /* Write block start address for each streams */475 writel(block_addr, mmu_hw->base +476 mmu_hw->l1_stream_id_reg_offset + 4 * j);477 }478 479 /* Configure MMU TLB stream configuration for L2 */480 for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams;481 block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {482 if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {483 dev_err(mmu->dev, "invalid L2 configuration\n");484 return -EINVAL;485 }486 487 writel(block_addr, mmu_hw->base +488 mmu_hw->l2_stream_id_reg_offset + 4 * j);489 }490 }491 492 if (!mmu->trash_page) {493 int ret;494 495 mmu->trash_page = alloc_page(GFP_KERNEL);496 if (!mmu->trash_page) {497 dev_err(mmu->dev, "insufficient memory for trash buffer\n");498 return -ENOMEM;499 }500 501 ret = allocate_trash_buffer(mmu);502 if (ret) {503 __free_page(mmu->trash_page);504 mmu->trash_page = NULL;505 dev_err(mmu->dev, "trash buffer allocation failed\n");506 return ret;507 }508 }509 510 spin_lock_irqsave(&mmu->ready_lock, flags);511 mmu->ready = true;512 spin_unlock_irqrestore(&mmu->ready_lock, flags);513 514 return 0;515}516EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_init, INTEL_IPU6);517 518static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp)519{520 struct ipu6_mmu_info *mmu_info;521 int ret;522 523 mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL);524 if (!mmu_info)525 return NULL;526 527 mmu_info->aperture_start = 0;528 mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ?529 IPU6_MMU_ADDR_BITS :530 IPU6_MMU_ADDR_BITS_NON_SECURE);531 mmu_info->pgsize_bitmap = SZ_4K;532 mmu_info->dev = &isp->pdev->dev;533 534 ret = get_dummy_page(mmu_info);535 if (ret)536 goto err_free_info;537 538 ret = alloc_dummy_l2_pt(mmu_info);539 if (ret)540 goto err_free_dummy_page;541 542 mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts));543 if (!mmu_info->l2_pts)544 goto err_free_dummy_l2_pt;545 546 /*547 * We always map the L1 page table (a single page as well as548 * the L2 page tables).549 */550 mmu_info->l1_pt = alloc_l1_pt(mmu_info);551 if (!mmu_info->l1_pt)552 goto err_free_l2_pts;553 554 spin_lock_init(&mmu_info->lock);555 556 dev_dbg(mmu_info->dev, "domain initialised\n");557 558 return mmu_info;559 560err_free_l2_pts:561 vfree(mmu_info->l2_pts);562err_free_dummy_l2_pt:563 free_dummy_l2_pt(mmu_info);564err_free_dummy_page:565 free_dummy_page(mmu_info);566err_free_info:567 kfree(mmu_info);568 569 return NULL;570}571 572void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu)573{574 unsigned long flags;575 576 spin_lock_irqsave(&mmu->ready_lock, flags);577 mmu->ready = false;578 spin_unlock_irqrestore(&mmu->ready_lock, flags);579}580EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_cleanup, INTEL_IPU6);581 582static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)583{584 struct ipu6_dma_mapping *dmap;585 586 dmap = kzalloc(sizeof(*dmap), GFP_KERNEL);587 if (!dmap)588 return NULL;589 590 dmap->mmu_info = ipu6_mmu_alloc(isp);591 if (!dmap->mmu_info) {592 kfree(dmap);593 return NULL;594 }595 596 init_iova_domain(&dmap->iovad, SZ_4K, 1);597 dmap->mmu_info->dmap = dmap;598 599 dev_dbg(&isp->pdev->dev, "alloc mapping\n");600 601 iova_cache_get();602 603 return dmap;604}605 606phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info,607 dma_addr_t iova)608{609 phys_addr_t phy_addr;610 unsigned long flags;611 u32 *l2_pt;612 613 spin_lock_irqsave(&mmu_info->lock, flags);614 l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT];615 phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT];616 phy_addr <<= ISP_PAGE_SHIFT;617 spin_unlock_irqrestore(&mmu_info->lock, flags);618 619 return phy_addr;620}621 622static size_t ipu6_mmu_pgsize(unsigned long pgsize_bitmap,623 unsigned long addr_merge, size_t size)624{625 unsigned int pgsize_idx;626 size_t pgsize;627 628 /* Max page size that still fits into 'size' */629 pgsize_idx = __fls(size);630 631 if (likely(addr_merge)) {632 /* Max page size allowed by address */633 unsigned int align_pgsize_idx = __ffs(addr_merge);634 635 pgsize_idx = min(pgsize_idx, align_pgsize_idx);636 }637 638 pgsize = (1UL << (pgsize_idx + 1)) - 1;639 pgsize &= pgsize_bitmap;640 641 WARN_ON(!pgsize);642 643 /* pick the biggest page */644 pgsize_idx = __fls(pgsize);645 pgsize = 1UL << pgsize_idx;646 647 return pgsize;648}649 650size_t ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,651 size_t size)652{653 size_t unmapped_page, unmapped = 0;654 unsigned int min_pagesz;655 656 /* find out the minimum page size supported */657 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);658 659 /*660 * The virtual address and the size of the mapping must be661 * aligned (at least) to the size of the smallest page supported662 * by the hardware663 */664 if (!IS_ALIGNED(iova | size, min_pagesz)) {665 dev_err(NULL, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",666 iova, size, min_pagesz);667 return -EINVAL;668 }669 670 /*671 * Keep iterating until we either unmap 'size' bytes (or more)672 * or we hit an area that isn't mapped.673 */674 while (unmapped < size) {675 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap,676 iova, size - unmapped);677 678 unmapped_page = __ipu6_mmu_unmap(mmu_info, iova, pgsize);679 if (!unmapped_page)680 break;681 682 dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n",683 iova, unmapped_page);684 685 iova += unmapped_page;686 unmapped += unmapped_page;687 }688 689 return unmapped;690}691 692int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,693 phys_addr_t paddr, size_t size)694{695 unsigned long orig_iova = iova;696 unsigned int min_pagesz;697 size_t orig_size = size;698 int ret = 0;699 700 if (mmu_info->pgsize_bitmap == 0UL)701 return -ENODEV;702 703 /* find out the minimum page size supported */704 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);705 706 /*707 * both the virtual address and the physical one, as well as708 * the size of the mapping, must be aligned (at least) to the709 * size of the smallest page supported by the hardware710 */711 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {712 dev_err(mmu_info->dev,713 "unaligned: iova %lx pa %pa size %zx min_pagesz %x\n",714 iova, &paddr, size, min_pagesz);715 return -EINVAL;716 }717 718 dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",719 iova, &paddr, size);720 721 while (size) {722 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap,723 iova | paddr, size);724 725 dev_dbg(mmu_info->dev,726 "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",727 iova, &paddr, pgsize);728 729 ret = __ipu6_mmu_map(mmu_info, iova, paddr, pgsize);730 if (ret)731 break;732 733 iova += pgsize;734 paddr += pgsize;735 size -= pgsize;736 }737 738 /* unroll mapping in case something went wrong */739 if (ret)740 ipu6_mmu_unmap(mmu_info, orig_iova, orig_size - size);741 742 return ret;743}744 745static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)746{747 struct ipu6_dma_mapping *dmap = mmu->dmap;748 struct ipu6_mmu_info *mmu_info = dmap->mmu_info;749 struct iova *iova;750 u32 l1_idx;751 752 if (mmu->iova_trash_page) {753 iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page));754 if (iova) {755 /* unmap and free the trash buffer iova */756 ipu6_mmu_unmap(mmu_info, PFN_PHYS(iova->pfn_lo),757 PFN_PHYS(iova_size(iova)));758 __free_iova(&dmap->iovad, iova);759 } else {760 dev_err(mmu->dev, "trash buffer iova not found.\n");761 }762 763 mmu->iova_trash_page = 0;764 dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,765 PAGE_SIZE, DMA_BIDIRECTIONAL);766 mmu->pci_trash_page = 0;767 __free_page(mmu->trash_page);768 }769 770 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {771 if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) {772 dma_unmap_single(mmu_info->dev,773 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]),774 PAGE_SIZE, DMA_BIDIRECTIONAL);775 free_page((unsigned long)mmu_info->l2_pts[l1_idx]);776 }777 }778 779 vfree(mmu_info->l2_pts);780 free_dummy_page(mmu_info);781 dma_unmap_single(mmu_info->dev, TBL_PHYS_ADDR(mmu_info->l1_pt_dma),782 PAGE_SIZE, DMA_BIDIRECTIONAL);783 free_page((unsigned long)mmu_info->dummy_l2_pt);784 free_page((unsigned long)mmu_info->l1_pt);785 kfree(mmu_info);786}787 788struct ipu6_mmu *ipu6_mmu_init(struct device *dev,789 void __iomem *base, int mmid,790 const struct ipu6_hw_variants *hw)791{792 struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));793 struct ipu6_mmu_pdata *pdata;794 struct ipu6_mmu *mmu;795 unsigned int i;796 797 if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES)798 return ERR_PTR(-EINVAL);799 800 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);801 if (!pdata)802 return ERR_PTR(-ENOMEM);803 804 for (i = 0; i < hw->nr_mmus; i++) {805 struct ipu6_mmu_hw *pdata_mmu = &pdata->mmu_hw[i];806 const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i];807 808 if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||809 src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)810 return ERR_PTR(-EINVAL);811 812 *pdata_mmu = *src_mmu;813 pdata_mmu->base = base + src_mmu->offset;814 }815 816 mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);817 if (!mmu)818 return ERR_PTR(-ENOMEM);819 820 mmu->mmid = mmid;821 mmu->mmu_hw = pdata->mmu_hw;822 mmu->nr_mmus = hw->nr_mmus;823 mmu->tlb_invalidate = tlb_invalidate;824 mmu->ready = false;825 INIT_LIST_HEAD(&mmu->vma_list);826 spin_lock_init(&mmu->ready_lock);827 828 mmu->dmap = alloc_dma_mapping(isp);829 if (!mmu->dmap) {830 dev_err(dev, "can't alloc dma mapping\n");831 return ERR_PTR(-ENOMEM);832 }833 834 return mmu;835}836 837void ipu6_mmu_cleanup(struct ipu6_mmu *mmu)838{839 struct ipu6_dma_mapping *dmap = mmu->dmap;840 841 ipu6_mmu_destroy(mmu);842 mmu->dmap = NULL;843 iova_cache_put();844 put_iova_domain(&dmap->iovad);845 kfree(dmap);846}847