1380 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VFIO: IOMMU DMA mapping support for TCE on POWER4 *5 * Copyright (C) 2013 IBM Corp. All rights reserved.6 * Author: Alexey Kardashevskiy <aik@ozlabs.ru>7 * Copyright Gavin Shan, IBM Corporation 2014.8 *9 * Derived from original vfio_iommu_type1.c:10 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.11 * Author: Alex Williamson <alex.williamson@redhat.com>12 */13 14#include <linux/module.h>15#include <linux/pci.h>16#include <linux/slab.h>17#include <linux/uaccess.h>18#include <linux/err.h>19#include <linux/vfio.h>20#include <linux/vmalloc.h>21#include <linux/sched/mm.h>22#include <linux/sched/signal.h>23#include <linux/mm.h>24#include "vfio.h"25 26#include <asm/iommu.h>27#include <asm/tce.h>28#include <asm/mmu_context.h>29 30#define DRIVER_VERSION "0.1"31#define DRIVER_AUTHOR "aik@ozlabs.ru"32#define DRIVER_DESC "VFIO IOMMU SPAPR TCE"33 34static void tce_iommu_detach_group(void *iommu_data,35 struct iommu_group *iommu_group);36 37/*38 * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation39 *40 * This code handles mapping and unmapping of user data buffers41 * into DMA'ble space using the IOMMU42 */43 44struct tce_iommu_group {45 struct list_head next;46 struct iommu_group *grp;47};48 49/*50 * A container needs to remember which preregistered region it has51 * referenced to do proper cleanup at the userspace process exit.52 */53struct tce_iommu_prereg {54 struct list_head next;55 struct mm_iommu_table_group_mem_t *mem;56};57 58/*59 * The container descriptor supports only a single group per container.60 * Required by the API as the container is not supplied with the IOMMU group61 * at the moment of initialization.62 */63struct tce_container {64 struct mutex lock;65 bool enabled;66 bool v2;67 bool def_window_pending;68 unsigned long locked_pages;69 struct mm_struct *mm;70 struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];71 struct list_head group_list;72 struct list_head prereg_list;73};74 75static long tce_iommu_mm_set(struct tce_container *container)76{77 if (container->mm) {78 if (container->mm == current->mm)79 return 0;80 return -EPERM;81 }82 BUG_ON(!current->mm);83 container->mm = current->mm;84 mmgrab(container->mm);85 86 return 0;87}88 89static long tce_iommu_prereg_free(struct tce_container *container,90 struct tce_iommu_prereg *tcemem)91{92 long ret;93 94 ret = mm_iommu_put(container->mm, tcemem->mem);95 if (ret)96 return ret;97 98 list_del(&tcemem->next);99 kfree(tcemem);100 101 return 0;102}103 104static long tce_iommu_unregister_pages(struct tce_container *container,105 __u64 vaddr, __u64 size)106{107 struct mm_iommu_table_group_mem_t *mem;108 struct tce_iommu_prereg *tcemem;109 bool found = false;110 long ret;111 112 if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))113 return -EINVAL;114 115 mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);116 if (!mem)117 return -ENOENT;118 119 list_for_each_entry(tcemem, &container->prereg_list, next) {120 if (tcemem->mem == mem) {121 found = true;122 break;123 }124 }125 126 if (!found)127 ret = -ENOENT;128 else129 ret = tce_iommu_prereg_free(container, tcemem);130 131 mm_iommu_put(container->mm, mem);132 133 return ret;134}135 136static long tce_iommu_register_pages(struct tce_container *container,137 __u64 vaddr, __u64 size)138{139 long ret = 0;140 struct mm_iommu_table_group_mem_t *mem = NULL;141 struct tce_iommu_prereg *tcemem;142 unsigned long entries = size >> PAGE_SHIFT;143 144 if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||145 ((vaddr + size) < vaddr))146 return -EINVAL;147 148 mem = mm_iommu_get(container->mm, vaddr, entries);149 if (mem) {150 list_for_each_entry(tcemem, &container->prereg_list, next) {151 if (tcemem->mem == mem) {152 ret = -EBUSY;153 goto put_exit;154 }155 }156 } else {157 ret = mm_iommu_new(container->mm, vaddr, entries, &mem);158 if (ret)159 return ret;160 }161 162 tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);163 if (!tcemem) {164 ret = -ENOMEM;165 goto put_exit;166 }167 168 tcemem->mem = mem;169 list_add(&tcemem->next, &container->prereg_list);170 171 container->enabled = true;172 173 return 0;174 175put_exit:176 mm_iommu_put(container->mm, mem);177 return ret;178}179 180static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,181 unsigned int it_page_shift)182{183 struct page *page;184 unsigned long size = 0;185 186 if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))187 return size == (1UL << it_page_shift);188 189 page = pfn_to_page(hpa >> PAGE_SHIFT);190 /*191 * Check that the TCE table granularity is not bigger than the size of192 * a page we just found. Otherwise the hardware can get access to193 * a bigger memory chunk that it should.194 */195 return page_shift(compound_head(page)) >= it_page_shift;196}197 198static inline bool tce_groups_attached(struct tce_container *container)199{200 return !list_empty(&container->group_list);201}202 203static long tce_iommu_find_table(struct tce_container *container,204 phys_addr_t ioba, struct iommu_table **ptbl)205{206 long i;207 208 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {209 struct iommu_table *tbl = container->tables[i];210 211 if (tbl) {212 unsigned long entry = ioba >> tbl->it_page_shift;213 unsigned long start = tbl->it_offset;214 unsigned long end = start + tbl->it_size;215 216 if ((start <= entry) && (entry < end)) {217 *ptbl = tbl;218 return i;219 }220 }221 }222 223 return -1;224}225 226static int tce_iommu_find_free_table(struct tce_container *container)227{228 int i;229 230 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {231 if (!container->tables[i])232 return i;233 }234 235 return -ENOSPC;236}237 238static int tce_iommu_enable(struct tce_container *container)239{240 int ret = 0;241 unsigned long locked;242 struct iommu_table_group *table_group;243 struct tce_iommu_group *tcegrp;244 245 if (container->enabled)246 return -EBUSY;247 248 /*249 * When userspace pages are mapped into the IOMMU, they are effectively250 * locked memory, so, theoretically, we need to update the accounting251 * of locked pages on each map and unmap. For powerpc, the map unmap252 * paths can be very hot, though, and the accounting would kill253 * performance, especially since it would be difficult to impossible254 * to handle the accounting in real mode only.255 *256 * To address that, rather than precisely accounting every page, we257 * instead account for a worst case on locked memory when the iommu is258 * enabled and disabled. The worst case upper bound on locked memory259 * is the size of the whole iommu window, which is usually relatively260 * small (compared to total memory sizes) on POWER hardware.261 *262 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,263 * that would effectively kill the guest at random points, much better264 * enforcing the limit based on the max that the guest can map.265 *266 * Unfortunately at the moment it counts whole tables, no matter how267 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups268 * each with 2GB DMA window, 8GB will be counted here. The reason for269 * this is that we cannot tell here the amount of RAM used by the guest270 * as this information is only available from KVM and VFIO is271 * KVM agnostic.272 *273 * So we do not allow enabling a container without a group attached274 * as there is no way to know how much we should increment275 * the locked_vm counter.276 */277 if (!tce_groups_attached(container))278 return -ENODEV;279 280 tcegrp = list_first_entry(&container->group_list,281 struct tce_iommu_group, next);282 table_group = iommu_group_get_iommudata(tcegrp->grp);283 if (!table_group)284 return -ENODEV;285 286 if (!table_group->tce32_size)287 return -EPERM;288 289 ret = tce_iommu_mm_set(container);290 if (ret)291 return ret;292 293 locked = table_group->tce32_size >> PAGE_SHIFT;294 ret = account_locked_vm(container->mm, locked, true);295 if (ret)296 return ret;297 298 container->locked_pages = locked;299 300 container->enabled = true;301 302 return ret;303}304 305static void tce_iommu_disable(struct tce_container *container)306{307 if (!container->enabled)308 return;309 310 container->enabled = false;311 312 BUG_ON(!container->mm);313 account_locked_vm(container->mm, container->locked_pages, false);314}315 316static void *tce_iommu_open(unsigned long arg)317{318 struct tce_container *container;319 320 if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {321 pr_err("tce_vfio: Wrong IOMMU type\n");322 return ERR_PTR(-EINVAL);323 }324 325 container = kzalloc(sizeof(*container), GFP_KERNEL);326 if (!container)327 return ERR_PTR(-ENOMEM);328 329 mutex_init(&container->lock);330 INIT_LIST_HEAD_RCU(&container->group_list);331 INIT_LIST_HEAD_RCU(&container->prereg_list);332 333 container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;334 335 return container;336}337 338static int tce_iommu_clear(struct tce_container *container,339 struct iommu_table *tbl,340 unsigned long entry, unsigned long pages);341static void tce_iommu_free_table(struct tce_container *container,342 struct iommu_table *tbl);343 344static void tce_iommu_release(void *iommu_data)345{346 struct tce_container *container = iommu_data;347 struct tce_iommu_group *tcegrp;348 struct tce_iommu_prereg *tcemem, *tmtmp;349 long i;350 351 while (tce_groups_attached(container)) {352 tcegrp = list_first_entry(&container->group_list,353 struct tce_iommu_group, next);354 tce_iommu_detach_group(iommu_data, tcegrp->grp);355 }356 357 /*358 * If VFIO created a table, it was not disposed359 * by tce_iommu_detach_group() so do it now.360 */361 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {362 struct iommu_table *tbl = container->tables[i];363 364 if (!tbl)365 continue;366 367 tce_iommu_free_table(container, tbl);368 }369 370 list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)371 WARN_ON(tce_iommu_prereg_free(container, tcemem));372 373 tce_iommu_disable(container);374 if (container->mm)375 mmdrop(container->mm);376 mutex_destroy(&container->lock);377 378 kfree(container);379}380 381static void tce_iommu_unuse_page(unsigned long hpa)382{383 struct page *page;384 385 page = pfn_to_page(hpa >> PAGE_SHIFT);386 unpin_user_page(page);387}388 389static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,390 unsigned long tce, unsigned long shift,391 unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)392{393 long ret = 0;394 struct mm_iommu_table_group_mem_t *mem;395 396 mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);397 if (!mem)398 return -EINVAL;399 400 ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);401 if (ret)402 return -EINVAL;403 404 *pmem = mem;405 406 return 0;407}408 409static void tce_iommu_unuse_page_v2(struct tce_container *container,410 struct iommu_table *tbl, unsigned long entry)411{412 struct mm_iommu_table_group_mem_t *mem = NULL;413 int ret;414 unsigned long hpa = 0;415 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);416 417 if (!pua)418 return;419 420 ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),421 tbl->it_page_shift, &hpa, &mem);422 if (ret)423 pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",424 __func__, be64_to_cpu(*pua), entry, ret);425 if (mem)426 mm_iommu_mapped_dec(mem);427 428 *pua = cpu_to_be64(0);429}430 431static int tce_iommu_clear(struct tce_container *container,432 struct iommu_table *tbl,433 unsigned long entry, unsigned long pages)434{435 unsigned long oldhpa;436 long ret;437 enum dma_data_direction direction;438 unsigned long lastentry = entry + pages, firstentry = entry;439 440 for ( ; entry < lastentry; ++entry) {441 if (tbl->it_indirect_levels && tbl->it_userspace) {442 /*443 * For multilevel tables, we can take a shortcut here444 * and skip some TCEs as we know that the userspace445 * addresses cache is a mirror of the real TCE table446 * and if it is missing some indirect levels, then447 * the hardware table does not have them allocated448 * either and therefore does not require updating.449 */450 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,451 entry);452 if (!pua) {453 /* align to level_size which is power of two */454 entry |= tbl->it_level_size - 1;455 continue;456 }457 }458 459 cond_resched();460 461 direction = DMA_NONE;462 oldhpa = 0;463 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,464 &direction);465 if (ret)466 continue;467 468 if (direction == DMA_NONE)469 continue;470 471 if (container->v2) {472 tce_iommu_unuse_page_v2(container, tbl, entry);473 continue;474 }475 476 tce_iommu_unuse_page(oldhpa);477 }478 479 iommu_tce_kill(tbl, firstentry, pages);480 481 return 0;482}483 484static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)485{486 struct page *page = NULL;487 enum dma_data_direction direction = iommu_tce_direction(tce);488 489 if (pin_user_pages_fast(tce & PAGE_MASK, 1,490 direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,491 &page) != 1)492 return -EFAULT;493 494 *hpa = __pa((unsigned long) page_address(page));495 496 return 0;497}498 499static long tce_iommu_build(struct tce_container *container,500 struct iommu_table *tbl,501 unsigned long entry, unsigned long tce, unsigned long pages,502 enum dma_data_direction direction)503{504 long i, ret = 0;505 unsigned long hpa;506 enum dma_data_direction dirtmp;507 508 for (i = 0; i < pages; ++i) {509 unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;510 511 ret = tce_iommu_use_page(tce, &hpa);512 if (ret)513 break;514 515 if (!tce_page_is_contained(container->mm, hpa,516 tbl->it_page_shift)) {517 ret = -EPERM;518 break;519 }520 521 hpa |= offset;522 dirtmp = direction;523 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,524 &hpa, &dirtmp);525 if (ret) {526 tce_iommu_unuse_page(hpa);527 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",528 __func__, entry << tbl->it_page_shift,529 tce, ret);530 break;531 }532 533 if (dirtmp != DMA_NONE)534 tce_iommu_unuse_page(hpa);535 536 tce += IOMMU_PAGE_SIZE(tbl);537 }538 539 if (ret)540 tce_iommu_clear(container, tbl, entry, i);541 else542 iommu_tce_kill(tbl, entry, pages);543 544 return ret;545}546 547static long tce_iommu_build_v2(struct tce_container *container,548 struct iommu_table *tbl,549 unsigned long entry, unsigned long tce, unsigned long pages,550 enum dma_data_direction direction)551{552 long i, ret = 0;553 unsigned long hpa;554 enum dma_data_direction dirtmp;555 556 for (i = 0; i < pages; ++i) {557 struct mm_iommu_table_group_mem_t *mem = NULL;558 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);559 560 ret = tce_iommu_prereg_ua_to_hpa(container,561 tce, tbl->it_page_shift, &hpa, &mem);562 if (ret)563 break;564 565 if (!tce_page_is_contained(container->mm, hpa,566 tbl->it_page_shift)) {567 ret = -EPERM;568 break;569 }570 571 /* Preserve offset within IOMMU page */572 hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;573 dirtmp = direction;574 575 /* The registered region is being unregistered */576 if (mm_iommu_mapped_inc(mem))577 break;578 579 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,580 &hpa, &dirtmp);581 if (ret) {582 /* dirtmp cannot be DMA_NONE here */583 tce_iommu_unuse_page_v2(container, tbl, entry + i);584 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",585 __func__, entry << tbl->it_page_shift,586 tce, ret);587 break;588 }589 590 if (dirtmp != DMA_NONE)591 tce_iommu_unuse_page_v2(container, tbl, entry + i);592 593 *pua = cpu_to_be64(tce);594 595 tce += IOMMU_PAGE_SIZE(tbl);596 }597 598 if (ret)599 tce_iommu_clear(container, tbl, entry, i);600 else601 iommu_tce_kill(tbl, entry, pages);602 603 return ret;604}605 606static long tce_iommu_create_table(struct tce_container *container,607 struct iommu_table_group *table_group,608 int num,609 __u32 page_shift,610 __u64 window_size,611 __u32 levels,612 struct iommu_table **ptbl)613{614 long ret, table_size;615 616 table_size = table_group->ops->get_table_size(page_shift, window_size,617 levels);618 if (!table_size)619 return -EINVAL;620 621 ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);622 if (ret)623 return ret;624 625 ret = table_group->ops->create_table(table_group, num,626 page_shift, window_size, levels, ptbl);627 628 WARN_ON(!ret && !(*ptbl)->it_ops->free);629 WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));630 631 return ret;632}633 634static void tce_iommu_free_table(struct tce_container *container,635 struct iommu_table *tbl)636{637 unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;638 639 iommu_tce_table_put(tbl);640 account_locked_vm(container->mm, pages, false);641}642 643static long tce_iommu_create_window(struct tce_container *container,644 __u32 page_shift, __u64 window_size, __u32 levels,645 __u64 *start_addr)646{647 struct tce_iommu_group *tcegrp;648 struct iommu_table_group *table_group;649 struct iommu_table *tbl = NULL;650 long ret, num;651 652 num = tce_iommu_find_free_table(container);653 if (num < 0)654 return num;655 656 /* Get the first group for ops::create_table */657 tcegrp = list_first_entry(&container->group_list,658 struct tce_iommu_group, next);659 table_group = iommu_group_get_iommudata(tcegrp->grp);660 if (!table_group)661 return -EFAULT;662 663 if (!(table_group->pgsizes & (1ULL << page_shift)))664 return -EINVAL;665 666 if (!table_group->ops->set_window || !table_group->ops->unset_window ||667 !table_group->ops->get_table_size ||668 !table_group->ops->create_table)669 return -EPERM;670 671 /* Create TCE table */672 ret = tce_iommu_create_table(container, table_group, num,673 page_shift, window_size, levels, &tbl);674 if (ret)675 return ret;676 677 BUG_ON(!tbl->it_ops->free);678 679 /*680 * Program the table to every group.681 * Groups have been tested for compatibility at the attach time.682 */683 list_for_each_entry(tcegrp, &container->group_list, next) {684 table_group = iommu_group_get_iommudata(tcegrp->grp);685 686 ret = table_group->ops->set_window(table_group, num, tbl);687 if (ret)688 goto unset_exit;689 }690 691 container->tables[num] = tbl;692 693 /* Return start address assigned by platform in create_table() */694 *start_addr = tbl->it_offset << tbl->it_page_shift;695 696 return 0;697 698unset_exit:699 list_for_each_entry(tcegrp, &container->group_list, next) {700 table_group = iommu_group_get_iommudata(tcegrp->grp);701 table_group->ops->unset_window(table_group, num);702 }703 tce_iommu_free_table(container, tbl);704 705 return ret;706}707 708static long tce_iommu_remove_window(struct tce_container *container,709 __u64 start_addr)710{711 struct iommu_table_group *table_group = NULL;712 struct iommu_table *tbl;713 struct tce_iommu_group *tcegrp;714 int num;715 716 num = tce_iommu_find_table(container, start_addr, &tbl);717 if (num < 0)718 return -EINVAL;719 720 BUG_ON(!tbl->it_size);721 722 tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);723 724 /* Detach groups from IOMMUs */725 list_for_each_entry(tcegrp, &container->group_list, next) {726 table_group = iommu_group_get_iommudata(tcegrp->grp);727 728 /*729 * SPAPR TCE IOMMU exposes the default DMA window to730 * the guest via dma32_window_start/size of731 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow732 * the userspace to remove this window, some do not so733 * here we check for the platform capability.734 */735 if (!table_group->ops || !table_group->ops->unset_window)736 return -EPERM;737 738 table_group->ops->unset_window(table_group, num);739 }740 741 /* Free table */742 tce_iommu_free_table(container, tbl);743 container->tables[num] = NULL;744 745 return 0;746}747 748static long tce_iommu_create_default_window(struct tce_container *container)749{750 long ret;751 __u64 start_addr = 0;752 struct tce_iommu_group *tcegrp;753 struct iommu_table_group *table_group;754 755 if (!container->def_window_pending)756 return 0;757 758 if (!tce_groups_attached(container))759 return -ENODEV;760 761 tcegrp = list_first_entry(&container->group_list,762 struct tce_iommu_group, next);763 table_group = iommu_group_get_iommudata(tcegrp->grp);764 if (!table_group)765 return -ENODEV;766 767 ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,768 table_group->tce32_size, 1, &start_addr);769 WARN_ON_ONCE(!ret && start_addr);770 771 if (!ret)772 container->def_window_pending = false;773 774 return ret;775}776 777static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,778 unsigned long arg)779{780 struct eeh_pe *pe;781 struct vfio_eeh_pe_op op;782 unsigned long minsz;783 784 pe = eeh_iommu_group_to_pe(group);785 if (!pe)786 return -ENODEV;787 788 minsz = offsetofend(struct vfio_eeh_pe_op, op);789 if (copy_from_user(&op, (void __user *)arg, minsz))790 return -EFAULT;791 if (op.argsz < minsz || op.flags)792 return -EINVAL;793 794 switch (op.op) {795 case VFIO_EEH_PE_DISABLE:796 return eeh_pe_set_option(pe, EEH_OPT_DISABLE);797 case VFIO_EEH_PE_ENABLE:798 return eeh_pe_set_option(pe, EEH_OPT_ENABLE);799 case VFIO_EEH_PE_UNFREEZE_IO:800 return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);801 case VFIO_EEH_PE_UNFREEZE_DMA:802 return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);803 case VFIO_EEH_PE_GET_STATE:804 return eeh_pe_get_state(pe);805 break;806 case VFIO_EEH_PE_RESET_DEACTIVATE:807 return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);808 case VFIO_EEH_PE_RESET_HOT:809 return eeh_pe_reset(pe, EEH_RESET_HOT, true);810 case VFIO_EEH_PE_RESET_FUNDAMENTAL:811 return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);812 case VFIO_EEH_PE_CONFIGURE:813 return eeh_pe_configure(pe);814 case VFIO_EEH_PE_INJECT_ERR:815 minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);816 if (op.argsz < minsz)817 return -EINVAL;818 if (copy_from_user(&op, (void __user *)arg, minsz))819 return -EFAULT;820 821 return eeh_pe_inject_err(pe, op.err.type, op.err.func,822 op.err.addr, op.err.mask);823 default:824 return -EINVAL;825 }826}827 828static long tce_iommu_ioctl(void *iommu_data,829 unsigned int cmd, unsigned long arg)830{831 struct tce_container *container = iommu_data;832 unsigned long minsz, ddwsz;833 long ret;834 835 switch (cmd) {836 case VFIO_CHECK_EXTENSION:837 switch (arg) {838 case VFIO_SPAPR_TCE_IOMMU:839 case VFIO_SPAPR_TCE_v2_IOMMU:840 return 1;841 case VFIO_EEH:842 return eeh_enabled();843 default:844 return 0;845 }846 }847 848 /*849 * Sanity check to prevent one userspace from manipulating850 * another userspace mm.851 */852 BUG_ON(!container);853 if (container->mm && container->mm != current->mm)854 return -EPERM;855 856 switch (cmd) {857 case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {858 struct vfio_iommu_spapr_tce_info info;859 struct tce_iommu_group *tcegrp;860 struct iommu_table_group *table_group;861 862 if (!tce_groups_attached(container))863 return -ENXIO;864 865 tcegrp = list_first_entry(&container->group_list,866 struct tce_iommu_group, next);867 table_group = iommu_group_get_iommudata(tcegrp->grp);868 869 if (!table_group)870 return -ENXIO;871 872 minsz = offsetofend(struct vfio_iommu_spapr_tce_info,873 dma32_window_size);874 875 if (copy_from_user(&info, (void __user *)arg, minsz))876 return -EFAULT;877 878 if (info.argsz < minsz)879 return -EINVAL;880 881 info.dma32_window_start = table_group->tce32_start;882 info.dma32_window_size = table_group->tce32_size;883 info.flags = 0;884 memset(&info.ddw, 0, sizeof(info.ddw));885 886 if (table_group->max_dynamic_windows_supported &&887 container->v2) {888 info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;889 info.ddw.pgsizes = table_group->pgsizes;890 info.ddw.max_dynamic_windows_supported =891 table_group->max_dynamic_windows_supported;892 info.ddw.levels = table_group->max_levels;893 }894 895 ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);896 897 if (info.argsz >= ddwsz)898 minsz = ddwsz;899 900 if (copy_to_user((void __user *)arg, &info, minsz))901 return -EFAULT;902 903 return 0;904 }905 case VFIO_IOMMU_MAP_DMA: {906 struct vfio_iommu_type1_dma_map param;907 struct iommu_table *tbl = NULL;908 long num;909 enum dma_data_direction direction;910 911 if (!container->enabled)912 return -EPERM;913 914 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);915 916 if (copy_from_user(¶m, (void __user *)arg, minsz))917 return -EFAULT;918 919 if (param.argsz < minsz)920 return -EINVAL;921 922 if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |923 VFIO_DMA_MAP_FLAG_WRITE))924 return -EINVAL;925 926 ret = tce_iommu_create_default_window(container);927 if (ret)928 return ret;929 930 num = tce_iommu_find_table(container, param.iova, &tbl);931 if (num < 0)932 return -ENXIO;933 934 if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||935 (param.vaddr & ~IOMMU_PAGE_MASK(tbl)))936 return -EINVAL;937 938 /* iova is checked by the IOMMU API */939 if (param.flags & VFIO_DMA_MAP_FLAG_READ) {940 if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)941 direction = DMA_BIDIRECTIONAL;942 else943 direction = DMA_TO_DEVICE;944 } else {945 if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)946 direction = DMA_FROM_DEVICE;947 else948 return -EINVAL;949 }950 951 ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);952 if (ret)953 return ret;954 955 if (container->v2)956 ret = tce_iommu_build_v2(container, tbl,957 param.iova >> tbl->it_page_shift,958 param.vaddr,959 param.size >> tbl->it_page_shift,960 direction);961 else962 ret = tce_iommu_build(container, tbl,963 param.iova >> tbl->it_page_shift,964 param.vaddr,965 param.size >> tbl->it_page_shift,966 direction);967 968 iommu_flush_tce(tbl);969 970 return ret;971 }972 case VFIO_IOMMU_UNMAP_DMA: {973 struct vfio_iommu_type1_dma_unmap param;974 struct iommu_table *tbl = NULL;975 long num;976 977 if (!container->enabled)978 return -EPERM;979 980 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,981 size);982 983 if (copy_from_user(¶m, (void __user *)arg, minsz))984 return -EFAULT;985 986 if (param.argsz < minsz)987 return -EINVAL;988 989 /* No flag is supported now */990 if (param.flags)991 return -EINVAL;992 993 ret = tce_iommu_create_default_window(container);994 if (ret)995 return ret;996 997 num = tce_iommu_find_table(container, param.iova, &tbl);998 if (num < 0)999 return -ENXIO;1000 1001 if (param.size & ~IOMMU_PAGE_MASK(tbl))1002 return -EINVAL;1003 1004 ret = iommu_tce_clear_param_check(tbl, param.iova, 0,1005 param.size >> tbl->it_page_shift);1006 if (ret)1007 return ret;1008 1009 ret = tce_iommu_clear(container, tbl,1010 param.iova >> tbl->it_page_shift,1011 param.size >> tbl->it_page_shift);1012 iommu_flush_tce(tbl);1013 1014 return ret;1015 }1016 case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {1017 struct vfio_iommu_spapr_register_memory param;1018 1019 if (!container->v2)1020 break;1021 1022 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,1023 size);1024 1025 ret = tce_iommu_mm_set(container);1026 if (ret)1027 return ret;1028 1029 if (copy_from_user(¶m, (void __user *)arg, minsz))1030 return -EFAULT;1031 1032 if (param.argsz < minsz)1033 return -EINVAL;1034 1035 /* No flag is supported now */1036 if (param.flags)1037 return -EINVAL;1038 1039 mutex_lock(&container->lock);1040 ret = tce_iommu_register_pages(container, param.vaddr,1041 param.size);1042 mutex_unlock(&container->lock);1043 1044 return ret;1045 }1046 case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {1047 struct vfio_iommu_spapr_register_memory param;1048 1049 if (!container->v2)1050 break;1051 1052 if (!container->mm)1053 return -EPERM;1054 1055 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,1056 size);1057 1058 if (copy_from_user(¶m, (void __user *)arg, minsz))1059 return -EFAULT;1060 1061 if (param.argsz < minsz)1062 return -EINVAL;1063 1064 /* No flag is supported now */1065 if (param.flags)1066 return -EINVAL;1067 1068 mutex_lock(&container->lock);1069 ret = tce_iommu_unregister_pages(container, param.vaddr,1070 param.size);1071 mutex_unlock(&container->lock);1072 1073 return ret;1074 }1075 case VFIO_IOMMU_ENABLE:1076 if (container->v2)1077 break;1078 1079 mutex_lock(&container->lock);1080 ret = tce_iommu_enable(container);1081 mutex_unlock(&container->lock);1082 return ret;1083 1084 1085 case VFIO_IOMMU_DISABLE:1086 if (container->v2)1087 break;1088 1089 mutex_lock(&container->lock);1090 tce_iommu_disable(container);1091 mutex_unlock(&container->lock);1092 return 0;1093 1094 case VFIO_EEH_PE_OP: {1095 struct tce_iommu_group *tcegrp;1096 1097 ret = 0;1098 list_for_each_entry(tcegrp, &container->group_list, next) {1099 ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);1100 if (ret)1101 return ret;1102 }1103 return ret;1104 }1105 1106 case VFIO_IOMMU_SPAPR_TCE_CREATE: {1107 struct vfio_iommu_spapr_tce_create create;1108 1109 if (!container->v2)1110 break;1111 1112 ret = tce_iommu_mm_set(container);1113 if (ret)1114 return ret;1115 1116 if (!tce_groups_attached(container))1117 return -ENXIO;1118 1119 minsz = offsetofend(struct vfio_iommu_spapr_tce_create,1120 start_addr);1121 1122 if (copy_from_user(&create, (void __user *)arg, minsz))1123 return -EFAULT;1124 1125 if (create.argsz < minsz)1126 return -EINVAL;1127 1128 if (create.flags)1129 return -EINVAL;1130 1131 mutex_lock(&container->lock);1132 1133 ret = tce_iommu_create_default_window(container);1134 if (!ret)1135 ret = tce_iommu_create_window(container,1136 create.page_shift,1137 create.window_size, create.levels,1138 &create.start_addr);1139 1140 mutex_unlock(&container->lock);1141 1142 if (!ret && copy_to_user((void __user *)arg, &create, minsz))1143 ret = -EFAULT;1144 1145 return ret;1146 }1147 case VFIO_IOMMU_SPAPR_TCE_REMOVE: {1148 struct vfio_iommu_spapr_tce_remove remove;1149 1150 if (!container->v2)1151 break;1152 1153 ret = tce_iommu_mm_set(container);1154 if (ret)1155 return ret;1156 1157 if (!tce_groups_attached(container))1158 return -ENXIO;1159 1160 minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,1161 start_addr);1162 1163 if (copy_from_user(&remove, (void __user *)arg, minsz))1164 return -EFAULT;1165 1166 if (remove.argsz < minsz)1167 return -EINVAL;1168 1169 if (remove.flags)1170 return -EINVAL;1171 1172 if (container->def_window_pending && !remove.start_addr) {1173 container->def_window_pending = false;1174 return 0;1175 }1176 1177 mutex_lock(&container->lock);1178 1179 ret = tce_iommu_remove_window(container, remove.start_addr);1180 1181 mutex_unlock(&container->lock);1182 1183 return ret;1184 }1185 }1186 1187 return -ENOTTY;1188}1189 1190static void tce_iommu_release_ownership(struct tce_container *container,1191 struct iommu_table_group *table_group)1192{1193 long i;1194 1195 if (!table_group->ops->unset_window) {1196 WARN_ON_ONCE(1);1197 return;1198 }1199 1200 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {1201 if (container->tables[i]) {1202 tce_iommu_clear(container, container->tables[i],1203 container->tables[i]->it_offset,1204 container->tables[i]->it_size);1205 table_group->ops->unset_window(table_group, i);1206 }1207 }1208}1209 1210static long tce_iommu_take_ownership(struct tce_container *container,1211 struct iommu_table_group *table_group)1212{1213 long i, ret = 0;1214 1215 /* Set all windows to the new group */1216 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {1217 struct iommu_table *tbl = container->tables[i];1218 1219 if (!tbl)1220 continue;1221 1222 ret = table_group->ops->set_window(table_group, i, tbl);1223 if (ret)1224 goto release_exit;1225 }1226 1227 return 0;1228 1229release_exit:1230 for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)1231 table_group->ops->unset_window(table_group, i);1232 1233 return ret;1234}1235 1236static int tce_iommu_attach_group(void *iommu_data,1237 struct iommu_group *iommu_group, enum vfio_group_type type)1238{1239 int ret = 0;1240 struct tce_container *container = iommu_data;1241 struct iommu_table_group *table_group;1242 struct tce_iommu_group *tcegrp = NULL;1243 1244 if (type == VFIO_EMULATED_IOMMU)1245 return -EINVAL;1246 1247 mutex_lock(&container->lock);1248 1249 /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",1250 iommu_group_id(iommu_group), iommu_group); */1251 table_group = iommu_group_get_iommudata(iommu_group);1252 if (!table_group) {1253 ret = -ENODEV;1254 goto unlock_exit;1255 }1256 1257 /* v2 requires full support of dynamic DMA windows */1258 if (container->v2 && table_group->max_dynamic_windows_supported == 0) {1259 ret = -EINVAL;1260 goto unlock_exit;1261 }1262 1263 /* v1 reuses TCE tables and does not share them among PEs */1264 if (!container->v2 && tce_groups_attached(container)) {1265 ret = -EBUSY;1266 goto unlock_exit;1267 }1268 1269 /*1270 * Check if new group has the same iommu_table_group_ops1271 * (i.e. compatible)1272 */1273 list_for_each_entry(tcegrp, &container->group_list, next) {1274 struct iommu_table_group *table_group_tmp;1275 1276 if (tcegrp->grp == iommu_group) {1277 pr_warn("tce_vfio: Group %d is already attached\n",1278 iommu_group_id(iommu_group));1279 ret = -EBUSY;1280 goto unlock_exit;1281 }1282 table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);1283 if (table_group_tmp->ops->create_table !=1284 table_group->ops->create_table) {1285 pr_warn("tce_vfio: Group %d is incompatible with group %d\n",1286 iommu_group_id(iommu_group),1287 iommu_group_id(tcegrp->grp));1288 ret = -EPERM;1289 goto unlock_exit;1290 }1291 }1292 1293 tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);1294 if (!tcegrp) {1295 ret = -ENOMEM;1296 goto unlock_exit;1297 }1298 1299 ret = tce_iommu_take_ownership(container, table_group);1300 if (!tce_groups_attached(container) && !container->tables[0])1301 container->def_window_pending = true;1302 1303 if (!ret) {1304 tcegrp->grp = iommu_group;1305 list_add(&tcegrp->next, &container->group_list);1306 }1307 1308 if (ret && tcegrp)1309 kfree(tcegrp);1310 1311unlock_exit:1312 mutex_unlock(&container->lock);1313 1314 return ret;1315}1316 1317static void tce_iommu_detach_group(void *iommu_data,1318 struct iommu_group *iommu_group)1319{1320 struct tce_container *container = iommu_data;1321 struct iommu_table_group *table_group;1322 bool found = false;1323 struct tce_iommu_group *tcegrp;1324 1325 mutex_lock(&container->lock);1326 1327 list_for_each_entry(tcegrp, &container->group_list, next) {1328 if (tcegrp->grp == iommu_group) {1329 found = true;1330 break;1331 }1332 }1333 1334 if (!found) {1335 pr_warn("tce_vfio: detaching unattached group #%u\n",1336 iommu_group_id(iommu_group));1337 goto unlock_exit;1338 }1339 1340 list_del(&tcegrp->next);1341 kfree(tcegrp);1342 1343 table_group = iommu_group_get_iommudata(iommu_group);1344 BUG_ON(!table_group);1345 1346 tce_iommu_release_ownership(container, table_group);1347 1348unlock_exit:1349 mutex_unlock(&container->lock);1350}1351 1352static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {1353 .name = "iommu-vfio-powerpc",1354 .owner = THIS_MODULE,1355 .open = tce_iommu_open,1356 .release = tce_iommu_release,1357 .ioctl = tce_iommu_ioctl,1358 .attach_group = tce_iommu_attach_group,1359 .detach_group = tce_iommu_detach_group,1360};1361 1362static int __init tce_iommu_init(void)1363{1364 return vfio_register_iommu_driver(&tce_iommu_driver_ops);1365}1366 1367static void __exit tce_iommu_cleanup(void)1368{1369 vfio_unregister_iommu_driver(&tce_iommu_driver_ops);1370}1371 1372module_init(tce_iommu_init);1373module_exit(tce_iommu_cleanup);1374 1375MODULE_VERSION(DRIVER_VERSION);1376MODULE_LICENSE("GPL v2");1377MODULE_AUTHOR(DRIVER_AUTHOR);1378MODULE_DESCRIPTION(DRIVER_DESC);1379 1380