2588 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.4 * Author: Alex Williamson <alex.williamson@redhat.com>5 *6 * Derived from original vfio:7 * Copyright 2010 Cisco Systems, Inc. All rights reserved.8 * Author: Tom Lyon, pugs@cisco.com9 */10 11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt12 13#include <linux/aperture.h>14#include <linux/device.h>15#include <linux/eventfd.h>16#include <linux/file.h>17#include <linux/interrupt.h>18#include <linux/iommu.h>19#include <linux/module.h>20#include <linux/mutex.h>21#include <linux/notifier.h>22#include <linux/pci.h>23#include <linux/pfn_t.h>24#include <linux/pm_runtime.h>25#include <linux/slab.h>26#include <linux/types.h>27#include <linux/uaccess.h>28#include <linux/vgaarb.h>29#include <linux/nospec.h>30#include <linux/sched/mm.h>31#include <linux/iommufd.h>32#if IS_ENABLED(CONFIG_EEH)33#include <asm/eeh.h>34#endif35 36#include "vfio_pci_priv.h"37 38#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"39#define DRIVER_DESC "core driver for VFIO based PCI devices"40 41static bool nointxmask;42static bool disable_vga;43static bool disable_idle_d3;44 45/* List of PF's that vfio_pci_core_sriov_configure() has been called on */46static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);47static LIST_HEAD(vfio_pci_sriov_pfs);48 49struct vfio_pci_dummy_resource {50 struct resource resource;51 int index;52 struct list_head res_next;53};54 55struct vfio_pci_vf_token {56 struct mutex lock;57 uuid_t uuid;58 int users;59};60 61static inline bool vfio_vga_disabled(void)62{63#ifdef CONFIG_VFIO_PCI_VGA64 return disable_vga;65#else66 return true;67#endif68}69 70/*71 * Our VGA arbiter participation is limited since we don't know anything72 * about the device itself. However, if the device is the only VGA device73 * downstream of a bridge and VFIO VGA support is disabled, then we can74 * safely return legacy VGA IO and memory as not decoded since the user75 * has no way to get to it and routing can be disabled externally at the76 * bridge.77 */78static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga)79{80 struct pci_dev *tmp = NULL;81 unsigned char max_busnr;82 unsigned int decodes;83 84 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))85 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |86 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;87 88 max_busnr = pci_bus_max_busnr(pdev->bus);89 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;90 91 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {92 if (tmp == pdev ||93 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||94 pci_is_root_bus(tmp->bus))95 continue;96 97 if (tmp->bus->number >= pdev->bus->number &&98 tmp->bus->number <= max_busnr) {99 pci_dev_put(tmp);100 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;101 break;102 }103 }104 105 return decodes;106}107 108static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)109{110 struct resource *res;111 int i;112 struct vfio_pci_dummy_resource *dummy_res;113 114 for (i = 0; i < PCI_STD_NUM_BARS; i++) {115 int bar = i + PCI_STD_RESOURCES;116 117 res = &vdev->pdev->resource[bar];118 119 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))120 goto no_mmap;121 122 if (!(res->flags & IORESOURCE_MEM))123 goto no_mmap;124 125 /*126 * The PCI core shouldn't set up a resource with a127 * type but zero size. But there may be bugs that128 * cause us to do that.129 */130 if (!resource_size(res))131 goto no_mmap;132 133 if (resource_size(res) >= PAGE_SIZE) {134 vdev->bar_mmap_supported[bar] = true;135 continue;136 }137 138 if (!(res->start & ~PAGE_MASK)) {139 /*140 * Add a dummy resource to reserve the remainder141 * of the exclusive page in case that hot-add142 * device's bar is assigned into it.143 */144 dummy_res =145 kzalloc(sizeof(*dummy_res), GFP_KERNEL_ACCOUNT);146 if (dummy_res == NULL)147 goto no_mmap;148 149 dummy_res->resource.name = "vfio sub-page reserved";150 dummy_res->resource.start = res->end + 1;151 dummy_res->resource.end = res->start + PAGE_SIZE - 1;152 dummy_res->resource.flags = res->flags;153 if (request_resource(res->parent,154 &dummy_res->resource)) {155 kfree(dummy_res);156 goto no_mmap;157 }158 dummy_res->index = bar;159 list_add(&dummy_res->res_next,160 &vdev->dummy_resources_list);161 vdev->bar_mmap_supported[bar] = true;162 continue;163 }164 /*165 * Here we don't handle the case when the BAR is not page166 * aligned because we can't expect the BAR will be167 * assigned into the same location in a page in guest168 * when we passthrough the BAR. And it's hard to access169 * this BAR in userspace because we have no way to get170 * the BAR's location in a page.171 */172no_mmap:173 vdev->bar_mmap_supported[bar] = false;174 }175}176 177struct vfio_pci_group_info;178static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);179static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,180 struct vfio_pci_group_info *groups,181 struct iommufd_ctx *iommufd_ctx);182 183/*184 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND185 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.186 * If a device implements the former but not the latter we would typically187 * expect broken_intx_masking be set and require an exclusive interrupt.188 * However since we do have control of the device's ability to assert INTx,189 * we can instead pretend that the device does not implement INTx, virtualizing190 * the pin register to report zero and maintaining DisINTx set on the host.191 */192static bool vfio_pci_nointx(struct pci_dev *pdev)193{194 switch (pdev->vendor) {195 case PCI_VENDOR_ID_INTEL:196 switch (pdev->device) {197 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */198 case 0x1572:199 case 0x1574:200 case 0x1580 ... 0x1581:201 case 0x1583 ... 0x158b:202 case 0x37d0 ... 0x37d2:203 /* X550 */204 case 0x1563:205 return true;206 default:207 return false;208 }209 }210 211 return false;212}213 214static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)215{216 struct pci_dev *pdev = vdev->pdev;217 u16 pmcsr;218 219 if (!pdev->pm_cap)220 return;221 222 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);223 224 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);225}226 227/*228 * pci_set_power_state() wrapper handling devices which perform a soft reset on229 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,230 * restore when returned to D0. Saved separately from pci_saved_state for use231 * by PM capability emulation and separately from pci_dev internal saved state232 * to avoid it being overwritten and consumed around other resets.233 */234int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)235{236 struct pci_dev *pdev = vdev->pdev;237 bool needs_restore = false, needs_save = false;238 int ret;239 240 /* Prevent changing power state for PFs with VFs enabled */241 if (pci_num_vf(pdev) && state > PCI_D0)242 return -EBUSY;243 244 if (vdev->needs_pm_restore) {245 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {246 pci_save_state(pdev);247 needs_save = true;248 }249 250 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)251 needs_restore = true;252 }253 254 ret = pci_set_power_state(pdev, state);255 256 if (!ret) {257 /* D3 might be unsupported via quirk, skip unless in D3 */258 if (needs_save && pdev->current_state >= PCI_D3hot) {259 /*260 * The current PCI state will be saved locally in261 * 'pm_save' during the D3hot transition. When the262 * device state is changed to D0 again with the current263 * function, then pci_store_saved_state() will restore264 * the state and will free the memory pointed by265 * 'pm_save'. There are few cases where the PCI power266 * state can be changed to D0 without the involvement267 * of the driver. For these cases, free the earlier268 * allocated memory first before overwriting 'pm_save'269 * to prevent the memory leak.270 */271 kfree(vdev->pm_save);272 vdev->pm_save = pci_store_saved_state(pdev);273 } else if (needs_restore) {274 pci_load_and_free_saved_state(pdev, &vdev->pm_save);275 pci_restore_state(pdev);276 }277 }278 279 return ret;280}281 282static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,283 struct eventfd_ctx *efdctx)284{285 /*286 * The vdev power related flags are protected with 'memory_lock'287 * semaphore.288 */289 vfio_pci_zap_and_down_write_memory_lock(vdev);290 if (vdev->pm_runtime_engaged) {291 up_write(&vdev->memory_lock);292 return -EINVAL;293 }294 295 vdev->pm_runtime_engaged = true;296 vdev->pm_wake_eventfd_ctx = efdctx;297 pm_runtime_put_noidle(&vdev->pdev->dev);298 up_write(&vdev->memory_lock);299 300 return 0;301}302 303static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags,304 void __user *arg, size_t argsz)305{306 struct vfio_pci_core_device *vdev =307 container_of(device, struct vfio_pci_core_device, vdev);308 int ret;309 310 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);311 if (ret != 1)312 return ret;313 314 /*315 * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count316 * will be decremented. The pm_runtime_put() will be invoked again317 * while returning from the ioctl and then the device can go into318 * runtime suspended state.319 */320 return vfio_pci_runtime_pm_entry(vdev, NULL);321}322 323static int vfio_pci_core_pm_entry_with_wakeup(324 struct vfio_device *device, u32 flags,325 struct vfio_device_low_power_entry_with_wakeup __user *arg,326 size_t argsz)327{328 struct vfio_pci_core_device *vdev =329 container_of(device, struct vfio_pci_core_device, vdev);330 struct vfio_device_low_power_entry_with_wakeup entry;331 struct eventfd_ctx *efdctx;332 int ret;333 334 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,335 sizeof(entry));336 if (ret != 1)337 return ret;338 339 if (copy_from_user(&entry, arg, sizeof(entry)))340 return -EFAULT;341 342 if (entry.wakeup_eventfd < 0)343 return -EINVAL;344 345 efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd);346 if (IS_ERR(efdctx))347 return PTR_ERR(efdctx);348 349 ret = vfio_pci_runtime_pm_entry(vdev, efdctx);350 if (ret)351 eventfd_ctx_put(efdctx);352 353 return ret;354}355 356static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)357{358 if (vdev->pm_runtime_engaged) {359 vdev->pm_runtime_engaged = false;360 pm_runtime_get_noresume(&vdev->pdev->dev);361 362 if (vdev->pm_wake_eventfd_ctx) {363 eventfd_ctx_put(vdev->pm_wake_eventfd_ctx);364 vdev->pm_wake_eventfd_ctx = NULL;365 }366 }367}368 369static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)370{371 /*372 * The vdev power related flags are protected with 'memory_lock'373 * semaphore.374 */375 down_write(&vdev->memory_lock);376 __vfio_pci_runtime_pm_exit(vdev);377 up_write(&vdev->memory_lock);378}379 380static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags,381 void __user *arg, size_t argsz)382{383 struct vfio_pci_core_device *vdev =384 container_of(device, struct vfio_pci_core_device, vdev);385 int ret;386 387 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);388 if (ret != 1)389 return ret;390 391 /*392 * The device is always in the active state here due to pm wrappers393 * around ioctls. If the device had entered a low power state and394 * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has395 * already signaled the eventfd and exited low power mode itself.396 * pm_runtime_engaged protects the redundant call here.397 */398 vfio_pci_runtime_pm_exit(vdev);399 return 0;400}401 402#ifdef CONFIG_PM403static int vfio_pci_core_runtime_suspend(struct device *dev)404{405 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);406 407 down_write(&vdev->memory_lock);408 /*409 * The user can move the device into D3hot state before invoking410 * power management IOCTL. Move the device into D0 state here and then411 * the pci-driver core runtime PM suspend function will move the device412 * into the low power state. Also, for the devices which have413 * NoSoftRst-, it will help in restoring the original state414 * (saved locally in 'vdev->pm_save').415 */416 vfio_pci_set_power_state(vdev, PCI_D0);417 up_write(&vdev->memory_lock);418 419 /*420 * If INTx is enabled, then mask INTx before going into the runtime421 * suspended state and unmask the same in the runtime resume.422 * If INTx has already been masked by the user, then423 * vfio_pci_intx_mask() will return false and in that case, INTx424 * should not be unmasked in the runtime resume.425 */426 vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) &&427 vfio_pci_intx_mask(vdev));428 429 return 0;430}431 432static int vfio_pci_core_runtime_resume(struct device *dev)433{434 struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);435 436 /*437 * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit438 * low power mode.439 */440 down_write(&vdev->memory_lock);441 if (vdev->pm_wake_eventfd_ctx) {442 eventfd_signal(vdev->pm_wake_eventfd_ctx);443 __vfio_pci_runtime_pm_exit(vdev);444 }445 up_write(&vdev->memory_lock);446 447 if (vdev->pm_intx_masked)448 vfio_pci_intx_unmask(vdev);449 450 return 0;451}452#endif /* CONFIG_PM */453 454/*455 * The pci-driver core runtime PM routines always save the device state456 * before going into suspended state. If the device is going into low power457 * state with only with runtime PM ops, then no explicit handling is needed458 * for the devices which have NoSoftRst-.459 */460static const struct dev_pm_ops vfio_pci_core_pm_ops = {461 SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend,462 vfio_pci_core_runtime_resume,463 NULL)464};465 466int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)467{468 struct pci_dev *pdev = vdev->pdev;469 int ret;470 u16 cmd;471 u8 msix_pos;472 473 if (!disable_idle_d3) {474 ret = pm_runtime_resume_and_get(&pdev->dev);475 if (ret < 0)476 return ret;477 }478 479 /* Don't allow our initial saved state to include busmaster */480 pci_clear_master(pdev);481 482 ret = pci_enable_device(pdev);483 if (ret)484 goto out_power;485 486 /* If reset fails because of the device lock, fail this path entirely */487 ret = pci_try_reset_function(pdev);488 if (ret == -EAGAIN)489 goto out_disable_device;490 491 vdev->reset_works = !ret;492 pci_save_state(pdev);493 vdev->pci_saved_state = pci_store_saved_state(pdev);494 if (!vdev->pci_saved_state)495 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);496 497 if (likely(!nointxmask)) {498 if (vfio_pci_nointx(pdev)) {499 pci_info(pdev, "Masking broken INTx support\n");500 vdev->nointx = true;501 pci_intx(pdev, 0);502 } else503 vdev->pci_2_3 = pci_intx_mask_supported(pdev);504 }505 506 pci_read_config_word(pdev, PCI_COMMAND, &cmd);507 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {508 cmd &= ~PCI_COMMAND_INTX_DISABLE;509 pci_write_config_word(pdev, PCI_COMMAND, cmd);510 }511 512 ret = vfio_pci_zdev_open_device(vdev);513 if (ret)514 goto out_free_state;515 516 ret = vfio_config_init(vdev);517 if (ret)518 goto out_free_zdev;519 520 msix_pos = pdev->msix_cap;521 if (msix_pos) {522 u16 flags;523 u32 table;524 525 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);526 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);527 528 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;529 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;530 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;531 vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev);532 } else {533 vdev->msix_bar = 0xFF;534 vdev->has_dyn_msix = false;535 }536 537 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))538 vdev->has_vga = true;539 540 541 return 0;542 543out_free_zdev:544 vfio_pci_zdev_close_device(vdev);545out_free_state:546 kfree(vdev->pci_saved_state);547 vdev->pci_saved_state = NULL;548out_disable_device:549 pci_disable_device(pdev);550out_power:551 if (!disable_idle_d3)552 pm_runtime_put(&pdev->dev);553 return ret;554}555EXPORT_SYMBOL_GPL(vfio_pci_core_enable);556 557void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)558{559 struct pci_dev *pdev = vdev->pdev;560 struct vfio_pci_dummy_resource *dummy_res, *tmp;561 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp;562 int i, bar;563 564 /* For needs_reset */565 lockdep_assert_held(&vdev->vdev.dev_set->lock);566 567 /*568 * This function can be invoked while the power state is non-D0.569 * This non-D0 power state can be with or without runtime PM.570 * vfio_pci_runtime_pm_exit() will internally increment the usage571 * count corresponding to pm_runtime_put() called during low power572 * feature entry and then pm_runtime_resume() will wake up the device,573 * if the device has already gone into the suspended state. Otherwise,574 * the vfio_pci_set_power_state() will change the device power state575 * to D0.576 */577 vfio_pci_runtime_pm_exit(vdev);578 pm_runtime_resume(&pdev->dev);579 580 /*581 * This function calls __pci_reset_function_locked() which internally582 * can use pci_pm_reset() for the function reset. pci_pm_reset() will583 * fail if the power state is non-D0. Also, for the devices which584 * have NoSoftRst-, the reset function can cause the PCI config space585 * reset without restoring the original state (saved locally in586 * 'vdev->pm_save').587 */588 vfio_pci_set_power_state(vdev, PCI_D0);589 590 /* Stop the device from further DMA */591 pci_clear_master(pdev);592 593 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |594 VFIO_IRQ_SET_ACTION_TRIGGER,595 vdev->irq_type, 0, 0, NULL);596 597 /* Device closed, don't need mutex here */598 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,599 &vdev->ioeventfds_list, next) {600 vfio_virqfd_disable(&ioeventfd->virqfd);601 list_del(&ioeventfd->next);602 kfree(ioeventfd);603 }604 vdev->ioeventfds_nr = 0;605 606 vdev->virq_disabled = false;607 608 for (i = 0; i < vdev->num_regions; i++)609 vdev->region[i].ops->release(vdev, &vdev->region[i]);610 611 vdev->num_regions = 0;612 kfree(vdev->region);613 vdev->region = NULL; /* don't krealloc a freed pointer */614 615 vfio_config_free(vdev);616 617 for (i = 0; i < PCI_STD_NUM_BARS; i++) {618 bar = i + PCI_STD_RESOURCES;619 if (!vdev->barmap[bar])620 continue;621 pci_iounmap(pdev, vdev->barmap[bar]);622 pci_release_selected_regions(pdev, 1 << bar);623 vdev->barmap[bar] = NULL;624 }625 626 list_for_each_entry_safe(dummy_res, tmp,627 &vdev->dummy_resources_list, res_next) {628 list_del(&dummy_res->res_next);629 release_resource(&dummy_res->resource);630 kfree(dummy_res);631 }632 633 vdev->needs_reset = true;634 635 vfio_pci_zdev_close_device(vdev);636 637 /*638 * If we have saved state, restore it. If we can reset the device,639 * even better. Resetting with current state seems better than640 * nothing, but saving and restoring current state without reset641 * is just busy work.642 */643 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {644 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);645 646 if (!vdev->reset_works)647 goto out;648 649 pci_save_state(pdev);650 }651 652 /*653 * Disable INTx and MSI, presumably to avoid spurious interrupts654 * during reset. Stolen from pci_reset_function()655 */656 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);657 658 /*659 * Try to get the locks ourselves to prevent a deadlock. The660 * success of this is dependent on being able to lock the device,661 * which is not always possible.662 * We can not use the "try" reset interface here, which will663 * overwrite the previously restored configuration information.664 */665 if (vdev->reset_works && pci_dev_trylock(pdev)) {666 if (!__pci_reset_function_locked(pdev))667 vdev->needs_reset = false;668 pci_dev_unlock(pdev);669 }670 671 pci_restore_state(pdev);672out:673 pci_disable_device(pdev);674 675 vfio_pci_dev_set_try_reset(vdev->vdev.dev_set);676 677 /* Put the pm-runtime usage counter acquired during enable */678 if (!disable_idle_d3)679 pm_runtime_put(&pdev->dev);680}681EXPORT_SYMBOL_GPL(vfio_pci_core_disable);682 683void vfio_pci_core_close_device(struct vfio_device *core_vdev)684{685 struct vfio_pci_core_device *vdev =686 container_of(core_vdev, struct vfio_pci_core_device, vdev);687 688 if (vdev->sriov_pf_core_dev) {689 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);690 WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);691 vdev->sriov_pf_core_dev->vf_token->users--;692 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);693 }694#if IS_ENABLED(CONFIG_EEH)695 eeh_dev_release(vdev->pdev);696#endif697 vfio_pci_core_disable(vdev);698 699 mutex_lock(&vdev->igate);700 if (vdev->err_trigger) {701 eventfd_ctx_put(vdev->err_trigger);702 vdev->err_trigger = NULL;703 }704 if (vdev->req_trigger) {705 eventfd_ctx_put(vdev->req_trigger);706 vdev->req_trigger = NULL;707 }708 mutex_unlock(&vdev->igate);709}710EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);711 712void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)713{714 vfio_pci_probe_mmaps(vdev);715#if IS_ENABLED(CONFIG_EEH)716 eeh_dev_open(vdev->pdev);717#endif718 719 if (vdev->sriov_pf_core_dev) {720 mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);721 vdev->sriov_pf_core_dev->vf_token->users++;722 mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);723 }724}725EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);726 727static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)728{729 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {730 u8 pin;731 732 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) ||733 vdev->nointx || vdev->pdev->is_virtfn)734 return 0;735 736 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);737 738 return pin ? 1 : 0;739 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {740 u8 pos;741 u16 flags;742 743 pos = vdev->pdev->msi_cap;744 if (pos) {745 pci_read_config_word(vdev->pdev,746 pos + PCI_MSI_FLAGS, &flags);747 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);748 }749 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {750 u8 pos;751 u16 flags;752 753 pos = vdev->pdev->msix_cap;754 if (pos) {755 pci_read_config_word(vdev->pdev,756 pos + PCI_MSIX_FLAGS, &flags);757 758 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;759 }760 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {761 if (pci_is_pcie(vdev->pdev))762 return 1;763 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {764 return 1;765 }766 767 return 0;768}769 770static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)771{772 (*(int *)data)++;773 return 0;774}775 776struct vfio_pci_fill_info {777 struct vfio_device *vdev;778 struct vfio_pci_dependent_device *devices;779 int nr_devices;780 u32 count;781 u32 flags;782};783 784static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)785{786 struct vfio_pci_dependent_device *info;787 struct vfio_pci_fill_info *fill = data;788 789 /* The topology changed since we counted devices */790 if (fill->count >= fill->nr_devices)791 return -EAGAIN;792 793 info = &fill->devices[fill->count++];794 info->segment = pci_domain_nr(pdev->bus);795 info->bus = pdev->bus->number;796 info->devfn = pdev->devfn;797 798 if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) {799 struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev);800 struct vfio_device_set *dev_set = fill->vdev->dev_set;801 struct vfio_device *vdev;802 803 /*804 * hot-reset requires all affected devices be represented in805 * the dev_set.806 */807 vdev = vfio_find_device_in_devset(dev_set, &pdev->dev);808 if (!vdev) {809 info->devid = VFIO_PCI_DEVID_NOT_OWNED;810 } else {811 int id = vfio_iommufd_get_dev_id(vdev, iommufd);812 813 if (id > 0)814 info->devid = id;815 else if (id == -ENOENT)816 info->devid = VFIO_PCI_DEVID_OWNED;817 else818 info->devid = VFIO_PCI_DEVID_NOT_OWNED;819 }820 /* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */821 if (info->devid == VFIO_PCI_DEVID_NOT_OWNED)822 fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;823 } else {824 struct iommu_group *iommu_group;825 826 iommu_group = iommu_group_get(&pdev->dev);827 if (!iommu_group)828 return -EPERM; /* Cannot reset non-isolated devices */829 830 info->group_id = iommu_group_id(iommu_group);831 iommu_group_put(iommu_group);832 }833 834 return 0;835}836 837struct vfio_pci_group_info {838 int count;839 struct file **files;840};841 842static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)843{844 for (; pdev; pdev = pdev->bus->self)845 if (pdev->bus == slot->bus)846 return (pdev->slot == slot);847 return false;848}849 850struct vfio_pci_walk_info {851 int (*fn)(struct pci_dev *pdev, void *data);852 void *data;853 struct pci_dev *pdev;854 bool slot;855 int ret;856};857 858static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)859{860 struct vfio_pci_walk_info *walk = data;861 862 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))863 walk->ret = walk->fn(pdev, walk->data);864 865 return walk->ret;866}867 868static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,869 int (*fn)(struct pci_dev *,870 void *data), void *data,871 bool slot)872{873 struct vfio_pci_walk_info walk = {874 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,875 };876 877 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);878 879 return walk.ret;880}881 882static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,883 struct vfio_info_cap *caps)884{885 struct vfio_info_cap_header header = {886 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,887 .version = 1888 };889 890 return vfio_info_add_capability(caps, &header, sizeof(header));891}892 893int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,894 unsigned int type, unsigned int subtype,895 const struct vfio_pci_regops *ops,896 size_t size, u32 flags, void *data)897{898 struct vfio_pci_region *region;899 900 region = krealloc(vdev->region,901 (vdev->num_regions + 1) * sizeof(*region),902 GFP_KERNEL_ACCOUNT);903 if (!region)904 return -ENOMEM;905 906 vdev->region = region;907 vdev->region[vdev->num_regions].type = type;908 vdev->region[vdev->num_regions].subtype = subtype;909 vdev->region[vdev->num_regions].ops = ops;910 vdev->region[vdev->num_regions].size = size;911 vdev->region[vdev->num_regions].flags = flags;912 vdev->region[vdev->num_regions].data = data;913 914 vdev->num_regions++;915 916 return 0;917}918EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region);919 920static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,921 struct vfio_info_cap *caps)922{923 struct vfio_device_info_cap_pci_atomic_comp cap = {924 .header.id = VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP,925 .header.version = 1926 };927 struct pci_dev *pdev = pci_physfn(vdev->pdev);928 u32 devcap2;929 930 pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &devcap2);931 932 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&933 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32))934 cap.flags |= VFIO_PCI_ATOMIC_COMP32;935 936 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&937 !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))938 cap.flags |= VFIO_PCI_ATOMIC_COMP64;939 940 if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP128) &&941 !pci_enable_atomic_ops_to_root(pdev,942 PCI_EXP_DEVCAP2_ATOMIC_COMP128))943 cap.flags |= VFIO_PCI_ATOMIC_COMP128;944 945 if (!cap.flags)946 return -ENODEV;947 948 return vfio_info_add_capability(caps, &cap.header, sizeof(cap));949}950 951static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,952 struct vfio_device_info __user *arg)953{954 unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs);955 struct vfio_device_info info = {};956 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };957 int ret;958 959 if (copy_from_user(&info, arg, minsz))960 return -EFAULT;961 962 if (info.argsz < minsz)963 return -EINVAL;964 965 minsz = min_t(size_t, info.argsz, sizeof(info));966 967 info.flags = VFIO_DEVICE_FLAGS_PCI;968 969 if (vdev->reset_works)970 info.flags |= VFIO_DEVICE_FLAGS_RESET;971 972 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;973 info.num_irqs = VFIO_PCI_NUM_IRQS;974 975 ret = vfio_pci_info_zdev_add_caps(vdev, &caps);976 if (ret && ret != -ENODEV) {977 pci_warn(vdev->pdev,978 "Failed to setup zPCI info capabilities\n");979 return ret;980 }981 982 ret = vfio_pci_info_atomic_cap(vdev, &caps);983 if (ret && ret != -ENODEV) {984 pci_warn(vdev->pdev,985 "Failed to setup AtomicOps info capability\n");986 return ret;987 }988 989 if (caps.size) {990 info.flags |= VFIO_DEVICE_FLAGS_CAPS;991 if (info.argsz < sizeof(info) + caps.size) {992 info.argsz = sizeof(info) + caps.size;993 } else {994 vfio_info_cap_shift(&caps, sizeof(info));995 if (copy_to_user(arg + 1, caps.buf, caps.size)) {996 kfree(caps.buf);997 return -EFAULT;998 }999 info.cap_offset = sizeof(*arg);1000 }1001 1002 kfree(caps.buf);1003 }1004 1005 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;1006}1007 1008static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev,1009 struct vfio_region_info __user *arg)1010{1011 unsigned long minsz = offsetofend(struct vfio_region_info, offset);1012 struct pci_dev *pdev = vdev->pdev;1013 struct vfio_region_info info;1014 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };1015 int i, ret;1016 1017 if (copy_from_user(&info, arg, minsz))1018 return -EFAULT;1019 1020 if (info.argsz < minsz)1021 return -EINVAL;1022 1023 switch (info.index) {1024 case VFIO_PCI_CONFIG_REGION_INDEX:1025 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1026 info.size = pdev->cfg_size;1027 info.flags = VFIO_REGION_INFO_FLAG_READ |1028 VFIO_REGION_INFO_FLAG_WRITE;1029 break;1030 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:1031 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1032 info.size = pci_resource_len(pdev, info.index);1033 if (!info.size) {1034 info.flags = 0;1035 break;1036 }1037 1038 info.flags = VFIO_REGION_INFO_FLAG_READ |1039 VFIO_REGION_INFO_FLAG_WRITE;1040 if (vdev->bar_mmap_supported[info.index]) {1041 info.flags |= VFIO_REGION_INFO_FLAG_MMAP;1042 if (info.index == vdev->msix_bar) {1043 ret = msix_mmappable_cap(vdev, &caps);1044 if (ret)1045 return ret;1046 }1047 }1048 1049 break;1050 case VFIO_PCI_ROM_REGION_INDEX: {1051 void __iomem *io;1052 size_t size;1053 u16 cmd;1054 1055 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1056 info.flags = 0;1057 1058 /* Report the BAR size, not the ROM size */1059 info.size = pci_resource_len(pdev, info.index);1060 if (!info.size) {1061 /* Shadow ROMs appear as PCI option ROMs */1062 if (pdev->resource[PCI_ROM_RESOURCE].flags &1063 IORESOURCE_ROM_SHADOW)1064 info.size = 0x20000;1065 else1066 break;1067 }1068 1069 /*1070 * Is it really there? Enable memory decode for implicit access1071 * in pci_map_rom().1072 */1073 cmd = vfio_pci_memory_lock_and_enable(vdev);1074 io = pci_map_rom(pdev, &size);1075 if (io) {1076 info.flags = VFIO_REGION_INFO_FLAG_READ;1077 pci_unmap_rom(pdev, io);1078 } else {1079 info.size = 0;1080 }1081 vfio_pci_memory_unlock_and_restore(vdev, cmd);1082 1083 break;1084 }1085 case VFIO_PCI_VGA_REGION_INDEX:1086 if (!vdev->has_vga)1087 return -EINVAL;1088 1089 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1090 info.size = 0xc0000;1091 info.flags = VFIO_REGION_INFO_FLAG_READ |1092 VFIO_REGION_INFO_FLAG_WRITE;1093 1094 break;1095 default: {1096 struct vfio_region_info_cap_type cap_type = {1097 .header.id = VFIO_REGION_INFO_CAP_TYPE,1098 .header.version = 11099 };1100 1101 if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)1102 return -EINVAL;1103 info.index = array_index_nospec(1104 info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions);1105 1106 i = info.index - VFIO_PCI_NUM_REGIONS;1107 1108 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1109 info.size = vdev->region[i].size;1110 info.flags = vdev->region[i].flags;1111 1112 cap_type.type = vdev->region[i].type;1113 cap_type.subtype = vdev->region[i].subtype;1114 1115 ret = vfio_info_add_capability(&caps, &cap_type.header,1116 sizeof(cap_type));1117 if (ret)1118 return ret;1119 1120 if (vdev->region[i].ops->add_capability) {1121 ret = vdev->region[i].ops->add_capability(1122 vdev, &vdev->region[i], &caps);1123 if (ret)1124 return ret;1125 }1126 }1127 }1128 1129 if (caps.size) {1130 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;1131 if (info.argsz < sizeof(info) + caps.size) {1132 info.argsz = sizeof(info) + caps.size;1133 info.cap_offset = 0;1134 } else {1135 vfio_info_cap_shift(&caps, sizeof(info));1136 if (copy_to_user(arg + 1, caps.buf, caps.size)) {1137 kfree(caps.buf);1138 return -EFAULT;1139 }1140 info.cap_offset = sizeof(*arg);1141 }1142 1143 kfree(caps.buf);1144 }1145 1146 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;1147}1148 1149static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev,1150 struct vfio_irq_info __user *arg)1151{1152 unsigned long minsz = offsetofend(struct vfio_irq_info, count);1153 struct vfio_irq_info info;1154 1155 if (copy_from_user(&info, arg, minsz))1156 return -EFAULT;1157 1158 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)1159 return -EINVAL;1160 1161 switch (info.index) {1162 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:1163 case VFIO_PCI_REQ_IRQ_INDEX:1164 break;1165 case VFIO_PCI_ERR_IRQ_INDEX:1166 if (pci_is_pcie(vdev->pdev))1167 break;1168 fallthrough;1169 default:1170 return -EINVAL;1171 }1172 1173 info.flags = VFIO_IRQ_INFO_EVENTFD;1174 1175 info.count = vfio_pci_get_irq_count(vdev, info.index);1176 1177 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)1178 info.flags |=1179 (VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED);1180 else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix)1181 info.flags |= VFIO_IRQ_INFO_NORESIZE;1182 1183 return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;1184}1185 1186static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,1187 struct vfio_irq_set __user *arg)1188{1189 unsigned long minsz = offsetofend(struct vfio_irq_set, count);1190 struct vfio_irq_set hdr;1191 u8 *data = NULL;1192 int max, ret = 0;1193 size_t data_size = 0;1194 1195 if (copy_from_user(&hdr, arg, minsz))1196 return -EFAULT;1197 1198 max = vfio_pci_get_irq_count(vdev, hdr.index);1199 1200 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS,1201 &data_size);1202 if (ret)1203 return ret;1204 1205 if (data_size) {1206 data = memdup_user(&arg->data, data_size);1207 if (IS_ERR(data))1208 return PTR_ERR(data);1209 }1210 1211 mutex_lock(&vdev->igate);1212 1213 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start,1214 hdr.count, data);1215 1216 mutex_unlock(&vdev->igate);1217 kfree(data);1218 1219 return ret;1220}1221 1222static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,1223 void __user *arg)1224{1225 int ret;1226 1227 if (!vdev->reset_works)1228 return -EINVAL;1229 1230 vfio_pci_zap_and_down_write_memory_lock(vdev);1231 1232 /*1233 * This function can be invoked while the power state is non-D0. If1234 * pci_try_reset_function() has been called while the power state is1235 * non-D0, then pci_try_reset_function() will internally set the power1236 * state to D0 without vfio driver involvement. For the devices which1237 * have NoSoftRst-, the reset function can cause the PCI config space1238 * reset without restoring the original state (saved locally in1239 * 'vdev->pm_save').1240 */1241 vfio_pci_set_power_state(vdev, PCI_D0);1242 1243 ret = pci_try_reset_function(vdev->pdev);1244 up_write(&vdev->memory_lock);1245 1246 return ret;1247}1248 1249static int vfio_pci_ioctl_get_pci_hot_reset_info(1250 struct vfio_pci_core_device *vdev,1251 struct vfio_pci_hot_reset_info __user *arg)1252{1253 unsigned long minsz =1254 offsetofend(struct vfio_pci_hot_reset_info, count);1255 struct vfio_pci_dependent_device *devices = NULL;1256 struct vfio_pci_hot_reset_info hdr;1257 struct vfio_pci_fill_info fill = {};1258 bool slot = false;1259 int ret, count = 0;1260 1261 if (copy_from_user(&hdr, arg, minsz))1262 return -EFAULT;1263 1264 if (hdr.argsz < minsz)1265 return -EINVAL;1266 1267 hdr.flags = 0;1268 1269 /* Can we do a slot or bus reset or neither? */1270 if (!pci_probe_reset_slot(vdev->pdev->slot))1271 slot = true;1272 else if (pci_probe_reset_bus(vdev->pdev->bus))1273 return -ENODEV;1274 1275 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,1276 &count, slot);1277 if (ret)1278 return ret;1279 1280 if (WARN_ON(!count)) /* Should always be at least one */1281 return -ERANGE;1282 1283 if (count > (hdr.argsz - sizeof(hdr)) / sizeof(*devices)) {1284 hdr.count = count;1285 ret = -ENOSPC;1286 goto header;1287 }1288 1289 devices = kcalloc(count, sizeof(*devices), GFP_KERNEL);1290 if (!devices)1291 return -ENOMEM;1292 1293 fill.devices = devices;1294 fill.nr_devices = count;1295 fill.vdev = &vdev->vdev;1296 1297 if (vfio_device_cdev_opened(&vdev->vdev))1298 fill.flags |= VFIO_PCI_HOT_RESET_FLAG_DEV_ID |1299 VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;1300 1301 mutex_lock(&vdev->vdev.dev_set->lock);1302 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs,1303 &fill, slot);1304 mutex_unlock(&vdev->vdev.dev_set->lock);1305 if (ret)1306 goto out;1307 1308 if (copy_to_user(arg->devices, devices,1309 sizeof(*devices) * fill.count)) {1310 ret = -EFAULT;1311 goto out;1312 }1313 1314 hdr.count = fill.count;1315 hdr.flags = fill.flags;1316 1317header:1318 if (copy_to_user(arg, &hdr, minsz))1319 ret = -EFAULT;1320out:1321 kfree(devices);1322 return ret;1323}1324 1325static int1326vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev,1327 u32 array_count, bool slot,1328 struct vfio_pci_hot_reset __user *arg)1329{1330 int32_t *group_fds;1331 struct file **files;1332 struct vfio_pci_group_info info;1333 int file_idx, count = 0, ret = 0;1334 1335 /*1336 * We can't let userspace give us an arbitrarily large buffer to copy,1337 * so verify how many we think there could be. Note groups can have1338 * multiple devices so one group per device is the max.1339 */1340 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,1341 &count, slot);1342 if (ret)1343 return ret;1344 1345 if (array_count > count)1346 return -EINVAL;1347 1348 group_fds = kcalloc(array_count, sizeof(*group_fds), GFP_KERNEL);1349 files = kcalloc(array_count, sizeof(*files), GFP_KERNEL);1350 if (!group_fds || !files) {1351 kfree(group_fds);1352 kfree(files);1353 return -ENOMEM;1354 }1355 1356 if (copy_from_user(group_fds, arg->group_fds,1357 array_count * sizeof(*group_fds))) {1358 kfree(group_fds);1359 kfree(files);1360 return -EFAULT;1361 }1362 1363 /*1364 * Get the group file for each fd to ensure the group is held across1365 * the reset1366 */1367 for (file_idx = 0; file_idx < array_count; file_idx++) {1368 struct file *file = fget(group_fds[file_idx]);1369 1370 if (!file) {1371 ret = -EBADF;1372 break;1373 }1374 1375 /* Ensure the FD is a vfio group FD.*/1376 if (!vfio_file_is_group(file)) {1377 fput(file);1378 ret = -EINVAL;1379 break;1380 }1381 1382 files[file_idx] = file;1383 }1384 1385 kfree(group_fds);1386 1387 /* release reference to groups on error */1388 if (ret)1389 goto hot_reset_release;1390 1391 info.count = array_count;1392 info.files = files;1393 1394 ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL);1395 1396hot_reset_release:1397 for (file_idx--; file_idx >= 0; file_idx--)1398 fput(files[file_idx]);1399 1400 kfree(files);1401 return ret;1402}1403 1404static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev,1405 struct vfio_pci_hot_reset __user *arg)1406{1407 unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count);1408 struct vfio_pci_hot_reset hdr;1409 bool slot = false;1410 1411 if (copy_from_user(&hdr, arg, minsz))1412 return -EFAULT;1413 1414 if (hdr.argsz < minsz || hdr.flags)1415 return -EINVAL;1416 1417 /* zero-length array is only for cdev opened devices */1418 if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev))1419 return -EINVAL;1420 1421 /* Can we do a slot or bus reset or neither? */1422 if (!pci_probe_reset_slot(vdev->pdev->slot))1423 slot = true;1424 else if (pci_probe_reset_bus(vdev->pdev->bus))1425 return -ENODEV;1426 1427 if (hdr.count)1428 return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg);1429 1430 return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL,1431 vfio_iommufd_device_ictx(&vdev->vdev));1432}1433 1434static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev,1435 struct vfio_device_ioeventfd __user *arg)1436{1437 unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd);1438 struct vfio_device_ioeventfd ioeventfd;1439 int count;1440 1441 if (copy_from_user(&ioeventfd, arg, minsz))1442 return -EFAULT;1443 1444 if (ioeventfd.argsz < minsz)1445 return -EINVAL;1446 1447 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)1448 return -EINVAL;1449 1450 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;1451 1452 if (hweight8(count) != 1 || ioeventfd.fd < -1)1453 return -EINVAL;1454 1455 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count,1456 ioeventfd.fd);1457}1458 1459long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,1460 unsigned long arg)1461{1462 struct vfio_pci_core_device *vdev =1463 container_of(core_vdev, struct vfio_pci_core_device, vdev);1464 void __user *uarg = (void __user *)arg;1465 1466 switch (cmd) {1467 case VFIO_DEVICE_GET_INFO:1468 return vfio_pci_ioctl_get_info(vdev, uarg);1469 case VFIO_DEVICE_GET_IRQ_INFO:1470 return vfio_pci_ioctl_get_irq_info(vdev, uarg);1471 case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO:1472 return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg);1473 case VFIO_DEVICE_GET_REGION_INFO:1474 return vfio_pci_ioctl_get_region_info(vdev, uarg);1475 case VFIO_DEVICE_IOEVENTFD:1476 return vfio_pci_ioctl_ioeventfd(vdev, uarg);1477 case VFIO_DEVICE_PCI_HOT_RESET:1478 return vfio_pci_ioctl_pci_hot_reset(vdev, uarg);1479 case VFIO_DEVICE_RESET:1480 return vfio_pci_ioctl_reset(vdev, uarg);1481 case VFIO_DEVICE_SET_IRQS:1482 return vfio_pci_ioctl_set_irqs(vdev, uarg);1483 default:1484 return -ENOTTY;1485 }1486}1487EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);1488 1489static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags,1490 uuid_t __user *arg, size_t argsz)1491{1492 struct vfio_pci_core_device *vdev =1493 container_of(device, struct vfio_pci_core_device, vdev);1494 uuid_t uuid;1495 int ret;1496 1497 if (!vdev->vf_token)1498 return -ENOTTY;1499 /*1500 * We do not support GET of the VF Token UUID as this could1501 * expose the token of the previous device user.1502 */1503 ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,1504 sizeof(uuid));1505 if (ret != 1)1506 return ret;1507 1508 if (copy_from_user(&uuid, arg, sizeof(uuid)))1509 return -EFAULT;1510 1511 mutex_lock(&vdev->vf_token->lock);1512 uuid_copy(&vdev->vf_token->uuid, &uuid);1513 mutex_unlock(&vdev->vf_token->lock);1514 return 0;1515}1516 1517int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,1518 void __user *arg, size_t argsz)1519{1520 switch (flags & VFIO_DEVICE_FEATURE_MASK) {1521 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY:1522 return vfio_pci_core_pm_entry(device, flags, arg, argsz);1523 case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP:1524 return vfio_pci_core_pm_entry_with_wakeup(device, flags,1525 arg, argsz);1526 case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT:1527 return vfio_pci_core_pm_exit(device, flags, arg, argsz);1528 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:1529 return vfio_pci_core_feature_token(device, flags, arg, argsz);1530 default:1531 return -ENOTTY;1532 }1533}1534EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature);1535 1536static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf,1537 size_t count, loff_t *ppos, bool iswrite)1538{1539 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);1540 int ret;1541 1542 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)1543 return -EINVAL;1544 1545 ret = pm_runtime_resume_and_get(&vdev->pdev->dev);1546 if (ret) {1547 pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n",1548 ret);1549 return -EIO;1550 }1551 1552 switch (index) {1553 case VFIO_PCI_CONFIG_REGION_INDEX:1554 ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);1555 break;1556 1557 case VFIO_PCI_ROM_REGION_INDEX:1558 if (iswrite)1559 ret = -EINVAL;1560 else1561 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false);1562 break;1563 1564 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:1565 ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);1566 break;1567 1568 case VFIO_PCI_VGA_REGION_INDEX:1569 ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);1570 break;1571 1572 default:1573 index -= VFIO_PCI_NUM_REGIONS;1574 ret = vdev->region[index].ops->rw(vdev, buf,1575 count, ppos, iswrite);1576 break;1577 }1578 1579 pm_runtime_put(&vdev->pdev->dev);1580 return ret;1581}1582 1583ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,1584 size_t count, loff_t *ppos)1585{1586 struct vfio_pci_core_device *vdev =1587 container_of(core_vdev, struct vfio_pci_core_device, vdev);1588 1589 if (!count)1590 return 0;1591 1592 return vfio_pci_rw(vdev, buf, count, ppos, false);1593}1594EXPORT_SYMBOL_GPL(vfio_pci_core_read);1595 1596ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,1597 size_t count, loff_t *ppos)1598{1599 struct vfio_pci_core_device *vdev =1600 container_of(core_vdev, struct vfio_pci_core_device, vdev);1601 1602 if (!count)1603 return 0;1604 1605 return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);1606}1607EXPORT_SYMBOL_GPL(vfio_pci_core_write);1608 1609static void vfio_pci_zap_bars(struct vfio_pci_core_device *vdev)1610{1611 struct vfio_device *core_vdev = &vdev->vdev;1612 loff_t start = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);1613 loff_t end = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);1614 loff_t len = end - start;1615 1616 unmap_mapping_range(core_vdev->inode->i_mapping, start, len, true);1617}1618 1619void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)1620{1621 down_write(&vdev->memory_lock);1622 vfio_pci_zap_bars(vdev);1623}1624 1625u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)1626{1627 u16 cmd;1628 1629 down_write(&vdev->memory_lock);1630 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);1631 if (!(cmd & PCI_COMMAND_MEMORY))1632 pci_write_config_word(vdev->pdev, PCI_COMMAND,1633 cmd | PCI_COMMAND_MEMORY);1634 1635 return cmd;1636}1637 1638void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)1639{1640 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);1641 up_write(&vdev->memory_lock);1642}1643 1644static unsigned long vma_to_pfn(struct vm_area_struct *vma)1645{1646 struct vfio_pci_core_device *vdev = vma->vm_private_data;1647 int index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);1648 u64 pgoff;1649 1650 pgoff = vma->vm_pgoff &1651 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);1652 1653 return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff;1654}1655 1656static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,1657 unsigned int order)1658{1659 struct vm_area_struct *vma = vmf->vma;1660 struct vfio_pci_core_device *vdev = vma->vm_private_data;1661 unsigned long pfn, pgoff = vmf->pgoff - vma->vm_pgoff;1662 vm_fault_t ret = VM_FAULT_SIGBUS;1663 1664 if (order && (vmf->address & ((PAGE_SIZE << order) - 1) ||1665 vmf->address + (PAGE_SIZE << order) > vma->vm_end)) {1666 ret = VM_FAULT_FALLBACK;1667 goto out;1668 }1669 1670 pfn = vma_to_pfn(vma);1671 1672 down_read(&vdev->memory_lock);1673 1674 if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))1675 goto out_unlock;1676 1677 switch (order) {1678 case 0:1679 ret = vmf_insert_pfn(vma, vmf->address, pfn + pgoff);1680 break;1681#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP1682 case PMD_ORDER:1683 ret = vmf_insert_pfn_pmd(vmf, __pfn_to_pfn_t(pfn + pgoff,1684 PFN_DEV), false);1685 break;1686#endif1687#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP1688 case PUD_ORDER:1689 ret = vmf_insert_pfn_pud(vmf, __pfn_to_pfn_t(pfn + pgoff,1690 PFN_DEV), false);1691 break;1692#endif1693 default:1694 ret = VM_FAULT_FALLBACK;1695 }1696 1697out_unlock:1698 up_read(&vdev->memory_lock);1699out:1700 dev_dbg_ratelimited(&vdev->pdev->dev,1701 "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n",1702 __func__, order,1703 vma->vm_pgoff >>1704 (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT),1705 pgoff, (unsigned int)ret);1706 1707 return ret;1708}1709 1710static vm_fault_t vfio_pci_mmap_page_fault(struct vm_fault *vmf)1711{1712 return vfio_pci_mmap_huge_fault(vmf, 0);1713}1714 1715static const struct vm_operations_struct vfio_pci_mmap_ops = {1716 .fault = vfio_pci_mmap_page_fault,1717#ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP1718 .huge_fault = vfio_pci_mmap_huge_fault,1719#endif1720};1721 1722int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)1723{1724 struct vfio_pci_core_device *vdev =1725 container_of(core_vdev, struct vfio_pci_core_device, vdev);1726 struct pci_dev *pdev = vdev->pdev;1727 unsigned int index;1728 u64 phys_len, req_len, pgoff, req_start;1729 int ret;1730 1731 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);1732 1733 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)1734 return -EINVAL;1735 if (vma->vm_end < vma->vm_start)1736 return -EINVAL;1737 if ((vma->vm_flags & VM_SHARED) == 0)1738 return -EINVAL;1739 if (index >= VFIO_PCI_NUM_REGIONS) {1740 int regnum = index - VFIO_PCI_NUM_REGIONS;1741 struct vfio_pci_region *region = vdev->region + regnum;1742 1743 if (region->ops && region->ops->mmap &&1744 (region->flags & VFIO_REGION_INFO_FLAG_MMAP))1745 return region->ops->mmap(vdev, region, vma);1746 return -EINVAL;1747 }1748 if (index >= VFIO_PCI_ROM_REGION_INDEX)1749 return -EINVAL;1750 if (!vdev->bar_mmap_supported[index])1751 return -EINVAL;1752 1753 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));1754 req_len = vma->vm_end - vma->vm_start;1755 pgoff = vma->vm_pgoff &1756 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);1757 req_start = pgoff << PAGE_SHIFT;1758 1759 if (req_start + req_len > phys_len)1760 return -EINVAL;1761 1762 /*1763 * Even though we don't make use of the barmap for the mmap,1764 * we need to request the region and the barmap tracks that.1765 */1766 if (!vdev->barmap[index]) {1767 ret = pci_request_selected_regions(pdev,1768 1 << index, "vfio-pci");1769 if (ret)1770 return ret;1771 1772 vdev->barmap[index] = pci_iomap(pdev, index, 0);1773 if (!vdev->barmap[index]) {1774 pci_release_selected_regions(pdev, 1 << index);1775 return -ENOMEM;1776 }1777 }1778 1779 vma->vm_private_data = vdev;1780 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);1781 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);1782 1783 /*1784 * Set vm_flags now, they should not be changed in the fault handler.1785 * We want the same flags and page protection (decrypted above) as1786 * io_remap_pfn_range() would set.1787 *1788 * VM_ALLOW_ANY_UNCACHED: The VMA flag is implemented for ARM64,1789 * allowing KVM stage 2 device mapping attributes to use Normal-NC1790 * rather than DEVICE_nGnRE, which allows guest mappings1791 * supporting write-combining attributes (WC). ARM does not1792 * architecturally guarantee this is safe, and indeed some MMIO1793 * regions like the GICv2 VCPU interface can trigger uncontained1794 * faults if Normal-NC is used.1795 *1796 * To safely use VFIO in KVM the platform must guarantee full1797 * safety in the guest where no action taken against a MMIO1798 * mapping can trigger an uncontained failure. The assumption is1799 * that most VFIO PCI platforms support this for both mapping types,1800 * at least in common flows, based on some expectations of how1801 * PCI IP is integrated. Hence VM_ALLOW_ANY_UNCACHED is set in1802 * the VMA flags.1803 */1804 vm_flags_set(vma, VM_ALLOW_ANY_UNCACHED | VM_IO | VM_PFNMAP |1805 VM_DONTEXPAND | VM_DONTDUMP);1806 vma->vm_ops = &vfio_pci_mmap_ops;1807 1808 return 0;1809}1810EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);1811 1812void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)1813{1814 struct vfio_pci_core_device *vdev =1815 container_of(core_vdev, struct vfio_pci_core_device, vdev);1816 struct pci_dev *pdev = vdev->pdev;1817 1818 mutex_lock(&vdev->igate);1819 1820 if (vdev->req_trigger) {1821 if (!(count % 10))1822 pci_notice_ratelimited(pdev,1823 "Relaying device request to user (#%u)\n",1824 count);1825 eventfd_signal(vdev->req_trigger);1826 } else if (count == 0) {1827 pci_warn(pdev,1828 "No device request channel registered, blocked until released by user\n");1829 }1830 1831 mutex_unlock(&vdev->igate);1832}1833EXPORT_SYMBOL_GPL(vfio_pci_core_request);1834 1835static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,1836 bool vf_token, uuid_t *uuid)1837{1838 /*1839 * There's always some degree of trust or collaboration between SR-IOV1840 * PF and VFs, even if just that the PF hosts the SR-IOV capability and1841 * can disrupt VFs with a reset, but often the PF has more explicit1842 * access to deny service to the VF or access data passed through the1843 * VF. We therefore require an opt-in via a shared VF token (UUID) to1844 * represent this trust. This both prevents that a VF driver might1845 * assume the PF driver is a trusted, in-kernel driver, and also that1846 * a PF driver might be replaced with a rogue driver, unknown to in-use1847 * VF drivers.1848 *1849 * Therefore when presented with a VF, if the PF is a vfio device and1850 * it is bound to the vfio-pci driver, the user needs to provide a VF1851 * token to access the device, in the form of appending a vf_token to1852 * the device name, for example:1853 *1854 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"1855 *1856 * When presented with a PF which has VFs in use, the user must also1857 * provide the current VF token to prove collaboration with existing1858 * VF users. If VFs are not in use, the VF token provided for the PF1859 * device will act to set the VF token.1860 *1861 * If the VF token is provided but unused, an error is generated.1862 */1863 if (vdev->pdev->is_virtfn) {1864 struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev;1865 bool match;1866 1867 if (!pf_vdev) {1868 if (!vf_token)1869 return 0; /* PF is not vfio-pci, no VF token */1870 1871 pci_info_ratelimited(vdev->pdev,1872 "VF token incorrectly provided, PF not bound to vfio-pci\n");1873 return -EINVAL;1874 }1875 1876 if (!vf_token) {1877 pci_info_ratelimited(vdev->pdev,1878 "VF token required to access device\n");1879 return -EACCES;1880 }1881 1882 mutex_lock(&pf_vdev->vf_token->lock);1883 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);1884 mutex_unlock(&pf_vdev->vf_token->lock);1885 1886 if (!match) {1887 pci_info_ratelimited(vdev->pdev,1888 "Incorrect VF token provided for device\n");1889 return -EACCES;1890 }1891 } else if (vdev->vf_token) {1892 mutex_lock(&vdev->vf_token->lock);1893 if (vdev->vf_token->users) {1894 if (!vf_token) {1895 mutex_unlock(&vdev->vf_token->lock);1896 pci_info_ratelimited(vdev->pdev,1897 "VF token required to access device\n");1898 return -EACCES;1899 }1900 1901 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {1902 mutex_unlock(&vdev->vf_token->lock);1903 pci_info_ratelimited(vdev->pdev,1904 "Incorrect VF token provided for device\n");1905 return -EACCES;1906 }1907 } else if (vf_token) {1908 uuid_copy(&vdev->vf_token->uuid, uuid);1909 }1910 1911 mutex_unlock(&vdev->vf_token->lock);1912 } else if (vf_token) {1913 pci_info_ratelimited(vdev->pdev,1914 "VF token incorrectly provided, not a PF or VF\n");1915 return -EINVAL;1916 }1917 1918 return 0;1919}1920 1921#define VF_TOKEN_ARG "vf_token="1922 1923int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf)1924{1925 struct vfio_pci_core_device *vdev =1926 container_of(core_vdev, struct vfio_pci_core_device, vdev);1927 bool vf_token = false;1928 uuid_t uuid;1929 int ret;1930 1931 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))1932 return 0; /* No match */1933 1934 if (strlen(buf) > strlen(pci_name(vdev->pdev))) {1935 buf += strlen(pci_name(vdev->pdev));1936 1937 if (*buf != ' ')1938 return 0; /* No match: non-whitespace after name */1939 1940 while (*buf) {1941 if (*buf == ' ') {1942 buf++;1943 continue;1944 }1945 1946 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,1947 strlen(VF_TOKEN_ARG))) {1948 buf += strlen(VF_TOKEN_ARG);1949 1950 if (strlen(buf) < UUID_STRING_LEN)1951 return -EINVAL;1952 1953 ret = uuid_parse(buf, &uuid);1954 if (ret)1955 return ret;1956 1957 vf_token = true;1958 buf += UUID_STRING_LEN;1959 } else {1960 /* Unknown/duplicate option */1961 return -EINVAL;1962 }1963 }1964 }1965 1966 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);1967 if (ret)1968 return ret;1969 1970 return 1; /* Match */1971}1972EXPORT_SYMBOL_GPL(vfio_pci_core_match);1973 1974static int vfio_pci_bus_notifier(struct notifier_block *nb,1975 unsigned long action, void *data)1976{1977 struct vfio_pci_core_device *vdev = container_of(nb,1978 struct vfio_pci_core_device, nb);1979 struct device *dev = data;1980 struct pci_dev *pdev = to_pci_dev(dev);1981 struct pci_dev *physfn = pci_physfn(pdev);1982 1983 if (action == BUS_NOTIFY_ADD_DEVICE &&1984 pdev->is_virtfn && physfn == vdev->pdev) {1985 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",1986 pci_name(pdev));1987 pdev->driver_override = kasprintf(GFP_KERNEL, "%s",1988 vdev->vdev.ops->name);1989 WARN_ON(!pdev->driver_override);1990 } else if (action == BUS_NOTIFY_BOUND_DRIVER &&1991 pdev->is_virtfn && physfn == vdev->pdev) {1992 struct pci_driver *drv = pci_dev_driver(pdev);1993 1994 if (drv && drv != pci_dev_driver(vdev->pdev))1995 pci_warn(vdev->pdev,1996 "VF %s bound to driver %s while PF bound to driver %s\n",1997 pci_name(pdev), drv->name,1998 pci_dev_driver(vdev->pdev)->name);1999 }2000 2001 return 0;2002}2003 2004static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)2005{2006 struct pci_dev *pdev = vdev->pdev;2007 struct vfio_pci_core_device *cur;2008 struct pci_dev *physfn;2009 int ret;2010 2011 if (pdev->is_virtfn) {2012 /*2013 * If this VF was created by our vfio_pci_core_sriov_configure()2014 * then we can find the PF vfio_pci_core_device now, and due to2015 * the locking in pci_disable_sriov() it cannot change until2016 * this VF device driver is removed.2017 */2018 physfn = pci_physfn(vdev->pdev);2019 mutex_lock(&vfio_pci_sriov_pfs_mutex);2020 list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) {2021 if (cur->pdev == physfn) {2022 vdev->sriov_pf_core_dev = cur;2023 break;2024 }2025 }2026 mutex_unlock(&vfio_pci_sriov_pfs_mutex);2027 return 0;2028 }2029 2030 /* Not a SRIOV PF */2031 if (!pdev->is_physfn)2032 return 0;2033 2034 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);2035 if (!vdev->vf_token)2036 return -ENOMEM;2037 2038 mutex_init(&vdev->vf_token->lock);2039 uuid_gen(&vdev->vf_token->uuid);2040 2041 vdev->nb.notifier_call = vfio_pci_bus_notifier;2042 ret = bus_register_notifier(&pci_bus_type, &vdev->nb);2043 if (ret) {2044 kfree(vdev->vf_token);2045 return ret;2046 }2047 return 0;2048}2049 2050static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)2051{2052 if (!vdev->vf_token)2053 return;2054 2055 bus_unregister_notifier(&pci_bus_type, &vdev->nb);2056 WARN_ON(vdev->vf_token->users);2057 mutex_destroy(&vdev->vf_token->lock);2058 kfree(vdev->vf_token);2059}2060 2061static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)2062{2063 struct pci_dev *pdev = vdev->pdev;2064 int ret;2065 2066 if (!vfio_pci_is_vga(pdev))2067 return 0;2068 2069 ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name);2070 if (ret)2071 return ret;2072 2073 ret = vga_client_register(pdev, vfio_pci_set_decode);2074 if (ret)2075 return ret;2076 vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false));2077 return 0;2078}2079 2080static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)2081{2082 struct pci_dev *pdev = vdev->pdev;2083 2084 if (!vfio_pci_is_vga(pdev))2085 return;2086 vga_client_unregister(pdev);2087 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |2088 VGA_RSRC_LEGACY_IO |2089 VGA_RSRC_LEGACY_MEM);2090}2091 2092int vfio_pci_core_init_dev(struct vfio_device *core_vdev)2093{2094 struct vfio_pci_core_device *vdev =2095 container_of(core_vdev, struct vfio_pci_core_device, vdev);2096 2097 vdev->pdev = to_pci_dev(core_vdev->dev);2098 vdev->irq_type = VFIO_PCI_NUM_IRQS;2099 mutex_init(&vdev->igate);2100 spin_lock_init(&vdev->irqlock);2101 mutex_init(&vdev->ioeventfds_lock);2102 INIT_LIST_HEAD(&vdev->dummy_resources_list);2103 INIT_LIST_HEAD(&vdev->ioeventfds_list);2104 INIT_LIST_HEAD(&vdev->sriov_pfs_item);2105 init_rwsem(&vdev->memory_lock);2106 xa_init(&vdev->ctx);2107 2108 return 0;2109}2110EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);2111 2112void vfio_pci_core_release_dev(struct vfio_device *core_vdev)2113{2114 struct vfio_pci_core_device *vdev =2115 container_of(core_vdev, struct vfio_pci_core_device, vdev);2116 2117 mutex_destroy(&vdev->igate);2118 mutex_destroy(&vdev->ioeventfds_lock);2119 kfree(vdev->region);2120 kfree(vdev->pm_save);2121}2122EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev);2123 2124int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)2125{2126 struct pci_dev *pdev = vdev->pdev;2127 struct device *dev = &pdev->dev;2128 int ret;2129 2130 /* Drivers must set the vfio_pci_core_device to their drvdata */2131 if (WARN_ON(vdev != dev_get_drvdata(dev)))2132 return -EINVAL;2133 2134 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)2135 return -EINVAL;2136 2137 if (vdev->vdev.mig_ops) {2138 if (!(vdev->vdev.mig_ops->migration_get_state &&2139 vdev->vdev.mig_ops->migration_set_state &&2140 vdev->vdev.mig_ops->migration_get_data_size) ||2141 !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY))2142 return -EINVAL;2143 }2144 2145 if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start &&2146 vdev->vdev.log_ops->log_stop &&2147 vdev->vdev.log_ops->log_read_and_clear))2148 return -EINVAL;2149 2150 /*2151 * Prevent binding to PFs with VFs enabled, the VFs might be in use2152 * by the host or other users. We cannot capture the VFs if they2153 * already exist, nor can we track VF users. Disabling SR-IOV here2154 * would initiate removing the VFs, which would unbind the driver,2155 * which is prone to blocking if that VF is also in use by vfio-pci.2156 * Just reject these PFs and let the user sort it out.2157 */2158 if (pci_num_vf(pdev)) {2159 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");2160 return -EBUSY;2161 }2162 2163 if (pci_is_root_bus(pdev->bus)) {2164 ret = vfio_assign_device_set(&vdev->vdev, vdev);2165 } else if (!pci_probe_reset_slot(pdev->slot)) {2166 ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);2167 } else {2168 /*2169 * If there is no slot reset support for this device, the whole2170 * bus needs to be grouped together to support bus-wide resets.2171 */2172 ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);2173 }2174 2175 if (ret)2176 return ret;2177 ret = vfio_pci_vf_init(vdev);2178 if (ret)2179 return ret;2180 ret = vfio_pci_vga_init(vdev);2181 if (ret)2182 goto out_vf;2183 2184 vfio_pci_probe_power_state(vdev);2185 2186 /*2187 * pci-core sets the device power state to an unknown value at2188 * bootup and after being removed from a driver. The only2189 * transition it allows from this unknown state is to D0, which2190 * typically happens when a driver calls pci_enable_device().2191 * We're not ready to enable the device yet, but we do want to2192 * be able to get to D3. Therefore first do a D0 transition2193 * before enabling runtime PM.2194 */2195 vfio_pci_set_power_state(vdev, PCI_D0);2196 2197 dev->driver->pm = &vfio_pci_core_pm_ops;2198 pm_runtime_allow(dev);2199 if (!disable_idle_d3)2200 pm_runtime_put(dev);2201 2202 ret = vfio_register_group_dev(&vdev->vdev);2203 if (ret)2204 goto out_power;2205 return 0;2206 2207out_power:2208 if (!disable_idle_d3)2209 pm_runtime_get_noresume(dev);2210 2211 pm_runtime_forbid(dev);2212out_vf:2213 vfio_pci_vf_uninit(vdev);2214 return ret;2215}2216EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);2217 2218void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)2219{2220 vfio_pci_core_sriov_configure(vdev, 0);2221 2222 vfio_unregister_group_dev(&vdev->vdev);2223 2224 vfio_pci_vf_uninit(vdev);2225 vfio_pci_vga_uninit(vdev);2226 2227 if (!disable_idle_d3)2228 pm_runtime_get_noresume(&vdev->pdev->dev);2229 2230 pm_runtime_forbid(&vdev->pdev->dev);2231}2232EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);2233 2234pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,2235 pci_channel_state_t state)2236{2237 struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);2238 2239 mutex_lock(&vdev->igate);2240 2241 if (vdev->err_trigger)2242 eventfd_signal(vdev->err_trigger);2243 2244 mutex_unlock(&vdev->igate);2245 2246 return PCI_ERS_RESULT_CAN_RECOVER;2247}2248EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected);2249 2250int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,2251 int nr_virtfn)2252{2253 struct pci_dev *pdev = vdev->pdev;2254 int ret = 0;2255 2256 device_lock_assert(&pdev->dev);2257 2258 if (nr_virtfn) {2259 mutex_lock(&vfio_pci_sriov_pfs_mutex);2260 /*2261 * The thread that adds the vdev to the list is the only thread2262 * that gets to call pci_enable_sriov() and we will only allow2263 * it to be called once without going through2264 * pci_disable_sriov()2265 */2266 if (!list_empty(&vdev->sriov_pfs_item)) {2267 ret = -EINVAL;2268 goto out_unlock;2269 }2270 list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs);2271 mutex_unlock(&vfio_pci_sriov_pfs_mutex);2272 2273 /*2274 * The PF power state should always be higher than the VF power2275 * state. The PF can be in low power state either with runtime2276 * power management (when there is no user) or PCI_PM_CTRL2277 * register write by the user. If PF is in the low power state,2278 * then change the power state to D0 first before enabling2279 * SR-IOV. Also, this function can be called at any time, and2280 * userspace PCI_PM_CTRL write can race against this code path,2281 * so protect the same with 'memory_lock'.2282 */2283 ret = pm_runtime_resume_and_get(&pdev->dev);2284 if (ret)2285 goto out_del;2286 2287 down_write(&vdev->memory_lock);2288 vfio_pci_set_power_state(vdev, PCI_D0);2289 ret = pci_enable_sriov(pdev, nr_virtfn);2290 up_write(&vdev->memory_lock);2291 if (ret) {2292 pm_runtime_put(&pdev->dev);2293 goto out_del;2294 }2295 return nr_virtfn;2296 }2297 2298 if (pci_num_vf(pdev)) {2299 pci_disable_sriov(pdev);2300 pm_runtime_put(&pdev->dev);2301 }2302 2303out_del:2304 mutex_lock(&vfio_pci_sriov_pfs_mutex);2305 list_del_init(&vdev->sriov_pfs_item);2306out_unlock:2307 mutex_unlock(&vfio_pci_sriov_pfs_mutex);2308 return ret;2309}2310EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);2311 2312const struct pci_error_handlers vfio_pci_core_err_handlers = {2313 .error_detected = vfio_pci_core_aer_err_detected,2314};2315EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);2316 2317static bool vfio_dev_in_groups(struct vfio_device *vdev,2318 struct vfio_pci_group_info *groups)2319{2320 unsigned int i;2321 2322 if (!groups)2323 return false;2324 2325 for (i = 0; i < groups->count; i++)2326 if (vfio_file_has_dev(groups->files[i], vdev))2327 return true;2328 return false;2329}2330 2331static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data)2332{2333 struct vfio_device_set *dev_set = data;2334 2335 return vfio_find_device_in_devset(dev_set, &pdev->dev) ? 0 : -ENODEV;2336}2337 2338/*2339 * vfio-core considers a group to be viable and will create a vfio_device even2340 * if some devices are bound to drivers like pci-stub or pcieport. Here we2341 * require all PCI devices to be inside our dev_set since that ensures they stay2342 * put and that every driver controlling the device can co-ordinate with the2343 * device reset.2344 *2345 * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be2346 * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.2347 */2348static struct pci_dev *2349vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)2350{2351 struct pci_dev *pdev;2352 2353 lockdep_assert_held(&dev_set->lock);2354 2355 /*2356 * By definition all PCI devices in the dev_set share the same PCI2357 * reset, so any pci_dev will have the same outcomes for2358 * pci_probe_reset_*() and pci_reset_bus().2359 */2360 pdev = list_first_entry(&dev_set->device_list,2361 struct vfio_pci_core_device,2362 vdev.dev_set_list)->pdev;2363 2364 /* pci_reset_bus() is supported */2365 if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))2366 return NULL;2367 2368 if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,2369 dev_set,2370 !pci_probe_reset_slot(pdev->slot)))2371 return NULL;2372 return pdev;2373}2374 2375static int vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set *dev_set)2376{2377 struct vfio_pci_core_device *cur;2378 int ret;2379 2380 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {2381 ret = pm_runtime_resume_and_get(&cur->pdev->dev);2382 if (ret)2383 goto unwind;2384 }2385 2386 return 0;2387 2388unwind:2389 list_for_each_entry_continue_reverse(cur, &dev_set->device_list,2390 vdev.dev_set_list)2391 pm_runtime_put(&cur->pdev->dev);2392 2393 return ret;2394}2395 2396static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,2397 struct vfio_pci_group_info *groups,2398 struct iommufd_ctx *iommufd_ctx)2399{2400 struct vfio_pci_core_device *vdev;2401 struct pci_dev *pdev;2402 int ret;2403 2404 mutex_lock(&dev_set->lock);2405 2406 pdev = vfio_pci_dev_set_resettable(dev_set);2407 if (!pdev) {2408 ret = -EINVAL;2409 goto err_unlock;2410 }2411 2412 /*2413 * Some of the devices in the dev_set can be in the runtime suspended2414 * state. Increment the usage count for all the devices in the dev_set2415 * before reset and decrement the same after reset.2416 */2417 ret = vfio_pci_dev_set_pm_runtime_get(dev_set);2418 if (ret)2419 goto err_unlock;2420 2421 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list) {2422 bool owned;2423 2424 /*2425 * Test whether all the affected devices can be reset by the2426 * user.2427 *2428 * If called from a group opened device and the user provides2429 * a set of groups, all the devices in the dev_set should be2430 * contained by the set of groups provided by the user.2431 *2432 * If called from a cdev opened device and the user provides2433 * a zero-length array, all the devices in the dev_set must2434 * be bound to the same iommufd_ctx as the input iommufd_ctx.2435 * If there is any device that has not been bound to any2436 * iommufd_ctx yet, check if its iommu_group has any device2437 * bound to the input iommufd_ctx. Such devices can be2438 * considered owned by the input iommufd_ctx as the device2439 * cannot be owned by another iommufd_ctx when its iommu_group2440 * is owned.2441 *2442 * Otherwise, reset is not allowed.2443 */2444 if (iommufd_ctx) {2445 int devid = vfio_iommufd_get_dev_id(&vdev->vdev,2446 iommufd_ctx);2447 2448 owned = (devid > 0 || devid == -ENOENT);2449 } else {2450 owned = vfio_dev_in_groups(&vdev->vdev, groups);2451 }2452 2453 if (!owned) {2454 ret = -EINVAL;2455 break;2456 }2457 2458 /*2459 * Take the memory write lock for each device and zap BAR2460 * mappings to prevent the user accessing the device while in2461 * reset. Locking multiple devices is prone to deadlock,2462 * runaway and unwind if we hit contention.2463 */2464 if (!down_write_trylock(&vdev->memory_lock)) {2465 ret = -EBUSY;2466 break;2467 }2468 2469 vfio_pci_zap_bars(vdev);2470 }2471 2472 if (!list_entry_is_head(vdev,2473 &dev_set->device_list, vdev.dev_set_list)) {2474 vdev = list_prev_entry(vdev, vdev.dev_set_list);2475 goto err_undo;2476 }2477 2478 /*2479 * The pci_reset_bus() will reset all the devices in the bus.2480 * The power state can be non-D0 for some of the devices in the bus.2481 * For these devices, the pci_reset_bus() will internally set2482 * the power state to D0 without vfio driver involvement.2483 * For the devices which have NoSoftRst-, the reset function can2484 * cause the PCI config space reset without restoring the original2485 * state (saved locally in 'vdev->pm_save').2486 */2487 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list)2488 vfio_pci_set_power_state(vdev, PCI_D0);2489 2490 ret = pci_reset_bus(pdev);2491 2492 vdev = list_last_entry(&dev_set->device_list,2493 struct vfio_pci_core_device, vdev.dev_set_list);2494 2495err_undo:2496 list_for_each_entry_from_reverse(vdev, &dev_set->device_list,2497 vdev.dev_set_list)2498 up_write(&vdev->memory_lock);2499 2500 list_for_each_entry(vdev, &dev_set->device_list, vdev.dev_set_list)2501 pm_runtime_put(&vdev->pdev->dev);2502 2503err_unlock:2504 mutex_unlock(&dev_set->lock);2505 return ret;2506}2507 2508static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)2509{2510 struct vfio_pci_core_device *cur;2511 bool needs_reset = false;2512 2513 /* No other VFIO device in the set can be open. */2514 if (vfio_device_set_open_count(dev_set) > 1)2515 return false;2516 2517 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)2518 needs_reset |= cur->needs_reset;2519 return needs_reset;2520}2521 2522/*2523 * If a bus or slot reset is available for the provided dev_set and:2524 * - All of the devices affected by that bus or slot reset are unused2525 * - At least one of the affected devices is marked dirty via2526 * needs_reset (such as by lack of FLR support)2527 * Then attempt to perform that bus or slot reset.2528 */2529static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)2530{2531 struct vfio_pci_core_device *cur;2532 struct pci_dev *pdev;2533 bool reset_done = false;2534 2535 if (!vfio_pci_dev_set_needs_reset(dev_set))2536 return;2537 2538 pdev = vfio_pci_dev_set_resettable(dev_set);2539 if (!pdev)2540 return;2541 2542 /*2543 * Some of the devices in the bus can be in the runtime suspended2544 * state. Increment the usage count for all the devices in the dev_set2545 * before reset and decrement the same after reset.2546 */2547 if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set))2548 return;2549 2550 if (!pci_reset_bus(pdev))2551 reset_done = true;2552 2553 list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {2554 if (reset_done)2555 cur->needs_reset = false;2556 2557 if (!disable_idle_d3)2558 pm_runtime_put(&cur->pdev->dev);2559 }2560}2561 2562void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,2563 bool is_disable_idle_d3)2564{2565 nointxmask = is_nointxmask;2566 disable_vga = is_disable_vga;2567 disable_idle_d3 = is_disable_idle_d3;2568}2569EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);2570 2571static void vfio_pci_core_cleanup(void)2572{2573 vfio_pci_uninit_perm_bits();2574}2575 2576static int __init vfio_pci_core_init(void)2577{2578 /* Allocate shared config space permission data used by all devices */2579 return vfio_pci_init_perm_bits();2580}2581 2582module_init(vfio_pci_core_init);2583module_exit(vfio_pci_core_cleanup);2584 2585MODULE_LICENSE("GPL v2");2586MODULE_AUTHOR(DRIVER_AUTHOR);2587MODULE_DESCRIPTION(DRIVER_DESC);2588