608 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.4 *5 * VFIO container (/dev/vfio/vfio)6 */7#include <linux/file.h>8#include <linux/slab.h>9#include <linux/fs.h>10#include <linux/capability.h>11#include <linux/iommu.h>12#include <linux/miscdevice.h>13#include <linux/vfio.h>14#include <uapi/linux/vfio.h>15 16#include "vfio.h"17 18struct vfio_container {19 struct kref kref;20 struct list_head group_list;21 struct rw_semaphore group_lock;22 struct vfio_iommu_driver *iommu_driver;23 void *iommu_data;24 bool noiommu;25};26 27static struct vfio {28 struct list_head iommu_drivers_list;29 struct mutex iommu_drivers_lock;30} vfio;31 32static void *vfio_noiommu_open(unsigned long arg)33{34 if (arg != VFIO_NOIOMMU_IOMMU)35 return ERR_PTR(-EINVAL);36 if (!capable(CAP_SYS_RAWIO))37 return ERR_PTR(-EPERM);38 39 return NULL;40}41 42static void vfio_noiommu_release(void *iommu_data)43{44}45 46static long vfio_noiommu_ioctl(void *iommu_data,47 unsigned int cmd, unsigned long arg)48{49 if (cmd == VFIO_CHECK_EXTENSION)50 return vfio_noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;51 52 return -ENOTTY;53}54 55static int vfio_noiommu_attach_group(void *iommu_data,56 struct iommu_group *iommu_group, enum vfio_group_type type)57{58 return 0;59}60 61static void vfio_noiommu_detach_group(void *iommu_data,62 struct iommu_group *iommu_group)63{64}65 66static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {67 .name = "vfio-noiommu",68 .owner = THIS_MODULE,69 .open = vfio_noiommu_open,70 .release = vfio_noiommu_release,71 .ioctl = vfio_noiommu_ioctl,72 .attach_group = vfio_noiommu_attach_group,73 .detach_group = vfio_noiommu_detach_group,74};75 76/*77 * Only noiommu containers can use vfio-noiommu and noiommu containers can only78 * use vfio-noiommu.79 */80static bool vfio_iommu_driver_allowed(struct vfio_container *container,81 const struct vfio_iommu_driver *driver)82{83 if (!IS_ENABLED(CONFIG_VFIO_NOIOMMU))84 return true;85 return container->noiommu == (driver->ops == &vfio_noiommu_ops);86}87 88/*89 * IOMMU driver registration90 */91int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)92{93 struct vfio_iommu_driver *driver, *tmp;94 95 if (WARN_ON(!ops->register_device != !ops->unregister_device))96 return -EINVAL;97 98 driver = kzalloc(sizeof(*driver), GFP_KERNEL);99 if (!driver)100 return -ENOMEM;101 102 driver->ops = ops;103 104 mutex_lock(&vfio.iommu_drivers_lock);105 106 /* Check for duplicates */107 list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {108 if (tmp->ops == ops) {109 mutex_unlock(&vfio.iommu_drivers_lock);110 kfree(driver);111 return -EINVAL;112 }113 }114 115 list_add(&driver->vfio_next, &vfio.iommu_drivers_list);116 117 mutex_unlock(&vfio.iommu_drivers_lock);118 119 return 0;120}121EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);122 123void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)124{125 struct vfio_iommu_driver *driver;126 127 mutex_lock(&vfio.iommu_drivers_lock);128 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {129 if (driver->ops == ops) {130 list_del(&driver->vfio_next);131 mutex_unlock(&vfio.iommu_drivers_lock);132 kfree(driver);133 return;134 }135 }136 mutex_unlock(&vfio.iommu_drivers_lock);137}138EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);139 140/*141 * Container objects - containers are created when /dev/vfio/vfio is142 * opened, but their lifecycle extends until the last user is done, so143 * it's freed via kref. Must support container/group/device being144 * closed in any order.145 */146static void vfio_container_release(struct kref *kref)147{148 struct vfio_container *container;149 container = container_of(kref, struct vfio_container, kref);150 151 kfree(container);152}153 154static void vfio_container_get(struct vfio_container *container)155{156 kref_get(&container->kref);157}158 159static void vfio_container_put(struct vfio_container *container)160{161 kref_put(&container->kref, vfio_container_release);162}163 164void vfio_device_container_register(struct vfio_device *device)165{166 struct vfio_iommu_driver *iommu_driver =167 device->group->container->iommu_driver;168 169 if (iommu_driver && iommu_driver->ops->register_device)170 iommu_driver->ops->register_device(171 device->group->container->iommu_data, device);172}173 174void vfio_device_container_unregister(struct vfio_device *device)175{176 struct vfio_iommu_driver *iommu_driver =177 device->group->container->iommu_driver;178 179 if (iommu_driver && iommu_driver->ops->unregister_device)180 iommu_driver->ops->unregister_device(181 device->group->container->iommu_data, device);182}183 184static long185vfio_container_ioctl_check_extension(struct vfio_container *container,186 unsigned long arg)187{188 struct vfio_iommu_driver *driver;189 long ret = 0;190 191 down_read(&container->group_lock);192 193 driver = container->iommu_driver;194 195 switch (arg) {196 /* No base extensions yet */197 default:198 /*199 * If no driver is set, poll all registered drivers for200 * extensions and return the first positive result. If201 * a driver is already set, further queries will be passed202 * only to that driver.203 */204 if (!driver) {205 mutex_lock(&vfio.iommu_drivers_lock);206 list_for_each_entry(driver, &vfio.iommu_drivers_list,207 vfio_next) {208 209 if (!list_empty(&container->group_list) &&210 !vfio_iommu_driver_allowed(container,211 driver))212 continue;213 if (!try_module_get(driver->ops->owner))214 continue;215 216 ret = driver->ops->ioctl(NULL,217 VFIO_CHECK_EXTENSION,218 arg);219 module_put(driver->ops->owner);220 if (ret > 0)221 break;222 }223 mutex_unlock(&vfio.iommu_drivers_lock);224 } else225 ret = driver->ops->ioctl(container->iommu_data,226 VFIO_CHECK_EXTENSION, arg);227 }228 229 up_read(&container->group_lock);230 231 return ret;232}233 234/* hold write lock on container->group_lock */235static int __vfio_container_attach_groups(struct vfio_container *container,236 struct vfio_iommu_driver *driver,237 void *data)238{239 struct vfio_group *group;240 int ret = -ENODEV;241 242 list_for_each_entry(group, &container->group_list, container_next) {243 ret = driver->ops->attach_group(data, group->iommu_group,244 group->type);245 if (ret)246 goto unwind;247 }248 249 return ret;250 251unwind:252 list_for_each_entry_continue_reverse(group, &container->group_list,253 container_next) {254 driver->ops->detach_group(data, group->iommu_group);255 }256 257 return ret;258}259 260static long vfio_ioctl_set_iommu(struct vfio_container *container,261 unsigned long arg)262{263 struct vfio_iommu_driver *driver;264 long ret = -ENODEV;265 266 down_write(&container->group_lock);267 268 /*269 * The container is designed to be an unprivileged interface while270 * the group can be assigned to specific users. Therefore, only by271 * adding a group to a container does the user get the privilege of272 * enabling the iommu, which may allocate finite resources. There273 * is no unset_iommu, but by removing all the groups from a container,274 * the container is deprivileged and returns to an unset state.275 */276 if (list_empty(&container->group_list) || container->iommu_driver) {277 up_write(&container->group_lock);278 return -EINVAL;279 }280 281 mutex_lock(&vfio.iommu_drivers_lock);282 list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {283 void *data;284 285 if (!vfio_iommu_driver_allowed(container, driver))286 continue;287 if (!try_module_get(driver->ops->owner))288 continue;289 290 /*291 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,292 * so test which iommu driver reported support for this293 * extension and call open on them. We also pass them the294 * magic, allowing a single driver to support multiple295 * interfaces if they'd like.296 */297 if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {298 module_put(driver->ops->owner);299 continue;300 }301 302 data = driver->ops->open(arg);303 if (IS_ERR(data)) {304 ret = PTR_ERR(data);305 module_put(driver->ops->owner);306 continue;307 }308 309 ret = __vfio_container_attach_groups(container, driver, data);310 if (ret) {311 driver->ops->release(data);312 module_put(driver->ops->owner);313 continue;314 }315 316 container->iommu_driver = driver;317 container->iommu_data = data;318 break;319 }320 321 mutex_unlock(&vfio.iommu_drivers_lock);322 up_write(&container->group_lock);323 324 return ret;325}326 327static long vfio_fops_unl_ioctl(struct file *filep,328 unsigned int cmd, unsigned long arg)329{330 struct vfio_container *container = filep->private_data;331 struct vfio_iommu_driver *driver;332 void *data;333 long ret = -EINVAL;334 335 if (!container)336 return ret;337 338 switch (cmd) {339 case VFIO_GET_API_VERSION:340 ret = VFIO_API_VERSION;341 break;342 case VFIO_CHECK_EXTENSION:343 ret = vfio_container_ioctl_check_extension(container, arg);344 break;345 case VFIO_SET_IOMMU:346 ret = vfio_ioctl_set_iommu(container, arg);347 break;348 default:349 driver = container->iommu_driver;350 data = container->iommu_data;351 352 if (driver) /* passthrough all unrecognized ioctls */353 ret = driver->ops->ioctl(data, cmd, arg);354 }355 356 return ret;357}358 359static int vfio_fops_open(struct inode *inode, struct file *filep)360{361 struct vfio_container *container;362 363 container = kzalloc(sizeof(*container), GFP_KERNEL_ACCOUNT);364 if (!container)365 return -ENOMEM;366 367 INIT_LIST_HEAD(&container->group_list);368 init_rwsem(&container->group_lock);369 kref_init(&container->kref);370 371 filep->private_data = container;372 373 return 0;374}375 376static int vfio_fops_release(struct inode *inode, struct file *filep)377{378 struct vfio_container *container = filep->private_data;379 380 filep->private_data = NULL;381 382 vfio_container_put(container);383 384 return 0;385}386 387static const struct file_operations vfio_fops = {388 .owner = THIS_MODULE,389 .open = vfio_fops_open,390 .release = vfio_fops_release,391 .unlocked_ioctl = vfio_fops_unl_ioctl,392 .compat_ioctl = compat_ptr_ioctl,393};394 395struct vfio_container *vfio_container_from_file(struct file *file)396{397 struct vfio_container *container;398 399 /* Sanity check, is this really our fd? */400 if (file->f_op != &vfio_fops)401 return NULL;402 403 container = file->private_data;404 WARN_ON(!container); /* fget ensures we don't race vfio_release */405 return container;406}407 408static struct miscdevice vfio_dev = {409 .minor = VFIO_MINOR,410 .name = "vfio",411 .fops = &vfio_fops,412 .nodename = "vfio/vfio",413 .mode = S_IRUGO | S_IWUGO,414};415 416int vfio_container_attach_group(struct vfio_container *container,417 struct vfio_group *group)418{419 struct vfio_iommu_driver *driver;420 int ret = 0;421 422 lockdep_assert_held(&group->group_lock);423 424 if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))425 return -EPERM;426 427 down_write(&container->group_lock);428 429 /* Real groups and fake groups cannot mix */430 if (!list_empty(&container->group_list) &&431 container->noiommu != (group->type == VFIO_NO_IOMMU)) {432 ret = -EPERM;433 goto out_unlock_container;434 }435 436 if (group->type == VFIO_IOMMU) {437 ret = iommu_group_claim_dma_owner(group->iommu_group, group);438 if (ret)439 goto out_unlock_container;440 }441 442 driver = container->iommu_driver;443 if (driver) {444 ret = driver->ops->attach_group(container->iommu_data,445 group->iommu_group,446 group->type);447 if (ret) {448 if (group->type == VFIO_IOMMU)449 iommu_group_release_dma_owner(450 group->iommu_group);451 goto out_unlock_container;452 }453 }454 455 group->container = container;456 group->container_users = 1;457 container->noiommu = (group->type == VFIO_NO_IOMMU);458 list_add(&group->container_next, &container->group_list);459 460 /* Get a reference on the container and mark a user within the group */461 vfio_container_get(container);462 463out_unlock_container:464 up_write(&container->group_lock);465 return ret;466}467 468void vfio_group_detach_container(struct vfio_group *group)469{470 struct vfio_container *container = group->container;471 struct vfio_iommu_driver *driver;472 473 lockdep_assert_held(&group->group_lock);474 WARN_ON(group->container_users != 1);475 476 down_write(&container->group_lock);477 478 driver = container->iommu_driver;479 if (driver)480 driver->ops->detach_group(container->iommu_data,481 group->iommu_group);482 483 if (group->type == VFIO_IOMMU)484 iommu_group_release_dma_owner(group->iommu_group);485 486 group->container = NULL;487 group->container_users = 0;488 list_del(&group->container_next);489 490 /* Detaching the last group deprivileges a container, remove iommu */491 if (driver && list_empty(&container->group_list)) {492 driver->ops->release(container->iommu_data);493 module_put(driver->ops->owner);494 container->iommu_driver = NULL;495 container->iommu_data = NULL;496 }497 498 up_write(&container->group_lock);499 500 vfio_container_put(container);501}502 503int vfio_group_use_container(struct vfio_group *group)504{505 lockdep_assert_held(&group->group_lock);506 507 /*508 * The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but509 * VFIO_SET_IOMMU hasn't been done yet.510 */511 if (!group->container->iommu_driver)512 return -EINVAL;513 514 if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))515 return -EPERM;516 517 get_file(group->opened_file);518 group->container_users++;519 return 0;520}521 522void vfio_group_unuse_container(struct vfio_group *group)523{524 lockdep_assert_held(&group->group_lock);525 526 WARN_ON(group->container_users <= 1);527 group->container_users--;528 fput(group->opened_file);529}530 531int vfio_device_container_pin_pages(struct vfio_device *device,532 dma_addr_t iova, int npage,533 int prot, struct page **pages)534{535 struct vfio_container *container = device->group->container;536 struct iommu_group *iommu_group = device->group->iommu_group;537 struct vfio_iommu_driver *driver = container->iommu_driver;538 539 if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)540 return -E2BIG;541 542 if (unlikely(!driver || !driver->ops->pin_pages))543 return -ENOTTY;544 return driver->ops->pin_pages(container->iommu_data, iommu_group, iova,545 npage, prot, pages);546}547 548void vfio_device_container_unpin_pages(struct vfio_device *device,549 dma_addr_t iova, int npage)550{551 struct vfio_container *container = device->group->container;552 553 if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES))554 return;555 556 container->iommu_driver->ops->unpin_pages(container->iommu_data, iova,557 npage);558}559 560int vfio_device_container_dma_rw(struct vfio_device *device,561 dma_addr_t iova, void *data,562 size_t len, bool write)563{564 struct vfio_container *container = device->group->container;565 struct vfio_iommu_driver *driver = container->iommu_driver;566 567 if (unlikely(!driver || !driver->ops->dma_rw))568 return -ENOTTY;569 return driver->ops->dma_rw(container->iommu_data, iova, data, len,570 write);571}572 573int __init vfio_container_init(void)574{575 int ret;576 577 mutex_init(&vfio.iommu_drivers_lock);578 INIT_LIST_HEAD(&vfio.iommu_drivers_list);579 580 ret = misc_register(&vfio_dev);581 if (ret) {582 pr_err("vfio: misc device register failed\n");583 return ret;584 }585 586 if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) {587 ret = vfio_register_iommu_driver(&vfio_noiommu_ops);588 if (ret)589 goto err_misc;590 }591 return 0;592 593err_misc:594 misc_deregister(&vfio_dev);595 return ret;596}597 598void vfio_container_cleanup(void)599{600 if (IS_ENABLED(CONFIG_VFIO_NOIOMMU))601 vfio_unregister_iommu_driver(&vfio_noiommu_ops);602 misc_deregister(&vfio_dev);603 mutex_destroy(&vfio.iommu_drivers_lock);604}605 606MODULE_ALIAS_MISCDEV(VFIO_MINOR);607MODULE_ALIAS("devname:vfio/vfio");608