649 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Physical device callbacks for vfio_ccw4 *5 * Copyright IBM Corp. 20176 * Copyright Red Hat, Inc. 20197 *8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>10 * Cornelia Huck <cohuck@redhat.com>11 */12 13#include <linux/vfio.h>14#include <linux/nospec.h>15#include <linux/slab.h>16 17#include "vfio_ccw_private.h"18 19static const struct vfio_device_ops vfio_ccw_dev_ops;20 21static int vfio_ccw_mdev_reset(struct vfio_ccw_private *private)22{23 /*24 * If the FSM state is seen as Not Operational after closing25 * and re-opening the mdev, return an error.26 */27 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);28 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);29 if (private->state == VFIO_CCW_STATE_NOT_OPER)30 return -EINVAL;31 32 return 0;33}34 35static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)36{37 struct vfio_ccw_private *private =38 container_of(vdev, struct vfio_ccw_private, vdev);39 40 /* Drivers MUST unpin pages in response to an invalidation. */41 if (!cp_iova_pinned(&private->cp, iova, length))42 return;43 44 vfio_ccw_mdev_reset(private);45}46 47static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev)48{49 struct vfio_ccw_private *private =50 container_of(vdev, struct vfio_ccw_private, vdev);51 52 mutex_init(&private->io_mutex);53 private->state = VFIO_CCW_STATE_STANDBY;54 INIT_LIST_HEAD(&private->crw);55 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);56 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);57 58 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),59 GFP_KERNEL);60 if (!private->cp.guest_cp)61 goto out_free_private;62 63 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,64 GFP_KERNEL | GFP_DMA);65 if (!private->io_region)66 goto out_free_cp;67 68 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,69 GFP_KERNEL | GFP_DMA);70 if (!private->cmd_region)71 goto out_free_io;72 73 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,74 GFP_KERNEL | GFP_DMA);75 if (!private->schib_region)76 goto out_free_cmd;77 78 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,79 GFP_KERNEL | GFP_DMA);80 if (!private->crw_region)81 goto out_free_schib;82 83 return 0;84 85out_free_schib:86 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);87out_free_cmd:88 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);89out_free_io:90 kmem_cache_free(vfio_ccw_io_region, private->io_region);91out_free_cp:92 kfree(private->cp.guest_cp);93out_free_private:94 mutex_destroy(&private->io_mutex);95 return -ENOMEM;96}97 98static int vfio_ccw_mdev_probe(struct mdev_device *mdev)99{100 struct subchannel *sch = to_subchannel(mdev->dev.parent);101 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);102 struct vfio_ccw_private *private;103 int ret;104 105 private = vfio_alloc_device(vfio_ccw_private, vdev, &mdev->dev,106 &vfio_ccw_dev_ops);107 if (IS_ERR(private))108 return PTR_ERR(private);109 110 dev_set_drvdata(&parent->dev, private);111 112 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n",113 sch->schid.cssid,114 sch->schid.ssid,115 sch->schid.sch_no);116 117 ret = vfio_register_emulated_iommu_dev(&private->vdev);118 if (ret)119 goto err_put_vdev;120 dev_set_drvdata(&mdev->dev, private);121 return 0;122 123err_put_vdev:124 dev_set_drvdata(&parent->dev, NULL);125 vfio_put_device(&private->vdev);126 return ret;127}128 129static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev)130{131 struct vfio_ccw_private *private =132 container_of(vdev, struct vfio_ccw_private, vdev);133 struct vfio_ccw_crw *crw, *temp;134 135 list_for_each_entry_safe(crw, temp, &private->crw, next) {136 list_del(&crw->next);137 kfree(crw);138 }139 140 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);141 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);142 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);143 kmem_cache_free(vfio_ccw_io_region, private->io_region);144 kfree(private->cp.guest_cp);145 mutex_destroy(&private->io_mutex);146}147 148static void vfio_ccw_mdev_remove(struct mdev_device *mdev)149{150 struct subchannel *sch = to_subchannel(mdev->dev.parent);151 struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev);152 struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev);153 154 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: remove\n",155 sch->schid.cssid,156 sch->schid.ssid,157 sch->schid.sch_no);158 159 vfio_unregister_group_dev(&private->vdev);160 161 dev_set_drvdata(&parent->dev, NULL);162 vfio_put_device(&private->vdev);163}164 165static int vfio_ccw_mdev_open_device(struct vfio_device *vdev)166{167 struct vfio_ccw_private *private =168 container_of(vdev, struct vfio_ccw_private, vdev);169 int ret;170 171 /* Device cannot simply be opened again from this state */172 if (private->state == VFIO_CCW_STATE_NOT_OPER)173 return -EINVAL;174 175 ret = vfio_ccw_register_async_dev_regions(private);176 if (ret)177 return ret;178 179 ret = vfio_ccw_register_schib_dev_regions(private);180 if (ret)181 goto out_unregister;182 183 ret = vfio_ccw_register_crw_dev_regions(private);184 if (ret)185 goto out_unregister;186 187 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_OPEN);188 if (private->state == VFIO_CCW_STATE_NOT_OPER) {189 ret = -EINVAL;190 goto out_unregister;191 }192 193 return ret;194 195out_unregister:196 vfio_ccw_unregister_dev_regions(private);197 return ret;198}199 200static void vfio_ccw_mdev_close_device(struct vfio_device *vdev)201{202 struct vfio_ccw_private *private =203 container_of(vdev, struct vfio_ccw_private, vdev);204 205 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);206 vfio_ccw_unregister_dev_regions(private);207}208 209static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,210 char __user *buf, size_t count,211 loff_t *ppos)212{213 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;214 struct ccw_io_region *region;215 int ret;216 217 if (pos + count > sizeof(*region))218 return -EINVAL;219 220 mutex_lock(&private->io_mutex);221 region = private->io_region;222 if (copy_to_user(buf, (void *)region + pos, count))223 ret = -EFAULT;224 else225 ret = count;226 mutex_unlock(&private->io_mutex);227 return ret;228}229 230static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev,231 char __user *buf,232 size_t count,233 loff_t *ppos)234{235 struct vfio_ccw_private *private =236 container_of(vdev, struct vfio_ccw_private, vdev);237 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);238 239 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)240 return -EINVAL;241 242 switch (index) {243 case VFIO_CCW_CONFIG_REGION_INDEX:244 return vfio_ccw_mdev_read_io_region(private, buf, count, ppos);245 default:246 index -= VFIO_CCW_NUM_REGIONS;247 return private->region[index].ops->read(private, buf, count,248 ppos);249 }250 251 return -EINVAL;252}253 254static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private,255 const char __user *buf,256 size_t count, loff_t *ppos)257{258 loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;259 struct ccw_io_region *region;260 int ret;261 262 if (pos + count > sizeof(*region))263 return -EINVAL;264 265 if (!mutex_trylock(&private->io_mutex))266 return -EAGAIN;267 268 region = private->io_region;269 if (copy_from_user((void *)region + pos, buf, count)) {270 ret = -EFAULT;271 goto out_unlock;272 }273 274 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ);275 ret = (region->ret_code != 0) ? region->ret_code : count;276 277out_unlock:278 mutex_unlock(&private->io_mutex);279 return ret;280}281 282static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev,283 const char __user *buf,284 size_t count,285 loff_t *ppos)286{287 struct vfio_ccw_private *private =288 container_of(vdev, struct vfio_ccw_private, vdev);289 unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos);290 291 if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions)292 return -EINVAL;293 294 switch (index) {295 case VFIO_CCW_CONFIG_REGION_INDEX:296 return vfio_ccw_mdev_write_io_region(private, buf, count, ppos);297 default:298 index -= VFIO_CCW_NUM_REGIONS;299 return private->region[index].ops->write(private, buf, count,300 ppos);301 }302 303 return -EINVAL;304}305 306static int vfio_ccw_mdev_get_device_info(struct vfio_ccw_private *private,307 struct vfio_device_info *info)308{309 info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET;310 info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions;311 info->num_irqs = VFIO_CCW_NUM_IRQS;312 313 return 0;314}315 316static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private,317 struct vfio_region_info *info,318 unsigned long arg)319{320 int i;321 322 switch (info->index) {323 case VFIO_CCW_CONFIG_REGION_INDEX:324 info->offset = 0;325 info->size = sizeof(struct ccw_io_region);326 info->flags = VFIO_REGION_INFO_FLAG_READ327 | VFIO_REGION_INFO_FLAG_WRITE;328 return 0;329 default: /* all other regions are handled via capability chain */330 {331 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };332 struct vfio_region_info_cap_type cap_type = {333 .header.id = VFIO_REGION_INFO_CAP_TYPE,334 .header.version = 1 };335 int ret;336 337 if (info->index >=338 VFIO_CCW_NUM_REGIONS + private->num_regions)339 return -EINVAL;340 341 info->index = array_index_nospec(info->index,342 VFIO_CCW_NUM_REGIONS +343 private->num_regions);344 345 i = info->index - VFIO_CCW_NUM_REGIONS;346 347 info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index);348 info->size = private->region[i].size;349 info->flags = private->region[i].flags;350 351 cap_type.type = private->region[i].type;352 cap_type.subtype = private->region[i].subtype;353 354 ret = vfio_info_add_capability(&caps, &cap_type.header,355 sizeof(cap_type));356 if (ret)357 return ret;358 359 info->flags |= VFIO_REGION_INFO_FLAG_CAPS;360 if (info->argsz < sizeof(*info) + caps.size) {361 info->argsz = sizeof(*info) + caps.size;362 info->cap_offset = 0;363 } else {364 vfio_info_cap_shift(&caps, sizeof(*info));365 if (copy_to_user((void __user *)arg + sizeof(*info),366 caps.buf, caps.size)) {367 kfree(caps.buf);368 return -EFAULT;369 }370 info->cap_offset = sizeof(*info);371 }372 373 kfree(caps.buf);374 375 }376 }377 return 0;378}379 380static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)381{382 switch (info->index) {383 case VFIO_CCW_IO_IRQ_INDEX:384 case VFIO_CCW_CRW_IRQ_INDEX:385 case VFIO_CCW_REQ_IRQ_INDEX:386 info->count = 1;387 info->flags = VFIO_IRQ_INFO_EVENTFD;388 break;389 default:390 return -EINVAL;391 }392 393 return 0;394}395 396static int vfio_ccw_mdev_set_irqs(struct vfio_ccw_private *private,397 uint32_t flags,398 uint32_t index,399 void __user *data)400{401 struct eventfd_ctx **ctx;402 403 if (!(flags & VFIO_IRQ_SET_ACTION_TRIGGER))404 return -EINVAL;405 406 switch (index) {407 case VFIO_CCW_IO_IRQ_INDEX:408 ctx = &private->io_trigger;409 break;410 case VFIO_CCW_CRW_IRQ_INDEX:411 ctx = &private->crw_trigger;412 break;413 case VFIO_CCW_REQ_IRQ_INDEX:414 ctx = &private->req_trigger;415 break;416 default:417 return -EINVAL;418 }419 420 switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {421 case VFIO_IRQ_SET_DATA_NONE:422 {423 if (*ctx)424 eventfd_signal(*ctx);425 return 0;426 }427 case VFIO_IRQ_SET_DATA_BOOL:428 {429 uint8_t trigger;430 431 if (get_user(trigger, (uint8_t __user *)data))432 return -EFAULT;433 434 if (trigger && *ctx)435 eventfd_signal(*ctx);436 return 0;437 }438 case VFIO_IRQ_SET_DATA_EVENTFD:439 {440 int32_t fd;441 442 if (get_user(fd, (int32_t __user *)data))443 return -EFAULT;444 445 if (fd == -1) {446 if (*ctx)447 eventfd_ctx_put(*ctx);448 *ctx = NULL;449 } else if (fd >= 0) {450 struct eventfd_ctx *efdctx;451 452 efdctx = eventfd_ctx_fdget(fd);453 if (IS_ERR(efdctx))454 return PTR_ERR(efdctx);455 456 if (*ctx)457 eventfd_ctx_put(*ctx);458 459 *ctx = efdctx;460 } else461 return -EINVAL;462 463 return 0;464 }465 default:466 return -EINVAL;467 }468}469 470int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,471 unsigned int subtype,472 const struct vfio_ccw_regops *ops,473 size_t size, u32 flags, void *data)474{475 struct vfio_ccw_region *region;476 477 region = krealloc(private->region,478 (private->num_regions + 1) * sizeof(*region),479 GFP_KERNEL);480 if (!region)481 return -ENOMEM;482 483 private->region = region;484 private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW;485 private->region[private->num_regions].subtype = subtype;486 private->region[private->num_regions].ops = ops;487 private->region[private->num_regions].size = size;488 private->region[private->num_regions].flags = flags;489 private->region[private->num_regions].data = data;490 491 private->num_regions++;492 493 return 0;494}495 496void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)497{498 int i;499 500 for (i = 0; i < private->num_regions; i++)501 private->region[i].ops->release(private, &private->region[i]);502 private->num_regions = 0;503 kfree(private->region);504 private->region = NULL;505}506 507static ssize_t vfio_ccw_mdev_ioctl(struct vfio_device *vdev,508 unsigned int cmd,509 unsigned long arg)510{511 struct vfio_ccw_private *private =512 container_of(vdev, struct vfio_ccw_private, vdev);513 int ret = 0;514 unsigned long minsz;515 516 switch (cmd) {517 case VFIO_DEVICE_GET_INFO:518 {519 struct vfio_device_info info;520 521 minsz = offsetofend(struct vfio_device_info, num_irqs);522 523 if (copy_from_user(&info, (void __user *)arg, minsz))524 return -EFAULT;525 526 if (info.argsz < minsz)527 return -EINVAL;528 529 ret = vfio_ccw_mdev_get_device_info(private, &info);530 if (ret)531 return ret;532 533 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;534 }535 case VFIO_DEVICE_GET_REGION_INFO:536 {537 struct vfio_region_info info;538 539 minsz = offsetofend(struct vfio_region_info, offset);540 541 if (copy_from_user(&info, (void __user *)arg, minsz))542 return -EFAULT;543 544 if (info.argsz < minsz)545 return -EINVAL;546 547 ret = vfio_ccw_mdev_get_region_info(private, &info, arg);548 if (ret)549 return ret;550 551 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;552 }553 case VFIO_DEVICE_GET_IRQ_INFO:554 {555 struct vfio_irq_info info;556 557 minsz = offsetofend(struct vfio_irq_info, count);558 559 if (copy_from_user(&info, (void __user *)arg, minsz))560 return -EFAULT;561 562 if (info.argsz < minsz || info.index >= VFIO_CCW_NUM_IRQS)563 return -EINVAL;564 565 ret = vfio_ccw_mdev_get_irq_info(&info);566 if (ret)567 return ret;568 569 if (info.count == -1)570 return -EINVAL;571 572 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;573 }574 case VFIO_DEVICE_SET_IRQS:575 {576 struct vfio_irq_set hdr;577 size_t data_size;578 void __user *data;579 580 minsz = offsetofend(struct vfio_irq_set, count);581 582 if (copy_from_user(&hdr, (void __user *)arg, minsz))583 return -EFAULT;584 585 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1,586 VFIO_CCW_NUM_IRQS,587 &data_size);588 if (ret)589 return ret;590 591 data = (void __user *)(arg + minsz);592 return vfio_ccw_mdev_set_irqs(private, hdr.flags, hdr.index,593 data);594 }595 case VFIO_DEVICE_RESET:596 return vfio_ccw_mdev_reset(private);597 default:598 return -ENOTTY;599 }600}601 602/* Request removal of the device*/603static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count)604{605 struct vfio_ccw_private *private =606 container_of(vdev, struct vfio_ccw_private, vdev);607 struct device *dev = vdev->dev;608 609 if (private->req_trigger) {610 if (!(count % 10))611 dev_notice_ratelimited(dev,612 "Relaying device request to user (#%u)\n",613 count);614 615 eventfd_signal(private->req_trigger);616 } else if (count == 0) {617 dev_notice(dev,618 "No device request channel registered, blocked until released by user\n");619 }620}621 622static const struct vfio_device_ops vfio_ccw_dev_ops = {623 .init = vfio_ccw_mdev_init_dev,624 .release = vfio_ccw_mdev_release_dev,625 .open_device = vfio_ccw_mdev_open_device,626 .close_device = vfio_ccw_mdev_close_device,627 .read = vfio_ccw_mdev_read,628 .write = vfio_ccw_mdev_write,629 .ioctl = vfio_ccw_mdev_ioctl,630 .request = vfio_ccw_mdev_request,631 .dma_unmap = vfio_ccw_dma_unmap,632 .bind_iommufd = vfio_iommufd_emulated_bind,633 .unbind_iommufd = vfio_iommufd_emulated_unbind,634 .attach_ioas = vfio_iommufd_emulated_attach_ioas,635 .detach_ioas = vfio_iommufd_emulated_detach_ioas,636};637 638struct mdev_driver vfio_ccw_mdev_driver = {639 .device_api = VFIO_DEVICE_API_CCW_STRING,640 .max_instances = 1,641 .driver = {642 .name = "vfio_ccw_mdev",643 .owner = THIS_MODULE,644 .mod_name = KBUILD_MODNAME,645 },646 .probe = vfio_ccw_mdev_probe,647 .remove = vfio_ccw_mdev_remove,648};649