701 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2024 Intel Corporation */3 4#include <linux/anon_inodes.h>5#include <linux/container_of.h>6#include <linux/device.h>7#include <linux/file.h>8#include <linux/init.h>9#include <linux/kernel.h>10#include <linux/module.h>11#include <linux/mutex.h>12#include <linux/pci.h>13#include <linux/sizes.h>14#include <linux/types.h>15#include <linux/uaccess.h>16#include <linux/vfio_pci_core.h>17#include <linux/qat/qat_mig_dev.h>18 19/*20 * The migration data of each Intel QAT VF device is encapsulated into a21 * 4096 bytes block. The data consists of two parts.22 * The first is a pre-configured set of attributes of the VF being migrated,23 * which are only set when it is created. This can be migrated during pre-copy24 * stage and used for a device compatibility check.25 * The second is the VF state. This includes the required MMIO regions and26 * the shadow states maintained by the QAT PF driver. This part can only be27 * saved when the VF is fully quiesced and be migrated during stop-copy stage.28 * Both these 2 parts of data are saved in hierarchical structures including29 * a preamble section and several raw state sections.30 * When the pre-configured part of the migration data is fully retrieved from31 * user space, the preamble section are used to validate the correctness of32 * the data blocks and check the version compatibility. The raw state sections33 * are then used to do a device compatibility check.34 * When the device transits from RESUMING state, the VF states are extracted35 * from the raw state sections of the VF state part of the migration data and36 * then loaded into the device.37 */38 39struct qat_vf_migration_file {40 struct file *filp;41 /* protects migration region context */42 struct mutex lock;43 bool disabled;44 struct qat_vf_core_device *qat_vdev;45 ssize_t filled_size;46};47 48struct qat_vf_core_device {49 struct vfio_pci_core_device core_device;50 struct qat_mig_dev *mdev;51 /* protects migration state */52 struct mutex state_mutex;53 enum vfio_device_mig_state mig_state;54 struct qat_vf_migration_file *resuming_migf;55 struct qat_vf_migration_file *saving_migf;56};57 58static int qat_vf_pci_open_device(struct vfio_device *core_vdev)59{60 struct qat_vf_core_device *qat_vdev =61 container_of(core_vdev, struct qat_vf_core_device,62 core_device.vdev);63 struct vfio_pci_core_device *vdev = &qat_vdev->core_device;64 int ret;65 66 ret = vfio_pci_core_enable(vdev);67 if (ret)68 return ret;69 70 ret = qat_vfmig_open(qat_vdev->mdev);71 if (ret) {72 vfio_pci_core_disable(vdev);73 return ret;74 }75 qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;76 77 vfio_pci_core_finish_enable(vdev);78 79 return 0;80}81 82static void qat_vf_disable_fd(struct qat_vf_migration_file *migf)83{84 mutex_lock(&migf->lock);85 migf->disabled = true;86 migf->filp->f_pos = 0;87 migf->filled_size = 0;88 mutex_unlock(&migf->lock);89}90 91static void qat_vf_disable_fds(struct qat_vf_core_device *qat_vdev)92{93 if (qat_vdev->resuming_migf) {94 qat_vf_disable_fd(qat_vdev->resuming_migf);95 fput(qat_vdev->resuming_migf->filp);96 qat_vdev->resuming_migf = NULL;97 }98 99 if (qat_vdev->saving_migf) {100 qat_vf_disable_fd(qat_vdev->saving_migf);101 fput(qat_vdev->saving_migf->filp);102 qat_vdev->saving_migf = NULL;103 }104}105 106static void qat_vf_pci_close_device(struct vfio_device *core_vdev)107{108 struct qat_vf_core_device *qat_vdev = container_of(core_vdev,109 struct qat_vf_core_device, core_device.vdev);110 111 qat_vfmig_close(qat_vdev->mdev);112 qat_vf_disable_fds(qat_vdev);113 vfio_pci_core_close_device(core_vdev);114}115 116static long qat_vf_precopy_ioctl(struct file *filp, unsigned int cmd,117 unsigned long arg)118{119 struct qat_vf_migration_file *migf = filp->private_data;120 struct qat_vf_core_device *qat_vdev = migf->qat_vdev;121 struct qat_mig_dev *mig_dev = qat_vdev->mdev;122 struct vfio_precopy_info info;123 loff_t *pos = &filp->f_pos;124 unsigned long minsz;125 int ret = 0;126 127 if (cmd != VFIO_MIG_GET_PRECOPY_INFO)128 return -ENOTTY;129 130 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);131 132 if (copy_from_user(&info, (void __user *)arg, minsz))133 return -EFAULT;134 if (info.argsz < minsz)135 return -EINVAL;136 137 mutex_lock(&qat_vdev->state_mutex);138 if (qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY &&139 qat_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY_P2P) {140 mutex_unlock(&qat_vdev->state_mutex);141 return -EINVAL;142 }143 144 mutex_lock(&migf->lock);145 if (migf->disabled) {146 ret = -ENODEV;147 goto out;148 }149 150 if (*pos > mig_dev->setup_size) {151 ret = -EINVAL;152 goto out;153 }154 155 info.dirty_bytes = 0;156 info.initial_bytes = mig_dev->setup_size - *pos;157 158out:159 mutex_unlock(&migf->lock);160 mutex_unlock(&qat_vdev->state_mutex);161 if (ret)162 return ret;163 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;164}165 166static ssize_t qat_vf_save_read(struct file *filp, char __user *buf,167 size_t len, loff_t *pos)168{169 struct qat_vf_migration_file *migf = filp->private_data;170 struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev;171 ssize_t done = 0;172 loff_t *offs;173 int ret;174 175 if (pos)176 return -ESPIPE;177 offs = &filp->f_pos;178 179 mutex_lock(&migf->lock);180 if (*offs > migf->filled_size || *offs < 0) {181 done = -EINVAL;182 goto out_unlock;183 }184 185 if (migf->disabled) {186 done = -ENODEV;187 goto out_unlock;188 }189 190 len = min_t(size_t, migf->filled_size - *offs, len);191 if (len) {192 ret = copy_to_user(buf, mig_dev->state + *offs, len);193 if (ret) {194 done = -EFAULT;195 goto out_unlock;196 }197 *offs += len;198 done = len;199 }200 201out_unlock:202 mutex_unlock(&migf->lock);203 return done;204}205 206static int qat_vf_release_file(struct inode *inode, struct file *filp)207{208 struct qat_vf_migration_file *migf = filp->private_data;209 210 qat_vf_disable_fd(migf);211 mutex_destroy(&migf->lock);212 kfree(migf);213 214 return 0;215}216 217static const struct file_operations qat_vf_save_fops = {218 .owner = THIS_MODULE,219 .read = qat_vf_save_read,220 .unlocked_ioctl = qat_vf_precopy_ioctl,221 .compat_ioctl = compat_ptr_ioctl,222 .release = qat_vf_release_file,223};224 225static int qat_vf_save_state(struct qat_vf_core_device *qat_vdev,226 struct qat_vf_migration_file *migf)227{228 int ret;229 230 ret = qat_vfmig_save_state(qat_vdev->mdev);231 if (ret)232 return ret;233 migf->filled_size = qat_vdev->mdev->state_size;234 235 return 0;236}237 238static int qat_vf_save_setup(struct qat_vf_core_device *qat_vdev,239 struct qat_vf_migration_file *migf)240{241 int ret;242 243 ret = qat_vfmig_save_setup(qat_vdev->mdev);244 if (ret)245 return ret;246 migf->filled_size = qat_vdev->mdev->setup_size;247 248 return 0;249}250 251/*252 * Allocate a file handler for user space and then save the migration data for253 * the device being migrated. If this is called in the pre-copy stage, save the254 * pre-configured device data. Otherwise, if this is called in the stop-copy255 * stage, save the device state. In both cases, update the data size which can256 * then be read from user space.257 */258static struct qat_vf_migration_file *259qat_vf_save_device_data(struct qat_vf_core_device *qat_vdev, bool pre_copy)260{261 struct qat_vf_migration_file *migf;262 int ret;263 264 migf = kzalloc(sizeof(*migf), GFP_KERNEL);265 if (!migf)266 return ERR_PTR(-ENOMEM);267 268 migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_save_fops,269 migf, O_RDONLY);270 ret = PTR_ERR_OR_ZERO(migf->filp);271 if (ret) {272 kfree(migf);273 return ERR_PTR(ret);274 }275 276 stream_open(migf->filp->f_inode, migf->filp);277 mutex_init(&migf->lock);278 279 if (pre_copy)280 ret = qat_vf_save_setup(qat_vdev, migf);281 else282 ret = qat_vf_save_state(qat_vdev, migf);283 if (ret) {284 fput(migf->filp);285 return ERR_PTR(ret);286 }287 288 migf->qat_vdev = qat_vdev;289 290 return migf;291}292 293static ssize_t qat_vf_resume_write(struct file *filp, const char __user *buf,294 size_t len, loff_t *pos)295{296 struct qat_vf_migration_file *migf = filp->private_data;297 struct qat_mig_dev *mig_dev = migf->qat_vdev->mdev;298 loff_t end, *offs;299 ssize_t done = 0;300 int ret;301 302 if (pos)303 return -ESPIPE;304 offs = &filp->f_pos;305 306 if (*offs < 0 ||307 check_add_overflow((loff_t)len, *offs, &end))308 return -EOVERFLOW;309 310 if (end > mig_dev->state_size)311 return -ENOMEM;312 313 mutex_lock(&migf->lock);314 if (migf->disabled) {315 done = -ENODEV;316 goto out_unlock;317 }318 319 ret = copy_from_user(mig_dev->state + *offs, buf, len);320 if (ret) {321 done = -EFAULT;322 goto out_unlock;323 }324 *offs += len;325 migf->filled_size += len;326 327 /*328 * Load the pre-configured device data first to check if the target329 * device is compatible with the source device.330 */331 ret = qat_vfmig_load_setup(mig_dev, migf->filled_size);332 if (ret && ret != -EAGAIN) {333 done = ret;334 goto out_unlock;335 }336 done = len;337 338out_unlock:339 mutex_unlock(&migf->lock);340 return done;341}342 343static const struct file_operations qat_vf_resume_fops = {344 .owner = THIS_MODULE,345 .write = qat_vf_resume_write,346 .release = qat_vf_release_file,347};348 349static struct qat_vf_migration_file *350qat_vf_resume_device_data(struct qat_vf_core_device *qat_vdev)351{352 struct qat_vf_migration_file *migf;353 int ret;354 355 migf = kzalloc(sizeof(*migf), GFP_KERNEL);356 if (!migf)357 return ERR_PTR(-ENOMEM);358 359 migf->filp = anon_inode_getfile("qat_vf_mig", &qat_vf_resume_fops, migf, O_WRONLY);360 ret = PTR_ERR_OR_ZERO(migf->filp);361 if (ret) {362 kfree(migf);363 return ERR_PTR(ret);364 }365 366 migf->qat_vdev = qat_vdev;367 migf->filled_size = 0;368 stream_open(migf->filp->f_inode, migf->filp);369 mutex_init(&migf->lock);370 371 return migf;372}373 374static int qat_vf_load_device_data(struct qat_vf_core_device *qat_vdev)375{376 return qat_vfmig_load_state(qat_vdev->mdev);377}378 379static struct file *qat_vf_pci_step_device_state(struct qat_vf_core_device *qat_vdev, u32 new)380{381 u32 cur = qat_vdev->mig_state;382 int ret;383 384 /*385 * As the device is not capable of just stopping P2P DMAs, suspend the386 * device completely once any of the P2P states are reached.387 * When it is suspended, all its MMIO registers can still be operated388 * correctly, jobs submitted through ring are queued while no jobs are389 * processed by the device. The MMIO states can be safely migrated to390 * the target VF during stop-copy stage and restored correctly in the391 * target VF. All queued jobs can be resumed then.392 */393 if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) ||394 (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) {395 ret = qat_vfmig_suspend(qat_vdev->mdev);396 if (ret)397 return ERR_PTR(ret);398 return NULL;399 }400 401 if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) ||402 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_PRE_COPY)) {403 qat_vfmig_resume(qat_vdev->mdev);404 return NULL;405 }406 407 if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) ||408 (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P))409 return NULL;410 411 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {412 struct qat_vf_migration_file *migf;413 414 migf = qat_vf_save_device_data(qat_vdev, false);415 if (IS_ERR(migf))416 return ERR_CAST(migf);417 get_file(migf->filp);418 qat_vdev->saving_migf = migf;419 return migf->filp;420 }421 422 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {423 struct qat_vf_migration_file *migf;424 425 migf = qat_vf_resume_device_data(qat_vdev);426 if (IS_ERR(migf))427 return ERR_CAST(migf);428 get_file(migf->filp);429 qat_vdev->resuming_migf = migf;430 return migf->filp;431 }432 433 if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) ||434 (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) ||435 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_RUNNING_P2P)) {436 qat_vf_disable_fds(qat_vdev);437 return NULL;438 }439 440 if ((cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) ||441 (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_PRE_COPY_P2P)) {442 struct qat_vf_migration_file *migf;443 444 migf = qat_vf_save_device_data(qat_vdev, true);445 if (IS_ERR(migf))446 return ERR_CAST(migf);447 get_file(migf->filp);448 qat_vdev->saving_migf = migf;449 return migf->filp;450 }451 452 if (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && new == VFIO_DEVICE_STATE_STOP_COPY) {453 struct qat_vf_migration_file *migf = qat_vdev->saving_migf;454 455 if (!migf)456 return ERR_PTR(-EINVAL);457 ret = qat_vf_save_state(qat_vdev, migf);458 if (ret)459 return ERR_PTR(ret);460 return NULL;461 }462 463 if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {464 ret = qat_vf_load_device_data(qat_vdev);465 if (ret)466 return ERR_PTR(ret);467 468 qat_vf_disable_fds(qat_vdev);469 return NULL;470 }471 472 /* vfio_mig_get_next_state() does not use arcs other than the above */473 WARN_ON(true);474 return ERR_PTR(-EINVAL);475}476 477static void qat_vf_reset_done(struct qat_vf_core_device *qat_vdev)478{479 qat_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;480 qat_vfmig_reset(qat_vdev->mdev);481 qat_vf_disable_fds(qat_vdev);482}483 484static struct file *qat_vf_pci_set_device_state(struct vfio_device *vdev,485 enum vfio_device_mig_state new_state)486{487 struct qat_vf_core_device *qat_vdev = container_of(vdev,488 struct qat_vf_core_device, core_device.vdev);489 enum vfio_device_mig_state next_state;490 struct file *res = NULL;491 int ret;492 493 mutex_lock(&qat_vdev->state_mutex);494 while (new_state != qat_vdev->mig_state) {495 ret = vfio_mig_get_next_state(vdev, qat_vdev->mig_state,496 new_state, &next_state);497 if (ret) {498 res = ERR_PTR(ret);499 break;500 }501 res = qat_vf_pci_step_device_state(qat_vdev, next_state);502 if (IS_ERR(res))503 break;504 qat_vdev->mig_state = next_state;505 if (WARN_ON(res && new_state != qat_vdev->mig_state)) {506 fput(res);507 res = ERR_PTR(-EINVAL);508 break;509 }510 }511 mutex_unlock(&qat_vdev->state_mutex);512 513 return res;514}515 516static int qat_vf_pci_get_device_state(struct vfio_device *vdev,517 enum vfio_device_mig_state *curr_state)518{519 struct qat_vf_core_device *qat_vdev = container_of(vdev,520 struct qat_vf_core_device, core_device.vdev);521 522 mutex_lock(&qat_vdev->state_mutex);523 *curr_state = qat_vdev->mig_state;524 mutex_unlock(&qat_vdev->state_mutex);525 526 return 0;527}528 529static int qat_vf_pci_get_data_size(struct vfio_device *vdev,530 unsigned long *stop_copy_length)531{532 struct qat_vf_core_device *qat_vdev = container_of(vdev,533 struct qat_vf_core_device, core_device.vdev);534 535 mutex_lock(&qat_vdev->state_mutex);536 *stop_copy_length = qat_vdev->mdev->state_size;537 mutex_unlock(&qat_vdev->state_mutex);538 539 return 0;540}541 542static const struct vfio_migration_ops qat_vf_pci_mig_ops = {543 .migration_set_state = qat_vf_pci_set_device_state,544 .migration_get_state = qat_vf_pci_get_device_state,545 .migration_get_data_size = qat_vf_pci_get_data_size,546};547 548static void qat_vf_pci_release_dev(struct vfio_device *core_vdev)549{550 struct qat_vf_core_device *qat_vdev = container_of(core_vdev,551 struct qat_vf_core_device, core_device.vdev);552 553 qat_vfmig_cleanup(qat_vdev->mdev);554 qat_vfmig_destroy(qat_vdev->mdev);555 mutex_destroy(&qat_vdev->state_mutex);556 vfio_pci_core_release_dev(core_vdev);557}558 559static int qat_vf_pci_init_dev(struct vfio_device *core_vdev)560{561 struct qat_vf_core_device *qat_vdev = container_of(core_vdev,562 struct qat_vf_core_device, core_device.vdev);563 struct qat_mig_dev *mdev;564 struct pci_dev *parent;565 int ret, vf_id;566 567 core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P |568 VFIO_MIGRATION_PRE_COPY;569 core_vdev->mig_ops = &qat_vf_pci_mig_ops;570 571 ret = vfio_pci_core_init_dev(core_vdev);572 if (ret)573 return ret;574 575 mutex_init(&qat_vdev->state_mutex);576 577 parent = pci_physfn(qat_vdev->core_device.pdev);578 vf_id = pci_iov_vf_id(qat_vdev->core_device.pdev);579 if (vf_id < 0) {580 ret = -ENODEV;581 goto err_rel;582 }583 584 mdev = qat_vfmig_create(parent, vf_id);585 if (IS_ERR(mdev)) {586 ret = PTR_ERR(mdev);587 goto err_rel;588 }589 590 ret = qat_vfmig_init(mdev);591 if (ret)592 goto err_destroy;593 594 qat_vdev->mdev = mdev;595 596 return 0;597 598err_destroy:599 qat_vfmig_destroy(mdev);600err_rel:601 vfio_pci_core_release_dev(core_vdev);602 return ret;603}604 605static const struct vfio_device_ops qat_vf_pci_ops = {606 .name = "qat-vf-vfio-pci",607 .init = qat_vf_pci_init_dev,608 .release = qat_vf_pci_release_dev,609 .open_device = qat_vf_pci_open_device,610 .close_device = qat_vf_pci_close_device,611 .ioctl = vfio_pci_core_ioctl,612 .read = vfio_pci_core_read,613 .write = vfio_pci_core_write,614 .mmap = vfio_pci_core_mmap,615 .request = vfio_pci_core_request,616 .match = vfio_pci_core_match,617 .bind_iommufd = vfio_iommufd_physical_bind,618 .unbind_iommufd = vfio_iommufd_physical_unbind,619 .attach_ioas = vfio_iommufd_physical_attach_ioas,620 .detach_ioas = vfio_iommufd_physical_detach_ioas,621};622 623static struct qat_vf_core_device *qat_vf_drvdata(struct pci_dev *pdev)624{625 struct vfio_pci_core_device *core_device = pci_get_drvdata(pdev);626 627 return container_of(core_device, struct qat_vf_core_device, core_device);628}629 630static void qat_vf_pci_aer_reset_done(struct pci_dev *pdev)631{632 struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev);633 634 if (!qat_vdev->mdev)635 return;636 637 mutex_lock(&qat_vdev->state_mutex);638 qat_vf_reset_done(qat_vdev);639 mutex_unlock(&qat_vdev->state_mutex);640}641 642static int643qat_vf_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)644{645 struct device *dev = &pdev->dev;646 struct qat_vf_core_device *qat_vdev;647 int ret;648 649 qat_vdev = vfio_alloc_device(qat_vf_core_device, core_device.vdev, dev, &qat_vf_pci_ops);650 if (IS_ERR(qat_vdev))651 return PTR_ERR(qat_vdev);652 653 pci_set_drvdata(pdev, &qat_vdev->core_device);654 ret = vfio_pci_core_register_device(&qat_vdev->core_device);655 if (ret)656 goto out_put_device;657 658 return 0;659 660out_put_device:661 vfio_put_device(&qat_vdev->core_device.vdev);662 return ret;663}664 665static void qat_vf_vfio_pci_remove(struct pci_dev *pdev)666{667 struct qat_vf_core_device *qat_vdev = qat_vf_drvdata(pdev);668 669 vfio_pci_core_unregister_device(&qat_vdev->core_device);670 vfio_put_device(&qat_vdev->core_device.vdev);671}672 673static const struct pci_device_id qat_vf_vfio_pci_table[] = {674 /* Intel QAT GEN4 4xxx VF device */675 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4941) },676 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4943) },677 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, 0x4945) },678 {}679};680MODULE_DEVICE_TABLE(pci, qat_vf_vfio_pci_table);681 682static const struct pci_error_handlers qat_vf_err_handlers = {683 .reset_done = qat_vf_pci_aer_reset_done,684 .error_detected = vfio_pci_core_aer_err_detected,685};686 687static struct pci_driver qat_vf_vfio_pci_driver = {688 .name = "qat_vfio_pci",689 .id_table = qat_vf_vfio_pci_table,690 .probe = qat_vf_vfio_pci_probe,691 .remove = qat_vf_vfio_pci_remove,692 .err_handler = &qat_vf_err_handlers,693 .driver_managed_dma = true,694};695module_pci_driver(qat_vf_vfio_pci_driver);696 697MODULE_LICENSE("GPL");698MODULE_AUTHOR("Xin Zeng <xin.zeng@intel.com>");699MODULE_DESCRIPTION("QAT VFIO PCI - VFIO PCI driver with live migration support for Intel(R) QAT GEN4 device family");700MODULE_IMPORT_NS(CRYPTO_QAT);701