194 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2023 Advanced Micro Devices, Inc. */3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt5 6#include <linux/module.h>7#include <linux/pci.h>8#include <linux/types.h>9#include <linux/vfio.h>10 11#include <linux/pds/pds_common.h>12#include <linux/pds/pds_core_if.h>13#include <linux/pds/pds_adminq.h>14 15#include "vfio_dev.h"16#include "pci_drv.h"17#include "cmds.h"18 19#define PDS_VFIO_DRV_DESCRIPTION "AMD/Pensando VFIO Device Driver"20#define PCI_VENDOR_ID_PENSANDO 0x1dd821 22static void pds_vfio_recovery(struct pds_vfio_pci_device *pds_vfio)23{24 /*25 * Documentation states that the kernel migration driver must not26 * generate asynchronous device state transitions outside of27 * manipulation by the user or the VFIO_DEVICE_RESET ioctl.28 *29 * Since recovery is an asynchronous event received from the device,30 * initiate a reset in the following situations:31 * 1. Migration is in progress, which will cause the next step of32 * the migration to fail.33 * 2. If the device is in a state that will be set to34 * VFIO_DEVICE_STATE_RUNNING on the next action (i.e. VM is35 * shutdown and device is in VFIO_DEVICE_STATE_STOP).36 */37 mutex_lock(&pds_vfio->state_mutex);38 if ((pds_vfio->state != VFIO_DEVICE_STATE_RUNNING &&39 pds_vfio->state != VFIO_DEVICE_STATE_ERROR) ||40 (pds_vfio->state == VFIO_DEVICE_STATE_RUNNING &&41 pds_vfio_dirty_is_enabled(pds_vfio)))42 pds_vfio_reset(pds_vfio, VFIO_DEVICE_STATE_ERROR);43 mutex_unlock(&pds_vfio->state_mutex);44}45 46static int pds_vfio_pci_notify_handler(struct notifier_block *nb,47 unsigned long ecode, void *data)48{49 struct pds_vfio_pci_device *pds_vfio =50 container_of(nb, struct pds_vfio_pci_device, nb);51 struct device *dev = pds_vfio_to_dev(pds_vfio);52 union pds_core_notifyq_comp *event = data;53 54 dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);55 56 /*57 * We don't need to do anything for RESET state==0 as there is no notify58 * or feedback mechanism available, and it is possible that we won't59 * even see a state==0 event since the pds_core recovery is pending.60 *61 * Any requests from VFIO while state==0 will fail, which will return62 * error and may cause migration to fail.63 */64 if (ecode == PDS_EVENT_RESET) {65 dev_info(dev, "%s: PDS_EVENT_RESET event received, state==%d\n",66 __func__, event->reset.state);67 /*68 * pds_core device finished recovery and sent us the69 * notification (state == 1) to allow us to recover70 */71 if (event->reset.state == 1)72 pds_vfio_recovery(pds_vfio);73 }74 75 return 0;76}77 78static int79pds_vfio_pci_register_event_handler(struct pds_vfio_pci_device *pds_vfio)80{81 struct device *dev = pds_vfio_to_dev(pds_vfio);82 struct notifier_block *nb = &pds_vfio->nb;83 int err;84 85 if (!nb->notifier_call) {86 nb->notifier_call = pds_vfio_pci_notify_handler;87 err = pdsc_register_notify(nb);88 if (err) {89 nb->notifier_call = NULL;90 dev_err(dev,91 "failed to register pds event handler: %pe\n",92 ERR_PTR(err));93 return -EINVAL;94 }95 dev_dbg(dev, "pds event handler registered\n");96 }97 98 return 0;99}100 101static void102pds_vfio_pci_unregister_event_handler(struct pds_vfio_pci_device *pds_vfio)103{104 if (pds_vfio->nb.notifier_call) {105 pdsc_unregister_notify(&pds_vfio->nb);106 pds_vfio->nb.notifier_call = NULL;107 }108}109 110static int pds_vfio_pci_probe(struct pci_dev *pdev,111 const struct pci_device_id *id)112{113 struct pds_vfio_pci_device *pds_vfio;114 int err;115 116 pds_vfio = vfio_alloc_device(pds_vfio_pci_device, vfio_coredev.vdev,117 &pdev->dev, pds_vfio_ops_info());118 if (IS_ERR(pds_vfio))119 return PTR_ERR(pds_vfio);120 121 dev_set_drvdata(&pdev->dev, &pds_vfio->vfio_coredev);122 123 err = vfio_pci_core_register_device(&pds_vfio->vfio_coredev);124 if (err)125 goto out_put_vdev;126 127 err = pds_vfio_register_client_cmd(pds_vfio);128 if (err) {129 dev_err(&pdev->dev, "failed to register as client: %pe\n",130 ERR_PTR(err));131 goto out_unregister_coredev;132 }133 134 err = pds_vfio_pci_register_event_handler(pds_vfio);135 if (err)136 goto out_unregister_client;137 138 return 0;139 140out_unregister_client:141 pds_vfio_unregister_client_cmd(pds_vfio);142out_unregister_coredev:143 vfio_pci_core_unregister_device(&pds_vfio->vfio_coredev);144out_put_vdev:145 vfio_put_device(&pds_vfio->vfio_coredev.vdev);146 return err;147}148 149static void pds_vfio_pci_remove(struct pci_dev *pdev)150{151 struct pds_vfio_pci_device *pds_vfio = pds_vfio_pci_drvdata(pdev);152 153 pds_vfio_pci_unregister_event_handler(pds_vfio);154 pds_vfio_unregister_client_cmd(pds_vfio);155 vfio_pci_core_unregister_device(&pds_vfio->vfio_coredev);156 vfio_put_device(&pds_vfio->vfio_coredev.vdev);157}158 159static const struct pci_device_id pds_vfio_pci_table[] = {160 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_PENSANDO, 0x1003) }, /* Ethernet VF */161 { 0, }162};163MODULE_DEVICE_TABLE(pci, pds_vfio_pci_table);164 165static void pds_vfio_pci_aer_reset_done(struct pci_dev *pdev)166{167 struct pds_vfio_pci_device *pds_vfio = pds_vfio_pci_drvdata(pdev);168 169 mutex_lock(&pds_vfio->state_mutex);170 pds_vfio_reset(pds_vfio, VFIO_DEVICE_STATE_RUNNING);171 mutex_unlock(&pds_vfio->state_mutex);172}173 174static const struct pci_error_handlers pds_vfio_pci_err_handlers = {175 .reset_done = pds_vfio_pci_aer_reset_done,176 .error_detected = vfio_pci_core_aer_err_detected,177};178 179static struct pci_driver pds_vfio_pci_driver = {180 .name = KBUILD_MODNAME,181 .id_table = pds_vfio_pci_table,182 .probe = pds_vfio_pci_probe,183 .remove = pds_vfio_pci_remove,184 .err_handler = &pds_vfio_pci_err_handlers,185 .driver_managed_dma = true,186};187 188module_pci_driver(pds_vfio_pci_driver);189 190MODULE_IMPORT_NS(IOMMUFD);191MODULE_DESCRIPTION(PDS_VFIO_DRV_DESCRIPTION);192MODULE_AUTHOR("Brett Creeley <brett.creeley@amd.com>");193MODULE_LICENSE("GPL");194