595 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2023 Advanced Micro Devices, Inc */3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt5 6#include <linux/pci.h>7 8#include <linux/pds/pds_common.h>9 10#include "core.h"11 12MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);13MODULE_AUTHOR("Advanced Micro Devices, Inc");14MODULE_LICENSE("GPL");15 16/* Supported devices */17static const struct pci_device_id pdsc_id_table[] = {18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },20 { 0, } /* end of table */21};22MODULE_DEVICE_TABLE(pci, pdsc_id_table);23 24static void pdsc_wdtimer_cb(struct timer_list *t)25{26 struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);27 28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);29 mod_timer(&pdsc->wdtimer,30 round_jiffies(jiffies + pdsc->wdtimer_period));31 32 queue_work(pdsc->wq, &pdsc->health_work);33}34 35static void pdsc_unmap_bars(struct pdsc *pdsc)36{37 struct pdsc_dev_bar *bars = pdsc->bars;38 unsigned int i;39 40 pdsc->info_regs = NULL;41 pdsc->cmd_regs = NULL;42 pdsc->intr_status = NULL;43 pdsc->intr_ctrl = NULL;44 45 for (i = 0; i < PDS_CORE_BARS_MAX; i++) {46 if (bars[i].vaddr)47 pci_iounmap(pdsc->pdev, bars[i].vaddr);48 bars[i].vaddr = NULL;49 }50}51 52static int pdsc_map_bars(struct pdsc *pdsc)53{54 struct pdsc_dev_bar *bar = pdsc->bars;55 struct pci_dev *pdev = pdsc->pdev;56 struct device *dev = pdsc->dev;57 struct pdsc_dev_bar *bars;58 unsigned int i, j;59 int num_bars = 0;60 int err;61 u32 sig;62 63 bars = pdsc->bars;64 65 /* Since the PCI interface in the hardware is configurable,66 * we need to poke into all the bars to find the set we're67 * expecting.68 */69 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {70 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))71 continue;72 73 bars[j].len = pci_resource_len(pdev, i);74 bars[j].bus_addr = pci_resource_start(pdev, i);75 bars[j].res_index = i;76 77 /* only map the whole bar 0 */78 if (j > 0) {79 bars[j].vaddr = NULL;80 } else {81 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);82 if (!bars[j].vaddr) {83 dev_err(dev, "Cannot map BAR %d, aborting\n", i);84 return -ENODEV;85 }86 }87 88 j++;89 }90 num_bars = j;91 92 /* BAR0: dev_cmd and interrupts */93 if (num_bars < 1) {94 dev_err(dev, "No bars found\n");95 err = -EFAULT;96 goto err_out;97 }98 99 if (bar->len < PDS_CORE_BAR0_SIZE) {100 dev_err(dev, "Resource bar size %lu too small\n", bar->len);101 err = -EFAULT;102 goto err_out;103 }104 105 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;106 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;107 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;108 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;109 110 sig = ioread32(&pdsc->info_regs->signature);111 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {112 dev_err(dev, "Incompatible firmware signature %x", sig);113 err = -EFAULT;114 goto err_out;115 }116 117 /* BAR1: doorbells */118 bar++;119 if (num_bars < 2) {120 dev_err(dev, "Doorbell bar missing\n");121 err = -EFAULT;122 goto err_out;123 }124 125 pdsc->db_pages = bar->vaddr;126 pdsc->phy_db_pages = bar->bus_addr;127 128 return 0;129 130err_out:131 pdsc_unmap_bars(pdsc);132 return err;133}134 135void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)136{137 return pci_iomap_range(pdsc->pdev,138 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,139 (u64)page_num << PAGE_SHIFT, PAGE_SIZE);140}141 142static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)143{144 struct pdsc *pdsc = pci_get_drvdata(pdev);145 struct device *dev = pdsc->dev;146 int ret = 0;147 148 if (num_vfs > 0) {149 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),150 GFP_KERNEL);151 if (!pdsc->vfs)152 return -ENOMEM;153 pdsc->num_vfs = num_vfs;154 155 ret = pci_enable_sriov(pdev, num_vfs);156 if (ret) {157 dev_err(dev, "Cannot enable SRIOV: %pe\n",158 ERR_PTR(ret));159 goto no_vfs;160 }161 162 return num_vfs;163 }164 165no_vfs:166 pci_disable_sriov(pdev);167 168 kfree(pdsc->vfs);169 pdsc->vfs = NULL;170 pdsc->num_vfs = 0;171 172 return ret;173}174 175static int pdsc_init_vf(struct pdsc *vf)176{177 struct devlink *dl;178 struct pdsc *pf;179 int err;180 181 pf = pdsc_get_pf_struct(vf->pdev);182 if (IS_ERR_OR_NULL(pf))183 return PTR_ERR(pf) ?: -1;184 185 vf->vf_id = pci_iov_vf_id(vf->pdev);186 187 dl = priv_to_devlink(vf);188 devl_lock(dl);189 devl_register(dl);190 devl_unlock(dl);191 192 pf->vfs[vf->vf_id].vf = vf;193 err = pdsc_auxbus_dev_add(vf, pf);194 if (err) {195 devl_lock(dl);196 devl_unregister(dl);197 devl_unlock(dl);198 }199 200 return err;201}202 203static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {204 .name = "fw",205 .diagnose = pdsc_fw_reporter_diagnose,206};207 208static const struct devlink_param pdsc_dl_params[] = {209 DEVLINK_PARAM_GENERIC(ENABLE_VNET,210 BIT(DEVLINK_PARAM_CMODE_RUNTIME),211 pdsc_dl_enable_get,212 pdsc_dl_enable_set,213 pdsc_dl_enable_validate),214};215 216#define PDSC_WQ_NAME_LEN 24217 218static int pdsc_init_pf(struct pdsc *pdsc)219{220 struct devlink_health_reporter *hr;221 char wq_name[PDSC_WQ_NAME_LEN];222 struct devlink *dl;223 int err;224 225 pcie_print_link_status(pdsc->pdev);226 227 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);228 if (err) {229 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",230 ERR_PTR(err));231 return err;232 }233 234 err = pdsc_map_bars(pdsc);235 if (err)236 goto err_out_release_regions;237 238 /* General workqueue and timer, but don't start timer yet */239 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);240 pdsc->wq = create_singlethread_workqueue(wq_name);241 INIT_WORK(&pdsc->health_work, pdsc_health_thread);242 INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);243 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);244 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;245 246 mutex_init(&pdsc->devcmd_lock);247 mutex_init(&pdsc->config_lock);248 spin_lock_init(&pdsc->adminq_lock);249 250 mutex_lock(&pdsc->config_lock);251 set_bit(PDSC_S_FW_DEAD, &pdsc->state);252 253 err = pdsc_setup(pdsc, PDSC_SETUP_INIT);254 if (err) {255 mutex_unlock(&pdsc->config_lock);256 goto err_out_unmap_bars;257 }258 259 err = pdsc_start(pdsc);260 if (err) {261 mutex_unlock(&pdsc->config_lock);262 goto err_out_teardown;263 }264 265 mutex_unlock(&pdsc->config_lock);266 267 dl = priv_to_devlink(pdsc);268 devl_lock(dl);269 err = devl_params_register(dl, pdsc_dl_params,270 ARRAY_SIZE(pdsc_dl_params));271 if (err) {272 devl_unlock(dl);273 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",274 ERR_PTR(err));275 goto err_out_stop;276 }277 278 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);279 if (IS_ERR(hr)) {280 devl_unlock(dl);281 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);282 err = PTR_ERR(hr);283 goto err_out_unreg_params;284 }285 pdsc->fw_reporter = hr;286 287 devl_register(dl);288 devl_unlock(dl);289 290 /* Lastly, start the health check timer */291 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));292 293 return 0;294 295err_out_unreg_params:296 devlink_params_unregister(dl, pdsc_dl_params,297 ARRAY_SIZE(pdsc_dl_params));298err_out_stop:299 pdsc_stop(pdsc);300err_out_teardown:301 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);302err_out_unmap_bars:303 timer_shutdown_sync(&pdsc->wdtimer);304 if (pdsc->wq)305 destroy_workqueue(pdsc->wq);306 mutex_destroy(&pdsc->config_lock);307 mutex_destroy(&pdsc->devcmd_lock);308 pci_free_irq_vectors(pdsc->pdev);309 pdsc_unmap_bars(pdsc);310err_out_release_regions:311 pci_release_regions(pdsc->pdev);312 313 return err;314}315 316static const struct devlink_ops pdsc_dl_ops = {317 .info_get = pdsc_dl_info_get,318 .flash_update = pdsc_dl_flash_update,319};320 321static const struct devlink_ops pdsc_dl_vf_ops = {322};323 324static DEFINE_IDA(pdsc_ida);325 326static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)327{328 struct device *dev = &pdev->dev;329 const struct devlink_ops *ops;330 struct devlink *dl;331 struct pdsc *pdsc;332 bool is_pf;333 int err;334 335 is_pf = !pdev->is_virtfn;336 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;337 dl = devlink_alloc(ops, sizeof(struct pdsc), dev);338 if (!dl)339 return -ENOMEM;340 pdsc = devlink_priv(dl);341 342 pdsc->pdev = pdev;343 pdsc->dev = &pdev->dev;344 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);345 pci_set_drvdata(pdev, pdsc);346 pdsc_debugfs_add_dev(pdsc);347 348 err = ida_alloc(&pdsc_ida, GFP_KERNEL);349 if (err < 0) {350 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",351 __func__, ERR_PTR(err));352 goto err_out_free_devlink;353 }354 pdsc->uid = err;355 356 /* Query system for DMA addressing limitation for the device. */357 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));358 if (err) {359 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",360 ERR_PTR(err));361 goto err_out_free_ida;362 }363 364 err = pci_enable_device(pdev);365 if (err) {366 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));367 goto err_out_free_ida;368 }369 pci_set_master(pdev);370 371 if (is_pf)372 err = pdsc_init_pf(pdsc);373 else374 err = pdsc_init_vf(pdsc);375 if (err) {376 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));377 goto err_out_disable_device;378 }379 380 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);381 return 0;382 383err_out_disable_device:384 pci_disable_device(pdev);385err_out_free_ida:386 ida_free(&pdsc_ida, pdsc->uid);387err_out_free_devlink:388 pdsc_debugfs_del_dev(pdsc);389 devlink_free(dl);390 391 return err;392}393 394static void pdsc_remove(struct pci_dev *pdev)395{396 struct pdsc *pdsc = pci_get_drvdata(pdev);397 struct devlink *dl;398 399 /* Unhook the registrations first to be sure there400 * are no requests while we're stopping.401 */402 dl = priv_to_devlink(pdsc);403 devl_lock(dl);404 devl_unregister(dl);405 if (!pdev->is_virtfn) {406 if (pdsc->fw_reporter) {407 devl_health_reporter_destroy(pdsc->fw_reporter);408 pdsc->fw_reporter = NULL;409 }410 devl_params_unregister(dl, pdsc_dl_params,411 ARRAY_SIZE(pdsc_dl_params));412 }413 devl_unlock(dl);414 415 if (pdev->is_virtfn) {416 struct pdsc *pf;417 418 pf = pdsc_get_pf_struct(pdsc->pdev);419 if (!IS_ERR(pf)) {420 pdsc_auxbus_dev_del(pdsc, pf);421 pf->vfs[pdsc->vf_id].vf = NULL;422 }423 } else {424 /* Remove the VFs and their aux_bus connections before other425 * cleanup so that the clients can use the AdminQ to cleanly426 * shut themselves down.427 */428 pdsc_sriov_configure(pdev, 0);429 430 timer_shutdown_sync(&pdsc->wdtimer);431 if (pdsc->wq)432 destroy_workqueue(pdsc->wq);433 434 mutex_lock(&pdsc->config_lock);435 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);436 437 pdsc_stop(pdsc);438 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);439 mutex_unlock(&pdsc->config_lock);440 mutex_destroy(&pdsc->config_lock);441 mutex_destroy(&pdsc->devcmd_lock);442 443 pdsc_unmap_bars(pdsc);444 pci_release_regions(pdev);445 }446 447 pci_disable_device(pdev);448 449 ida_free(&pdsc_ida, pdsc->uid);450 pdsc_debugfs_del_dev(pdsc);451 devlink_free(dl);452}453 454static void pdsc_stop_health_thread(struct pdsc *pdsc)455{456 if (pdsc->pdev->is_virtfn)457 return;458 459 timer_shutdown_sync(&pdsc->wdtimer);460 if (pdsc->health_work.func)461 cancel_work_sync(&pdsc->health_work);462}463 464static void pdsc_restart_health_thread(struct pdsc *pdsc)465{466 if (pdsc->pdev->is_virtfn)467 return;468 469 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);470 mod_timer(&pdsc->wdtimer, jiffies + 1);471}472 473static void pdsc_reset_prepare(struct pci_dev *pdev)474{475 struct pdsc *pdsc = pci_get_drvdata(pdev);476 477 pdsc_stop_health_thread(pdsc);478 pdsc_fw_down(pdsc);479 480 if (pdev->is_virtfn) {481 struct pdsc *pf;482 483 pf = pdsc_get_pf_struct(pdsc->pdev);484 if (!IS_ERR(pf))485 pdsc_auxbus_dev_del(pdsc, pf);486 }487 488 pdsc_unmap_bars(pdsc);489 pci_release_regions(pdev);490 if (pci_is_enabled(pdev))491 pci_disable_device(pdev);492}493 494static void pdsc_reset_done(struct pci_dev *pdev)495{496 struct pdsc *pdsc = pci_get_drvdata(pdev);497 struct device *dev = pdsc->dev;498 int err;499 500 err = pci_enable_device(pdev);501 if (err) {502 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));503 return;504 }505 pci_set_master(pdev);506 507 if (!pdev->is_virtfn) {508 pcie_print_link_status(pdsc->pdev);509 510 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);511 if (err) {512 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",513 ERR_PTR(err));514 return;515 }516 517 err = pdsc_map_bars(pdsc);518 if (err)519 return;520 }521 522 pdsc_fw_up(pdsc);523 pdsc_restart_health_thread(pdsc);524 525 if (pdev->is_virtfn) {526 struct pdsc *pf;527 528 pf = pdsc_get_pf_struct(pdsc->pdev);529 if (!IS_ERR(pf))530 pdsc_auxbus_dev_add(pdsc, pf);531 }532}533 534static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,535 pci_channel_state_t error)536{537 if (error == pci_channel_io_frozen) {538 pdsc_reset_prepare(pdev);539 return PCI_ERS_RESULT_NEED_RESET;540 }541 542 return PCI_ERS_RESULT_NONE;543}544 545static void pdsc_pci_error_resume(struct pci_dev *pdev)546{547 struct pdsc *pdsc = pci_get_drvdata(pdev);548 549 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))550 pci_reset_function_locked(pdev);551}552 553static const struct pci_error_handlers pdsc_err_handler = {554 /* FLR handling */555 .reset_prepare = pdsc_reset_prepare,556 .reset_done = pdsc_reset_done,557 558 /* AER handling */559 .error_detected = pdsc_pci_error_detected,560 .resume = pdsc_pci_error_resume,561};562 563static struct pci_driver pdsc_driver = {564 .name = PDS_CORE_DRV_NAME,565 .id_table = pdsc_id_table,566 .probe = pdsc_probe,567 .remove = pdsc_remove,568 .sriov_configure = pdsc_sriov_configure,569 .err_handler = &pdsc_err_handler,570};571 572void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)573{574 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);575}576EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);577 578static int __init pdsc_init_module(void)579{580 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))581 return -EINVAL;582 583 pdsc_debugfs_create();584 return pci_register_driver(&pdsc_driver);585}586 587static void __exit pdsc_cleanup_module(void)588{589 pci_unregister_driver(&pdsc_driver);590 pdsc_debugfs_destroy();591}592 593module_init(pdsc_init_module);594module_exit(pdsc_cleanup_module);595