565 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) Meta Platforms, Inc. and affiliates. */3 4#include <linux/init.h>5#include <linux/module.h>6#include <linux/pci.h>7#include <linux/rtnetlink.h>8#include <linux/types.h>9 10#include "fbnic.h"11#include "fbnic_drvinfo.h"12#include "fbnic_netdev.h"13 14char fbnic_driver_name[] = DRV_NAME;15 16MODULE_DESCRIPTION(DRV_SUMMARY);17MODULE_LICENSE("GPL");18 19static const struct fbnic_info fbnic_asic_info = {20 .max_num_queues = FBNIC_MAX_QUEUES,21 .bar_mask = BIT(0) | BIT(4)22};23 24static const struct fbnic_info *fbnic_info_tbl[] = {25 [fbnic_board_asic] = &fbnic_asic_info,26};27 28static const struct pci_device_id fbnic_pci_tbl[] = {29 { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) },30 /* Required last entry */31 {0, }32};33MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl);34 35u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg)36{37 u32 __iomem *csr = READ_ONCE(fbd->uc_addr0);38 u32 value;39 40 if (!csr)41 return ~0U;42 43 value = readl(csr + reg);44 45 /* If any bits are 0 value should be valid */46 if (~value)47 return value;48 49 /* All 1's may be valid if ZEROs register still works */50 if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0))51 return value;52 53 /* Hardware is giving us all 1's reads, assume it is gone */54 WRITE_ONCE(fbd->uc_addr0, NULL);55 WRITE_ONCE(fbd->uc_addr4, NULL);56 57 dev_err(fbd->dev,58 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",59 reg, reg << 2);60 61 /* Notify stack that device has lost (PCIe) link */62 if (!fbnic_init_failure(fbd))63 netif_device_detach(fbd->netdev);64 65 return ~0U;66}67 68bool fbnic_fw_present(struct fbnic_dev *fbd)69{70 return !!READ_ONCE(fbd->uc_addr4);71}72 73void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val)74{75 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);76 77 if (csr)78 writel(val, csr + reg);79}80 81u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg)82{83 u32 __iomem *csr = READ_ONCE(fbd->uc_addr4);84 u32 value;85 86 if (!csr)87 return ~0U;88 89 value = readl(csr + reg);90 91 /* If any bits are 0 value should be valid */92 if (~value)93 return value;94 95 /* All 1's may be valid if ZEROs register still works */96 if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG))97 return value;98 99 /* Hardware is giving us all 1's reads, assume it is gone */100 WRITE_ONCE(fbd->uc_addr0, NULL);101 WRITE_ONCE(fbd->uc_addr4, NULL);102 103 dev_err(fbd->dev,104 "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",105 reg, reg << 2);106 107 /* Notify stack that device has lost (PCIe) link */108 if (!fbnic_init_failure(fbd))109 netif_device_detach(fbd->netdev);110 111 return ~0U;112}113 114static void fbnic_service_task_start(struct fbnic_net *fbn)115{116 struct fbnic_dev *fbd = fbn->fbd;117 118 schedule_delayed_work(&fbd->service_task, HZ);119 phylink_resume(fbn->phylink);120}121 122static void fbnic_service_task_stop(struct fbnic_net *fbn)123{124 struct fbnic_dev *fbd = fbn->fbd;125 126 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbd));127 cancel_delayed_work(&fbd->service_task);128}129 130void fbnic_up(struct fbnic_net *fbn)131{132 fbnic_enable(fbn);133 134 fbnic_fill(fbn);135 136 fbnic_rss_reinit_hw(fbn->fbd, fbn);137 138 __fbnic_set_rx_mode(fbn->netdev);139 140 /* Enable Tx/Rx processing */141 fbnic_napi_enable(fbn);142 netif_tx_start_all_queues(fbn->netdev);143 144 fbnic_service_task_start(fbn);145}146 147static void fbnic_down_noidle(struct fbnic_net *fbn)148{149 fbnic_service_task_stop(fbn);150 151 /* Disable Tx/Rx Processing */152 fbnic_napi_disable(fbn);153 netif_tx_disable(fbn->netdev);154 155 fbnic_clear_rx_mode(fbn->netdev);156 fbnic_clear_rules(fbn->fbd);157 fbnic_rss_disable_hw(fbn->fbd);158 fbnic_disable(fbn);159}160 161void fbnic_down(struct fbnic_net *fbn)162{163 fbnic_down_noidle(fbn);164 165 fbnic_wait_all_queues_idle(fbn->fbd, false);166 167 fbnic_flush(fbn);168}169 170static void fbnic_health_check(struct fbnic_dev *fbd)171{172 struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];173 174 /* As long as the heart is beating the FW is healty */175 if (fbd->fw_heartbeat_enabled)176 return;177 178 /* If the Tx mailbox still has messages sitting in it then there likely179 * isn't anything we can do. We will wait until the mailbox is empty to180 * report the fault so we can collect the crashlog.181 */182 if (tx_mbx->head != tx_mbx->tail)183 return;184 185 /* TBD: Need to add a more thorough recovery here.186 * Specifically I need to verify what all the firmware will have187 * changed since we had setup and it rebooted. May just need to188 * perform a down/up. For now we will just reclaim ownership so189 * the heartbeat can catch the next fault.190 */191 fbnic_fw_xmit_ownership_msg(fbd, true);192}193 194static void fbnic_service_task(struct work_struct *work)195{196 struct fbnic_dev *fbd = container_of(to_delayed_work(work),197 struct fbnic_dev, service_task);198 199 rtnl_lock();200 201 fbnic_fw_check_heartbeat(fbd);202 203 fbnic_health_check(fbd);204 205 if (netif_carrier_ok(fbd->netdev))206 fbnic_napi_depletion_check(fbd->netdev);207 208 if (netif_running(fbd->netdev))209 schedule_delayed_work(&fbd->service_task, HZ);210 211 rtnl_unlock();212}213 214/**215 * fbnic_probe - Device Initialization Routine216 * @pdev: PCI device information struct217 * @ent: entry in fbnic_pci_tbl218 *219 * Initializes a PCI device identified by a pci_dev structure.220 * The OS initialization, configuring of the adapter private structure,221 * and a hardware reset occur.222 *223 * Return: 0 on success, negative on failure224 **/225static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)226{227 const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data];228 struct net_device *netdev;229 struct fbnic_dev *fbd;230 int err;231 232 if (pdev->error_state != pci_channel_io_normal) {233 dev_err(&pdev->dev,234 "PCI device still in an error state. Unable to load...\n");235 return -EIO;236 }237 238 err = pcim_enable_device(pdev);239 if (err) {240 dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);241 return err;242 }243 244 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));245 if (err)246 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));247 if (err) {248 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);249 return err;250 }251 252 err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name);253 if (err) {254 dev_err(&pdev->dev,255 "pci_request_selected_regions failed: %d\n", err);256 return err;257 }258 259 fbd = fbnic_devlink_alloc(pdev);260 if (!fbd) {261 dev_err(&pdev->dev, "Devlink allocation failed\n");262 return -ENOMEM;263 }264 265 /* Populate driver with hardware-specific info and handlers */266 fbd->max_num_queues = info->max_num_queues;267 268 pci_set_master(pdev);269 pci_save_state(pdev);270 271 INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task);272 273 err = fbnic_alloc_irqs(fbd);274 if (err)275 goto free_fbd;276 277 err = fbnic_mac_init(fbd);278 if (err) {279 dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err);280 goto free_irqs;281 }282 283 err = fbnic_fw_enable_mbx(fbd);284 if (err) {285 dev_err(&pdev->dev,286 "Firmware mailbox initialization failure\n");287 goto free_irqs;288 }289 290 fbnic_devlink_register(fbd);291 292 if (!fbd->dsn) {293 dev_warn(&pdev->dev, "Reading serial number failed\n");294 goto init_failure_mode;295 }296 297 netdev = fbnic_netdev_alloc(fbd);298 if (!netdev) {299 dev_err(&pdev->dev, "Netdev allocation failed\n");300 goto init_failure_mode;301 }302 303 err = fbnic_netdev_register(netdev);304 if (err) {305 dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);306 goto ifm_free_netdev;307 }308 309 return 0;310 311ifm_free_netdev:312 fbnic_netdev_free(fbd);313init_failure_mode:314 dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n");315 /* Always return 0 even on error so devlink is registered to allow316 * firmware updates for fixes.317 */318 return 0;319free_irqs:320 fbnic_free_irqs(fbd);321free_fbd:322 pci_disable_device(pdev);323 fbnic_devlink_free(fbd);324 325 return err;326}327 328/**329 * fbnic_remove - Device Removal Routine330 * @pdev: PCI device information struct331 *332 * Called by the PCI subsystem to alert the driver that it should release333 * a PCI device. The could be caused by a Hot-Plug event, or because the334 * driver is going to be removed from memory.335 **/336static void fbnic_remove(struct pci_dev *pdev)337{338 struct fbnic_dev *fbd = pci_get_drvdata(pdev);339 340 if (!fbnic_init_failure(fbd)) {341 struct net_device *netdev = fbd->netdev;342 343 fbnic_netdev_unregister(netdev);344 cancel_delayed_work_sync(&fbd->service_task);345 fbnic_netdev_free(fbd);346 }347 348 fbnic_devlink_unregister(fbd);349 fbnic_fw_disable_mbx(fbd);350 fbnic_free_irqs(fbd);351 352 pci_disable_device(pdev);353 fbnic_devlink_free(fbd);354}355 356static int fbnic_pm_suspend(struct device *dev)357{358 struct fbnic_dev *fbd = dev_get_drvdata(dev);359 struct net_device *netdev = fbd->netdev;360 361 if (fbnic_init_failure(fbd))362 goto null_uc_addr;363 364 rtnl_lock();365 366 netif_device_detach(netdev);367 368 if (netif_running(netdev))369 netdev->netdev_ops->ndo_stop(netdev);370 371 rtnl_unlock();372 373null_uc_addr:374 fbnic_fw_disable_mbx(fbd);375 376 /* Free the IRQs so they aren't trying to occupy sleeping CPUs */377 fbnic_free_irqs(fbd);378 379 /* Hardware is about to go away, so switch off MMIO access internally */380 WRITE_ONCE(fbd->uc_addr0, NULL);381 WRITE_ONCE(fbd->uc_addr4, NULL);382 383 return 0;384}385 386static int __fbnic_pm_resume(struct device *dev)387{388 struct fbnic_dev *fbd = dev_get_drvdata(dev);389 struct net_device *netdev = fbd->netdev;390 void __iomem * const *iomap_table;391 struct fbnic_net *fbn;392 int err;393 394 /* Restore MMIO access */395 iomap_table = pcim_iomap_table(to_pci_dev(dev));396 fbd->uc_addr0 = iomap_table[0];397 fbd->uc_addr4 = iomap_table[4];398 399 /* Rerequest the IRQs */400 err = fbnic_alloc_irqs(fbd);401 if (err)402 goto err_invalidate_uc_addr;403 404 fbd->mac->init_regs(fbd);405 406 /* Re-enable mailbox */407 err = fbnic_fw_enable_mbx(fbd);408 if (err)409 goto err_free_irqs;410 411 /* No netdev means there isn't a network interface to bring up */412 if (fbnic_init_failure(fbd))413 return 0;414 415 fbn = netdev_priv(netdev);416 417 /* Reset the queues if needed */418 fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);419 420 rtnl_lock();421 422 if (netif_running(netdev)) {423 err = __fbnic_open(fbn);424 if (err)425 goto err_disable_mbx;426 }427 428 rtnl_unlock();429 430 return 0;431err_disable_mbx:432 rtnl_unlock();433 fbnic_fw_disable_mbx(fbd);434err_free_irqs:435 fbnic_free_irqs(fbd);436err_invalidate_uc_addr:437 WRITE_ONCE(fbd->uc_addr0, NULL);438 WRITE_ONCE(fbd->uc_addr4, NULL);439 return err;440}441 442static void __fbnic_pm_attach(struct device *dev)443{444 struct fbnic_dev *fbd = dev_get_drvdata(dev);445 struct net_device *netdev = fbd->netdev;446 struct fbnic_net *fbn;447 448 if (fbnic_init_failure(fbd))449 return;450 451 fbn = netdev_priv(netdev);452 453 if (netif_running(netdev))454 fbnic_up(fbn);455 456 netif_device_attach(netdev);457}458 459static int __maybe_unused fbnic_pm_resume(struct device *dev)460{461 int err;462 463 err = __fbnic_pm_resume(dev);464 if (!err)465 __fbnic_pm_attach(dev);466 467 return err;468}469 470static const struct dev_pm_ops fbnic_pm_ops = {471 SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume)472};473 474static void fbnic_shutdown(struct pci_dev *pdev)475{476 fbnic_pm_suspend(&pdev->dev);477}478 479static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev,480 pci_channel_state_t state)481{482 /* Disconnect device if failure is not recoverable via reset */483 if (state == pci_channel_io_perm_failure)484 return PCI_ERS_RESULT_DISCONNECT;485 486 fbnic_pm_suspend(&pdev->dev);487 488 /* Request a slot reset */489 return PCI_ERS_RESULT_NEED_RESET;490}491 492static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev)493{494 int err;495 496 pci_set_power_state(pdev, PCI_D0);497 pci_restore_state(pdev);498 pci_save_state(pdev);499 500 if (pci_enable_device_mem(pdev)) {501 dev_err(&pdev->dev,502 "Cannot re-enable PCI device after reset.\n");503 return PCI_ERS_RESULT_DISCONNECT;504 }505 506 /* Restore device to previous state */507 err = __fbnic_pm_resume(&pdev->dev);508 509 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;510}511 512static void fbnic_err_resume(struct pci_dev *pdev)513{514 __fbnic_pm_attach(&pdev->dev);515}516 517static const struct pci_error_handlers fbnic_err_handler = {518 .error_detected = fbnic_err_error_detected,519 .slot_reset = fbnic_err_slot_reset,520 .resume = fbnic_err_resume,521};522 523static struct pci_driver fbnic_driver = {524 .name = fbnic_driver_name,525 .id_table = fbnic_pci_tbl,526 .probe = fbnic_probe,527 .remove = fbnic_remove,528 .driver.pm = &fbnic_pm_ops,529 .shutdown = fbnic_shutdown,530 .err_handler = &fbnic_err_handler,531};532 533/**534 * fbnic_init_module - Driver Registration Routine535 *536 * The first routine called when the driver is loaded. All it does is537 * register with the PCI subsystem.538 *539 * Return: 0 on success, negative on failure540 **/541static int __init fbnic_init_module(void)542{543 int err;544 545 err = pci_register_driver(&fbnic_driver);546 if (err)547 goto out;548 549 pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name);550out:551 return err;552}553module_init(fbnic_init_module);554 555/**556 * fbnic_exit_module - Driver Exit Cleanup Routine557 *558 * Called just before the driver is removed from memory.559 **/560static void __exit fbnic_exit_module(void)561{562 pci_unregister_driver(&fbnic_driver);563}564module_exit(fbnic_exit_module);565