533 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */3 4#include <linux/module.h>5#include <linux/netdevice.h>6#include <linux/etherdevice.h>7#include <linux/pci.h>8 9#include "ionic.h"10#include "ionic_bus.h"11#include "ionic_lif.h"12#include "ionic_debugfs.h"13 14/* Supported devices */15static const struct pci_device_id ionic_id_table[] = {16 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) },17 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) },18 { 0, } /* end of table */19};20MODULE_DEVICE_TABLE(pci, ionic_id_table);21 22int ionic_bus_get_irq(struct ionic *ionic, unsigned int num)23{24 return pci_irq_vector(ionic->pdev, num);25}26 27const char *ionic_bus_info(struct ionic *ionic)28{29 return pci_name(ionic->pdev);30}31 32int ionic_bus_alloc_irq_vectors(struct ionic *ionic, unsigned int nintrs)33{34 return pci_alloc_irq_vectors(ionic->pdev, nintrs, nintrs,35 PCI_IRQ_MSIX);36}37 38void ionic_bus_free_irq_vectors(struct ionic *ionic)39{40 if (!ionic->nintrs)41 return;42 43 pci_free_irq_vectors(ionic->pdev);44}45 46static int ionic_map_bars(struct ionic *ionic)47{48 struct pci_dev *pdev = ionic->pdev;49 struct device *dev = ionic->dev;50 struct ionic_dev_bar *bars;51 unsigned int i, j;52 53 bars = ionic->bars;54 ionic->num_bars = 0;55 56 for (i = 0, j = 0; i < IONIC_BARS_MAX; i++) {57 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))58 continue;59 bars[j].len = pci_resource_len(pdev, i);60 61 /* only map the whole bar 0 */62 if (j > 0) {63 bars[j].vaddr = NULL;64 } else {65 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);66 if (!bars[j].vaddr) {67 dev_err(dev,68 "Cannot memory-map BAR %d, aborting\n",69 i);70 return -ENODEV;71 }72 }73 74 bars[j].bus_addr = pci_resource_start(pdev, i);75 bars[j].res_index = i;76 ionic->num_bars++;77 j++;78 }79 80 return 0;81}82 83static void ionic_unmap_bars(struct ionic *ionic)84{85 struct ionic_dev_bar *bars = ionic->bars;86 unsigned int i;87 88 for (i = 0; i < IONIC_BARS_MAX; i++) {89 if (bars[i].vaddr) {90 iounmap(bars[i].vaddr);91 bars[i].bus_addr = 0;92 bars[i].vaddr = NULL;93 bars[i].len = 0;94 }95 }96 ionic->num_bars = 0;97}98 99void __iomem *ionic_bus_map_dbpage(struct ionic *ionic, int page_num)100{101 return pci_iomap_range(ionic->pdev,102 ionic->bars[IONIC_PCI_BAR_DBELL].res_index,103 (u64)page_num << PAGE_SHIFT, PAGE_SIZE);104}105 106void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page)107{108 iounmap(page);109}110 111static void ionic_vf_dealloc_locked(struct ionic *ionic)112{113 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };114 struct ionic_vf *v;115 int i;116 117 if (!ionic->vfs)118 return;119 120 for (i = ionic->num_vfs - 1; i >= 0; i--) {121 v = &ionic->vfs[i];122 123 if (v->stats_pa) {124 vfc.stats_pa = 0;125 ionic_set_vf_config(ionic, i, &vfc);126 dma_unmap_single(ionic->dev, v->stats_pa,127 sizeof(v->stats), DMA_FROM_DEVICE);128 v->stats_pa = 0;129 }130 }131 132 kfree(ionic->vfs);133 ionic->vfs = NULL;134 ionic->num_vfs = 0;135}136 137static void ionic_vf_dealloc(struct ionic *ionic)138{139 down_write(&ionic->vf_op_lock);140 ionic_vf_dealloc_locked(ionic);141 up_write(&ionic->vf_op_lock);142}143 144static int ionic_vf_alloc(struct ionic *ionic, int num_vfs)145{146 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };147 struct ionic_vf *v;148 int err = 0;149 int i;150 151 down_write(&ionic->vf_op_lock);152 153 ionic->vfs = kcalloc(num_vfs, sizeof(struct ionic_vf), GFP_KERNEL);154 if (!ionic->vfs) {155 err = -ENOMEM;156 goto out;157 }158 159 for (i = 0; i < num_vfs; i++) {160 v = &ionic->vfs[i];161 v->stats_pa = dma_map_single(ionic->dev, &v->stats,162 sizeof(v->stats), DMA_FROM_DEVICE);163 if (dma_mapping_error(ionic->dev, v->stats_pa)) {164 v->stats_pa = 0;165 err = -ENODEV;166 goto out;167 }168 169 ionic->num_vfs++;170 171 /* ignore failures from older FW, we just won't get stats */172 vfc.stats_pa = cpu_to_le64(v->stats_pa);173 ionic_set_vf_config(ionic, i, &vfc);174 }175 176out:177 if (err)178 ionic_vf_dealloc_locked(ionic);179 up_write(&ionic->vf_op_lock);180 return err;181}182 183static int ionic_sriov_configure(struct pci_dev *pdev, int num_vfs)184{185 struct ionic *ionic = pci_get_drvdata(pdev);186 struct device *dev = ionic->dev;187 int ret = 0;188 189 if (ionic->lif &&190 test_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state))191 return -EBUSY;192 193 if (num_vfs > 0) {194 ret = pci_enable_sriov(pdev, num_vfs);195 if (ret) {196 dev_err(dev, "Cannot enable SRIOV: %d\n", ret);197 goto out;198 }199 200 ret = ionic_vf_alloc(ionic, num_vfs);201 if (ret) {202 dev_err(dev, "Cannot alloc VFs: %d\n", ret);203 pci_disable_sriov(pdev);204 goto out;205 }206 207 ret = num_vfs;208 } else {209 pci_disable_sriov(pdev);210 ionic_vf_dealloc(ionic);211 }212 213out:214 return ret;215}216 217static void ionic_clear_pci(struct ionic *ionic)218{219 if (ionic->num_bars) {220 ionic->idev.dev_info_regs = NULL;221 ionic->idev.dev_cmd_regs = NULL;222 ionic->idev.intr_status = NULL;223 ionic->idev.intr_ctrl = NULL;224 225 ionic_unmap_bars(ionic);226 pci_release_regions(ionic->pdev);227 }228 229 if (pci_is_enabled(ionic->pdev))230 pci_disable_device(ionic->pdev);231}232 233static int ionic_setup_one(struct ionic *ionic)234{235 struct pci_dev *pdev = ionic->pdev;236 struct device *dev = ionic->dev;237 int err;238 239 ionic_debugfs_add_dev(ionic);240 241 /* Setup PCI device */242 err = pci_enable_device_mem(pdev);243 if (err) {244 dev_err(dev, "Cannot enable PCI device: %d, aborting\n", err);245 goto err_out_debugfs_del_dev;246 }247 248 err = pci_request_regions(pdev, IONIC_DRV_NAME);249 if (err) {250 dev_err(dev, "Cannot request PCI regions: %d, aborting\n", err);251 goto err_out_clear_pci;252 }253 pcie_print_link_status(pdev);254 255 err = ionic_map_bars(ionic);256 if (err)257 goto err_out_clear_pci;258 259 /* Configure the device */260 err = ionic_setup(ionic);261 if (err) {262 dev_err(dev, "Cannot setup device: %d, aborting\n", err);263 goto err_out_clear_pci;264 }265 pci_set_master(pdev);266 267 err = ionic_identify(ionic);268 if (err) {269 dev_err(dev, "Cannot identify device: %d, aborting\n", err);270 goto err_out_teardown;271 }272 ionic_debugfs_add_ident(ionic);273 274 err = ionic_init(ionic);275 if (err) {276 dev_err(dev, "Cannot init device: %d, aborting\n", err);277 goto err_out_teardown;278 }279 280 /* Configure the port */281 err = ionic_port_identify(ionic);282 if (err) {283 dev_err(dev, "Cannot identify port: %d, aborting\n", err);284 goto err_out_teardown;285 }286 287 err = ionic_port_init(ionic);288 if (err) {289 dev_err(dev, "Cannot init port: %d, aborting\n", err);290 goto err_out_teardown;291 }292 293 return 0;294 295err_out_teardown:296 ionic_dev_teardown(ionic);297err_out_clear_pci:298 ionic_clear_pci(ionic);299err_out_debugfs_del_dev:300 ionic_debugfs_del_dev(ionic);301 302 return err;303}304 305static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)306{307 struct device *dev = &pdev->dev;308 struct ionic *ionic;309 int num_vfs;310 int err;311 312 ionic = ionic_devlink_alloc(dev);313 if (!ionic)314 return -ENOMEM;315 316 ionic->pdev = pdev;317 ionic->dev = dev;318 pci_set_drvdata(pdev, ionic);319 mutex_init(&ionic->dev_cmd_lock);320 321 /* Query system for DMA addressing limitation for the device. */322 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN));323 if (err) {324 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting. err=%d\n",325 err);326 goto err_out;327 }328 329#ifdef CONFIG_PPC64330 /* Ensure MSI/MSI-X interrupts lie within addressable physical memory */331 pdev->no_64bit_msi = 1;332#endif333 334 err = ionic_setup_one(ionic);335 if (err)336 goto err_out;337 338 /* Allocate and init the LIF */339 err = ionic_lif_size(ionic);340 if (err) {341 dev_err(dev, "Cannot size LIF: %d, aborting\n", err);342 goto err_out_pci;343 }344 345 err = ionic_lif_alloc(ionic);346 if (err) {347 dev_err(dev, "Cannot allocate LIF: %d, aborting\n", err);348 goto err_out_free_irqs;349 }350 351 err = ionic_lif_init(ionic->lif);352 if (err) {353 dev_err(dev, "Cannot init LIF: %d, aborting\n", err);354 goto err_out_free_lifs;355 }356 357 init_rwsem(&ionic->vf_op_lock);358 num_vfs = pci_num_vf(pdev);359 if (num_vfs) {360 dev_info(dev, "%d VFs found already enabled\n", num_vfs);361 err = ionic_vf_alloc(ionic, num_vfs);362 if (err)363 dev_err(dev, "Cannot enable existing VFs: %d\n", err);364 }365 366 err = ionic_devlink_register(ionic);367 if (err) {368 dev_err(dev, "Cannot register devlink: %d\n", err);369 goto err_out_deinit_lifs;370 }371 372 err = ionic_lif_register(ionic->lif);373 if (err) {374 dev_err(dev, "Cannot register LIF: %d, aborting\n", err);375 goto err_out_deregister_devlink;376 }377 378 mod_timer(&ionic->watchdog_timer,379 round_jiffies(jiffies + ionic->watchdog_period));380 ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);381 382 return 0;383 384err_out_deregister_devlink:385 ionic_devlink_unregister(ionic);386err_out_deinit_lifs:387 ionic_vf_dealloc(ionic);388 ionic_lif_deinit(ionic->lif);389err_out_free_lifs:390 ionic_lif_free(ionic->lif);391 ionic->lif = NULL;392err_out_free_irqs:393 ionic_bus_free_irq_vectors(ionic);394err_out_pci:395 ionic_dev_teardown(ionic);396 ionic_clear_pci(ionic);397 ionic_debugfs_del_dev(ionic);398err_out:399 mutex_destroy(&ionic->dev_cmd_lock);400 ionic_devlink_free(ionic);401 402 return err;403}404 405static void ionic_remove(struct pci_dev *pdev)406{407 struct ionic *ionic = pci_get_drvdata(pdev);408 409 timer_shutdown_sync(&ionic->watchdog_timer);410 411 if (ionic->lif) {412 /* prevent adminq cmds if already known as down */413 if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state))414 set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state);415 416 if (ionic->lif->doorbell_wa)417 cancel_delayed_work_sync(&ionic->doorbell_check_dwork);418 ionic_lif_unregister(ionic->lif);419 ionic_devlink_unregister(ionic);420 ionic_lif_deinit(ionic->lif);421 ionic_lif_free(ionic->lif);422 ionic->lif = NULL;423 ionic_bus_free_irq_vectors(ionic);424 }425 426 ionic_port_reset(ionic);427 ionic_reset(ionic);428 ionic_dev_teardown(ionic);429 ionic_clear_pci(ionic);430 ionic_debugfs_del_dev(ionic);431 mutex_destroy(&ionic->dev_cmd_lock);432 ionic_devlink_free(ionic);433}434 435static void ionic_reset_prepare(struct pci_dev *pdev)436{437 struct ionic *ionic = pci_get_drvdata(pdev);438 struct ionic_lif *lif = ionic->lif;439 440 dev_dbg(ionic->dev, "%s: device stopping\n", __func__);441 442 set_bit(IONIC_LIF_F_FW_RESET, lif->state);443 444 del_timer_sync(&ionic->watchdog_timer);445 cancel_work_sync(&lif->deferred.work);446 447 mutex_lock(&lif->queue_lock);448 ionic_stop_queues_reconfig(lif);449 ionic_txrx_free(lif);450 ionic_lif_deinit(lif);451 ionic_qcqs_free(lif);452 ionic_debugfs_del_lif(lif);453 mutex_unlock(&lif->queue_lock);454 455 ionic_dev_teardown(ionic);456 ionic_clear_pci(ionic);457 ionic_debugfs_del_dev(ionic);458}459 460static void ionic_reset_done(struct pci_dev *pdev)461{462 struct ionic *ionic = pci_get_drvdata(pdev);463 struct ionic_lif *lif = ionic->lif;464 int err;465 466 err = ionic_setup_one(ionic);467 if (err)468 goto err_out;469 470 ionic_debugfs_add_sizes(ionic);471 ionic_debugfs_add_lif(ionic->lif);472 473 err = ionic_restart_lif(lif);474 if (err)475 goto err_out;476 477 mod_timer(&ionic->watchdog_timer, jiffies + 1);478 479err_out:480 dev_dbg(ionic->dev, "%s: device recovery %s\n",481 __func__, err ? "failed" : "done");482}483 484static pci_ers_result_t ionic_pci_error_detected(struct pci_dev *pdev,485 pci_channel_state_t error)486{487 if (error == pci_channel_io_frozen) {488 ionic_reset_prepare(pdev);489 return PCI_ERS_RESULT_NEED_RESET;490 }491 492 return PCI_ERS_RESULT_NONE;493}494 495static void ionic_pci_error_resume(struct pci_dev *pdev)496{497 struct ionic *ionic = pci_get_drvdata(pdev);498 struct ionic_lif *lif = ionic->lif;499 500 if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state))501 pci_reset_function_locked(pdev);502}503 504static const struct pci_error_handlers ionic_err_handler = {505 /* FLR handling */506 .reset_prepare = ionic_reset_prepare,507 .reset_done = ionic_reset_done,508 509 /* PCI bus error detected on this device */510 .error_detected = ionic_pci_error_detected,511 .resume = ionic_pci_error_resume,512 513};514 515static struct pci_driver ionic_driver = {516 .name = IONIC_DRV_NAME,517 .id_table = ionic_id_table,518 .probe = ionic_probe,519 .remove = ionic_remove,520 .sriov_configure = ionic_sriov_configure,521 .err_handler = &ionic_err_handler522};523 524int ionic_bus_register_driver(void)525{526 return pci_register_driver(&ionic_driver);527}528 529void ionic_bus_unregister_driver(void)530{531 pci_unregister_driver(&ionic_driver);532}533