1403 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * IBM Accelerator Family 'GenWQE'4 *5 * (C) Copyright IBM Corp. 20136 *7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>9 * Author: Michael Jung <mijung@gmx.net>10 * Author: Michael Ruettger <michael@ibmra.de>11 */12 13/*14 * Module initialization and PCIe setup. Card health monitoring and15 * recovery functionality. Character device creation and deletion are16 * controlled from here.17 */18 19#include <linux/types.h>20#include <linux/pci.h>21#include <linux/err.h>22#include <linux/string.h>23#include <linux/sched.h>24#include <linux/wait.h>25#include <linux/delay.h>26#include <linux/dma-mapping.h>27#include <linux/module.h>28#include <linux/notifier.h>29#include <linux/device.h>30#include <linux/log2.h>31 32#include "card_base.h"33#include "card_ddcb.h"34 35MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");36MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");37MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");38MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");39 40MODULE_DESCRIPTION("GenWQE Card");41MODULE_VERSION(DRV_VERSION);42MODULE_LICENSE("GPL");43 44static char genwqe_driver_name[] = GENWQE_DEVNAME;45 46static struct dentry *debugfs_genwqe;47static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX];48 49/* PCI structure for identifying device by PCI vendor and device ID */50static const struct pci_device_id genwqe_device_table[] = {51 { .vendor = PCI_VENDOR_ID_IBM,52 .device = PCI_DEVICE_GENWQE,53 .subvendor = PCI_SUBVENDOR_ID_IBM,54 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5,55 .class = (PCI_CLASSCODE_GENWQE5 << 8),56 .class_mask = ~0,57 .driver_data = 0 },58 59 /* Initial SR-IOV bring-up image */60 { .vendor = PCI_VENDOR_ID_IBM,61 .device = PCI_DEVICE_GENWQE,62 .subvendor = PCI_SUBVENDOR_ID_IBM_SRIOV,63 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,64 .class = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),65 .class_mask = ~0,66 .driver_data = 0 },67 68 { .vendor = PCI_VENDOR_ID_IBM, /* VF Vendor ID */69 .device = 0x0000, /* VF Device ID */70 .subvendor = PCI_SUBVENDOR_ID_IBM_SRIOV,71 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,72 .class = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),73 .class_mask = ~0,74 .driver_data = 0 },75 76 /* Fixed up image */77 { .vendor = PCI_VENDOR_ID_IBM,78 .device = PCI_DEVICE_GENWQE,79 .subvendor = PCI_SUBVENDOR_ID_IBM_SRIOV,80 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5,81 .class = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),82 .class_mask = ~0,83 .driver_data = 0 },84 85 { .vendor = PCI_VENDOR_ID_IBM, /* VF Vendor ID */86 .device = 0x0000, /* VF Device ID */87 .subvendor = PCI_SUBVENDOR_ID_IBM_SRIOV,88 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5,89 .class = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),90 .class_mask = ~0,91 .driver_data = 0 },92 93 /* Even one more ... */94 { .vendor = PCI_VENDOR_ID_IBM,95 .device = PCI_DEVICE_GENWQE,96 .subvendor = PCI_SUBVENDOR_ID_IBM,97 .subdevice = PCI_SUBSYSTEM_ID_GENWQE5_NEW,98 .class = (PCI_CLASSCODE_GENWQE5 << 8),99 .class_mask = ~0,100 .driver_data = 0 },101 102 { 0, } /* 0 terminated list. */103};104 105MODULE_DEVICE_TABLE(pci, genwqe_device_table);106 107/**108 * genwqe_devnode() - Set default access mode for genwqe devices.109 * @dev: Pointer to device (unused)110 * @mode: Carrier to pass-back given mode (permissions)111 *112 * Default mode should be rw for everybody. Do not change default113 * device name.114 */115static char *genwqe_devnode(const struct device *dev, umode_t *mode)116{117 if (mode)118 *mode = 0666;119 return NULL;120}121 122static const struct class class_genwqe = {123 .name = GENWQE_DEVNAME,124 .devnode = genwqe_devnode,125};126 127/**128 * genwqe_dev_alloc() - Create and prepare a new card descriptor129 *130 * Return: Pointer to card descriptor, or ERR_PTR(err) on error131 */132static struct genwqe_dev *genwqe_dev_alloc(void)133{134 unsigned int i = 0, j;135 struct genwqe_dev *cd;136 137 for (i = 0; i < GENWQE_CARD_NO_MAX; i++) {138 if (genwqe_devices[i] == NULL)139 break;140 }141 if (i >= GENWQE_CARD_NO_MAX)142 return ERR_PTR(-ENODEV);143 144 cd = kzalloc(sizeof(struct genwqe_dev), GFP_KERNEL);145 if (!cd)146 return ERR_PTR(-ENOMEM);147 148 cd->card_idx = i;149 cd->class_genwqe = &class_genwqe;150 cd->debugfs_genwqe = debugfs_genwqe;151 152 /*153 * This comes from kernel config option and can be overritten via154 * debugfs.155 */156 cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY;157 158 init_waitqueue_head(&cd->queue_waitq);159 160 spin_lock_init(&cd->file_lock);161 INIT_LIST_HEAD(&cd->file_list);162 163 cd->card_state = GENWQE_CARD_UNUSED;164 spin_lock_init(&cd->print_lock);165 166 cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;167 cd->kill_timeout = GENWQE_KILL_TIMEOUT;168 169 for (j = 0; j < GENWQE_MAX_VFS; j++)170 cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;171 172 genwqe_devices[i] = cd;173 return cd;174}175 176static void genwqe_dev_free(struct genwqe_dev *cd)177{178 if (!cd)179 return;180 181 genwqe_devices[cd->card_idx] = NULL;182 kfree(cd);183}184 185/**186 * genwqe_bus_reset() - Card recovery187 * @cd: GenWQE device information188 *189 * pci_reset_function() will recover the device and ensure that the190 * registers are accessible again when it completes with success. If191 * not, the card will stay dead and registers will be unaccessible192 * still.193 */194static int genwqe_bus_reset(struct genwqe_dev *cd)195{196 int rc = 0;197 struct pci_dev *pci_dev = cd->pci_dev;198 void __iomem *mmio;199 200 if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE)201 return -EIO;202 203 mmio = cd->mmio;204 cd->mmio = NULL;205 pci_iounmap(pci_dev, mmio);206 207 pci_release_mem_regions(pci_dev);208 209 /*210 * Firmware/BIOS might change memory mapping during bus reset.211 * Settings like enable bus-mastering, ... are backuped and212 * restored by the pci_reset_function().213 */214 dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__);215 rc = pci_reset_function(pci_dev);216 if (rc) {217 dev_err(&pci_dev->dev,218 "[%s] err: failed reset func (rc %d)\n", __func__, rc);219 return rc;220 }221 dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc);222 223 /*224 * Here is the right spot to clear the register read225 * failure. pci_bus_reset() does this job in real systems.226 */227 cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |228 GENWQE_INJECT_GFIR_FATAL |229 GENWQE_INJECT_GFIR_INFO);230 231 rc = pci_request_mem_regions(pci_dev, genwqe_driver_name);232 if (rc) {233 dev_err(&pci_dev->dev,234 "[%s] err: request bars failed (%d)\n", __func__, rc);235 return -EIO;236 }237 238 cd->mmio = pci_iomap(pci_dev, 0, 0);239 if (cd->mmio == NULL) {240 dev_err(&pci_dev->dev,241 "[%s] err: mapping BAR0 failed\n", __func__);242 return -ENOMEM;243 }244 return 0;245}246 247/*248 * Hardware circumvention section. Certain bitstreams in our test-lab249 * had different kinds of problems. Here is where we adjust those250 * bitstreams to function will with this version of our device driver.251 *252 * Thise circumventions are applied to the physical function only.253 * The magical numbers below are identifying development/manufacturing254 * versions of the bitstream used on the card.255 *256 * Turn off error reporting for old/manufacturing images.257 */258 259bool genwqe_need_err_masking(struct genwqe_dev *cd)260{261 return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;262}263 264static void genwqe_tweak_hardware(struct genwqe_dev *cd)265{266 struct pci_dev *pci_dev = cd->pci_dev;267 268 /* Mask FIRs for development images */269 if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) &&270 ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) {271 dev_warn(&pci_dev->dev,272 "FIRs masked due to bitstream %016llx.%016llx\n",273 cd->slu_unitcfg, cd->app_unitcfg);274 275 __genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR,276 0xFFFFFFFFFFFFFFFFull);277 278 __genwqe_writeq(cd, IO_APP_ERR_ACT_MASK,279 0x0000000000000000ull);280 }281}282 283/**284 * genwqe_recovery_on_fatal_gfir_required() - Version depended actions285 * @cd: GenWQE device information286 *287 * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must288 * be ignored. This is e.g. true for the bitstream we gave to the card289 * manufacturer, but also for some old bitstreams we released to our290 * test-lab.291 */292int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd)293{294 return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull;295}296 297int genwqe_flash_readback_fails(struct genwqe_dev *cd)298{299 return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;300}301 302/**303 * genwqe_T_psec() - Calculate PF/VF timeout register content304 * @cd: GenWQE device information305 *306 * Note: From a design perspective it turned out to be a bad idea to307 * use codes here to specifiy the frequency/speed values. An old308 * driver cannot understand new codes and is therefore always a309 * problem. Better is to measure out the value or put the310 * speed/frequency directly into a register which is always a valid311 * value for old as well as for new software.312 */313/* T = 1/f */314static int genwqe_T_psec(struct genwqe_dev *cd)315{316 u16 speed; /* 1/f -> 250, 200, 166, 175 */317 static const int T[] = { 4000, 5000, 6000, 5714 };318 319 speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);320 if (speed >= ARRAY_SIZE(T))321 return -1; /* illegal value */322 323 return T[speed];324}325 326/**327 * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution328 * @cd: GenWQE device information329 *330 * Do this _after_ card_reset() is called. Otherwise the values will331 * vanish. The settings need to be done when the queues are inactive.332 *333 * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.334 * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.335 */336static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)337{338 u32 T = genwqe_T_psec(cd);339 u64 x;340 341 if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)342 return false;343 344 /* PF: large value needed, flash update 2sec per block */345 x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *346 16000000000uL/(T * 15)) - 10;347 348 genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,349 0xff00 | (x & 0xff), 0);350 return true;351}352 353/**354 * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution355 * @cd: GenWQE device information356 */357static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd)358{359 struct pci_dev *pci_dev = cd->pci_dev;360 unsigned int vf;361 u32 T = genwqe_T_psec(cd);362 u64 x;363 int totalvfs;364 365 totalvfs = pci_sriov_get_totalvfs(pci_dev);366 if (totalvfs <= 0)367 return false;368 369 for (vf = 0; vf < totalvfs; vf++) {370 371 if (cd->vf_jobtimeout_msec[vf] == 0)372 continue;373 374 x = ilog2(cd->vf_jobtimeout_msec[vf] *375 16000000000uL/(T * 15)) - 10;376 377 genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,378 0xff00 | (x & 0xff), vf + 1);379 }380 return true;381}382 383static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd)384{385 unsigned int type, e = 0;386 387 for (type = 0; type < GENWQE_DBG_UNITS; type++) {388 switch (type) {389 case GENWQE_DBG_UNIT0:390 e = genwqe_ffdc_buff_size(cd, 0);391 break;392 case GENWQE_DBG_UNIT1:393 e = genwqe_ffdc_buff_size(cd, 1);394 break;395 case GENWQE_DBG_UNIT2:396 e = genwqe_ffdc_buff_size(cd, 2);397 break;398 case GENWQE_DBG_REGS:399 e = GENWQE_FFDC_REGS;400 break;401 }402 403 /* currently support only the debug units mentioned here */404 cd->ffdc[type].entries = e;405 cd->ffdc[type].regs =406 kmalloc_array(e, sizeof(struct genwqe_reg),407 GFP_KERNEL);408 /*409 * regs == NULL is ok, the using code treats this as no regs,410 * Printing warning is ok in this case.411 */412 }413 return 0;414}415 416static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd)417{418 unsigned int type;419 420 for (type = 0; type < GENWQE_DBG_UNITS; type++) {421 kfree(cd->ffdc[type].regs);422 cd->ffdc[type].regs = NULL;423 }424}425 426static int genwqe_read_ids(struct genwqe_dev *cd)427{428 int err = 0;429 int slu_id;430 struct pci_dev *pci_dev = cd->pci_dev;431 432 cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);433 if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) {434 dev_err(&pci_dev->dev,435 "err: SLUID=%016llx\n", cd->slu_unitcfg);436 err = -EIO;437 goto out_err;438 }439 440 slu_id = genwqe_get_slu_id(cd);441 if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) {442 dev_err(&pci_dev->dev,443 "err: incompatible SLU Architecture %u\n", slu_id);444 err = -ENOENT;445 goto out_err;446 }447 448 cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);449 if (cd->app_unitcfg == IO_ILLEGAL_VALUE) {450 dev_err(&pci_dev->dev,451 "err: APPID=%016llx\n", cd->app_unitcfg);452 err = -EIO;453 goto out_err;454 }455 genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name));456 457 /*458 * Is access to all registers possible? If we are a VF the459 * answer is obvious. If we run fully virtualized, we need to460 * check if we can access all registers. If we do not have461 * full access we will cause an UR and some informational FIRs462 * in the PF, but that should not harm.463 */464 if (pci_dev->is_virtfn)465 cd->is_privileged = 0;466 else467 cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM)468 != IO_ILLEGAL_VALUE);469 470 out_err:471 return err;472}473 474static int genwqe_start(struct genwqe_dev *cd)475{476 int err;477 struct pci_dev *pci_dev = cd->pci_dev;478 479 err = genwqe_read_ids(cd);480 if (err)481 return err;482 483 if (genwqe_is_privileged(cd)) {484 /* do this after the tweaks. alloc fail is acceptable */485 genwqe_ffdc_buffs_alloc(cd);486 genwqe_stop_traps(cd);487 488 /* Collect registers e.g. FIRs, UNITIDs, traces ... */489 genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs,490 cd->ffdc[GENWQE_DBG_REGS].entries, 0);491 492 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0,493 cd->ffdc[GENWQE_DBG_UNIT0].regs,494 cd->ffdc[GENWQE_DBG_UNIT0].entries);495 496 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1,497 cd->ffdc[GENWQE_DBG_UNIT1].regs,498 cd->ffdc[GENWQE_DBG_UNIT1].entries);499 500 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2,501 cd->ffdc[GENWQE_DBG_UNIT2].regs,502 cd->ffdc[GENWQE_DBG_UNIT2].entries);503 504 genwqe_start_traps(cd);505 506 if (cd->card_state == GENWQE_CARD_FATAL_ERROR) {507 dev_warn(&pci_dev->dev,508 "[%s] chip reload/recovery!\n", __func__);509 510 /*511 * Stealth Mode: Reload chip on either hot512 * reset or PERST.513 */514 cd->softreset = 0x7Cull;515 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,516 cd->softreset);517 518 err = genwqe_bus_reset(cd);519 if (err != 0) {520 dev_err(&pci_dev->dev,521 "[%s] err: bus reset failed!\n",522 __func__);523 goto out;524 }525 526 /*527 * Re-read the IDs because528 * it could happen that the bitstream load529 * failed!530 */531 err = genwqe_read_ids(cd);532 if (err)533 goto out;534 }535 }536 537 err = genwqe_setup_service_layer(cd); /* does a reset to the card */538 if (err != 0) {539 dev_err(&pci_dev->dev,540 "[%s] err: could not setup servicelayer!\n", __func__);541 err = -ENODEV;542 goto out;543 }544 545 if (genwqe_is_privileged(cd)) { /* code is running _after_ reset */546 genwqe_tweak_hardware(cd);547 548 genwqe_setup_pf_jtimer(cd);549 genwqe_setup_vf_jtimer(cd);550 }551 552 err = genwqe_device_create(cd);553 if (err < 0) {554 dev_err(&pci_dev->dev,555 "err: chdev init failed! (err=%d)\n", err);556 goto out_release_service_layer;557 }558 return 0;559 560 out_release_service_layer:561 genwqe_release_service_layer(cd);562 out:563 if (genwqe_is_privileged(cd))564 genwqe_ffdc_buffs_free(cd);565 return -EIO;566}567 568/**569 * genwqe_stop() - Stop card operation570 * @cd: GenWQE device information571 *572 * Recovery notes:573 * As long as genwqe_thread runs we might access registers during574 * error data capture. Same is with the genwqe_health_thread.575 * When genwqe_bus_reset() fails this function might called two times:576 * first by the genwqe_health_thread() and later by genwqe_remove() to577 * unbind the device. We must be able to survive that.578 *579 * This function must be robust enough to be called twice.580 */581static int genwqe_stop(struct genwqe_dev *cd)582{583 genwqe_finish_queue(cd); /* no register access */584 genwqe_device_remove(cd); /* device removed, procs killed */585 genwqe_release_service_layer(cd); /* here genwqe_thread is stopped */586 587 if (genwqe_is_privileged(cd)) {588 pci_disable_sriov(cd->pci_dev); /* access pci config space */589 genwqe_ffdc_buffs_free(cd);590 }591 592 return 0;593}594 595/**596 * genwqe_recover_card() - Try to recover the card if it is possible597 * @cd: GenWQE device information598 * @fatal_err: Indicate whether to attempt soft reset599 *600 * If fatal_err is set no register access is possible anymore. It is601 * likely that genwqe_start fails in that situation. Proper error602 * handling is required in this case.603 *604 * genwqe_bus_reset() will cause the pci code to call genwqe_remove()605 * and later genwqe_probe() for all virtual functions.606 */607static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err)608{609 int rc;610 struct pci_dev *pci_dev = cd->pci_dev;611 612 genwqe_stop(cd);613 614 /*615 * Make sure chip is not reloaded to maintain FFDC. Write SLU616 * Reset Register, CPLDReset field to 0.617 */618 if (!fatal_err) {619 cd->softreset = 0x70ull;620 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset);621 }622 623 rc = genwqe_bus_reset(cd);624 if (rc != 0) {625 dev_err(&pci_dev->dev,626 "[%s] err: card recovery impossible!\n", __func__);627 return rc;628 }629 630 rc = genwqe_start(cd);631 if (rc < 0) {632 dev_err(&pci_dev->dev,633 "[%s] err: failed to launch device!\n", __func__);634 return rc;635 }636 return 0;637}638 639static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir)640{641 *gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);642 return (*gfir & GFIR_ERR_TRIGGER) &&643 genwqe_recovery_on_fatal_gfir_required(cd);644}645 646/**647 * genwqe_fir_checking() - Check the fault isolation registers of the card648 * @cd: GenWQE device information649 *650 * If this code works ok, can be tried out with help of the genwqe_poke tool:651 * sudo ./tools/genwqe_poke 0x8 0xfefefefefef652 *653 * Now the relevant FIRs/sFIRs should be printed out and the driver should654 * invoke recovery (devices are removed and readded).655 */656static u64 genwqe_fir_checking(struct genwqe_dev *cd)657{658 int j, iterations = 0;659 u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec;660 u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr;661 struct pci_dev *pci_dev = cd->pci_dev;662 663 healthMonitor:664 iterations++;665 if (iterations > 16) {666 dev_err(&pci_dev->dev, "* exit looping after %d times\n",667 iterations);668 goto fatal_error;669 }670 671 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);672 if (gfir != 0x0)673 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n",674 IO_SLC_CFGREG_GFIR, gfir);675 if (gfir == IO_ILLEGAL_VALUE)676 goto fatal_error;677 678 /*679 * Avoid printing when to GFIR bit is on prevents contignous680 * printout e.g. for the following bug:681 * FIR set without a 2ndary FIR/FIR cannot be cleared682 * Comment out the following if to get the prints:683 */684 if (gfir == 0)685 return 0;686 687 gfir_masked = gfir & GFIR_ERR_TRIGGER; /* fatal errors */688 689 for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */690 691 /* read the primary FIR (pfir) */692 fir_addr = (uid << 24) + 0x08;693 fir = __genwqe_readq(cd, fir_addr);694 if (fir == 0x0)695 continue; /* no error in this unit */696 697 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir);698 if (fir == IO_ILLEGAL_VALUE)699 goto fatal_error;700 701 /* read primary FEC */702 fec_addr = (uid << 24) + 0x18;703 fec = __genwqe_readq(cd, fec_addr);704 705 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec);706 if (fec == IO_ILLEGAL_VALUE)707 goto fatal_error;708 709 for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) {710 711 /* secondary fir empty, skip it */712 if ((fir & mask) == 0x0)713 continue;714 715 sfir_addr = (uid << 24) + 0x100 + 0x08 * j;716 sfir = __genwqe_readq(cd, sfir_addr);717 718 if (sfir == IO_ILLEGAL_VALUE)719 goto fatal_error;720 dev_err(&pci_dev->dev,721 "* 0x%08x 0x%016llx\n", sfir_addr, sfir);722 723 sfec_addr = (uid << 24) + 0x300 + 0x08 * j;724 sfec = __genwqe_readq(cd, sfec_addr);725 726 if (sfec == IO_ILLEGAL_VALUE)727 goto fatal_error;728 dev_err(&pci_dev->dev,729 "* 0x%08x 0x%016llx\n", sfec_addr, sfec);730 731 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);732 if (gfir == IO_ILLEGAL_VALUE)733 goto fatal_error;734 735 /* gfir turned on during routine! get out and736 start over. */737 if ((gfir_masked == 0x0) &&738 (gfir & GFIR_ERR_TRIGGER)) {739 goto healthMonitor;740 }741 742 /* do not clear if we entered with a fatal gfir */743 if (gfir_masked == 0x0) {744 745 /* NEW clear by mask the logged bits */746 sfir_addr = (uid << 24) + 0x100 + 0x08 * j;747 __genwqe_writeq(cd, sfir_addr, sfir);748 749 dev_dbg(&pci_dev->dev,750 "[HM] Clearing 2ndary FIR 0x%08x with 0x%016llx\n",751 sfir_addr, sfir);752 753 /*754 * note, these cannot be error-Firs755 * since gfir_masked is 0 after sfir756 * was read. Also, it is safe to do757 * this write if sfir=0. Still need to758 * clear the primary. This just means759 * there is no secondary FIR.760 */761 762 /* clear by mask the logged bit. */763 fir_clr_addr = (uid << 24) + 0x10;764 __genwqe_writeq(cd, fir_clr_addr, mask);765 766 dev_dbg(&pci_dev->dev,767 "[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",768 fir_clr_addr, mask);769 }770 }771 }772 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);773 if (gfir == IO_ILLEGAL_VALUE)774 goto fatal_error;775 776 if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) {777 /*778 * Check once more that it didn't go on after all the779 * FIRS were cleared.780 */781 dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n",782 iterations);783 goto healthMonitor;784 }785 return gfir_masked;786 787 fatal_error:788 return IO_ILLEGAL_VALUE;789}790 791/**792 * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot793 * @pci_dev: PCI device information struct794 *795 * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this796 * reset method will not work in all cases.797 *798 * Return: 0 on success or error code from pci_set_pcie_reset_state()799 */800static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev)801{802 int rc;803 804 /*805 * lock pci config space access from userspace,806 * save state and issue PCIe fundamental reset807 */808 pci_cfg_access_lock(pci_dev);809 pci_save_state(pci_dev);810 rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset);811 if (!rc) {812 /* keep PCIe reset asserted for 250ms */813 msleep(250);814 pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset);815 /* Wait for 2s to reload flash and train the link */816 msleep(2000);817 }818 pci_restore_state(pci_dev);819 pci_cfg_access_unlock(pci_dev);820 return rc;821}822 823 824static int genwqe_platform_recovery(struct genwqe_dev *cd)825{826 struct pci_dev *pci_dev = cd->pci_dev;827 int rc;828 829 dev_info(&pci_dev->dev,830 "[%s] resetting card for error recovery\n", __func__);831 832 /* Clear out error injection flags */833 cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |834 GENWQE_INJECT_GFIR_FATAL |835 GENWQE_INJECT_GFIR_INFO);836 837 genwqe_stop(cd);838 839 /* Try recoverying the card with fundamental reset */840 rc = genwqe_pci_fundamental_reset(pci_dev);841 if (!rc) {842 rc = genwqe_start(cd);843 if (!rc)844 dev_info(&pci_dev->dev,845 "[%s] card recovered\n", __func__);846 else847 dev_err(&pci_dev->dev,848 "[%s] err: cannot start card services! (err=%d)\n",849 __func__, rc);850 } else {851 dev_err(&pci_dev->dev,852 "[%s] card reset failed\n", __func__);853 }854 855 return rc;856}857 858/**859 * genwqe_reload_bistream() - reload card bitstream860 * @cd: GenWQE device information861 *862 * Set the appropriate register and call fundamental reset to reaload the card863 * bitstream.864 *865 * Return: 0 on success, error code otherwise866 */867static int genwqe_reload_bistream(struct genwqe_dev *cd)868{869 struct pci_dev *pci_dev = cd->pci_dev;870 int rc;871 872 dev_info(&pci_dev->dev,873 "[%s] resetting card for bitstream reload\n",874 __func__);875 876 genwqe_stop(cd);877 878 /*879 * Cause a CPLD reprogram with the 'next_bitstream'880 * partition on PCIe hot or fundamental reset881 */882 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,883 (cd->softreset & 0xcull) | 0x70ull);884 885 rc = genwqe_pci_fundamental_reset(pci_dev);886 if (rc) {887 /*888 * A fundamental reset failure can be caused889 * by lack of support on the arch, so we just890 * log the error and try to start the card891 * again.892 */893 dev_err(&pci_dev->dev,894 "[%s] err: failed to reset card for bitstream reload\n",895 __func__);896 }897 898 rc = genwqe_start(cd);899 if (rc) {900 dev_err(&pci_dev->dev,901 "[%s] err: cannot start card services! (err=%d)\n",902 __func__, rc);903 return rc;904 }905 dev_info(&pci_dev->dev,906 "[%s] card reloaded\n", __func__);907 return 0;908}909 910 911/**912 * genwqe_health_thread() - Health checking thread913 * @data: GenWQE device information914 *915 * This thread is only started for the PF of the card.916 *917 * This thread monitors the health of the card. A critical situation918 * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In919 * this case we need to be recovered from outside. Writing to920 * registers will very likely not work either.921 *922 * This thread must only exit if kthread_should_stop() becomes true.923 *924 * Condition for the health-thread to trigger:925 * a) when a kthread_stop() request comes in or926 * b) a critical GFIR occured927 *928 * Informational GFIRs are checked and potentially printed in929 * GENWQE_HEALTH_CHECK_INTERVAL seconds.930 */931static int genwqe_health_thread(void *data)932{933 int rc, should_stop = 0;934 struct genwqe_dev *cd = data;935 struct pci_dev *pci_dev = cd->pci_dev;936 u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg;937 938 health_thread_begin:939 while (!kthread_should_stop()) {940 rc = wait_event_interruptible_timeout(cd->health_waitq,941 (genwqe_health_check_cond(cd, &gfir) ||942 (should_stop = kthread_should_stop())),943 GENWQE_HEALTH_CHECK_INTERVAL * HZ);944 945 if (should_stop)946 break;947 948 if (gfir == IO_ILLEGAL_VALUE) {949 dev_err(&pci_dev->dev,950 "[%s] GFIR=%016llx\n", __func__, gfir);951 goto fatal_error;952 }953 954 slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);955 if (slu_unitcfg == IO_ILLEGAL_VALUE) {956 dev_err(&pci_dev->dev,957 "[%s] SLU_UNITCFG=%016llx\n",958 __func__, slu_unitcfg);959 goto fatal_error;960 }961 962 app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);963 if (app_unitcfg == IO_ILLEGAL_VALUE) {964 dev_err(&pci_dev->dev,965 "[%s] APP_UNITCFG=%016llx\n",966 __func__, app_unitcfg);967 goto fatal_error;968 }969 970 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);971 if (gfir == IO_ILLEGAL_VALUE) {972 dev_err(&pci_dev->dev,973 "[%s] %s: GFIR=%016llx\n", __func__,974 (gfir & GFIR_ERR_TRIGGER) ? "err" : "info",975 gfir);976 goto fatal_error;977 }978 979 gfir_masked = genwqe_fir_checking(cd);980 if (gfir_masked == IO_ILLEGAL_VALUE)981 goto fatal_error;982 983 /*984 * GFIR ErrorTrigger bits set => reset the card!985 * Never do this for old/manufacturing images!986 */987 if ((gfir_masked) && !cd->skip_recovery &&988 genwqe_recovery_on_fatal_gfir_required(cd)) {989 990 cd->card_state = GENWQE_CARD_FATAL_ERROR;991 992 rc = genwqe_recover_card(cd, 0);993 if (rc < 0) {994 /* FIXME Card is unusable and needs unbind! */995 goto fatal_error;996 }997 }998 999 if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) {1000 /* Userspace requested card bitstream reload */1001 rc = genwqe_reload_bistream(cd);1002 if (rc)1003 goto fatal_error;1004 }1005 1006 cd->last_gfir = gfir;1007 cond_resched();1008 }1009 1010 return 0;1011 1012 fatal_error:1013 if (cd->use_platform_recovery) {1014 /*1015 * Since we use raw accessors, EEH errors won't be detected1016 * by the platform until we do a non-raw MMIO or config space1017 * read1018 */1019 readq(cd->mmio + IO_SLC_CFGREG_GFIR);1020 1021 /* We do nothing if the card is going over PCI recovery */1022 if (pci_channel_offline(pci_dev))1023 return -EIO;1024 1025 /*1026 * If it's supported by the platform, we try a fundamental reset1027 * to recover from a fatal error. Otherwise, we continue to wait1028 * for an external recovery procedure to take care of it.1029 */1030 rc = genwqe_platform_recovery(cd);1031 if (!rc)1032 goto health_thread_begin;1033 }1034 1035 dev_err(&pci_dev->dev,1036 "[%s] card unusable. Please trigger unbind!\n", __func__);1037 1038 /* Bring down logical devices to inform user space via udev remove. */1039 cd->card_state = GENWQE_CARD_FATAL_ERROR;1040 genwqe_stop(cd);1041 1042 /* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */1043 while (!kthread_should_stop())1044 cond_resched();1045 1046 return -EIO;1047}1048 1049static int genwqe_health_check_start(struct genwqe_dev *cd)1050{1051 int rc;1052 1053 if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)1054 return 0; /* valid for disabling the service */1055 1056 /* moved before request_irq() */1057 /* init_waitqueue_head(&cd->health_waitq); */1058 1059 cd->health_thread = kthread_run(genwqe_health_thread, cd,1060 GENWQE_DEVNAME "%d_health",1061 cd->card_idx);1062 if (IS_ERR(cd->health_thread)) {1063 rc = PTR_ERR(cd->health_thread);1064 cd->health_thread = NULL;1065 return rc;1066 }1067 return 0;1068}1069 1070static int genwqe_health_thread_running(struct genwqe_dev *cd)1071{1072 return cd->health_thread != NULL;1073}1074 1075static int genwqe_health_check_stop(struct genwqe_dev *cd)1076{1077 if (!genwqe_health_thread_running(cd))1078 return -EIO;1079 1080 kthread_stop(cd->health_thread);1081 cd->health_thread = NULL;1082 return 0;1083}1084 1085/**1086 * genwqe_pci_setup() - Allocate PCIe related resources for our card1087 * @cd: GenWQE device information1088 */1089static int genwqe_pci_setup(struct genwqe_dev *cd)1090{1091 int err;1092 struct pci_dev *pci_dev = cd->pci_dev;1093 1094 err = pci_enable_device_mem(pci_dev);1095 if (err) {1096 dev_err(&pci_dev->dev,1097 "err: failed to enable pci memory (err=%d)\n", err);1098 goto err_out;1099 }1100 1101 /* Reserve PCI I/O and memory resources */1102 err = pci_request_mem_regions(pci_dev, genwqe_driver_name);1103 if (err) {1104 dev_err(&pci_dev->dev,1105 "[%s] err: request bars failed (%d)\n", __func__, err);1106 err = -EIO;1107 goto err_disable_device;1108 }1109 1110 /* check for 64-bit DMA address supported (DAC) */1111 /* check for 32-bit DMA address supported (SAC) */1112 if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) &&1113 dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) {1114 dev_err(&pci_dev->dev,1115 "err: neither DMA32 nor DMA64 supported\n");1116 err = -EIO;1117 goto out_release_resources;1118 }1119 1120 pci_set_master(pci_dev);1121 1122 /* EEH recovery requires PCIe fundamental reset */1123 pci_dev->needs_freset = 1;1124 1125 /* request complete BAR-0 space (length = 0) */1126 cd->mmio_len = pci_resource_len(pci_dev, 0);1127 cd->mmio = pci_iomap(pci_dev, 0, 0);1128 if (cd->mmio == NULL) {1129 dev_err(&pci_dev->dev,1130 "[%s] err: mapping BAR0 failed\n", __func__);1131 err = -ENOMEM;1132 goto out_release_resources;1133 }1134 1135 cd->num_vfs = pci_sriov_get_totalvfs(pci_dev);1136 if (cd->num_vfs < 0)1137 cd->num_vfs = 0;1138 1139 err = genwqe_read_ids(cd);1140 if (err)1141 goto out_iounmap;1142 1143 return 0;1144 1145 out_iounmap:1146 pci_iounmap(pci_dev, cd->mmio);1147 out_release_resources:1148 pci_release_mem_regions(pci_dev);1149 err_disable_device:1150 pci_disable_device(pci_dev);1151 err_out:1152 return err;1153}1154 1155/**1156 * genwqe_pci_remove() - Free PCIe related resources for our card1157 * @cd: GenWQE device information1158 */1159static void genwqe_pci_remove(struct genwqe_dev *cd)1160{1161 struct pci_dev *pci_dev = cd->pci_dev;1162 1163 if (cd->mmio)1164 pci_iounmap(pci_dev, cd->mmio);1165 1166 pci_release_mem_regions(pci_dev);1167 pci_disable_device(pci_dev);1168}1169 1170/**1171 * genwqe_probe() - Device initialization1172 * @pci_dev: PCI device information struct1173 * @id: PCI device ID1174 *1175 * Callable for multiple cards. This function is called on bind.1176 *1177 * Return: 0 if succeeded, < 0 when failed1178 */1179static int genwqe_probe(struct pci_dev *pci_dev,1180 const struct pci_device_id *id)1181{1182 int err;1183 struct genwqe_dev *cd;1184 1185 genwqe_init_crc32();1186 1187 cd = genwqe_dev_alloc();1188 if (IS_ERR(cd)) {1189 dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n",1190 (int)PTR_ERR(cd));1191 return PTR_ERR(cd);1192 }1193 1194 dev_set_drvdata(&pci_dev->dev, cd);1195 cd->pci_dev = pci_dev;1196 1197 err = genwqe_pci_setup(cd);1198 if (err < 0) {1199 dev_err(&pci_dev->dev,1200 "err: problems with PCI setup (err=%d)\n", err);1201 goto out_free_dev;1202 }1203 1204 err = genwqe_start(cd);1205 if (err < 0) {1206 dev_err(&pci_dev->dev,1207 "err: cannot start card services! (err=%d)\n", err);1208 goto out_pci_remove;1209 }1210 1211 if (genwqe_is_privileged(cd)) {1212 err = genwqe_health_check_start(cd);1213 if (err < 0) {1214 dev_err(&pci_dev->dev,1215 "err: cannot start health checking! (err=%d)\n",1216 err);1217 goto out_stop_services;1218 }1219 }1220 return 0;1221 1222 out_stop_services:1223 genwqe_stop(cd);1224 out_pci_remove:1225 genwqe_pci_remove(cd);1226 out_free_dev:1227 genwqe_dev_free(cd);1228 return err;1229}1230 1231/**1232 * genwqe_remove() - Called when device is removed (hot-plugable)1233 * @pci_dev: PCI device information struct1234 *1235 * Or when driver is unloaded respecitively when unbind is done.1236 */1237static void genwqe_remove(struct pci_dev *pci_dev)1238{1239 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);1240 1241 genwqe_health_check_stop(cd);1242 1243 /*1244 * genwqe_stop() must survive if it is called twice1245 * sequentially. This happens when the health thread calls it1246 * and fails on genwqe_bus_reset().1247 */1248 genwqe_stop(cd);1249 genwqe_pci_remove(cd);1250 genwqe_dev_free(cd);1251}1252 1253/**1254 * genwqe_err_error_detected() - Error detection callback1255 * @pci_dev: PCI device information struct1256 * @state: PCI channel state1257 *1258 * This callback is called by the PCI subsystem whenever a PCI bus1259 * error is detected.1260 */1261static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev,1262 pci_channel_state_t state)1263{1264 struct genwqe_dev *cd;1265 1266 dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state);1267 1268 cd = dev_get_drvdata(&pci_dev->dev);1269 if (cd == NULL)1270 return PCI_ERS_RESULT_DISCONNECT;1271 1272 /* Stop the card */1273 genwqe_health_check_stop(cd);1274 genwqe_stop(cd);1275 1276 /*1277 * On permanent failure, the PCI code will call device remove1278 * after the return of this function.1279 * genwqe_stop() can be called twice.1280 */1281 if (state == pci_channel_io_perm_failure) {1282 return PCI_ERS_RESULT_DISCONNECT;1283 } else {1284 genwqe_pci_remove(cd);1285 return PCI_ERS_RESULT_NEED_RESET;1286 }1287}1288 1289static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev)1290{1291 int rc;1292 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);1293 1294 rc = genwqe_pci_setup(cd);1295 if (!rc) {1296 return PCI_ERS_RESULT_RECOVERED;1297 } else {1298 dev_err(&pci_dev->dev,1299 "err: problems with PCI setup (err=%d)\n", rc);1300 return PCI_ERS_RESULT_DISCONNECT;1301 }1302}1303 1304static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev)1305{1306 return PCI_ERS_RESULT_NONE;1307}1308 1309static void genwqe_err_resume(struct pci_dev *pci_dev)1310{1311 int rc;1312 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);1313 1314 rc = genwqe_start(cd);1315 if (!rc) {1316 rc = genwqe_health_check_start(cd);1317 if (rc)1318 dev_err(&pci_dev->dev,1319 "err: cannot start health checking! (err=%d)\n",1320 rc);1321 } else {1322 dev_err(&pci_dev->dev,1323 "err: cannot start card services! (err=%d)\n", rc);1324 }1325}1326 1327static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs)1328{1329 int rc;1330 struct genwqe_dev *cd = dev_get_drvdata(&dev->dev);1331 1332 if (numvfs > 0) {1333 genwqe_setup_vf_jtimer(cd);1334 rc = pci_enable_sriov(dev, numvfs);1335 if (rc < 0)1336 return rc;1337 return numvfs;1338 }1339 if (numvfs == 0) {1340 pci_disable_sriov(dev);1341 return 0;1342 }1343 return 0;1344}1345 1346static const struct pci_error_handlers genwqe_err_handler = {1347 .error_detected = genwqe_err_error_detected,1348 .mmio_enabled = genwqe_err_result_none,1349 .slot_reset = genwqe_err_slot_reset,1350 .resume = genwqe_err_resume,1351};1352 1353static struct pci_driver genwqe_driver = {1354 .name = genwqe_driver_name,1355 .id_table = genwqe_device_table,1356 .probe = genwqe_probe,1357 .remove = genwqe_remove,1358 .sriov_configure = genwqe_sriov_configure,1359 .err_handler = &genwqe_err_handler,1360};1361 1362/**1363 * genwqe_init_module() - Driver registration and initialization1364 */1365static int __init genwqe_init_module(void)1366{1367 int rc;1368 1369 rc = class_register(&class_genwqe);1370 if (rc) {1371 pr_err("[%s] create class failed\n", __func__);1372 return -ENOMEM;1373 }1374 1375 debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);1376 1377 rc = pci_register_driver(&genwqe_driver);1378 if (rc != 0) {1379 pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc);1380 goto err_out0;1381 }1382 1383 return rc;1384 1385 err_out0:1386 debugfs_remove(debugfs_genwqe);1387 class_unregister(&class_genwqe);1388 return rc;1389}1390 1391/**1392 * genwqe_exit_module() - Driver exit1393 */1394static void __exit genwqe_exit_module(void)1395{1396 pci_unregister_driver(&genwqe_driver);1397 debugfs_remove(debugfs_genwqe);1398 class_unregister(&class_genwqe);1399}1400 1401module_init(genwqe_init_module);1402module_exit(genwqe_exit_module);1403