1383 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/*3 * Copyright(c) 2015 - 2019 Intel Corporation.4 */5 6#include <linux/bitfield.h>7#include <linux/pci.h>8#include <linux/io.h>9#include <linux/delay.h>10#include <linux/vmalloc.h>11#include <linux/module.h>12 13#include "hfi.h"14#include "chip_registers.h"15#include "aspm.h"16 17/*18 * This file contains PCIe utility routines.19 */20 21/*22 * Do all the common PCIe setup and initialization.23 */24int hfi1_pcie_init(struct hfi1_devdata *dd)25{26 int ret;27 struct pci_dev *pdev = dd->pcidev;28 29 ret = pci_enable_device(pdev);30 if (ret) {31 /*32 * This can happen (in theory) iff:33 * We did a chip reset, and then failed to reprogram the34 * BAR, or the chip reset due to an internal error. We then35 * unloaded the driver and reloaded it.36 *37 * Both reset cases set the BAR back to initial state. For38 * the latter case, the AER sticky error bit at offset 0x71839 * should be set, but the Linux kernel doesn't yet know40 * about that, it appears. If the original BAR was retained41 * in the kernel data structures, this may be OK.42 */43 dd_dev_err(dd, "pci enable failed: error %d\n", -ret);44 return ret;45 }46 47 ret = pci_request_regions(pdev, DRIVER_NAME);48 if (ret) {49 dd_dev_err(dd, "pci_request_regions fails: err %d\n", -ret);50 goto bail;51 }52 53 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));54 if (ret) {55 /*56 * If the 64 bit setup fails, try 32 bit. Some systems57 * do not setup 64 bit maps on systems with 2GB or less58 * memory installed.59 */60 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));61 if (ret) {62 dd_dev_err(dd, "Unable to set DMA mask: %d\n", ret);63 goto bail;64 }65 }66 67 pci_set_master(pdev);68 return 0;69 70bail:71 hfi1_pcie_cleanup(pdev);72 return ret;73}74 75/*76 * Clean what was done in hfi1_pcie_init()77 */78void hfi1_pcie_cleanup(struct pci_dev *pdev)79{80 pci_disable_device(pdev);81 /*82 * Release regions should be called after the disable. OK to83 * call if request regions has not been called or failed.84 */85 pci_release_regions(pdev);86}87 88/*89 * Do remaining PCIe setup, once dd is allocated, and save away90 * fields required to re-initialize after a chip reset, or for91 * various other purposes92 */93int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)94{95 unsigned long len;96 resource_size_t addr;97 int ret = 0;98 u32 rcv_array_count;99 100 addr = pci_resource_start(pdev, 0);101 len = pci_resource_len(pdev, 0);102 103 /*104 * The TXE PIO buffers are at the tail end of the chip space.105 * Cut them off and map them separately.106 */107 108 /* sanity check vs expectations */109 if (len != TXE_PIO_SEND + TXE_PIO_SIZE) {110 dd_dev_err(dd, "chip PIO range does not match\n");111 return -EINVAL;112 }113 114 dd->kregbase1 = ioremap(addr, RCV_ARRAY);115 if (!dd->kregbase1) {116 dd_dev_err(dd, "UC mapping of kregbase1 failed\n");117 return -ENOMEM;118 }119 dd_dev_info(dd, "UC base1: %p for %x\n", dd->kregbase1, RCV_ARRAY);120 121 /* verify that reads actually work, save revision for reset check */122 dd->revision = readq(dd->kregbase1 + CCE_REVISION);123 if (dd->revision == ~(u64)0) {124 dd_dev_err(dd, "Cannot read chip CSRs\n");125 goto nomem;126 }127 128 rcv_array_count = readq(dd->kregbase1 + RCV_ARRAY_CNT);129 dd_dev_info(dd, "RcvArray count: %u\n", rcv_array_count);130 dd->base2_start = RCV_ARRAY + rcv_array_count * 8;131 132 dd->kregbase2 = ioremap(133 addr + dd->base2_start,134 TXE_PIO_SEND - dd->base2_start);135 if (!dd->kregbase2) {136 dd_dev_err(dd, "UC mapping of kregbase2 failed\n");137 goto nomem;138 }139 dd_dev_info(dd, "UC base2: %p for %x\n", dd->kregbase2,140 TXE_PIO_SEND - dd->base2_start);141 142 dd->piobase = ioremap_wc(addr + TXE_PIO_SEND, TXE_PIO_SIZE);143 if (!dd->piobase) {144 dd_dev_err(dd, "WC mapping of send buffers failed\n");145 goto nomem;146 }147 dd_dev_info(dd, "WC piobase: %p for %x\n", dd->piobase, TXE_PIO_SIZE);148 149 dd->physaddr = addr; /* used for io_remap, etc. */150 151 /*152 * Map the chip's RcvArray as write-combining to allow us153 * to write an entire cacheline worth of entries in one shot.154 */155 dd->rcvarray_wc = ioremap_wc(addr + RCV_ARRAY,156 rcv_array_count * 8);157 if (!dd->rcvarray_wc) {158 dd_dev_err(dd, "WC mapping of receive array failed\n");159 goto nomem;160 }161 dd_dev_info(dd, "WC RcvArray: %p for %x\n",162 dd->rcvarray_wc, rcv_array_count * 8);163 164 dd->flags |= HFI1_PRESENT; /* chip.c CSR routines now work */165 return 0;166nomem:167 ret = -ENOMEM;168 hfi1_pcie_ddcleanup(dd);169 return ret;170}171 172/*173 * Do PCIe cleanup related to dd, after chip-specific cleanup, etc. Just prior174 * to releasing the dd memory.175 * Void because all of the core pcie cleanup functions are void.176 */177void hfi1_pcie_ddcleanup(struct hfi1_devdata *dd)178{179 dd->flags &= ~HFI1_PRESENT;180 if (dd->kregbase1)181 iounmap(dd->kregbase1);182 dd->kregbase1 = NULL;183 if (dd->kregbase2)184 iounmap(dd->kregbase2);185 dd->kregbase2 = NULL;186 if (dd->rcvarray_wc)187 iounmap(dd->rcvarray_wc);188 dd->rcvarray_wc = NULL;189 if (dd->piobase)190 iounmap(dd->piobase);191 dd->piobase = NULL;192}193 194/* return the PCIe link speed from the given link status */195static u32 extract_speed(u16 linkstat)196{197 u32 speed;198 199 switch (linkstat & PCI_EXP_LNKSTA_CLS) {200 default: /* not defined, assume Gen1 */201 case PCI_EXP_LNKSTA_CLS_2_5GB:202 speed = 2500; /* Gen 1, 2.5GHz */203 break;204 case PCI_EXP_LNKSTA_CLS_5_0GB:205 speed = 5000; /* Gen 2, 5GHz */206 break;207 case PCI_EXP_LNKSTA_CLS_8_0GB:208 speed = 8000; /* Gen 3, 8GHz */209 break;210 }211 return speed;212}213 214/* read the link status and set dd->{lbus_width,lbus_speed,lbus_info} */215static void update_lbus_info(struct hfi1_devdata *dd)216{217 u16 linkstat;218 int ret;219 220 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat);221 if (ret) {222 dd_dev_err(dd, "Unable to read from PCI config\n");223 return;224 }225 226 dd->lbus_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, linkstat);227 dd->lbus_speed = extract_speed(linkstat);228 snprintf(dd->lbus_info, sizeof(dd->lbus_info),229 "PCIe,%uMHz,x%u", dd->lbus_speed, dd->lbus_width);230}231 232/*233 * Read in the current PCIe link width and speed. Find if the link is234 * Gen3 capable.235 */236int pcie_speeds(struct hfi1_devdata *dd)237{238 u32 linkcap;239 struct pci_dev *parent = dd->pcidev->bus->self;240 int ret;241 242 if (!pci_is_pcie(dd->pcidev)) {243 dd_dev_err(dd, "Can't find PCI Express capability!\n");244 return -EINVAL;245 }246 247 /* find if our max speed is Gen3 and parent supports Gen3 speeds */248 dd->link_gen3_capable = 1;249 250 ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);251 if (ret) {252 dd_dev_err(dd, "Unable to read from PCI config\n");253 return pcibios_err_to_errno(ret);254 }255 256 if ((linkcap & PCI_EXP_LNKCAP_SLS) != PCI_EXP_LNKCAP_SLS_8_0GB) {257 dd_dev_info(dd,258 "This HFI is not Gen3 capable, max speed 0x%x, need 0x3\n",259 linkcap & PCI_EXP_LNKCAP_SLS);260 dd->link_gen3_capable = 0;261 }262 263 /*264 * bus->max_bus_speed is set from the bridge's linkcap Max Link Speed265 */266 if (parent &&267 (dd->pcidev->bus->max_bus_speed == PCIE_SPEED_2_5GT ||268 dd->pcidev->bus->max_bus_speed == PCIE_SPEED_5_0GT)) {269 dd_dev_info(dd, "Parent PCIe bridge does not support Gen3\n");270 dd->link_gen3_capable = 0;271 }272 273 /* obtain the link width and current speed */274 update_lbus_info(dd);275 276 dd_dev_info(dd, "%s\n", dd->lbus_info);277 278 return 0;279}280 281/*282 * Restore command and BARs after a reset has wiped them out283 *284 * Returns 0 on success, otherwise a negative error value285 */286int restore_pci_variables(struct hfi1_devdata *dd)287{288 int ret;289 290 ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);291 if (ret)292 goto error;293 294 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,295 dd->pcibar0);296 if (ret)297 goto error;298 299 ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,300 dd->pcibar1);301 if (ret)302 goto error;303 304 ret = pci_write_config_dword(dd->pcidev, PCI_ROM_ADDRESS, dd->pci_rom);305 if (ret)306 goto error;307 308 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL,309 dd->pcie_devctl);310 if (ret)311 goto error;312 313 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL,314 dd->pcie_lnkctl);315 if (ret)316 goto error;317 318 ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL2,319 dd->pcie_devctl2);320 if (ret)321 goto error;322 323 ret = pci_write_config_dword(dd->pcidev, PCI_CFG_MSIX0, dd->pci_msix0);324 if (ret)325 goto error;326 327 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) {328 ret = pci_write_config_dword(dd->pcidev, PCIE_CFG_TPH2,329 dd->pci_tph2);330 if (ret)331 goto error;332 }333 return 0;334 335error:336 dd_dev_err(dd, "Unable to write to PCI config\n");337 return pcibios_err_to_errno(ret);338}339 340/*341 * Save BARs and command to rewrite after device reset342 *343 * Returns 0 on success, otherwise a negative error value344 */345int save_pci_variables(struct hfi1_devdata *dd)346{347 int ret;348 349 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,350 &dd->pcibar0);351 if (ret)352 goto error;353 354 ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,355 &dd->pcibar1);356 if (ret)357 goto error;358 359 ret = pci_read_config_dword(dd->pcidev, PCI_ROM_ADDRESS, &dd->pci_rom);360 if (ret)361 goto error;362 363 ret = pci_read_config_word(dd->pcidev, PCI_COMMAND, &dd->pci_command);364 if (ret)365 goto error;366 367 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL,368 &dd->pcie_devctl);369 if (ret)370 goto error;371 372 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL,373 &dd->pcie_lnkctl);374 if (ret)375 goto error;376 377 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL2,378 &dd->pcie_devctl2);379 if (ret)380 goto error;381 382 ret = pci_read_config_dword(dd->pcidev, PCI_CFG_MSIX0, &dd->pci_msix0);383 if (ret)384 goto error;385 386 if (pci_find_ext_capability(dd->pcidev, PCI_EXT_CAP_ID_TPH)) {387 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_TPH2,388 &dd->pci_tph2);389 if (ret)390 goto error;391 }392 return 0;393 394error:395 dd_dev_err(dd, "Unable to read from PCI config\n");396 return pcibios_err_to_errno(ret);397}398 399/*400 * BIOS may not set PCIe bus-utilization parameters for best performance.401 * Check and optionally adjust them to maximize our throughput.402 */403static int hfi1_pcie_caps;404module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444);405MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)");406 407/**408 * tune_pcie_caps() - Code to adjust PCIe capabilities.409 * @dd: Valid device data structure410 *411 */412void tune_pcie_caps(struct hfi1_devdata *dd)413{414 struct pci_dev *parent;415 u16 rc_mpss, rc_mps, ep_mpss, ep_mps;416 u16 rc_mrrs, ep_mrrs, max_mrrs, ectl;417 int ret;418 419 /*420 * Turn on extended tags in DevCtl in case the BIOS has turned it off421 * to improve WFR SDMA bandwidth422 */423 ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, &ectl);424 if ((!ret) && !(ectl & PCI_EXP_DEVCTL_EXT_TAG)) {425 dd_dev_info(dd, "Enabling PCIe extended tags\n");426 ectl |= PCI_EXP_DEVCTL_EXT_TAG;427 ret = pcie_capability_write_word(dd->pcidev,428 PCI_EXP_DEVCTL, ectl);429 if (ret)430 dd_dev_info(dd, "Unable to write to PCI config\n");431 }432 /* Find out supported and configured values for parent (root) */433 parent = dd->pcidev->bus->self;434 /*435 * The driver cannot perform the tuning if it does not have436 * access to the upstream component.437 */438 if (!parent) {439 dd_dev_info(dd, "Parent not found\n");440 return;441 }442 if (!pci_is_root_bus(parent->bus)) {443 dd_dev_info(dd, "Parent not root\n");444 return;445 }446 if (!pci_is_pcie(parent)) {447 dd_dev_info(dd, "Parent is not PCI Express capable\n");448 return;449 }450 if (!pci_is_pcie(dd->pcidev)) {451 dd_dev_info(dd, "PCI device is not PCI Express capable\n");452 return;453 }454 rc_mpss = parent->pcie_mpss;455 rc_mps = ffs(pcie_get_mps(parent)) - 8;456 /* Find out supported and configured values for endpoint (us) */457 ep_mpss = dd->pcidev->pcie_mpss;458 ep_mps = ffs(pcie_get_mps(dd->pcidev)) - 8;459 460 /* Find max payload supported by root, endpoint */461 if (rc_mpss > ep_mpss)462 rc_mpss = ep_mpss;463 464 /* If Supported greater than limit in module param, limit it */465 if (rc_mpss > (hfi1_pcie_caps & 7))466 rc_mpss = hfi1_pcie_caps & 7;467 /* If less than (allowed, supported), bump root payload */468 if (rc_mpss > rc_mps) {469 rc_mps = rc_mpss;470 pcie_set_mps(parent, 128 << rc_mps);471 }472 /* If less than (allowed, supported), bump endpoint payload */473 if (rc_mpss > ep_mps) {474 ep_mps = rc_mpss;475 pcie_set_mps(dd->pcidev, 128 << ep_mps);476 }477 478 /*479 * Now the Read Request size.480 * No field for max supported, but PCIe spec limits it to 4096,481 * which is code '5' (log2(4096) - 7)482 */483 max_mrrs = 5;484 if (max_mrrs > ((hfi1_pcie_caps >> 4) & 7))485 max_mrrs = (hfi1_pcie_caps >> 4) & 7;486 487 max_mrrs = 128 << max_mrrs;488 rc_mrrs = pcie_get_readrq(parent);489 ep_mrrs = pcie_get_readrq(dd->pcidev);490 491 if (max_mrrs > rc_mrrs) {492 rc_mrrs = max_mrrs;493 pcie_set_readrq(parent, rc_mrrs);494 }495 if (max_mrrs > ep_mrrs) {496 ep_mrrs = max_mrrs;497 pcie_set_readrq(dd->pcidev, ep_mrrs);498 }499}500 501/* End of PCIe capability tuning */502 503/*504 * From here through hfi1_pci_err_handler definition is invoked via505 * PCI error infrastructure, registered via pci506 */507static pci_ers_result_t508pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)509{510 struct hfi1_devdata *dd = pci_get_drvdata(pdev);511 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED;512 513 switch (state) {514 case pci_channel_io_normal:515 dd_dev_info(dd, "State Normal, ignoring\n");516 break;517 518 case pci_channel_io_frozen:519 dd_dev_info(dd, "State Frozen, requesting reset\n");520 pci_disable_device(pdev);521 ret = PCI_ERS_RESULT_NEED_RESET;522 break;523 524 case pci_channel_io_perm_failure:525 if (dd) {526 dd_dev_info(dd, "State Permanent Failure, disabling\n");527 /* no more register accesses! */528 dd->flags &= ~HFI1_PRESENT;529 hfi1_disable_after_error(dd);530 }531 /* else early, or other problem */532 ret = PCI_ERS_RESULT_DISCONNECT;533 break;534 535 default: /* shouldn't happen */536 dd_dev_info(dd, "HFI1 PCI errors detected (state %d)\n",537 state);538 break;539 }540 return ret;541}542 543static pci_ers_result_t544pci_mmio_enabled(struct pci_dev *pdev)545{546 u64 words = 0U;547 struct hfi1_devdata *dd = pci_get_drvdata(pdev);548 pci_ers_result_t ret = PCI_ERS_RESULT_RECOVERED;549 550 if (dd && dd->pport) {551 words = read_port_cntr(dd->pport, C_RX_WORDS, CNTR_INVALID_VL);552 if (words == ~0ULL)553 ret = PCI_ERS_RESULT_NEED_RESET;554 dd_dev_info(dd,555 "HFI1 mmio_enabled function called, read wordscntr %llx, returning %d\n",556 words, ret);557 }558 return ret;559}560 561static pci_ers_result_t562pci_slot_reset(struct pci_dev *pdev)563{564 struct hfi1_devdata *dd = pci_get_drvdata(pdev);565 566 dd_dev_info(dd, "HFI1 slot_reset function called, ignored\n");567 return PCI_ERS_RESULT_CAN_RECOVER;568}569 570static void571pci_resume(struct pci_dev *pdev)572{573 struct hfi1_devdata *dd = pci_get_drvdata(pdev);574 575 dd_dev_info(dd, "HFI1 resume function called\n");576 /*577 * Running jobs will fail, since it's asynchronous578 * unlike sysfs-requested reset. Better than579 * doing nothing.580 */581 hfi1_init(dd, 1); /* same as re-init after reset */582}583 584const struct pci_error_handlers hfi1_pci_err_handler = {585 .error_detected = pci_error_detected,586 .mmio_enabled = pci_mmio_enabled,587 .slot_reset = pci_slot_reset,588 .resume = pci_resume,589};590 591/*============================================================================*/592/* PCIe Gen3 support */593 594/*595 * This code is separated out because it is expected to be removed in the596 * final shipping product. If not, then it will be revisited and items597 * will be moved to more standard locations.598 */599 600/* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_STS field values */601#define DL_STATUS_HFI0 0x1 /* hfi0 firmware download complete */602#define DL_STATUS_HFI1 0x2 /* hfi1 firmware download complete */603#define DL_STATUS_BOTH 0x3 /* hfi0 and hfi1 firmware download complete */604 605/* ASIC_PCI_SD_HOST_STATUS.FW_DNLD_ERR field values */606#define DL_ERR_NONE 0x0 /* no error */607#define DL_ERR_SWAP_PARITY 0x1 /* parity error in SerDes interrupt */608 /* or response data */609#define DL_ERR_DISABLED 0x2 /* hfi disabled */610#define DL_ERR_SECURITY 0x3 /* security check failed */611#define DL_ERR_SBUS 0x4 /* SBus status error */612#define DL_ERR_XFR_PARITY 0x5 /* parity error during ROM transfer*/613 614/* gasket block secondary bus reset delay */615#define SBR_DELAY_US 200000 /* 200ms */616 617static uint pcie_target = 3;618module_param(pcie_target, uint, S_IRUGO);619MODULE_PARM_DESC(pcie_target, "PCIe target speed (0 skip, 1-3 Gen1-3)");620 621static uint pcie_force;622module_param(pcie_force, uint, S_IRUGO);623MODULE_PARM_DESC(pcie_force, "Force driver to do a PCIe firmware download even if already at target speed");624 625static uint pcie_retry = 5;626module_param(pcie_retry, uint, S_IRUGO);627MODULE_PARM_DESC(pcie_retry, "Driver will try this many times to reach requested speed");628 629#define UNSET_PSET 255630#define DEFAULT_DISCRETE_PSET 2 /* discrete HFI */631#define DEFAULT_MCP_PSET 6 /* MCP HFI */632static uint pcie_pset = UNSET_PSET;633module_param(pcie_pset, uint, S_IRUGO);634MODULE_PARM_DESC(pcie_pset, "PCIe Eq Pset value to use, range is 0-10");635 636static uint pcie_ctle = 3; /* discrete on, integrated on */637module_param(pcie_ctle, uint, S_IRUGO);638MODULE_PARM_DESC(pcie_ctle, "PCIe static CTLE mode, bit 0 - discrete on/off, bit 1 - integrated on/off");639 640/* equalization columns */641#define PREC 0642#define ATTN 1643#define POST 2644 645/* discrete silicon preliminary equalization values */646static const u8 discrete_preliminary_eq[11][3] = {647 /* prec attn post */648 { 0x00, 0x00, 0x12 }, /* p0 */649 { 0x00, 0x00, 0x0c }, /* p1 */650 { 0x00, 0x00, 0x0f }, /* p2 */651 { 0x00, 0x00, 0x09 }, /* p3 */652 { 0x00, 0x00, 0x00 }, /* p4 */653 { 0x06, 0x00, 0x00 }, /* p5 */654 { 0x09, 0x00, 0x00 }, /* p6 */655 { 0x06, 0x00, 0x0f }, /* p7 */656 { 0x09, 0x00, 0x09 }, /* p8 */657 { 0x0c, 0x00, 0x00 }, /* p9 */658 { 0x00, 0x00, 0x18 }, /* p10 */659};660 661/* integrated silicon preliminary equalization values */662static const u8 integrated_preliminary_eq[11][3] = {663 /* prec attn post */664 { 0x00, 0x1e, 0x07 }, /* p0 */665 { 0x00, 0x1e, 0x05 }, /* p1 */666 { 0x00, 0x1e, 0x06 }, /* p2 */667 { 0x00, 0x1e, 0x04 }, /* p3 */668 { 0x00, 0x1e, 0x00 }, /* p4 */669 { 0x03, 0x1e, 0x00 }, /* p5 */670 { 0x04, 0x1e, 0x00 }, /* p6 */671 { 0x03, 0x1e, 0x06 }, /* p7 */672 { 0x03, 0x1e, 0x04 }, /* p8 */673 { 0x05, 0x1e, 0x00 }, /* p9 */674 { 0x00, 0x1e, 0x0a }, /* p10 */675};676 677static const u8 discrete_ctle_tunings[11][4] = {678 /* DC LF HF BW */679 { 0x48, 0x0b, 0x04, 0x04 }, /* p0 */680 { 0x60, 0x05, 0x0f, 0x0a }, /* p1 */681 { 0x50, 0x09, 0x06, 0x06 }, /* p2 */682 { 0x68, 0x05, 0x0f, 0x0a }, /* p3 */683 { 0x80, 0x05, 0x0f, 0x0a }, /* p4 */684 { 0x70, 0x05, 0x0f, 0x0a }, /* p5 */685 { 0x68, 0x05, 0x0f, 0x0a }, /* p6 */686 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */687 { 0x48, 0x09, 0x06, 0x06 }, /* p8 */688 { 0x60, 0x05, 0x0f, 0x0a }, /* p9 */689 { 0x38, 0x0f, 0x00, 0x00 }, /* p10 */690};691 692static const u8 integrated_ctle_tunings[11][4] = {693 /* DC LF HF BW */694 { 0x38, 0x0f, 0x00, 0x00 }, /* p0 */695 { 0x38, 0x0f, 0x00, 0x00 }, /* p1 */696 { 0x38, 0x0f, 0x00, 0x00 }, /* p2 */697 { 0x38, 0x0f, 0x00, 0x00 }, /* p3 */698 { 0x58, 0x0a, 0x05, 0x05 }, /* p4 */699 { 0x48, 0x0a, 0x05, 0x05 }, /* p5 */700 { 0x40, 0x0a, 0x05, 0x05 }, /* p6 */701 { 0x38, 0x0f, 0x00, 0x00 }, /* p7 */702 { 0x38, 0x0f, 0x00, 0x00 }, /* p8 */703 { 0x38, 0x09, 0x06, 0x06 }, /* p9 */704 { 0x38, 0x0e, 0x01, 0x01 }, /* p10 */705};706 707/* helper to format the value to write to hardware */708#define eq_value(pre, curr, post) \709 ((((u32)(pre)) << \710 PCIE_CFG_REG_PL102_GEN3_EQ_PRE_CURSOR_PSET_SHIFT) \711 | (((u32)(curr)) << PCIE_CFG_REG_PL102_GEN3_EQ_CURSOR_PSET_SHIFT) \712 | (((u32)(post)) << \713 PCIE_CFG_REG_PL102_GEN3_EQ_POST_CURSOR_PSET_SHIFT))714 715/*716 * Load the given EQ preset table into the PCIe hardware.717 */718static int load_eq_table(struct hfi1_devdata *dd, const u8 eq[11][3], u8 fs,719 u8 div)720{721 struct pci_dev *pdev = dd->pcidev;722 u32 hit_error = 0;723 u32 violation;724 u32 i;725 u8 c_minus1, c0, c_plus1;726 int ret;727 728 for (i = 0; i < 11; i++) {729 /* set index */730 pci_write_config_dword(pdev, PCIE_CFG_REG_PL103, i);731 /* write the value */732 c_minus1 = eq[i][PREC] / div;733 c0 = fs - (eq[i][PREC] / div) - (eq[i][POST] / div);734 c_plus1 = eq[i][POST] / div;735 pci_write_config_dword(pdev, PCIE_CFG_REG_PL102,736 eq_value(c_minus1, c0, c_plus1));737 /* check if these coefficients violate EQ rules */738 ret = pci_read_config_dword(dd->pcidev,739 PCIE_CFG_REG_PL105, &violation);740 if (ret) {741 dd_dev_err(dd, "Unable to read from PCI config\n");742 hit_error = 1;743 break;744 }745 746 if (violation747 & PCIE_CFG_REG_PL105_GEN3_EQ_VIOLATE_COEF_RULES_SMASK){748 if (hit_error == 0) {749 dd_dev_err(dd,750 "Gen3 EQ Table Coefficient rule violations\n");751 dd_dev_err(dd, " prec attn post\n");752 }753 dd_dev_err(dd, " p%02d: %02x %02x %02x\n",754 i, (u32)eq[i][0], (u32)eq[i][1],755 (u32)eq[i][2]);756 dd_dev_err(dd, " %02x %02x %02x\n",757 (u32)c_minus1, (u32)c0, (u32)c_plus1);758 hit_error = 1;759 }760 }761 if (hit_error)762 return -EINVAL;763 return 0;764}765 766/*767 * Steps to be done after the PCIe firmware is downloaded and768 * before the SBR for the Pcie Gen3.769 * The SBus resource is already being held.770 */771static void pcie_post_steps(struct hfi1_devdata *dd)772{773 int i;774 775 set_sbus_fast_mode(dd);776 /*777 * Write to the PCIe PCSes to set the G3_LOCKED_NEXT bits to 1.778 * This avoids a spurious framing error that can otherwise be779 * generated by the MAC layer.780 *781 * Use individual addresses since no broadcast is set up.782 */783 for (i = 0; i < NUM_PCIE_SERDES; i++) {784 sbus_request(dd, pcie_pcs_addrs[dd->hfi1_id][i],785 0x03, WRITE_SBUS_RECEIVER, 0x00022132);786 }787 788 clear_sbus_fast_mode(dd);789}790 791/*792 * Trigger a secondary bus reset (SBR) on ourselves using our parent.793 *794 * Based on pci_parent_bus_reset() which is not exported by the795 * kernel core.796 */797static int trigger_sbr(struct hfi1_devdata *dd)798{799 struct pci_dev *dev = dd->pcidev;800 struct pci_dev *pdev;801 802 /* need a parent */803 if (!dev->bus->self) {804 dd_dev_err(dd, "%s: no parent device\n", __func__);805 return -ENOTTY;806 }807 808 /* should not be anyone else on the bus */809 list_for_each_entry(pdev, &dev->bus->devices, bus_list)810 if (pdev != dev) {811 dd_dev_err(dd,812 "%s: another device is on the same bus\n",813 __func__);814 return -ENOTTY;815 }816 817 /*818 * This is an end around to do an SBR during probe time. A new API needs819 * to be implemented to have cleaner interface but this fixes the820 * current brokenness821 */822 return pci_bridge_secondary_bus_reset(dev->bus->self);823}824 825/*826 * Write the given gasket interrupt register.827 */828static void write_gasket_interrupt(struct hfi1_devdata *dd, int index,829 u16 code, u16 data)830{831 write_csr(dd, ASIC_PCIE_SD_INTRPT_LIST + (index * 8),832 (((u64)code << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_CODE_SHIFT) |833 ((u64)data << ASIC_PCIE_SD_INTRPT_LIST_INTRPT_DATA_SHIFT)));834}835 836/*837 * Tell the gasket logic how to react to the reset.838 */839static void arm_gasket_logic(struct hfi1_devdata *dd)840{841 u64 reg;842 843 reg = (((u64)1 << dd->hfi1_id) <<844 ASIC_PCIE_SD_HOST_CMD_INTRPT_CMD_SHIFT) |845 ((u64)pcie_serdes_broadcast[dd->hfi1_id] <<846 ASIC_PCIE_SD_HOST_CMD_SBUS_RCVR_ADDR_SHIFT |847 ASIC_PCIE_SD_HOST_CMD_SBR_MODE_SMASK |848 ((u64)SBR_DELAY_US & ASIC_PCIE_SD_HOST_CMD_TIMER_MASK) <<849 ASIC_PCIE_SD_HOST_CMD_TIMER_SHIFT);850 write_csr(dd, ASIC_PCIE_SD_HOST_CMD, reg);851 /* read back to push the write */852 read_csr(dd, ASIC_PCIE_SD_HOST_CMD);853}854 855/*856 * CCE_PCIE_CTRL long name helpers857 * We redefine these shorter macros to use in the code while leaving858 * chip_registers.h to be autogenerated from the hardware spec.859 */860#define LANE_BUNDLE_MASK CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_MASK861#define LANE_BUNDLE_SHIFT CCE_PCIE_CTRL_PCIE_LANE_BUNDLE_SHIFT862#define LANE_DELAY_MASK CCE_PCIE_CTRL_PCIE_LANE_DELAY_MASK863#define LANE_DELAY_SHIFT CCE_PCIE_CTRL_PCIE_LANE_DELAY_SHIFT864#define MARGIN_OVERWRITE_ENABLE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_OVERWRITE_ENABLE_SHIFT865#define MARGIN_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_SHIFT866#define MARGIN_G1_G2_OVERWRITE_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_MASK867#define MARGIN_G1_G2_OVERWRITE_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_OVERWRITE_ENABLE_SHIFT868#define MARGIN_GEN1_GEN2_MASK CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_MASK869#define MARGIN_GEN1_GEN2_SHIFT CCE_PCIE_CTRL_XMT_MARGIN_GEN1_GEN2_SHIFT870 871 /*872 * Write xmt_margin for full-swing (WFR-B) or half-swing (WFR-C).873 */874static void write_xmt_margin(struct hfi1_devdata *dd, const char *fname)875{876 u64 pcie_ctrl;877 u64 xmt_margin;878 u64 xmt_margin_oe;879 u64 lane_delay;880 u64 lane_bundle;881 882 pcie_ctrl = read_csr(dd, CCE_PCIE_CTRL);883 884 /*885 * For Discrete, use full-swing.886 * - PCIe TX defaults to full-swing.887 * Leave this register as default.888 * For Integrated, use half-swing889 * - Copy xmt_margin and xmt_margin_oe890 * from Gen1/Gen2 to Gen3.891 */892 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL1) { /* integrated */893 /* extract initial fields */894 xmt_margin = (pcie_ctrl >> MARGIN_GEN1_GEN2_SHIFT)895 & MARGIN_GEN1_GEN2_MASK;896 xmt_margin_oe = (pcie_ctrl >> MARGIN_G1_G2_OVERWRITE_SHIFT)897 & MARGIN_G1_G2_OVERWRITE_MASK;898 lane_delay = (pcie_ctrl >> LANE_DELAY_SHIFT) & LANE_DELAY_MASK;899 lane_bundle = (pcie_ctrl >> LANE_BUNDLE_SHIFT)900 & LANE_BUNDLE_MASK;901 902 /*903 * For A0, EFUSE values are not set. Override with the904 * correct values.905 */906 if (is_ax(dd)) {907 /*908 * xmt_margin and OverwiteEnabel should be the909 * same for Gen1/Gen2 and Gen3910 */911 xmt_margin = 0x5;912 xmt_margin_oe = 0x1;913 lane_delay = 0xF; /* Delay 240ns. */914 lane_bundle = 0x0; /* Set to 1 lane. */915 }916 917 /* overwrite existing values */918 pcie_ctrl = (xmt_margin << MARGIN_GEN1_GEN2_SHIFT)919 | (xmt_margin_oe << MARGIN_G1_G2_OVERWRITE_SHIFT)920 | (xmt_margin << MARGIN_SHIFT)921 | (xmt_margin_oe << MARGIN_OVERWRITE_ENABLE_SHIFT)922 | (lane_delay << LANE_DELAY_SHIFT)923 | (lane_bundle << LANE_BUNDLE_SHIFT);924 925 write_csr(dd, CCE_PCIE_CTRL, pcie_ctrl);926 }927 928 dd_dev_dbg(dd, "%s: program XMT margin, CcePcieCtrl 0x%llx\n",929 fname, pcie_ctrl);930}931 932/*933 * Do all the steps needed to transition the PCIe link to Gen3 speed.934 */935int do_pcie_gen3_transition(struct hfi1_devdata *dd)936{937 struct pci_dev *parent = dd->pcidev->bus->self;938 u64 fw_ctrl;939 u64 reg, therm;940 u32 reg32, fs, lf;941 u32 status, err;942 int ret;943 int do_retry, retry_count = 0;944 int intnum = 0;945 uint default_pset;946 uint pset = pcie_pset;947 u16 target_vector, target_speed;948 u16 lnkctl2, vendor;949 u8 div;950 const u8 (*eq)[3];951 const u8 (*ctle_tunings)[4];952 uint static_ctle_mode;953 int return_error = 0;954 u32 target_width;955 956 /* PCIe Gen3 is for the ASIC only */957 if (dd->icode != ICODE_RTL_SILICON)958 return 0;959 960 if (pcie_target == 1) { /* target Gen1 */961 target_vector = PCI_EXP_LNKCTL2_TLS_2_5GT;962 target_speed = 2500;963 } else if (pcie_target == 2) { /* target Gen2 */964 target_vector = PCI_EXP_LNKCTL2_TLS_5_0GT;965 target_speed = 5000;966 } else if (pcie_target == 3) { /* target Gen3 */967 target_vector = PCI_EXP_LNKCTL2_TLS_8_0GT;968 target_speed = 8000;969 } else {970 /* off or invalid target - skip */971 dd_dev_info(dd, "%s: Skipping PCIe transition\n", __func__);972 return 0;973 }974 975 /* if already at target speed, done (unless forced) */976 if (dd->lbus_speed == target_speed) {977 dd_dev_info(dd, "%s: PCIe already at gen%d, %s\n", __func__,978 pcie_target,979 pcie_force ? "re-doing anyway" : "skipping");980 if (!pcie_force)981 return 0;982 }983 984 /*985 * The driver cannot do the transition if it has no access to the986 * upstream component987 */988 if (!parent) {989 dd_dev_info(dd, "%s: No upstream, Can't do gen3 transition\n",990 __func__);991 return 0;992 }993 994 /* Previous Gen1/Gen2 bus width */995 target_width = dd->lbus_width;996 997 /*998 * Do the Gen3 transition. Steps are those of the PCIe Gen3999 * recipe.1000 */1001 1002 /* step 1: pcie link working in gen1/gen2 */1003 1004 /* step 2: if either side is not capable of Gen3, done */1005 if (pcie_target == 3 && !dd->link_gen3_capable) {1006 dd_dev_err(dd, "The PCIe link is not Gen3 capable\n");1007 ret = -ENOSYS;1008 goto done_no_mutex;1009 }1010 1011 /* hold the SBus resource across the firmware download and SBR */1012 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);1013 if (ret) {1014 dd_dev_err(dd, "%s: unable to acquire SBus resource\n",1015 __func__);1016 return ret;1017 }1018 1019 /* make sure thermal polling is not causing interrupts */1020 therm = read_csr(dd, ASIC_CFG_THERM_POLL_EN);1021 if (therm) {1022 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);1023 msleep(100);1024 dd_dev_info(dd, "%s: Disabled therm polling\n",1025 __func__);1026 }1027 1028retry:1029 /* the SBus download will reset the spico for thermal */1030 1031 /* step 3: download SBus Master firmware */1032 /* step 4: download PCIe Gen3 SerDes firmware */1033 dd_dev_info(dd, "%s: downloading firmware\n", __func__);1034 ret = load_pcie_firmware(dd);1035 if (ret) {1036 /* do not proceed if the firmware cannot be downloaded */1037 return_error = 1;1038 goto done;1039 }1040 1041 /* step 5: set up device parameter settings */1042 dd_dev_info(dd, "%s: setting PCIe registers\n", __func__);1043 1044 /*1045 * PcieCfgSpcie1 - Link Control 31046 * Leave at reset value. No need to set PerfEq - link equalization1047 * will be performed automatically after the SBR when the target1048 * speed is 8GT/s.1049 */1050 1051 /* clear all 16 per-lane error bits (PCIe: Lane Error Status) */1052 pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, 0xffff);1053 1054 /* step 5a: Set Synopsys Port Logic registers */1055 1056 /*1057 * PcieCfgRegPl2 - Port Force Link1058 *1059 * Set the low power field to 0x10 to avoid unnecessary power1060 * management messages. All other fields are zero.1061 */1062 reg32 = 0x10ul << PCIE_CFG_REG_PL2_LOW_PWR_ENT_CNT_SHIFT;1063 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL2, reg32);1064 1065 /*1066 * PcieCfgRegPl100 - Gen3 Control1067 *1068 * turn off PcieCfgRegPl100.Gen3ZRxDcNonCompl1069 * turn on PcieCfgRegPl100.EqEieosCnt1070 * Everything else zero.1071 */1072 reg32 = PCIE_CFG_REG_PL100_EQ_EIEOS_CNT_SMASK;1073 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL100, reg32);1074 1075 /*1076 * PcieCfgRegPl101 - Gen3 EQ FS and LF1077 * PcieCfgRegPl102 - Gen3 EQ Presets to Coefficients Mapping1078 * PcieCfgRegPl103 - Gen3 EQ Preset Index1079 * PcieCfgRegPl105 - Gen3 EQ Status1080 *1081 * Give initial EQ settings.1082 */1083 if (dd->pcidev->device == PCI_DEVICE_ID_INTEL0) { /* discrete */1084 /* 1000mV, FS=24, LF = 8 */1085 fs = 24;1086 lf = 8;1087 div = 3;1088 eq = discrete_preliminary_eq;1089 default_pset = DEFAULT_DISCRETE_PSET;1090 ctle_tunings = discrete_ctle_tunings;1091 /* bit 0 - discrete on/off */1092 static_ctle_mode = pcie_ctle & 0x1;1093 } else {1094 /* 400mV, FS=29, LF = 9 */1095 fs = 29;1096 lf = 9;1097 div = 1;1098 eq = integrated_preliminary_eq;1099 default_pset = DEFAULT_MCP_PSET;1100 ctle_tunings = integrated_ctle_tunings;1101 /* bit 1 - integrated on/off */1102 static_ctle_mode = (pcie_ctle >> 1) & 0x1;1103 }1104 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL101,1105 (fs <<1106 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_FS_SHIFT) |1107 (lf <<1108 PCIE_CFG_REG_PL101_GEN3_EQ_LOCAL_LF_SHIFT));1109 ret = load_eq_table(dd, eq, fs, div);1110 if (ret)1111 goto done;1112 1113 /*1114 * PcieCfgRegPl106 - Gen3 EQ Control1115 *1116 * Set Gen3EqPsetReqVec, leave other fields 0.1117 */1118 if (pset == UNSET_PSET)1119 pset = default_pset;1120 if (pset > 10) { /* valid range is 0-10, inclusive */1121 dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n",1122 __func__, pset, default_pset);1123 pset = default_pset;1124 }1125 dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset);1126 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106,1127 ((1 << pset) <<1128 PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) |1129 PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK |1130 PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK);1131 1132 /*1133 * step 5b: Do post firmware download steps via SBus1134 */1135 dd_dev_info(dd, "%s: doing pcie post steps\n", __func__);1136 pcie_post_steps(dd);1137 1138 /*1139 * step 5c: Program gasket interrupts1140 */1141 /* set the Rx Bit Rate to REFCLK ratio */1142 write_gasket_interrupt(dd, intnum++, 0x0006, 0x0050);1143 /* disable pCal for PCIe Gen3 RX equalization */1144 /* select adaptive or static CTLE */1145 write_gasket_interrupt(dd, intnum++, 0x0026,1146 0x5b01 | (static_ctle_mode << 3));1147 /*1148 * Enable iCal for PCIe Gen3 RX equalization, and set which1149 * evaluation of RX_EQ_EVAL will launch the iCal procedure.1150 */1151 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5202);1152 1153 if (static_ctle_mode) {1154 /* apply static CTLE tunings */1155 u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;1156 1157 pcie_dc = ctle_tunings[pset][0];1158 pcie_lf = ctle_tunings[pset][1];1159 pcie_hf = ctle_tunings[pset][2];1160 pcie_bw = ctle_tunings[pset][3];1161 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc);1162 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf);1163 write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf);1164 write_gasket_interrupt(dd, intnum++, 0x0026, 0x5500 | pcie_bw);1165 }1166 1167 /* terminate list */1168 write_gasket_interrupt(dd, intnum++, 0x0000, 0x0000);1169 1170 /*1171 * step 5d: program XMT margin1172 */1173 write_xmt_margin(dd, __func__);1174 1175 /*1176 * step 5e: disable active state power management (ASPM). It1177 * will be enabled if required later1178 */1179 dd_dev_info(dd, "%s: clearing ASPM\n", __func__);1180 aspm_hw_disable_l1(dd);1181 1182 /*1183 * step 5f: clear DirectSpeedChange1184 * PcieCfgRegPl67.DirectSpeedChange must be zero to prevent the1185 * change in the speed target from starting before we are ready.1186 * This field defaults to 0 and we are not changing it, so nothing1187 * needs to be done.1188 */1189 1190 /* step 5g: Set target link speed */1191 /*1192 * Set target link speed to be target on both device and parent.1193 * On setting the parent: Some system BIOSs "helpfully" set the1194 * parent target speed to Gen2 to match the ASIC's initial speed.1195 * We can set the target Gen3 because we have already checked1196 * that it is Gen3 capable earlier.1197 */1198 dd_dev_info(dd, "%s: setting parent target link speed\n", __func__);1199 ret = pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2);1200 if (ret) {1201 dd_dev_err(dd, "Unable to read from PCI config\n");1202 return_error = 1;1203 goto done;1204 }1205 1206 dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,1207 (u32)lnkctl2);1208 /* only write to parent if target is not as high as ours */1209 if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) < target_vector) {1210 ret = pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL2,1211 PCI_EXP_LNKCTL2_TLS,1212 target_vector);1213 if (ret) {1214 dd_dev_err(dd, "Unable to change parent PCI target speed\n");1215 return_error = 1;1216 goto done;1217 }1218 } else {1219 dd_dev_info(dd, "%s: ..target speed is OK\n", __func__);1220 }1221 1222 dd_dev_info(dd, "%s: setting target link speed\n", __func__);1223 ret = pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL2,1224 PCI_EXP_LNKCTL2_TLS,1225 target_vector);1226 if (ret) {1227 dd_dev_err(dd, "Unable to change device PCI target speed\n");1228 return_error = 1;1229 goto done;1230 }1231 1232 /* step 5h: arm gasket logic */1233 /* hold DC in reset across the SBR */1234 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);1235 (void)read_csr(dd, CCE_DC_CTRL); /* DC reset hold */1236 /* save firmware control across the SBR */1237 fw_ctrl = read_csr(dd, MISC_CFG_FW_CTRL);1238 1239 dd_dev_info(dd, "%s: arming gasket logic\n", __func__);1240 arm_gasket_logic(dd);1241 1242 /*1243 * step 6: quiesce PCIe link1244 * The chip has already been reset, so there will be no traffic1245 * from the chip. Linux has no easy way to enforce that it will1246 * not try to access the device, so we just need to hope it doesn't1247 * do it while we are doing the reset.1248 */1249 1250 /*1251 * step 7: initiate the secondary bus reset (SBR)1252 * step 8: hardware brings the links back up1253 * step 9: wait for link speed transition to be complete1254 */1255 dd_dev_info(dd, "%s: calling trigger_sbr\n", __func__);1256 ret = trigger_sbr(dd);1257 if (ret)1258 goto done;1259 1260 /* step 10: decide what to do next */1261 1262 /* check if we can read PCI space */1263 ret = pci_read_config_word(dd->pcidev, PCI_VENDOR_ID, &vendor);1264 if (ret) {1265 dd_dev_info(dd,1266 "%s: read of VendorID failed after SBR, err %d\n",1267 __func__, ret);1268 return_error = 1;1269 goto done;1270 }1271 if (vendor == 0xffff) {1272 dd_dev_info(dd, "%s: VendorID is all 1s after SBR\n", __func__);1273 return_error = 1;1274 ret = -EIO;1275 goto done;1276 }1277 1278 /* restore PCI space registers we know were reset */1279 dd_dev_info(dd, "%s: calling restore_pci_variables\n", __func__);1280 ret = restore_pci_variables(dd);1281 if (ret) {1282 dd_dev_err(dd, "%s: Could not restore PCI variables\n",1283 __func__);1284 return_error = 1;1285 goto done;1286 }1287 1288 /* restore firmware control */1289 write_csr(dd, MISC_CFG_FW_CTRL, fw_ctrl);1290 1291 /*1292 * Check the gasket block status.1293 *1294 * This is the first CSR read after the SBR. If the read returns1295 * all 1s (fails), the link did not make it back.1296 *1297 * Once we're sure we can read and write, clear the DC reset after1298 * the SBR. Then check for any per-lane errors. Then look over1299 * the status.1300 */1301 reg = read_csr(dd, ASIC_PCIE_SD_HOST_STATUS);1302 dd_dev_info(dd, "%s: gasket block status: 0x%llx\n", __func__, reg);1303 if (reg == ~0ull) { /* PCIe read failed/timeout */1304 dd_dev_err(dd, "SBR failed - unable to read from device\n");1305 return_error = 1;1306 ret = -ENOSYS;1307 goto done;1308 }1309 1310 /* clear the DC reset */1311 write_csr(dd, CCE_DC_CTRL, 0);1312 1313 /* Set the LED off */1314 setextled(dd, 0);1315 1316 /* check for any per-lane errors */1317 ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, ®32);1318 if (ret) {1319 dd_dev_err(dd, "Unable to read from PCI config\n");1320 return_error = 1;1321 goto done;1322 }1323 1324 dd_dev_info(dd, "%s: per-lane errors: 0x%x\n", __func__, reg32);1325 1326 /* extract status, look for our HFI */1327 status = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_SHIFT)1328 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_STS_MASK;1329 if ((status & (1 << dd->hfi1_id)) == 0) {1330 dd_dev_err(dd,1331 "%s: gasket status 0x%x, expecting 0x%x\n",1332 __func__, status, 1 << dd->hfi1_id);1333 ret = -EIO;1334 goto done;1335 }1336 1337 /* extract error */1338 err = (reg >> ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_SHIFT)1339 & ASIC_PCIE_SD_HOST_STATUS_FW_DNLD_ERR_MASK;1340 if (err) {1341 dd_dev_err(dd, "%s: gasket error %d\n", __func__, err);1342 ret = -EIO;1343 goto done;1344 }1345 1346 /* update our link information cache */1347 update_lbus_info(dd);1348 dd_dev_info(dd, "%s: new speed and width: %s\n", __func__,1349 dd->lbus_info);1350 1351 if (dd->lbus_speed != target_speed ||1352 dd->lbus_width < target_width) { /* not target */1353 /* maybe retry */1354 do_retry = retry_count < pcie_retry;1355 dd_dev_err(dd, "PCIe link speed or width did not match target%s\n",1356 do_retry ? ", retrying" : "");1357 retry_count++;1358 if (do_retry) {1359 msleep(100); /* allow time to settle */1360 goto retry;1361 }1362 ret = -EIO;1363 }1364 1365done:1366 if (therm) {1367 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);1368 msleep(100);1369 dd_dev_info(dd, "%s: Re-enable therm polling\n",1370 __func__);1371 }1372 release_chip_resource(dd, CR_SBUS);1373done_no_mutex:1374 /* return no error if it is OK to be at current speed */1375 if (ret && !return_error) {1376 dd_dev_err(dd, "Proceeding at current speed PCIe speed\n");1377 ret = 0;1378 }1379 1380 dd_dev_info(dd, "%s: done\n", __func__);1381 return ret;1382}1383