1599 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de>4 * Copyright (C) 2015 Broadcom Corporation5 */6 7#include <linux/kernel.h>8#include <linux/pci.h>9#include <linux/pci-ecam.h>10#include <linux/msi.h>11#include <linux/clk.h>12#include <linux/module.h>13#include <linux/mbus.h>14#include <linux/slab.h>15#include <linux/delay.h>16#include <linux/interrupt.h>17#include <linux/irqchip/arm-gic-v3.h>18#include <linux/platform_device.h>19#include <linux/of_address.h>20#include <linux/of_pci.h>21#include <linux/of_platform.h>22#include <linux/phy/phy.h>23 24#include "pcie-iproc.h"25 26#define EP_PERST_SOURCE_SELECT_SHIFT 227#define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT)28#define EP_MODE_SURVIVE_PERST_SHIFT 129#define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT)30#define RC_PCIE_RST_OUTPUT_SHIFT 031#define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT)32#define PAXC_RESET_MASK 0x7f33 34#define GIC_V3_CFG_SHIFT 035#define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT)36 37#define MSI_ENABLE_CFG_SHIFT 038#define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT)39 40#define CFG_IND_ADDR_MASK 0x00001ffc41 42#define CFG_ADDR_REG_NUM_MASK 0x00000ffc43#define CFG_ADDR_CFG_TYPE_1 144 45#define SYS_RC_INTX_MASK 0xf46 47#define PCIE_PHYLINKUP_SHIFT 348#define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT)49#define PCIE_DL_ACTIVE_SHIFT 250#define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT)51 52#define APB_ERR_EN_SHIFT 053#define APB_ERR_EN BIT(APB_ERR_EN_SHIFT)54 55#define CFG_RD_SUCCESS 056#define CFG_RD_UR 157#define CFG_RD_RRS 258#define CFG_RD_CA 359#define CFG_RETRY_STATUS 0xffff000160#define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */61 62/* derive the enum index of the outbound/inbound mapping registers */63#define MAP_REG(base_reg, index) ((base_reg) + (index) * 2)64 65/*66 * Maximum number of outbound mapping window sizes that can be supported by any67 * OARR/OMAP mapping pair68 */69#define MAX_NUM_OB_WINDOW_SIZES 470 71#define OARR_VALID_SHIFT 072#define OARR_VALID BIT(OARR_VALID_SHIFT)73#define OARR_SIZE_CFG_SHIFT 174 75/*76 * Maximum number of inbound mapping region sizes that can be supported by an77 * IARR78 */79#define MAX_NUM_IB_REGION_SIZES 980 81#define IMAP_VALID_SHIFT 082#define IMAP_VALID BIT(IMAP_VALID_SHIFT)83 84#define IPROC_PCI_PM_CAP 0x4885#define IPROC_PCI_PM_CAP_MASK 0xffff86#define IPROC_PCI_EXP_CAP 0xac87 88#define IPROC_PCIE_REG_INVALID 0xffff89 90/**91 * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific92 * parameters93 * @window_sizes: list of supported outbound mapping window sizes in MB94 * @nr_sizes: number of supported outbound mapping window sizes95 */96struct iproc_pcie_ob_map {97 resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES];98 unsigned int nr_sizes;99};100 101static const struct iproc_pcie_ob_map paxb_ob_map[] = {102 {103 /* OARR0/OMAP0 */104 .window_sizes = { 128, 256 },105 .nr_sizes = 2,106 },107 {108 /* OARR1/OMAP1 */109 .window_sizes = { 128, 256 },110 .nr_sizes = 2,111 },112};113 114static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = {115 {116 /* OARR0/OMAP0 */117 .window_sizes = { 128, 256 },118 .nr_sizes = 2,119 },120 {121 /* OARR1/OMAP1 */122 .window_sizes = { 128, 256 },123 .nr_sizes = 2,124 },125 {126 /* OARR2/OMAP2 */127 .window_sizes = { 128, 256, 512, 1024 },128 .nr_sizes = 4,129 },130 {131 /* OARR3/OMAP3 */132 .window_sizes = { 128, 256, 512, 1024 },133 .nr_sizes = 4,134 },135};136 137/**138 * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type139 * @IPROC_PCIE_IB_MAP_MEM: DDR memory140 * @IPROC_PCIE_IB_MAP_IO: device I/O memory141 * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused142 */143enum iproc_pcie_ib_map_type {144 IPROC_PCIE_IB_MAP_MEM = 0,145 IPROC_PCIE_IB_MAP_IO,146 IPROC_PCIE_IB_MAP_INVALID147};148 149/**150 * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific151 * parameters152 * @type: inbound mapping region type153 * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or154 * SZ_1G155 * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or156 * GB, depending on the size unit157 * @nr_sizes: number of supported inbound mapping region sizes158 * @nr_windows: number of supported inbound mapping windows for the region159 * @imap_addr_offset: register offset between the upper and lower 32-bit160 * IMAP address registers161 * @imap_window_offset: register offset between each IMAP window162 */163struct iproc_pcie_ib_map {164 enum iproc_pcie_ib_map_type type;165 unsigned int size_unit;166 resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES];167 unsigned int nr_sizes;168 unsigned int nr_windows;169 u16 imap_addr_offset;170 u16 imap_window_offset;171};172 173static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = {174 {175 /* IARR0/IMAP0 */176 .type = IPROC_PCIE_IB_MAP_IO,177 .size_unit = SZ_1K,178 .region_sizes = { 32 },179 .nr_sizes = 1,180 .nr_windows = 8,181 .imap_addr_offset = 0x40,182 .imap_window_offset = 0x4,183 },184 {185 /* IARR1/IMAP1 */186 .type = IPROC_PCIE_IB_MAP_MEM,187 .size_unit = SZ_1M,188 .region_sizes = { 8 },189 .nr_sizes = 1,190 .nr_windows = 8,191 .imap_addr_offset = 0x4,192 .imap_window_offset = 0x8,193 194 },195 {196 /* IARR2/IMAP2 */197 .type = IPROC_PCIE_IB_MAP_MEM,198 .size_unit = SZ_1M,199 .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192,200 16384 },201 .nr_sizes = 9,202 .nr_windows = 1,203 .imap_addr_offset = 0x4,204 .imap_window_offset = 0x8,205 },206 {207 /* IARR3/IMAP3 */208 .type = IPROC_PCIE_IB_MAP_MEM,209 .size_unit = SZ_1G,210 .region_sizes = { 1, 2, 4, 8, 16, 32 },211 .nr_sizes = 6,212 .nr_windows = 8,213 .imap_addr_offset = 0x4,214 .imap_window_offset = 0x8,215 },216 {217 /* IARR4/IMAP4 */218 .type = IPROC_PCIE_IB_MAP_MEM,219 .size_unit = SZ_1G,220 .region_sizes = { 32, 64, 128, 256, 512 },221 .nr_sizes = 5,222 .nr_windows = 8,223 .imap_addr_offset = 0x4,224 .imap_window_offset = 0x8,225 },226};227 228/*229 * iProc PCIe host registers230 */231enum iproc_pcie_reg {232 /* clock/reset signal control */233 IPROC_PCIE_CLK_CTRL = 0,234 235 /*236 * To allow MSI to be steered to an external MSI controller (e.g., ARM237 * GICv3 ITS)238 */239 IPROC_PCIE_MSI_GIC_MODE,240 241 /*242 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the243 * window where the MSI posted writes are written, for the writes to be244 * interpreted as MSI writes.245 */246 IPROC_PCIE_MSI_BASE_ADDR,247 IPROC_PCIE_MSI_WINDOW_SIZE,248 249 /*250 * To hold the address of the register where the MSI writes are251 * programmed. When ARM GICv3 ITS is used, this should be programmed252 * with the address of the GITS_TRANSLATER register.253 */254 IPROC_PCIE_MSI_ADDR_LO,255 IPROC_PCIE_MSI_ADDR_HI,256 257 /* enable MSI */258 IPROC_PCIE_MSI_EN_CFG,259 260 /* allow access to root complex configuration space */261 IPROC_PCIE_CFG_IND_ADDR,262 IPROC_PCIE_CFG_IND_DATA,263 264 /* allow access to device configuration space */265 IPROC_PCIE_CFG_ADDR,266 IPROC_PCIE_CFG_DATA,267 268 /* enable INTx */269 IPROC_PCIE_INTX_EN,270 271 /* outbound address mapping */272 IPROC_PCIE_OARR0,273 IPROC_PCIE_OMAP0,274 IPROC_PCIE_OARR1,275 IPROC_PCIE_OMAP1,276 IPROC_PCIE_OARR2,277 IPROC_PCIE_OMAP2,278 IPROC_PCIE_OARR3,279 IPROC_PCIE_OMAP3,280 281 /* inbound address mapping */282 IPROC_PCIE_IARR0,283 IPROC_PCIE_IMAP0,284 IPROC_PCIE_IARR1,285 IPROC_PCIE_IMAP1,286 IPROC_PCIE_IARR2,287 IPROC_PCIE_IMAP2,288 IPROC_PCIE_IARR3,289 IPROC_PCIE_IMAP3,290 IPROC_PCIE_IARR4,291 IPROC_PCIE_IMAP4,292 293 /* config read status */294 IPROC_PCIE_CFG_RD_STATUS,295 296 /* link status */297 IPROC_PCIE_LINK_STATUS,298 299 /* enable APB error for unsupported requests */300 IPROC_PCIE_APB_ERR_EN,301 302 /* total number of core registers */303 IPROC_PCIE_MAX_NUM_REG,304};305 306/* iProc PCIe PAXB BCMA registers */307static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = {308 [IPROC_PCIE_CLK_CTRL] = 0x000,309 [IPROC_PCIE_CFG_IND_ADDR] = 0x120,310 [IPROC_PCIE_CFG_IND_DATA] = 0x124,311 [IPROC_PCIE_CFG_ADDR] = 0x1f8,312 [IPROC_PCIE_CFG_DATA] = 0x1fc,313 [IPROC_PCIE_INTX_EN] = 0x330,314 [IPROC_PCIE_LINK_STATUS] = 0xf0c,315};316 317/* iProc PCIe PAXB registers */318static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = {319 [IPROC_PCIE_CLK_CTRL] = 0x000,320 [IPROC_PCIE_CFG_IND_ADDR] = 0x120,321 [IPROC_PCIE_CFG_IND_DATA] = 0x124,322 [IPROC_PCIE_CFG_ADDR] = 0x1f8,323 [IPROC_PCIE_CFG_DATA] = 0x1fc,324 [IPROC_PCIE_INTX_EN] = 0x330,325 [IPROC_PCIE_OARR0] = 0xd20,326 [IPROC_PCIE_OMAP0] = 0xd40,327 [IPROC_PCIE_OARR1] = 0xd28,328 [IPROC_PCIE_OMAP1] = 0xd48,329 [IPROC_PCIE_LINK_STATUS] = 0xf0c,330 [IPROC_PCIE_APB_ERR_EN] = 0xf40,331};332 333/* iProc PCIe PAXB v2 registers */334static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = {335 [IPROC_PCIE_CLK_CTRL] = 0x000,336 [IPROC_PCIE_CFG_IND_ADDR] = 0x120,337 [IPROC_PCIE_CFG_IND_DATA] = 0x124,338 [IPROC_PCIE_CFG_ADDR] = 0x1f8,339 [IPROC_PCIE_CFG_DATA] = 0x1fc,340 [IPROC_PCIE_INTX_EN] = 0x330,341 [IPROC_PCIE_OARR0] = 0xd20,342 [IPROC_PCIE_OMAP0] = 0xd40,343 [IPROC_PCIE_OARR1] = 0xd28,344 [IPROC_PCIE_OMAP1] = 0xd48,345 [IPROC_PCIE_OARR2] = 0xd60,346 [IPROC_PCIE_OMAP2] = 0xd68,347 [IPROC_PCIE_OARR3] = 0xdf0,348 [IPROC_PCIE_OMAP3] = 0xdf8,349 [IPROC_PCIE_IARR0] = 0xd00,350 [IPROC_PCIE_IMAP0] = 0xc00,351 [IPROC_PCIE_IARR1] = 0xd08,352 [IPROC_PCIE_IMAP1] = 0xd70,353 [IPROC_PCIE_IARR2] = 0xd10,354 [IPROC_PCIE_IMAP2] = 0xcc0,355 [IPROC_PCIE_IARR3] = 0xe00,356 [IPROC_PCIE_IMAP3] = 0xe08,357 [IPROC_PCIE_IARR4] = 0xe68,358 [IPROC_PCIE_IMAP4] = 0xe70,359 [IPROC_PCIE_CFG_RD_STATUS] = 0xee0,360 [IPROC_PCIE_LINK_STATUS] = 0xf0c,361 [IPROC_PCIE_APB_ERR_EN] = 0xf40,362};363 364/* iProc PCIe PAXC v1 registers */365static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = {366 [IPROC_PCIE_CLK_CTRL] = 0x000,367 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,368 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4,369 [IPROC_PCIE_CFG_ADDR] = 0x1f8,370 [IPROC_PCIE_CFG_DATA] = 0x1fc,371};372 373/* iProc PCIe PAXC v2 registers */374static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = {375 [IPROC_PCIE_MSI_GIC_MODE] = 0x050,376 [IPROC_PCIE_MSI_BASE_ADDR] = 0x074,377 [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078,378 [IPROC_PCIE_MSI_ADDR_LO] = 0x07c,379 [IPROC_PCIE_MSI_ADDR_HI] = 0x080,380 [IPROC_PCIE_MSI_EN_CFG] = 0x09c,381 [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,382 [IPROC_PCIE_CFG_IND_DATA] = 0x1f4,383 [IPROC_PCIE_CFG_ADDR] = 0x1f8,384 [IPROC_PCIE_CFG_DATA] = 0x1fc,385};386 387/*388 * List of device IDs of controllers that have corrupted capability list that389 * require SW fixup390 */391static const u16 iproc_pcie_corrupt_cap_did[] = {392 0x16cd,393 0x16f0,394 0xd802,395 0xd804396};397 398static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)399{400 struct iproc_pcie *pcie = bus->sysdata;401 return pcie;402}403 404static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset)405{406 return !!(reg_offset == IPROC_PCIE_REG_INVALID);407}408 409static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,410 enum iproc_pcie_reg reg)411{412 return pcie->reg_offsets[reg];413}414 415static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,416 enum iproc_pcie_reg reg)417{418 u16 offset = iproc_pcie_reg_offset(pcie, reg);419 420 if (iproc_pcie_reg_is_invalid(offset))421 return 0;422 423 return readl(pcie->base + offset);424}425 426static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,427 enum iproc_pcie_reg reg, u32 val)428{429 u16 offset = iproc_pcie_reg_offset(pcie, reg);430 431 if (iproc_pcie_reg_is_invalid(offset))432 return;433 434 writel(val, pcie->base + offset);435}436 437/*438 * APB error forwarding can be disabled during access of configuration439 * registers of the endpoint device, to prevent unsupported requests440 * (typically seen during enumeration with multi-function devices) from441 * triggering a system exception.442 */443static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus,444 bool disable)445{446 struct iproc_pcie *pcie = iproc_data(bus);447 u32 val;448 449 if (bus->number && pcie->has_apb_err_disable) {450 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);451 if (disable)452 val &= ~APB_ERR_EN;453 else454 val |= APB_ERR_EN;455 iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);456 }457}458 459static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,460 unsigned int busno,461 unsigned int devfn,462 int where)463{464 u16 offset;465 u32 val;466 467 /* EP device access */468 val = ALIGN_DOWN(PCIE_ECAM_OFFSET(busno, devfn, where), 4) |469 CFG_ADDR_CFG_TYPE_1;470 471 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);472 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);473 474 if (iproc_pcie_reg_is_invalid(offset))475 return NULL;476 477 return (pcie->base + offset);478}479 480static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie,481 void __iomem *cfg_data_p)482{483 int timeout = CFG_RETRY_STATUS_TIMEOUT_US;484 unsigned int data;485 u32 status;486 487 /*488 * As per PCIe r6.0, sec 2.3.2, Config RRS Software Visibility only489 * affects config reads of the Vendor ID. For config writes or any490 * other config reads, the Root may automatically reissue the491 * configuration request again as a new request.492 *493 * For config reads, this hardware returns CFG_RETRY_STATUS data494 * when it receives a RRS completion, regardless of the address of495 * the read or the RRS Software Visibility Enable bit. As a496 * partial workaround for this, we retry in software any read that497 * returns CFG_RETRY_STATUS.498 *499 * Note that a non-Vendor ID config register may have a value of500 * CFG_RETRY_STATUS. If we read that, we can't distinguish it from501 * a RRS completion, so we will incorrectly retry the read and502 * eventually return the wrong data (0xffffffff).503 */504 data = readl(cfg_data_p);505 while (data == CFG_RETRY_STATUS && timeout--) {506 /*507 * RRS state is set in CFG_RD status register508 * This will handle the case where CFG_RETRY_STATUS is509 * valid config data.510 */511 status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS);512 if (status != CFG_RD_RRS)513 return data;514 515 udelay(1);516 data = readl(cfg_data_p);517 }518 519 if (data == CFG_RETRY_STATUS)520 data = 0xffffffff;521 522 return data;523}524 525static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val)526{527 u32 i, dev_id;528 529 switch (where & ~0x3) {530 case PCI_VENDOR_ID:531 dev_id = *val >> 16;532 533 /*534 * Activate fixup for those controllers that have corrupted535 * capability list registers536 */537 for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++)538 if (dev_id == iproc_pcie_corrupt_cap_did[i])539 pcie->fix_paxc_cap = true;540 break;541 542 case IPROC_PCI_PM_CAP:543 if (pcie->fix_paxc_cap) {544 /* advertise PM, force next capability to PCIe */545 *val &= ~IPROC_PCI_PM_CAP_MASK;546 *val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM;547 }548 break;549 550 case IPROC_PCI_EXP_CAP:551 if (pcie->fix_paxc_cap) {552 /* advertise root port, version 2, terminate here */553 *val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 |554 PCI_CAP_ID_EXP;555 }556 break;557 558 case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL:559 /* Don't advertise RRS SV support */560 *val &= ~(PCI_EXP_RTCAP_RRS_SV << 16);561 break;562 563 default:564 break;565 }566}567 568static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn,569 int where, int size, u32 *val)570{571 struct iproc_pcie *pcie = iproc_data(bus);572 unsigned int busno = bus->number;573 void __iomem *cfg_data_p;574 unsigned int data;575 int ret;576 577 /* root complex access */578 if (busno == 0) {579 ret = pci_generic_config_read32(bus, devfn, where, size, val);580 if (ret == PCIBIOS_SUCCESSFUL)581 iproc_pcie_fix_cap(pcie, where, val);582 583 return ret;584 }585 586 cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);587 588 if (!cfg_data_p)589 return PCIBIOS_DEVICE_NOT_FOUND;590 591 data = iproc_pcie_cfg_retry(pcie, cfg_data_p);592 593 *val = data;594 if (size <= 2)595 *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);596 597 /*598 * For PAXC and PAXCv2, the total number of PFs that one can enumerate599 * depends on the firmware configuration. Unfortunately, due to an ASIC600 * bug, unconfigured PFs cannot be properly hidden from the root601 * complex. As a result, write access to these PFs will cause bus lock602 * up on the embedded processor603 *604 * Since all unconfigured PFs are left with an incorrect, staled device605 * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access606 * early here and reject them all607 */608#define DEVICE_ID_MASK 0xffff0000609#define DEVICE_ID_SHIFT 16610 if (pcie->rej_unconfig_pf &&611 (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID)612 if ((*val & DEVICE_ID_MASK) ==613 (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT))614 return PCIBIOS_FUNC_NOT_SUPPORTED;615 616 return PCIBIOS_SUCCESSFUL;617}618 619/*620 * Note access to the configuration registers are protected at the higher layer621 * by 'pci_lock' in drivers/pci/access.c622 */623static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,624 int busno, unsigned int devfn,625 int where)626{627 u16 offset;628 629 /* root complex access */630 if (busno == 0) {631 if (PCIE_ECAM_DEVFN(devfn) > 0)632 return NULL;633 634 iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,635 where & CFG_IND_ADDR_MASK);636 offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);637 if (iproc_pcie_reg_is_invalid(offset))638 return NULL;639 else640 return (pcie->base + offset);641 }642 643 return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);644}645 646static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus,647 unsigned int devfn,648 int where)649{650 return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn,651 where);652}653 654static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie,655 unsigned int devfn, int where,656 int size, u32 *val)657{658 void __iomem *addr;659 660 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);661 if (!addr)662 return PCIBIOS_DEVICE_NOT_FOUND;663 664 *val = readl(addr);665 666 if (size <= 2)667 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);668 669 return PCIBIOS_SUCCESSFUL;670}671 672static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie,673 unsigned int devfn, int where,674 int size, u32 val)675{676 void __iomem *addr;677 u32 mask, tmp;678 679 addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);680 if (!addr)681 return PCIBIOS_DEVICE_NOT_FOUND;682 683 if (size == 4) {684 writel(val, addr);685 return PCIBIOS_SUCCESSFUL;686 }687 688 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));689 tmp = readl(addr) & mask;690 tmp |= val << ((where & 0x3) * 8);691 writel(tmp, addr);692 693 return PCIBIOS_SUCCESSFUL;694}695 696static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,697 int where, int size, u32 *val)698{699 int ret;700 struct iproc_pcie *pcie = iproc_data(bus);701 702 iproc_pcie_apb_err_disable(bus, true);703 if (pcie->iproc_cfg_read)704 ret = iproc_pcie_config_read(bus, devfn, where, size, val);705 else706 ret = pci_generic_config_read32(bus, devfn, where, size, val);707 iproc_pcie_apb_err_disable(bus, false);708 709 return ret;710}711 712static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn,713 int where, int size, u32 val)714{715 int ret;716 717 iproc_pcie_apb_err_disable(bus, true);718 ret = pci_generic_config_write32(bus, devfn, where, size, val);719 iproc_pcie_apb_err_disable(bus, false);720 721 return ret;722}723 724static struct pci_ops iproc_pcie_ops = {725 .map_bus = iproc_pcie_bus_map_cfg_bus,726 .read = iproc_pcie_config_read32,727 .write = iproc_pcie_config_write32,728};729 730static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)731{732 u32 val;733 734 /*735 * PAXC and the internal emulated endpoint device downstream should not736 * be reset. If firmware has been loaded on the endpoint device at an737 * earlier boot stage, reset here causes issues.738 */739 if (pcie->ep_is_internal)740 return;741 742 if (assert) {743 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);744 val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &745 ~RC_PCIE_RST_OUTPUT;746 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);747 udelay(250);748 } else {749 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);750 val |= RC_PCIE_RST_OUTPUT;751 iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);752 msleep(100);753 }754}755 756int iproc_pcie_shutdown(struct iproc_pcie *pcie)757{758 iproc_pcie_perst_ctrl(pcie, true);759 msleep(500);760 761 return 0;762}763EXPORT_SYMBOL_GPL(iproc_pcie_shutdown);764 765static int iproc_pcie_check_link(struct iproc_pcie *pcie)766{767 struct device *dev = pcie->dev;768 u32 hdr_type, link_ctrl, link_status, class, val;769 bool link_is_active = false;770 771 /*772 * PAXC connects to emulated endpoint devices directly and does not773 * have a Serdes. Therefore skip the link detection logic here.774 */775 if (pcie->ep_is_internal)776 return 0;777 778 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);779 if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) {780 dev_err(dev, "PHY or data link is INACTIVE!\n");781 return -ENODEV;782 }783 784 /* make sure we are not in EP mode */785 iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);786 if ((hdr_type & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_BRIDGE) {787 dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type);788 return -EFAULT;789 }790 791 /* force class to PCI_CLASS_BRIDGE_PCI_NORMAL (0x060400) */792#define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c793#define PCI_BRIDGE_CTRL_REG_CLASS_MASK 0xffffff794 iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,795 4, &class);796 class &= ~PCI_BRIDGE_CTRL_REG_CLASS_MASK;797 class |= PCI_CLASS_BRIDGE_PCI_NORMAL;798 iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,799 4, class);800 801 /* check link status to see if link is active */802 iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,803 2, &link_status);804 if (link_status & PCI_EXP_LNKSTA_NLW)805 link_is_active = true;806 807 if (!link_is_active) {808 /* try GEN 1 link speed */809#define PCI_TARGET_LINK_SPEED_MASK 0xf810#define PCI_TARGET_LINK_SPEED_GEN2 0x2811#define PCI_TARGET_LINK_SPEED_GEN1 0x1812 iproc_pci_raw_config_read32(pcie, 0,813 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,814 4, &link_ctrl);815 if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) ==816 PCI_TARGET_LINK_SPEED_GEN2) {817 link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK;818 link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1;819 iproc_pci_raw_config_write32(pcie, 0,820 IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,821 4, link_ctrl);822 msleep(100);823 824 iproc_pci_raw_config_read32(pcie, 0,825 IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,826 2, &link_status);827 if (link_status & PCI_EXP_LNKSTA_NLW)828 link_is_active = true;829 }830 }831 832 dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN");833 834 return link_is_active ? 0 : -ENODEV;835}836 837static void iproc_pcie_enable(struct iproc_pcie *pcie)838{839 iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);840}841 842static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,843 int window_idx)844{845 u32 val;846 847 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));848 849 return !!(val & OARR_VALID);850}851 852static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,853 int size_idx, u64 axi_addr, u64 pci_addr)854{855 struct device *dev = pcie->dev;856 u16 oarr_offset, omap_offset;857 858 /*859 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based860 * on window index.861 */862 oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,863 window_idx));864 omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,865 window_idx));866 if (iproc_pcie_reg_is_invalid(oarr_offset) ||867 iproc_pcie_reg_is_invalid(omap_offset))868 return -EINVAL;869 870 /*871 * Program the OARR registers. The upper 32-bit OARR register is872 * always right after the lower 32-bit OARR register.873 */874 writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) |875 OARR_VALID, pcie->base + oarr_offset);876 writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);877 878 /* now program the OMAP registers */879 writel(lower_32_bits(pci_addr), pcie->base + omap_offset);880 writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);881 882 dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n",883 window_idx, oarr_offset, &axi_addr, &pci_addr);884 dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n",885 readl(pcie->base + oarr_offset),886 readl(pcie->base + oarr_offset + 4));887 dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n",888 readl(pcie->base + omap_offset),889 readl(pcie->base + omap_offset + 4));890 891 return 0;892}893 894/*895 * Some iProc SoCs require the SW to configure the outbound address mapping896 *897 * Outbound address translation:898 *899 * iproc_pcie_address = axi_address - axi_offset900 * OARR = iproc_pcie_address901 * OMAP = pci_addr902 *903 * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address904 */905static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,906 u64 pci_addr, resource_size_t size)907{908 struct iproc_pcie_ob *ob = &pcie->ob;909 struct device *dev = pcie->dev;910 int ret = -EINVAL, window_idx, size_idx;911 912 if (axi_addr < ob->axi_offset) {913 dev_err(dev, "axi address %pap less than offset %pap\n",914 &axi_addr, &ob->axi_offset);915 return -EINVAL;916 }917 918 /*919 * Translate the AXI address to the internal address used by the iProc920 * PCIe core before programming the OARR921 */922 axi_addr -= ob->axi_offset;923 924 /* iterate through all OARR/OMAP mapping windows */925 for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) {926 const struct iproc_pcie_ob_map *ob_map =927 &pcie->ob_map[window_idx];928 929 /*930 * If current outbound window is already in use, move on to the931 * next one.932 */933 if (iproc_pcie_ob_is_valid(pcie, window_idx))934 continue;935 936 /*937 * Iterate through all supported window sizes within the938 * OARR/OMAP pair to find a match. Go through the window sizes939 * in a descending order.940 */941 for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0;942 size_idx--) {943 resource_size_t window_size =944 ob_map->window_sizes[size_idx] * SZ_1M;945 946 /*947 * Keep iterating until we reach the last window and948 * with the minimal window size at index zero. In this949 * case, we take a compromise by mapping it using the950 * minimum window size that can be supported951 */952 if (size < window_size) {953 if (size_idx > 0 || window_idx > 0)954 continue;955 956 /*957 * For the corner case of reaching the minimal958 * window size that can be supported on the959 * last window960 */961 axi_addr = ALIGN_DOWN(axi_addr, window_size);962 pci_addr = ALIGN_DOWN(pci_addr, window_size);963 size = window_size;964 }965 966 if (!IS_ALIGNED(axi_addr, window_size) ||967 !IS_ALIGNED(pci_addr, window_size)) {968 dev_err(dev,969 "axi %pap or pci %pap not aligned\n",970 &axi_addr, &pci_addr);971 return -EINVAL;972 }973 974 /*975 * Match found! Program both OARR and OMAP and mark976 * them as a valid entry.977 */978 ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,979 axi_addr, pci_addr);980 if (ret)981 goto err_ob;982 983 size -= window_size;984 if (size == 0)985 return 0;986 987 /*988 * If we are here, we are done with the current window,989 * but not yet finished all mappings. Need to move on990 * to the next window.991 */992 axi_addr += window_size;993 pci_addr += window_size;994 break;995 }996 }997 998err_ob:999 dev_err(dev, "unable to configure outbound mapping\n");1000 dev_err(dev,1001 "axi %pap, axi offset %pap, pci %pap, res size %pap\n",1002 &axi_addr, &ob->axi_offset, &pci_addr, &size);1003 1004 return ret;1005}1006 1007static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,1008 struct list_head *resources)1009{1010 struct device *dev = pcie->dev;1011 struct resource_entry *window;1012 int ret;1013 1014 resource_list_for_each_entry(window, resources) {1015 struct resource *res = window->res;1016 u64 res_type = resource_type(res);1017 1018 switch (res_type) {1019 case IORESOURCE_IO:1020 case IORESOURCE_BUS:1021 break;1022 case IORESOURCE_MEM:1023 ret = iproc_pcie_setup_ob(pcie, res->start,1024 res->start - window->offset,1025 resource_size(res));1026 if (ret)1027 return ret;1028 break;1029 default:1030 dev_err(dev, "invalid resource %pR\n", res);1031 return -EINVAL;1032 }1033 }1034 1035 return 0;1036}1037 1038static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,1039 int region_idx)1040{1041 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];1042 u32 val;1043 1044 val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));1045 1046 return !!(val & (BIT(ib_map->nr_sizes) - 1));1047}1048 1049static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map,1050 enum iproc_pcie_ib_map_type type)1051{1052 return !!(ib_map->type == type);1053}1054 1055static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,1056 int size_idx, int nr_windows, u64 axi_addr,1057 u64 pci_addr, resource_size_t size)1058{1059 struct device *dev = pcie->dev;1060 const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];1061 u16 iarr_offset, imap_offset;1062 u32 val;1063 int window_idx;1064 1065 iarr_offset = iproc_pcie_reg_offset(pcie,1066 MAP_REG(IPROC_PCIE_IARR0, region_idx));1067 imap_offset = iproc_pcie_reg_offset(pcie,1068 MAP_REG(IPROC_PCIE_IMAP0, region_idx));1069 if (iproc_pcie_reg_is_invalid(iarr_offset) ||1070 iproc_pcie_reg_is_invalid(imap_offset))1071 return -EINVAL;1072 1073 dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n",1074 region_idx, iarr_offset, &axi_addr, &pci_addr);1075 1076 /*1077 * Program the IARR registers. The upper 32-bit IARR register is1078 * always right after the lower 32-bit IARR register.1079 */1080 writel(lower_32_bits(pci_addr) | BIT(size_idx),1081 pcie->base + iarr_offset);1082 writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);1083 1084 dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n",1085 readl(pcie->base + iarr_offset),1086 readl(pcie->base + iarr_offset + 4));1087 1088 /*1089 * Now program the IMAP registers. Each IARR region may have one or1090 * more IMAP windows.1091 */1092 size >>= ilog2(nr_windows);1093 for (window_idx = 0; window_idx < nr_windows; window_idx++) {1094 val = readl(pcie->base + imap_offset);1095 val |= lower_32_bits(axi_addr) | IMAP_VALID;1096 writel(val, pcie->base + imap_offset);1097 writel(upper_32_bits(axi_addr),1098 pcie->base + imap_offset + ib_map->imap_addr_offset);1099 1100 dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n",1101 window_idx, readl(pcie->base + imap_offset),1102 readl(pcie->base + imap_offset +1103 ib_map->imap_addr_offset));1104 1105 imap_offset += ib_map->imap_window_offset;1106 axi_addr += size;1107 }1108 1109 return 0;1110}1111 1112static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,1113 struct resource_entry *entry,1114 enum iproc_pcie_ib_map_type type)1115{1116 struct device *dev = pcie->dev;1117 struct iproc_pcie_ib *ib = &pcie->ib;1118 int ret;1119 unsigned int region_idx, size_idx;1120 u64 axi_addr = entry->res->start;1121 u64 pci_addr = entry->res->start - entry->offset;1122 resource_size_t size = resource_size(entry->res);1123 1124 /* iterate through all IARR mapping regions */1125 for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) {1126 const struct iproc_pcie_ib_map *ib_map =1127 &pcie->ib_map[region_idx];1128 1129 /*1130 * If current inbound region is already in use or not a1131 * compatible type, move on to the next.1132 */1133 if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||1134 !iproc_pcie_ib_check_type(ib_map, type))1135 continue;1136 1137 /* iterate through all supported region sizes to find a match */1138 for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) {1139 resource_size_t region_size =1140 ib_map->region_sizes[size_idx] * ib_map->size_unit;1141 1142 if (size != region_size)1143 continue;1144 1145 if (!IS_ALIGNED(axi_addr, region_size) ||1146 !IS_ALIGNED(pci_addr, region_size)) {1147 dev_err(dev,1148 "axi %pap or pci %pap not aligned\n",1149 &axi_addr, &pci_addr);1150 return -EINVAL;1151 }1152 1153 /* Match found! Program IARR and all IMAP windows. */1154 ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,1155 ib_map->nr_windows, axi_addr,1156 pci_addr, size);1157 if (ret)1158 goto err_ib;1159 else1160 return 0;1161 1162 }1163 }1164 ret = -EINVAL;1165 1166err_ib:1167 dev_err(dev, "unable to configure inbound mapping\n");1168 dev_err(dev, "axi %pap, pci %pap, res size %pap\n",1169 &axi_addr, &pci_addr, &size);1170 1171 return ret;1172}1173 1174static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)1175{1176 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);1177 struct resource_entry *entry;1178 int ret = 0;1179 1180 resource_list_for_each_entry(entry, &host->dma_ranges) {1181 /* Each range entry corresponds to an inbound mapping region */1182 ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM);1183 if (ret)1184 break;1185 }1186 1187 return ret;1188}1189 1190static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie)1191{1192 struct iproc_pcie_ib *ib = &pcie->ib;1193 struct iproc_pcie_ob *ob = &pcie->ob;1194 int idx;1195 1196 if (pcie->ep_is_internal)1197 return;1198 1199 if (pcie->need_ob_cfg) {1200 /* iterate through all OARR mapping regions */1201 for (idx = ob->nr_windows - 1; idx >= 0; idx--) {1202 iproc_pcie_write_reg(pcie,1203 MAP_REG(IPROC_PCIE_OARR0, idx), 0);1204 }1205 }1206 1207 if (pcie->need_ib_cfg) {1208 /* iterate through all IARR mapping regions */1209 for (idx = 0; idx < ib->nr_regions; idx++) {1210 iproc_pcie_write_reg(pcie,1211 MAP_REG(IPROC_PCIE_IARR0, idx), 0);1212 }1213 }1214}1215 1216static int iproce_pcie_get_msi(struct iproc_pcie *pcie,1217 struct device_node *msi_node,1218 u64 *msi_addr)1219{1220 struct device *dev = pcie->dev;1221 int ret;1222 struct resource res;1223 1224 /*1225 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only1226 * supported external MSI controller that requires steering.1227 */1228 if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) {1229 dev_err(dev, "unable to find compatible MSI controller\n");1230 return -ENODEV;1231 }1232 1233 /* derive GITS_TRANSLATER address from GICv3 */1234 ret = of_address_to_resource(msi_node, 0, &res);1235 if (ret < 0) {1236 dev_err(dev, "unable to obtain MSI controller resources\n");1237 return ret;1238 }1239 1240 *msi_addr = res.start + GITS_TRANSLATER;1241 return 0;1242}1243 1244static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)1245{1246 int ret;1247 struct resource_entry entry;1248 1249 memset(&entry, 0, sizeof(entry));1250 entry.res = &entry.__res;1251 1252 msi_addr &= ~(SZ_32K - 1);1253 entry.res->start = msi_addr;1254 entry.res->end = msi_addr + SZ_32K - 1;1255 1256 ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO);1257 return ret;1258}1259 1260static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr,1261 bool enable)1262{1263 u32 val;1264 1265 if (!enable) {1266 /*1267 * Disable PAXC MSI steering. All write transfers will be1268 * treated as non-MSI transfers1269 */1270 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);1271 val &= ~MSI_ENABLE_CFG;1272 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);1273 return;1274 }1275 1276 /*1277 * Program bits [43:13] of address of GITS_TRANSLATER register into1278 * bits [30:0] of the MSI base address register. In fact, in all iProc1279 * based SoCs, all I/O register bases are well below the 32-bit1280 * boundary, so we can safely assume bits [43:32] are always zeros.1281 */1282 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,1283 (u32)(msi_addr >> 13));1284 1285 /* use a default 8K window size */1286 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);1287 1288 /* steering MSI to GICv3 ITS */1289 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);1290 val |= GIC_V3_CFG;1291 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);1292 1293 /*1294 * Program bits [43:2] of address of GITS_TRANSLATER register into the1295 * iProc MSI address registers.1296 */1297 msi_addr >>= 2;1298 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,1299 upper_32_bits(msi_addr));1300 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,1301 lower_32_bits(msi_addr));1302 1303 /* enable MSI */1304 val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);1305 val |= MSI_ENABLE_CFG;1306 iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);1307}1308 1309static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,1310 struct device_node *msi_node)1311{1312 struct device *dev = pcie->dev;1313 int ret;1314 u64 msi_addr;1315 1316 ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);1317 if (ret < 0) {1318 dev_err(dev, "msi steering failed\n");1319 return ret;1320 }1321 1322 switch (pcie->type) {1323 case IPROC_PCIE_PAXB_V2:1324 ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);1325 if (ret)1326 return ret;1327 break;1328 case IPROC_PCIE_PAXC_V2:1329 iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true);1330 break;1331 default:1332 return -EINVAL;1333 }1334 1335 return 0;1336}1337 1338static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)1339{1340 struct device_node *msi_node;1341 int ret;1342 1343 /*1344 * Either the "msi-parent" or the "msi-map" phandle needs to exist1345 * for us to obtain the MSI node.1346 */1347 1348 msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);1349 if (!msi_node) {1350 const __be32 *msi_map = NULL;1351 int len;1352 u32 phandle;1353 1354 msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);1355 if (!msi_map)1356 return -ENODEV;1357 1358 phandle = be32_to_cpup(msi_map + 1);1359 msi_node = of_find_node_by_phandle(phandle);1360 if (!msi_node)1361 return -ENODEV;1362 }1363 1364 /*1365 * Certain revisions of the iProc PCIe controller require additional1366 * configurations to steer the MSI writes towards an external MSI1367 * controller.1368 */1369 if (pcie->need_msi_steer) {1370 ret = iproc_pcie_msi_steer(pcie, msi_node);1371 if (ret)1372 goto out_put_node;1373 }1374 1375 /*1376 * If another MSI controller is being used, the call below should fail1377 * but that is okay1378 */1379 ret = iproc_msi_init(pcie, msi_node);1380 1381out_put_node:1382 of_node_put(msi_node);1383 return ret;1384}1385 1386static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)1387{1388 iproc_msi_exit(pcie);1389}1390 1391static int iproc_pcie_rev_init(struct iproc_pcie *pcie)1392{1393 struct device *dev = pcie->dev;1394 unsigned int reg_idx;1395 const u16 *regs;1396 1397 switch (pcie->type) {1398 case IPROC_PCIE_PAXB_BCMA:1399 regs = iproc_pcie_reg_paxb_bcma;1400 break;1401 case IPROC_PCIE_PAXB:1402 regs = iproc_pcie_reg_paxb;1403 pcie->has_apb_err_disable = true;1404 if (pcie->need_ob_cfg) {1405 pcie->ob_map = paxb_ob_map;1406 pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);1407 }1408 break;1409 case IPROC_PCIE_PAXB_V2:1410 regs = iproc_pcie_reg_paxb_v2;1411 pcie->iproc_cfg_read = true;1412 pcie->has_apb_err_disable = true;1413 if (pcie->need_ob_cfg) {1414 pcie->ob_map = paxb_v2_ob_map;1415 pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);1416 }1417 pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);1418 pcie->ib_map = paxb_v2_ib_map;1419 pcie->need_msi_steer = true;1420 dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n",1421 CFG_RETRY_STATUS);1422 break;1423 case IPROC_PCIE_PAXC:1424 regs = iproc_pcie_reg_paxc;1425 pcie->ep_is_internal = true;1426 pcie->iproc_cfg_read = true;1427 pcie->rej_unconfig_pf = true;1428 break;1429 case IPROC_PCIE_PAXC_V2:1430 regs = iproc_pcie_reg_paxc_v2;1431 pcie->ep_is_internal = true;1432 pcie->iproc_cfg_read = true;1433 pcie->rej_unconfig_pf = true;1434 pcie->need_msi_steer = true;1435 break;1436 default:1437 dev_err(dev, "incompatible iProc PCIe interface\n");1438 return -EINVAL;1439 }1440 1441 pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,1442 sizeof(*pcie->reg_offsets),1443 GFP_KERNEL);1444 if (!pcie->reg_offsets)1445 return -ENOMEM;1446 1447 /* go through the register table and populate all valid registers */1448 pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?1449 IPROC_PCIE_REG_INVALID : regs[0];1450 for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++)1451 pcie->reg_offsets[reg_idx] = regs[reg_idx] ?1452 regs[reg_idx] : IPROC_PCIE_REG_INVALID;1453 1454 return 0;1455}1456 1457int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)1458{1459 struct device *dev;1460 int ret;1461 struct pci_dev *pdev;1462 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);1463 1464 dev = pcie->dev;1465 1466 ret = iproc_pcie_rev_init(pcie);1467 if (ret) {1468 dev_err(dev, "unable to initialize controller parameters\n");1469 return ret;1470 }1471 1472 ret = phy_init(pcie->phy);1473 if (ret) {1474 dev_err(dev, "unable to initialize PCIe PHY\n");1475 return ret;1476 }1477 1478 ret = phy_power_on(pcie->phy);1479 if (ret) {1480 dev_err(dev, "unable to power on PCIe PHY\n");1481 goto err_exit_phy;1482 }1483 1484 iproc_pcie_perst_ctrl(pcie, true);1485 iproc_pcie_perst_ctrl(pcie, false);1486 1487 iproc_pcie_invalidate_mapping(pcie);1488 1489 if (pcie->need_ob_cfg) {1490 ret = iproc_pcie_map_ranges(pcie, res);1491 if (ret) {1492 dev_err(dev, "map failed\n");1493 goto err_power_off_phy;1494 }1495 }1496 1497 if (pcie->need_ib_cfg) {1498 ret = iproc_pcie_map_dma_ranges(pcie);1499 if (ret && ret != -ENOENT)1500 goto err_power_off_phy;1501 }1502 1503 ret = iproc_pcie_check_link(pcie);1504 if (ret) {1505 dev_err(dev, "no PCIe EP device detected\n");1506 goto err_power_off_phy;1507 }1508 1509 iproc_pcie_enable(pcie);1510 1511 if (IS_ENABLED(CONFIG_PCI_MSI))1512 if (iproc_pcie_msi_enable(pcie))1513 dev_info(dev, "not using iProc MSI\n");1514 1515 host->ops = &iproc_pcie_ops;1516 host->sysdata = pcie;1517 host->map_irq = pcie->map_irq;1518 1519 ret = pci_host_probe(host);1520 if (ret < 0) {1521 dev_err(dev, "failed to scan host: %d\n", ret);1522 goto err_power_off_phy;1523 }1524 1525 for_each_pci_bridge(pdev, host->bus) {1526 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT)1527 pcie_print_link_status(pdev);1528 }1529 1530 return 0;1531 1532err_power_off_phy:1533 phy_power_off(pcie->phy);1534err_exit_phy:1535 phy_exit(pcie->phy);1536 return ret;1537}1538EXPORT_SYMBOL(iproc_pcie_setup);1539 1540void iproc_pcie_remove(struct iproc_pcie *pcie)1541{1542 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);1543 1544 pci_stop_root_bus(host->bus);1545 pci_remove_root_bus(host->bus);1546 1547 iproc_pcie_msi_disable(pcie);1548 1549 phy_power_off(pcie->phy);1550 phy_exit(pcie->phy);1551}1552EXPORT_SYMBOL(iproc_pcie_remove);1553 1554/*1555 * The MSI parsing logic in certain revisions of Broadcom PAXC based root1556 * complex does not work and needs to be disabled1557 */1558static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev)1559{1560 struct iproc_pcie *pcie = iproc_data(pdev->bus);1561 1562 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)1563 iproc_pcie_paxc_v2_msi_steer(pcie, 0, false);1564}1565DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0,1566 quirk_paxc_disable_msi_parsing);1567DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802,1568 quirk_paxc_disable_msi_parsing);1569DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804,1570 quirk_paxc_disable_msi_parsing);1571 1572static void quirk_paxc_bridge(struct pci_dev *pdev)1573{1574 /*1575 * The PCI config space is shared with the PAXC root port and the first1576 * Ethernet device. So, we need to workaround this by telling the PCI1577 * code that the bridge is not an Ethernet device.1578 */1579 if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)1580 pdev->class = PCI_CLASS_BRIDGE_PCI_NORMAL;1581 1582 /*1583 * MPSS is not being set properly (as it is currently 0). This is1584 * because that area of the PCI config space is hard coded to zero, and1585 * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS)1586 * so that the MPS can be set to the real max value.1587 */1588 pdev->pcie_mpss = 2;1589}1590DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge);1591DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge);1592DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge);1593DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge);1594DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge);1595 1596MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>");1597MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver");1598MODULE_LICENSE("GPL v2");1599