584 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2017 Cadence3// Cadence PCIe host controller driver.4// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>5 6#include <linux/delay.h>7#include <linux/kernel.h>8#include <linux/list_sort.h>9#include <linux/of_address.h>10#include <linux/of_pci.h>11#include <linux/platform_device.h>12 13#include "pcie-cadence.h"14 15#define LINK_RETRAIN_TIMEOUT HZ16 17static u64 bar_max_size[] = {18 [RP_BAR0] = _ULL(128 * SZ_2G),19 [RP_BAR1] = SZ_2G,20 [RP_NO_BAR] = _BITULL(63),21};22 23static u8 bar_aperture_mask[] = {24 [RP_BAR0] = 0x1F,25 [RP_BAR1] = 0xF,26};27 28void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn,29 int where)30{31 struct pci_host_bridge *bridge = pci_find_host_bridge(bus);32 struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge);33 struct cdns_pcie *pcie = &rc->pcie;34 unsigned int busn = bus->number;35 u32 addr0, desc0;36 37 if (pci_is_root_bus(bus)) {38 /*39 * Only the root port (devfn == 0) is connected to this bus.40 * All other PCI devices are behind some bridge hence on another41 * bus.42 */43 if (devfn)44 return NULL;45 46 return pcie->reg_base + (where & 0xfff);47 }48 /* Check that the link is up */49 if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1))50 return NULL;51 /* Clear AXI link-down status */52 cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0);53 54 /* Update Output registers for AXI region 0. */55 addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) |56 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) |57 CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn);58 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0);59 60 /* Configuration Type 0 or Type 1 access. */61 desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID |62 CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0);63 /*64 * The bus number was already set once for all in desc1 by65 * cdns_pcie_host_init_address_translation().66 */67 if (busn == bridge->busnr + 1)68 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0;69 else70 desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1;71 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0);72 73 return rc->cfg_base + (where & 0xfff);74}75 76static struct pci_ops cdns_pcie_host_ops = {77 .map_bus = cdns_pci_map_bus,78 .read = pci_generic_config_read,79 .write = pci_generic_config_write,80};81 82static int cdns_pcie_host_training_complete(struct cdns_pcie *pcie)83{84 u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;85 unsigned long end_jiffies;86 u16 lnk_stat;87 88 /* Wait for link training to complete. Exit after timeout. */89 end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;90 do {91 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);92 if (!(lnk_stat & PCI_EXP_LNKSTA_LT))93 break;94 usleep_range(0, 1000);95 } while (time_before(jiffies, end_jiffies));96 97 if (!(lnk_stat & PCI_EXP_LNKSTA_LT))98 return 0;99 100 return -ETIMEDOUT;101}102 103static int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie)104{105 struct device *dev = pcie->dev;106 int retries;107 108 /* Check if the link is up or not */109 for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {110 if (cdns_pcie_link_up(pcie)) {111 dev_info(dev, "Link up\n");112 return 0;113 }114 usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);115 }116 117 return -ETIMEDOUT;118}119 120static int cdns_pcie_retrain(struct cdns_pcie *pcie)121{122 u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET;123 u16 lnk_stat, lnk_ctl;124 int ret = 0;125 126 /*127 * Set retrain bit if current speed is 2.5 GB/s,128 * but the PCIe root port support is > 2.5 GB/s.129 */130 131 lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off +132 PCI_EXP_LNKCAP));133 if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)134 return ret;135 136 lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA);137 if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) {138 lnk_ctl = cdns_pcie_rp_readw(pcie,139 pcie_cap_off + PCI_EXP_LNKCTL);140 lnk_ctl |= PCI_EXP_LNKCTL_RL;141 cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL,142 lnk_ctl);143 144 ret = cdns_pcie_host_training_complete(pcie);145 if (ret)146 return ret;147 148 ret = cdns_pcie_host_wait_for_link(pcie);149 }150 return ret;151}152 153static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie)154{155 u32 val;156 157 val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL);158 cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN);159}160 161static int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc)162{163 struct cdns_pcie *pcie = &rc->pcie;164 int ret;165 166 ret = cdns_pcie_host_wait_for_link(pcie);167 168 /*169 * Retrain link for Gen2 training defect170 * if quirk flag is set.171 */172 if (!ret && rc->quirk_retrain_flag)173 ret = cdns_pcie_retrain(pcie);174 175 return ret;176}177 178static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc)179{180 struct cdns_pcie *pcie = &rc->pcie;181 u32 value, ctrl;182 u32 id;183 184 /*185 * Set the root complex BAR configuration register:186 * - disable both BAR0 and BAR1.187 * - enable Prefetchable Memory Base and Limit registers in type 1188 * config space (64 bits).189 * - enable IO Base and Limit registers in type 1 config190 * space (32 bits).191 */192 ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;193 value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) |194 CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) |195 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE |196 CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS |197 CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE |198 CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS;199 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);200 201 /* Set root port configuration space */202 if (rc->vendor_id != 0xffff) {203 id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) |204 CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id);205 cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);206 }207 208 if (rc->device_id != 0xffff)209 cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id);210 211 cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0);212 cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0);213 cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);214 215 return 0;216}217 218static int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc,219 enum cdns_pcie_rp_bar bar,220 u64 cpu_addr, u64 size,221 unsigned long flags)222{223 struct cdns_pcie *pcie = &rc->pcie;224 u32 addr0, addr1, aperture, value;225 226 if (!rc->avail_ib_bar[bar])227 return -EBUSY;228 229 rc->avail_ib_bar[bar] = false;230 231 aperture = ilog2(size);232 addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) |233 (lower_32_bits(cpu_addr) & GENMASK(31, 8));234 addr1 = upper_32_bits(cpu_addr);235 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0);236 cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1);237 238 if (bar == RP_NO_BAR)239 return 0;240 241 value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG);242 value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) |243 LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) |244 LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) |245 LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) |246 LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2));247 if (size + cpu_addr >= SZ_4G) {248 if (!(flags & IORESOURCE_PREFETCH))249 value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar);250 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar);251 } else {252 if (!(flags & IORESOURCE_PREFETCH))253 value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar);254 value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar);255 }256 257 value |= LM_RC_BAR_CFG_APERTURE(bar, aperture);258 cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value);259 260 return 0;261}262 263static enum cdns_pcie_rp_bar264cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size)265{266 enum cdns_pcie_rp_bar bar, sel_bar;267 268 sel_bar = RP_BAR_UNDEFINED;269 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {270 if (!rc->avail_ib_bar[bar])271 continue;272 273 if (size <= bar_max_size[bar]) {274 if (sel_bar == RP_BAR_UNDEFINED) {275 sel_bar = bar;276 continue;277 }278 279 if (bar_max_size[bar] < bar_max_size[sel_bar])280 sel_bar = bar;281 }282 }283 284 return sel_bar;285}286 287static enum cdns_pcie_rp_bar288cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size)289{290 enum cdns_pcie_rp_bar bar, sel_bar;291 292 sel_bar = RP_BAR_UNDEFINED;293 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) {294 if (!rc->avail_ib_bar[bar])295 continue;296 297 if (size >= bar_max_size[bar]) {298 if (sel_bar == RP_BAR_UNDEFINED) {299 sel_bar = bar;300 continue;301 }302 303 if (bar_max_size[bar] > bar_max_size[sel_bar])304 sel_bar = bar;305 }306 }307 308 return sel_bar;309}310 311static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,312 struct resource_entry *entry)313{314 u64 cpu_addr, pci_addr, size, winsize;315 struct cdns_pcie *pcie = &rc->pcie;316 struct device *dev = pcie->dev;317 enum cdns_pcie_rp_bar bar;318 unsigned long flags;319 int ret;320 321 cpu_addr = entry->res->start;322 pci_addr = entry->res->start - entry->offset;323 flags = entry->res->flags;324 size = resource_size(entry->res);325 326 if (entry->offset) {327 dev_err(dev, "PCI addr: %llx must be equal to CPU addr: %llx\n",328 pci_addr, cpu_addr);329 return -EINVAL;330 }331 332 while (size > 0) {333 /*334 * Try to find a minimum BAR whose size is greater than335 * or equal to the remaining resource_entry size. This will336 * fail if the size of each of the available BARs is less than337 * the remaining resource_entry size.338 * If a minimum BAR is found, IB ATU will be configured and339 * exited.340 */341 bar = cdns_pcie_host_find_min_bar(rc, size);342 if (bar != RP_BAR_UNDEFINED) {343 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr,344 size, flags);345 if (ret)346 dev_err(dev, "IB BAR: %d config failed\n", bar);347 return ret;348 }349 350 /*351 * If the control reaches here, it would mean the remaining352 * resource_entry size cannot be fitted in a single BAR. So we353 * find a maximum BAR whose size is less than or equal to the354 * remaining resource_entry size and split the resource entry355 * so that part of resource entry is fitted inside the maximum356 * BAR. The remaining size would be fitted during the next357 * iteration of the loop.358 * If a maximum BAR is not found, there is no way we can fit359 * this resource_entry, so we error out.360 */361 bar = cdns_pcie_host_find_max_bar(rc, size);362 if (bar == RP_BAR_UNDEFINED) {363 dev_err(dev, "No free BAR to map cpu_addr %llx\n",364 cpu_addr);365 return -EINVAL;366 }367 368 winsize = bar_max_size[bar];369 ret = cdns_pcie_host_bar_ib_config(rc, bar, cpu_addr, winsize,370 flags);371 if (ret) {372 dev_err(dev, "IB BAR: %d config failed\n", bar);373 return ret;374 }375 376 size -= winsize;377 cpu_addr += winsize;378 }379 380 return 0;381}382 383static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,384 const struct list_head *b)385{386 struct resource_entry *entry1, *entry2;387 388 entry1 = container_of(a, struct resource_entry, node);389 entry2 = container_of(b, struct resource_entry, node);390 391 return resource_size(entry2->res) - resource_size(entry1->res);392}393 394static int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc)395{396 struct cdns_pcie *pcie = &rc->pcie;397 struct device *dev = pcie->dev;398 struct device_node *np = dev->of_node;399 struct pci_host_bridge *bridge;400 struct resource_entry *entry;401 u32 no_bar_nbits = 32;402 int err;403 404 bridge = pci_host_bridge_from_priv(rc);405 if (!bridge)406 return -ENOMEM;407 408 if (list_empty(&bridge->dma_ranges)) {409 of_property_read_u32(np, "cdns,no-bar-match-nbits",410 &no_bar_nbits);411 err = cdns_pcie_host_bar_ib_config(rc, RP_NO_BAR, 0x0,412 (u64)1 << no_bar_nbits, 0);413 if (err)414 dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR);415 return err;416 }417 418 list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp);419 420 resource_list_for_each_entry(entry, &bridge->dma_ranges) {421 err = cdns_pcie_host_bar_config(rc, entry);422 if (err) {423 dev_err(dev, "Fail to configure IB using dma-ranges\n");424 return err;425 }426 }427 428 return 0;429}430 431static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)432{433 struct cdns_pcie *pcie = &rc->pcie;434 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc);435 struct resource *cfg_res = rc->cfg_res;436 struct resource_entry *entry;437 u64 cpu_addr = cfg_res->start;438 u32 addr0, addr1, desc1;439 int r, busnr = 0;440 441 entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS);442 if (entry)443 busnr = entry->res->start;444 445 /*446 * Reserve region 0 for PCI configure space accesses:447 * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by448 * cdns_pci_map_bus(), other region registers are set here once for all.449 */450 addr1 = 0; /* Should be programmed to zero. */451 desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr);452 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1);453 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1);454 455 if (pcie->ops->cpu_addr_fixup)456 cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr);457 458 addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) |459 (lower_32_bits(cpu_addr) & GENMASK(31, 8));460 addr1 = upper_32_bits(cpu_addr);461 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0);462 cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1);463 464 r = 1;465 resource_list_for_each_entry(entry, &bridge->windows) {466 struct resource *res = entry->res;467 u64 pci_addr = res->start - entry->offset;468 469 if (resource_type(res) == IORESOURCE_IO)470 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,471 true,472 pci_pio_to_address(res->start),473 pci_addr,474 resource_size(res));475 else476 cdns_pcie_set_outbound_region(pcie, busnr, 0, r,477 false,478 res->start,479 pci_addr,480 resource_size(res));481 482 r++;483 }484 485 return cdns_pcie_host_map_dma_ranges(rc);486}487 488int cdns_pcie_host_init(struct cdns_pcie_rc *rc)489{490 int err;491 492 err = cdns_pcie_host_init_root_port(rc);493 if (err)494 return err;495 496 return cdns_pcie_host_init_address_translation(rc);497}498 499int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc)500{501 struct cdns_pcie *pcie = &rc->pcie;502 struct device *dev = rc->pcie.dev;503 int ret;504 505 if (rc->quirk_detect_quiet_flag)506 cdns_pcie_detect_quiet_min_delay_set(&rc->pcie);507 508 cdns_pcie_host_enable_ptm_response(pcie);509 510 ret = cdns_pcie_start_link(pcie);511 if (ret) {512 dev_err(dev, "Failed to start link\n");513 return ret;514 }515 516 ret = cdns_pcie_host_start_link(rc);517 if (ret)518 dev_dbg(dev, "PCIe link never came up\n");519 520 return 0;521}522 523int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)524{525 struct device *dev = rc->pcie.dev;526 struct platform_device *pdev = to_platform_device(dev);527 struct device_node *np = dev->of_node;528 struct pci_host_bridge *bridge;529 enum cdns_pcie_rp_bar bar;530 struct cdns_pcie *pcie;531 struct resource *res;532 int ret;533 534 bridge = pci_host_bridge_from_priv(rc);535 if (!bridge)536 return -ENOMEM;537 538 pcie = &rc->pcie;539 pcie->is_rc = true;540 541 rc->vendor_id = 0xffff;542 of_property_read_u32(np, "vendor-id", &rc->vendor_id);543 544 rc->device_id = 0xffff;545 of_property_read_u32(np, "device-id", &rc->device_id);546 547 pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");548 if (IS_ERR(pcie->reg_base)) {549 dev_err(dev, "missing \"reg\"\n");550 return PTR_ERR(pcie->reg_base);551 }552 553 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg");554 rc->cfg_base = devm_pci_remap_cfg_resource(dev, res);555 if (IS_ERR(rc->cfg_base))556 return PTR_ERR(rc->cfg_base);557 rc->cfg_res = res;558 559 ret = cdns_pcie_host_link_setup(rc);560 if (ret)561 return ret;562 563 for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++)564 rc->avail_ib_bar[bar] = true;565 566 ret = cdns_pcie_host_init(rc);567 if (ret)568 return ret;569 570 if (!bridge->ops)571 bridge->ops = &cdns_pcie_host_ops;572 573 ret = pci_host_probe(bridge);574 if (ret < 0)575 goto err_init;576 577 return 0;578 579 err_init:580 pm_runtime_put_sync(dev);581 582 return ret;583}584