1060 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * Rockchip AXI PCIe host controller driver4 *5 * Copyright (c) 2016 Rockchip, Inc.6 *7 * Author: Shawn Lin <shawn.lin@rock-chips.com>8 * Wenrui Li <wenrui.li@rock-chips.com>9 *10 * Bits taken from Synopsys DesignWare Host controller driver and11 * ARM PCI Host generic driver.12 */13 14#include <linux/bitrev.h>15#include <linux/clk.h>16#include <linux/delay.h>17#include <linux/gpio/consumer.h>18#include <linux/init.h>19#include <linux/interrupt.h>20#include <linux/iopoll.h>21#include <linux/irq.h>22#include <linux/irqchip/chained_irq.h>23#include <linux/irqdomain.h>24#include <linux/kernel.h>25#include <linux/mfd/syscon.h>26#include <linux/module.h>27#include <linux/of.h>28#include <linux/of_pci.h>29#include <linux/pci.h>30#include <linux/pci_ids.h>31#include <linux/phy/phy.h>32#include <linux/platform_device.h>33#include <linux/reset.h>34#include <linux/regmap.h>35 36#include "../pci.h"37#include "pcie-rockchip.h"38 39static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)40{41 u32 status;42 43 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);44 status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);45 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);46}47 48static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)49{50 u32 status;51 52 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);53 status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;54 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);55}56 57static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)58{59 u32 val;60 61 /* Update Tx credit maximum update interval */62 val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);63 val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;64 val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */65 rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);66}67 68static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,69 struct pci_bus *bus, int dev)70{71 /*72 * Access only one slot on each root port.73 * Do not read more than one device on the bus directly attached74 * to RC's downstream side.75 */76 if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))77 return dev == 0;78 79 return 1;80}81 82static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)83{84 u32 val;85 u8 map;86 87 if (rockchip->legacy_phy)88 return GENMASK(MAX_LANE_NUM - 1, 0);89 90 val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);91 map = val & PCIE_CORE_LANE_MAP_MASK;92 93 /* The link may be using a reverse-indexed mapping. */94 if (val & PCIE_CORE_LANE_MAP_REVERSE)95 map = bitrev8(map) >> 4;96 97 return map;98}99 100static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,101 int where, int size, u32 *val)102{103 void __iomem *addr;104 105 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;106 107 if (!IS_ALIGNED((uintptr_t)addr, size)) {108 *val = 0;109 return PCIBIOS_BAD_REGISTER_NUMBER;110 }111 112 if (size == 4) {113 *val = readl(addr);114 } else if (size == 2) {115 *val = readw(addr);116 } else if (size == 1) {117 *val = readb(addr);118 } else {119 *val = 0;120 return PCIBIOS_BAD_REGISTER_NUMBER;121 }122 return PCIBIOS_SUCCESSFUL;123}124 125static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,126 int where, int size, u32 val)127{128 u32 mask, tmp, offset;129 void __iomem *addr;130 131 offset = where & ~0x3;132 addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;133 134 if (size == 4) {135 writel(val, addr);136 return PCIBIOS_SUCCESSFUL;137 }138 139 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));140 141 /*142 * N.B. This read/modify/write isn't safe in general because it can143 * corrupt RW1C bits in adjacent registers. But the hardware144 * doesn't support smaller writes.145 */146 tmp = readl(addr) & mask;147 tmp |= val << ((where & 0x3) * 8);148 writel(tmp, addr);149 150 return PCIBIOS_SUCCESSFUL;151}152 153static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,154 struct pci_bus *bus, u32 devfn,155 int where, int size, u32 *val)156{157 void __iomem *addr;158 159 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);160 161 if (!IS_ALIGNED((uintptr_t)addr, size)) {162 *val = 0;163 return PCIBIOS_BAD_REGISTER_NUMBER;164 }165 166 if (pci_is_root_bus(bus->parent))167 rockchip_pcie_cfg_configuration_accesses(rockchip,168 AXI_WRAPPER_TYPE0_CFG);169 else170 rockchip_pcie_cfg_configuration_accesses(rockchip,171 AXI_WRAPPER_TYPE1_CFG);172 173 if (size == 4) {174 *val = readl(addr);175 } else if (size == 2) {176 *val = readw(addr);177 } else if (size == 1) {178 *val = readb(addr);179 } else {180 *val = 0;181 return PCIBIOS_BAD_REGISTER_NUMBER;182 }183 return PCIBIOS_SUCCESSFUL;184}185 186static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,187 struct pci_bus *bus, u32 devfn,188 int where, int size, u32 val)189{190 void __iomem *addr;191 192 addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where);193 194 if (!IS_ALIGNED((uintptr_t)addr, size))195 return PCIBIOS_BAD_REGISTER_NUMBER;196 197 if (pci_is_root_bus(bus->parent))198 rockchip_pcie_cfg_configuration_accesses(rockchip,199 AXI_WRAPPER_TYPE0_CFG);200 else201 rockchip_pcie_cfg_configuration_accesses(rockchip,202 AXI_WRAPPER_TYPE1_CFG);203 204 if (size == 4)205 writel(val, addr);206 else if (size == 2)207 writew(val, addr);208 else if (size == 1)209 writeb(val, addr);210 else211 return PCIBIOS_BAD_REGISTER_NUMBER;212 213 return PCIBIOS_SUCCESSFUL;214}215 216static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,217 int size, u32 *val)218{219 struct rockchip_pcie *rockchip = bus->sysdata;220 221 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))222 return PCIBIOS_DEVICE_NOT_FOUND;223 224 if (pci_is_root_bus(bus))225 return rockchip_pcie_rd_own_conf(rockchip, where, size, val);226 227 return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,228 val);229}230 231static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,232 int where, int size, u32 val)233{234 struct rockchip_pcie *rockchip = bus->sysdata;235 236 if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))237 return PCIBIOS_DEVICE_NOT_FOUND;238 239 if (pci_is_root_bus(bus))240 return rockchip_pcie_wr_own_conf(rockchip, where, size, val);241 242 return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,243 val);244}245 246static struct pci_ops rockchip_pcie_ops = {247 .read = rockchip_pcie_rd_conf,248 .write = rockchip_pcie_wr_conf,249};250 251static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)252{253 int curr;254 u32 status, scale, power;255 256 if (IS_ERR(rockchip->vpcie3v3))257 return;258 259 /*260 * Set RC's captured slot power limit and scale if261 * vpcie3v3 available. The default values are both zero262 * which means the software should set these two according263 * to the actual power supply.264 */265 curr = regulator_get_current_limit(rockchip->vpcie3v3);266 if (curr <= 0)267 return;268 269 scale = 3; /* 0.001x */270 curr = curr / 1000; /* convert to mA */271 power = (curr * 3300) / 1000; /* milliwatt */272 while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {273 if (!scale) {274 dev_warn(rockchip->dev, "invalid power supply\n");275 return;276 }277 scale--;278 power = power / 10;279 }280 281 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);282 status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |283 (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);284 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);285}286 287/**288 * rockchip_pcie_host_init_port - Initialize hardware289 * @rockchip: PCIe port information290 */291static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)292{293 struct device *dev = rockchip->dev;294 int err, i = MAX_LANE_NUM;295 u32 status;296 297 gpiod_set_value_cansleep(rockchip->ep_gpio, 0);298 299 err = rockchip_pcie_init_port(rockchip);300 if (err)301 return err;302 303 /* Fix the transmitted FTS count desired to exit from L0s. */304 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);305 status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |306 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);307 rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);308 309 rockchip_pcie_set_power_limit(rockchip);310 311 /* Set RC's clock architecture as common clock */312 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);313 status |= PCI_EXP_LNKSTA_SLC << 16;314 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);315 316 /* Set RC's RCB to 128 */317 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);318 status |= PCI_EXP_LNKCTL_RCB;319 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);320 321 /* Enable Gen1 training */322 rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,323 PCIE_CLIENT_CONFIG);324 325 msleep(PCIE_T_PVPERL_MS);326 gpiod_set_value_cansleep(rockchip->ep_gpio, 1);327 328 msleep(PCIE_T_RRS_READY_MS);329 330 /* 500ms timeout value should be enough for Gen1/2 training */331 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,332 status, PCIE_LINK_UP(status), 20,333 500 * USEC_PER_MSEC);334 if (err) {335 dev_err(dev, "PCIe link training gen1 timeout!\n");336 goto err_power_off_phy;337 }338 339 if (rockchip->link_gen == 2) {340 /*341 * Enable retrain for gen2. This should be configured only after342 * gen1 finished.343 */344 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);345 status |= PCI_EXP_LNKCTL_RL;346 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);347 348 err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,349 status, PCIE_LINK_IS_GEN2(status), 20,350 500 * USEC_PER_MSEC);351 if (err)352 dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");353 }354 355 /* Check the final link width from negotiated lane counter from MGMT */356 status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);357 status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>358 PCIE_CORE_PL_CONF_LANE_SHIFT);359 dev_dbg(dev, "current link width is x%d\n", status);360 361 /* Power off unused lane(s) */362 rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);363 for (i = 0; i < MAX_LANE_NUM; i++) {364 if (!(rockchip->lanes_map & BIT(i))) {365 dev_dbg(dev, "idling lane %d\n", i);366 phy_power_off(rockchip->phys[i]);367 }368 }369 370 rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,371 PCIE_CORE_CONFIG_VENDOR);372 rockchip_pcie_write(rockchip,373 PCI_CLASS_BRIDGE_PCI_NORMAL << 8,374 PCIE_RC_CONFIG_RID_CCR);375 376 /* Clear THP cap's next cap pointer to remove L1 substate cap */377 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);378 status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;379 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);380 381 /* Clear L0s from RC's link cap */382 if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {383 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);384 status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;385 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);386 }387 388 status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);389 status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;390 status |= PCIE_RC_CONFIG_DCSR_MPS_256;391 rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);392 393 return 0;394err_power_off_phy:395 while (i--)396 phy_power_off(rockchip->phys[i]);397 i = MAX_LANE_NUM;398 while (i--)399 phy_exit(rockchip->phys[i]);400 return err;401}402 403static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)404{405 struct rockchip_pcie *rockchip = arg;406 struct device *dev = rockchip->dev;407 u32 reg;408 u32 sub_reg;409 410 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);411 if (reg & PCIE_CLIENT_INT_LOCAL) {412 dev_dbg(dev, "local interrupt received\n");413 sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);414 if (sub_reg & PCIE_CORE_INT_PRFPE)415 dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");416 417 if (sub_reg & PCIE_CORE_INT_CRFPE)418 dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");419 420 if (sub_reg & PCIE_CORE_INT_RRPE)421 dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");422 423 if (sub_reg & PCIE_CORE_INT_PRFO)424 dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");425 426 if (sub_reg & PCIE_CORE_INT_CRFO)427 dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");428 429 if (sub_reg & PCIE_CORE_INT_RT)430 dev_dbg(dev, "replay timer timed out\n");431 432 if (sub_reg & PCIE_CORE_INT_RTR)433 dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");434 435 if (sub_reg & PCIE_CORE_INT_PE)436 dev_dbg(dev, "phy error detected on receive side\n");437 438 if (sub_reg & PCIE_CORE_INT_MTR)439 dev_dbg(dev, "malformed TLP received from the link\n");440 441 if (sub_reg & PCIE_CORE_INT_UCR)442 dev_dbg(dev, "malformed TLP received from the link\n");443 444 if (sub_reg & PCIE_CORE_INT_FCE)445 dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");446 447 if (sub_reg & PCIE_CORE_INT_CT)448 dev_dbg(dev, "a request timed out waiting for completion\n");449 450 if (sub_reg & PCIE_CORE_INT_UTC)451 dev_dbg(dev, "unmapped TC error\n");452 453 if (sub_reg & PCIE_CORE_INT_MMVC)454 dev_dbg(dev, "MSI mask register changes\n");455 456 rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);457 } else if (reg & PCIE_CLIENT_INT_PHY) {458 dev_dbg(dev, "phy link changes\n");459 rockchip_pcie_update_txcredit_mui(rockchip);460 rockchip_pcie_clr_bw_int(rockchip);461 }462 463 rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,464 PCIE_CLIENT_INT_STATUS);465 466 return IRQ_HANDLED;467}468 469static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)470{471 struct rockchip_pcie *rockchip = arg;472 struct device *dev = rockchip->dev;473 u32 reg;474 475 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);476 if (reg & PCIE_CLIENT_INT_LEGACY_DONE)477 dev_dbg(dev, "legacy done interrupt received\n");478 479 if (reg & PCIE_CLIENT_INT_MSG)480 dev_dbg(dev, "message done interrupt received\n");481 482 if (reg & PCIE_CLIENT_INT_HOT_RST)483 dev_dbg(dev, "hot reset interrupt received\n");484 485 if (reg & PCIE_CLIENT_INT_DPA)486 dev_dbg(dev, "dpa interrupt received\n");487 488 if (reg & PCIE_CLIENT_INT_FATAL_ERR)489 dev_dbg(dev, "fatal error interrupt received\n");490 491 if (reg & PCIE_CLIENT_INT_NFATAL_ERR)492 dev_dbg(dev, "no fatal error interrupt received\n");493 494 if (reg & PCIE_CLIENT_INT_CORR_ERR)495 dev_dbg(dev, "correctable error interrupt received\n");496 497 if (reg & PCIE_CLIENT_INT_PHY)498 dev_dbg(dev, "phy interrupt received\n");499 500 rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |501 PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |502 PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |503 PCIE_CLIENT_INT_NFATAL_ERR |504 PCIE_CLIENT_INT_CORR_ERR |505 PCIE_CLIENT_INT_PHY),506 PCIE_CLIENT_INT_STATUS);507 508 return IRQ_HANDLED;509}510 511static void rockchip_pcie_intx_handler(struct irq_desc *desc)512{513 struct irq_chip *chip = irq_desc_get_chip(desc);514 struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);515 struct device *dev = rockchip->dev;516 u32 reg;517 u32 hwirq;518 int ret;519 520 chained_irq_enter(chip, desc);521 522 reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);523 reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;524 525 while (reg) {526 hwirq = ffs(reg) - 1;527 reg &= ~BIT(hwirq);528 529 ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq);530 if (ret)531 dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);532 }533 534 chained_irq_exit(chip, desc);535}536 537static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)538{539 int irq, err;540 struct device *dev = rockchip->dev;541 struct platform_device *pdev = to_platform_device(dev);542 543 irq = platform_get_irq_byname(pdev, "sys");544 if (irq < 0)545 return irq;546 547 err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,548 IRQF_SHARED, "pcie-sys", rockchip);549 if (err) {550 dev_err(dev, "failed to request PCIe subsystem IRQ\n");551 return err;552 }553 554 irq = platform_get_irq_byname(pdev, "legacy");555 if (irq < 0)556 return irq;557 558 irq_set_chained_handler_and_data(irq,559 rockchip_pcie_intx_handler,560 rockchip);561 562 irq = platform_get_irq_byname(pdev, "client");563 if (irq < 0)564 return irq;565 566 err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,567 IRQF_SHARED, "pcie-client", rockchip);568 if (err) {569 dev_err(dev, "failed to request PCIe client IRQ\n");570 return err;571 }572 573 return 0;574}575 576/**577 * rockchip_pcie_parse_host_dt - Parse Device Tree578 * @rockchip: PCIe port information579 *580 * Return: '0' on success and error value on failure581 */582static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)583{584 struct device *dev = rockchip->dev;585 int err;586 587 err = rockchip_pcie_parse_dt(rockchip);588 if (err)589 return err;590 591 rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");592 if (IS_ERR(rockchip->vpcie12v)) {593 if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)594 return PTR_ERR(rockchip->vpcie12v);595 dev_info(dev, "no vpcie12v regulator found\n");596 }597 598 rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");599 if (IS_ERR(rockchip->vpcie3v3)) {600 if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)601 return PTR_ERR(rockchip->vpcie3v3);602 dev_info(dev, "no vpcie3v3 regulator found\n");603 }604 605 rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8");606 if (IS_ERR(rockchip->vpcie1v8))607 return PTR_ERR(rockchip->vpcie1v8);608 609 rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9");610 if (IS_ERR(rockchip->vpcie0v9))611 return PTR_ERR(rockchip->vpcie0v9);612 613 return 0;614}615 616static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)617{618 struct device *dev = rockchip->dev;619 int err;620 621 if (!IS_ERR(rockchip->vpcie12v)) {622 err = regulator_enable(rockchip->vpcie12v);623 if (err) {624 dev_err(dev, "fail to enable vpcie12v regulator\n");625 goto err_out;626 }627 }628 629 if (!IS_ERR(rockchip->vpcie3v3)) {630 err = regulator_enable(rockchip->vpcie3v3);631 if (err) {632 dev_err(dev, "fail to enable vpcie3v3 regulator\n");633 goto err_disable_12v;634 }635 }636 637 err = regulator_enable(rockchip->vpcie1v8);638 if (err) {639 dev_err(dev, "fail to enable vpcie1v8 regulator\n");640 goto err_disable_3v3;641 }642 643 err = regulator_enable(rockchip->vpcie0v9);644 if (err) {645 dev_err(dev, "fail to enable vpcie0v9 regulator\n");646 goto err_disable_1v8;647 }648 649 return 0;650 651err_disable_1v8:652 regulator_disable(rockchip->vpcie1v8);653err_disable_3v3:654 if (!IS_ERR(rockchip->vpcie3v3))655 regulator_disable(rockchip->vpcie3v3);656err_disable_12v:657 if (!IS_ERR(rockchip->vpcie12v))658 regulator_disable(rockchip->vpcie12v);659err_out:660 return err;661}662 663static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)664{665 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &666 (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);667 rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),668 PCIE_CORE_INT_MASK);669 670 rockchip_pcie_enable_bw_int(rockchip);671}672 673static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,674 irq_hw_number_t hwirq)675{676 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);677 irq_set_chip_data(irq, domain->host_data);678 679 return 0;680}681 682static const struct irq_domain_ops intx_domain_ops = {683 .map = rockchip_pcie_intx_map,684};685 686static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)687{688 struct device *dev = rockchip->dev;689 struct device_node *intc = of_get_next_child(dev->of_node, NULL);690 691 if (!intc) {692 dev_err(dev, "missing child interrupt-controller node\n");693 return -EINVAL;694 }695 696 rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,697 &intx_domain_ops, rockchip);698 of_node_put(intc);699 if (!rockchip->irq_domain) {700 dev_err(dev, "failed to get a INTx IRQ domain\n");701 return -EINVAL;702 }703 704 return 0;705}706 707static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,708 int region_no, int type, u8 num_pass_bits,709 u32 lower_addr, u32 upper_addr)710{711 u32 ob_addr_0;712 u32 ob_addr_1;713 u32 ob_desc_0;714 u32 aw_offset;715 716 if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)717 return -EINVAL;718 if (num_pass_bits + 1 < 8)719 return -EINVAL;720 if (num_pass_bits > 63)721 return -EINVAL;722 if (region_no == 0) {723 if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))724 return -EINVAL;725 }726 if (region_no != 0) {727 if (AXI_REGION_SIZE < (2ULL << num_pass_bits))728 return -EINVAL;729 }730 731 aw_offset = (region_no << OB_REG_SIZE_SHIFT);732 733 ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;734 ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;735 ob_addr_1 = upper_addr;736 ob_desc_0 = (1 << 23 | type);737 738 rockchip_pcie_write(rockchip, ob_addr_0,739 PCIE_CORE_OB_REGION_ADDR0 + aw_offset);740 rockchip_pcie_write(rockchip, ob_addr_1,741 PCIE_CORE_OB_REGION_ADDR1 + aw_offset);742 rockchip_pcie_write(rockchip, ob_desc_0,743 PCIE_CORE_OB_REGION_DESC0 + aw_offset);744 rockchip_pcie_write(rockchip, 0,745 PCIE_CORE_OB_REGION_DESC1 + aw_offset);746 747 return 0;748}749 750static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,751 int region_no, u8 num_pass_bits,752 u32 lower_addr, u32 upper_addr)753{754 u32 ib_addr_0;755 u32 ib_addr_1;756 u32 aw_offset;757 758 if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)759 return -EINVAL;760 if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)761 return -EINVAL;762 if (num_pass_bits > 63)763 return -EINVAL;764 765 aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);766 767 ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;768 ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;769 ib_addr_1 = upper_addr;770 771 rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);772 rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);773 774 return 0;775}776 777static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)778{779 struct device *dev = rockchip->dev;780 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);781 struct resource_entry *entry;782 u64 pci_addr, size;783 int offset;784 int err;785 int reg_no;786 787 rockchip_pcie_cfg_configuration_accesses(rockchip,788 AXI_WRAPPER_TYPE0_CFG);789 entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM);790 if (!entry)791 return -ENODEV;792 793 size = resource_size(entry->res);794 pci_addr = entry->res->start - entry->offset;795 rockchip->msg_bus_addr = pci_addr;796 797 for (reg_no = 0; reg_no < (size >> 20); reg_no++) {798 err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,799 AXI_WRAPPER_MEM_WRITE,800 20 - 1,801 pci_addr + (reg_no << 20),802 0);803 if (err) {804 dev_err(dev, "program RC mem outbound ATU failed\n");805 return err;806 }807 }808 809 err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);810 if (err) {811 dev_err(dev, "program RC mem inbound ATU failed\n");812 return err;813 }814 815 entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO);816 if (!entry)817 return -ENODEV;818 819 /* store the register number offset to program RC io outbound ATU */820 offset = size >> 20;821 822 size = resource_size(entry->res);823 pci_addr = entry->res->start - entry->offset;824 825 for (reg_no = 0; reg_no < (size >> 20); reg_no++) {826 err = rockchip_pcie_prog_ob_atu(rockchip,827 reg_no + 1 + offset,828 AXI_WRAPPER_IO_WRITE,829 20 - 1,830 pci_addr + (reg_no << 20),831 0);832 if (err) {833 dev_err(dev, "program RC io outbound ATU failed\n");834 return err;835 }836 }837 838 /* assign message regions */839 rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,840 AXI_WRAPPER_NOR_MSG,841 20 - 1, 0, 0);842 843 rockchip->msg_bus_addr += ((reg_no + offset) << 20);844 return err;845}846 847static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)848{849 u32 value;850 int err;851 852 /* send PME_TURN_OFF message */853 writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);854 855 /* read LTSSM and wait for falling into L2 link state */856 err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,857 value, PCIE_LINK_IS_L2(value), 20,858 jiffies_to_usecs(5 * HZ));859 if (err) {860 dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");861 return err;862 }863 864 return 0;865}866 867static int rockchip_pcie_suspend_noirq(struct device *dev)868{869 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);870 int ret;871 872 /* disable core and cli int since we don't need to ack PME_ACK */873 rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |874 PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);875 rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);876 877 ret = rockchip_pcie_wait_l2(rockchip);878 if (ret) {879 rockchip_pcie_enable_interrupts(rockchip);880 return ret;881 }882 883 rockchip_pcie_deinit_phys(rockchip);884 885 rockchip_pcie_disable_clocks(rockchip);886 887 regulator_disable(rockchip->vpcie0v9);888 889 return ret;890}891 892static int rockchip_pcie_resume_noirq(struct device *dev)893{894 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);895 int err;896 897 err = regulator_enable(rockchip->vpcie0v9);898 if (err) {899 dev_err(dev, "fail to enable vpcie0v9 regulator\n");900 return err;901 }902 903 err = rockchip_pcie_enable_clocks(rockchip);904 if (err)905 goto err_disable_0v9;906 907 err = rockchip_pcie_host_init_port(rockchip);908 if (err)909 goto err_pcie_resume;910 911 err = rockchip_pcie_cfg_atu(rockchip);912 if (err)913 goto err_err_deinit_port;914 915 /* Need this to enter L1 again */916 rockchip_pcie_update_txcredit_mui(rockchip);917 rockchip_pcie_enable_interrupts(rockchip);918 919 return 0;920 921err_err_deinit_port:922 rockchip_pcie_deinit_phys(rockchip);923err_pcie_resume:924 rockchip_pcie_disable_clocks(rockchip);925err_disable_0v9:926 regulator_disable(rockchip->vpcie0v9);927 return err;928}929 930static int rockchip_pcie_probe(struct platform_device *pdev)931{932 struct rockchip_pcie *rockchip;933 struct device *dev = &pdev->dev;934 struct pci_host_bridge *bridge;935 int err;936 937 if (!dev->of_node)938 return -ENODEV;939 940 bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));941 if (!bridge)942 return -ENOMEM;943 944 rockchip = pci_host_bridge_priv(bridge);945 946 platform_set_drvdata(pdev, rockchip);947 rockchip->dev = dev;948 rockchip->is_rc = true;949 950 err = rockchip_pcie_parse_host_dt(rockchip);951 if (err)952 return err;953 954 err = rockchip_pcie_enable_clocks(rockchip);955 if (err)956 return err;957 958 err = rockchip_pcie_set_vpcie(rockchip);959 if (err) {960 dev_err(dev, "failed to set vpcie regulator\n");961 goto err_set_vpcie;962 }963 964 err = rockchip_pcie_host_init_port(rockchip);965 if (err)966 goto err_vpcie;967 968 err = rockchip_pcie_init_irq_domain(rockchip);969 if (err < 0)970 goto err_deinit_port;971 972 err = rockchip_pcie_cfg_atu(rockchip);973 if (err)974 goto err_remove_irq_domain;975 976 rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);977 if (!rockchip->msg_region) {978 err = -ENOMEM;979 goto err_remove_irq_domain;980 }981 982 bridge->sysdata = rockchip;983 bridge->ops = &rockchip_pcie_ops;984 985 err = rockchip_pcie_setup_irq(rockchip);986 if (err)987 goto err_remove_irq_domain;988 989 rockchip_pcie_enable_interrupts(rockchip);990 991 err = pci_host_probe(bridge);992 if (err < 0)993 goto err_remove_irq_domain;994 995 return 0;996 997err_remove_irq_domain:998 irq_domain_remove(rockchip->irq_domain);999err_deinit_port:1000 rockchip_pcie_deinit_phys(rockchip);1001err_vpcie:1002 if (!IS_ERR(rockchip->vpcie12v))1003 regulator_disable(rockchip->vpcie12v);1004 if (!IS_ERR(rockchip->vpcie3v3))1005 regulator_disable(rockchip->vpcie3v3);1006 regulator_disable(rockchip->vpcie1v8);1007 regulator_disable(rockchip->vpcie0v9);1008err_set_vpcie:1009 rockchip_pcie_disable_clocks(rockchip);1010 return err;1011}1012 1013static void rockchip_pcie_remove(struct platform_device *pdev)1014{1015 struct device *dev = &pdev->dev;1016 struct rockchip_pcie *rockchip = dev_get_drvdata(dev);1017 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);1018 1019 pci_stop_root_bus(bridge->bus);1020 pci_remove_root_bus(bridge->bus);1021 irq_domain_remove(rockchip->irq_domain);1022 1023 rockchip_pcie_deinit_phys(rockchip);1024 1025 rockchip_pcie_disable_clocks(rockchip);1026 1027 if (!IS_ERR(rockchip->vpcie12v))1028 regulator_disable(rockchip->vpcie12v);1029 if (!IS_ERR(rockchip->vpcie3v3))1030 regulator_disable(rockchip->vpcie3v3);1031 regulator_disable(rockchip->vpcie1v8);1032 regulator_disable(rockchip->vpcie0v9);1033}1034 1035static const struct dev_pm_ops rockchip_pcie_pm_ops = {1036 NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,1037 rockchip_pcie_resume_noirq)1038};1039 1040static const struct of_device_id rockchip_pcie_of_match[] = {1041 { .compatible = "rockchip,rk3399-pcie", },1042 {}1043};1044MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);1045 1046static struct platform_driver rockchip_pcie_driver = {1047 .driver = {1048 .name = "rockchip-pcie",1049 .of_match_table = rockchip_pcie_of_match,1050 .pm = &rockchip_pcie_pm_ops,1051 },1052 .probe = rockchip_pcie_probe,1053 .remove_new = rockchip_pcie_remove,1054};1055module_platform_driver(rockchip_pcie_driver);1056 1057MODULE_AUTHOR("Rockchip Inc");1058MODULE_DESCRIPTION("Rockchip AXI PCIe driver");1059MODULE_LICENSE("GPL v2");1060