2511 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * PCIe host controller driver for the following SoCs4 * Tegra1945 * Tegra2346 *7 * Copyright (C) 2019-2022 NVIDIA Corporation.8 *9 * Author: Vidya Sagar <vidyas@nvidia.com>10 */11 12#include <linux/bitfield.h>13#include <linux/clk.h>14#include <linux/debugfs.h>15#include <linux/delay.h>16#include <linux/gpio/consumer.h>17#include <linux/interconnect.h>18#include <linux/interrupt.h>19#include <linux/iopoll.h>20#include <linux/kernel.h>21#include <linux/module.h>22#include <linux/of.h>23#include <linux/of_pci.h>24#include <linux/pci.h>25#include <linux/phy/phy.h>26#include <linux/pinctrl/consumer.h>27#include <linux/platform_device.h>28#include <linux/pm_runtime.h>29#include <linux/random.h>30#include <linux/reset.h>31#include <linux/resource.h>32#include <linux/types.h>33#include "pcie-designware.h"34#include <soc/tegra/bpmp.h>35#include <soc/tegra/bpmp-abi.h>36#include "../../pci.h"37 38#define TEGRA194_DWC_IP_VER 0x490A39#define TEGRA234_DWC_IP_VER 0x562A40 41#define APPL_PINMUX 0x042#define APPL_PINMUX_PEX_RST BIT(0)43#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)44#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)45#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)46#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)47 48#define APPL_CTRL 0x449#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)50#define APPL_CTRL_LTSSM_EN BIT(7)51#define APPL_CTRL_HW_HOT_RST_EN BIT(20)52#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)53#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 2254#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x155#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x256 57#define APPL_INTR_EN_L0_0 0x858#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)59#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)60#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)61#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)62#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)63#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)64#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)65 66#define APPL_INTR_STATUS_L0 0xC67#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)68#define APPL_INTR_STATUS_L0_INT_INT BIT(8)69#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)70#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)71#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)72 73#define APPL_INTR_EN_L1_0_0 0x1C74#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)75#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)76#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)77 78#define APPL_INTR_STATUS_L1_0_0 0x2079#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)80#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)81#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)82 83#define APPL_INTR_STATUS_L1_1 0x2C84#define APPL_INTR_STATUS_L1_2 0x3085#define APPL_INTR_STATUS_L1_3 0x3486#define APPL_INTR_STATUS_L1_6 0x3C87#define APPL_INTR_STATUS_L1_7 0x4088#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)89 90#define APPL_INTR_EN_L1_8_0 0x4491#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)92#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)93#define APPL_INTR_EN_L1_8_INTX_EN BIT(11)94#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)95 96#define APPL_INTR_STATUS_L1_8_0 0x4C97#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)98#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)99#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)100 101#define APPL_INTR_STATUS_L1_9 0x54102#define APPL_INTR_STATUS_L1_10 0x58103#define APPL_INTR_STATUS_L1_11 0x64104#define APPL_INTR_STATUS_L1_13 0x74105#define APPL_INTR_STATUS_L1_14 0x78106#define APPL_INTR_STATUS_L1_15 0x7C107#define APPL_INTR_STATUS_L1_17 0x88108 109#define APPL_INTR_EN_L1_18 0x90110#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)111#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)112#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)113 114#define APPL_INTR_STATUS_L1_18 0x94115#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)116#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)117#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)118 119#define APPL_MSI_CTRL_1 0xAC120 121#define APPL_MSI_CTRL_2 0xB0122 123#define APPL_LEGACY_INTX 0xB8124 125#define APPL_LTR_MSG_1 0xC4126#define LTR_MSG_REQ BIT(15)127#define LTR_NOSNOOP_MSG_REQ BIT(31)128 129#define APPL_LTR_MSG_2 0xC8130#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)131 132#define APPL_LINK_STATUS 0xCC133#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)134 135#define APPL_DEBUG 0xD0136#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)137#define APPL_DEBUG_PM_LINKST_IN_L0 0x11138#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)139#define APPL_DEBUG_LTSSM_STATE_SHIFT 3140#define LTSSM_STATE_PRE_DETECT 5141 142#define APPL_RADM_STATUS 0xE4143#define APPL_PM_XMT_TURNOFF_STATE BIT(0)144 145#define APPL_DM_TYPE 0x100146#define APPL_DM_TYPE_MASK GENMASK(3, 0)147#define APPL_DM_TYPE_RP 0x4148#define APPL_DM_TYPE_EP 0x0149 150#define APPL_CFG_BASE_ADDR 0x104151#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)152 153#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108154#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)155 156#define APPL_CFG_MISC 0x110157#define APPL_CFG_MISC_SLV_EP_MODE BIT(14)158#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)159#define APPL_CFG_MISC_ARCACHE_SHIFT 10160#define APPL_CFG_MISC_ARCACHE_VAL 3161 162#define APPL_CFG_SLCG_OVERRIDE 0x114163#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)164 165#define APPL_CAR_RESET_OVRD 0x12C166#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)167 168#define IO_BASE_IO_DECODE BIT(0)169#define IO_BASE_IO_DECODE_BIT8 BIT(8)170 171#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)172#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)173 174#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718175#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)176 177#define N_FTS_VAL 52178#define FTS_VAL 52179 180#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0181#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3182#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0)183#define AMBA_ERROR_RESPONSE_RRS_OKAY 0184#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1185#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2186 187#define MSIX_ADDR_MATCH_LOW_OFF 0x940188#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)189#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)190 191#define MSIX_ADDR_MATCH_HIGH_OFF 0x944192#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)193 194#define PORT_LOGIC_MSIX_DOORBELL 0x948195 196#define CAP_SPCIE_CAP_OFF 0x154197#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)198#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)199#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8200 201#define PME_ACK_TIMEOUT 10000202 203#define LTSSM_TIMEOUT 50000 /* 50ms */204 205#define GEN3_GEN4_EQ_PRESET_INIT 5206 207#define GEN1_CORE_CLK_FREQ 62500000208#define GEN2_CORE_CLK_FREQ 125000000209#define GEN3_CORE_CLK_FREQ 250000000210#define GEN4_CORE_CLK_FREQ 500000000211 212#define LTR_MSG_TIMEOUT (100 * 1000)213 214#define PERST_DEBOUNCE_TIME (5 * 1000)215 216#define EP_STATE_DISABLED 0217#define EP_STATE_ENABLED 1218 219static const unsigned int pcie_gen_freq[] = {220 GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */221 GEN1_CORE_CLK_FREQ,222 GEN2_CORE_CLK_FREQ,223 GEN3_CORE_CLK_FREQ,224 GEN4_CORE_CLK_FREQ225};226 227struct tegra_pcie_dw_of_data {228 u32 version;229 enum dw_pcie_device_mode mode;230 bool has_msix_doorbell_access_fix;231 bool has_sbr_reset_fix;232 bool has_l1ss_exit_fix;233 bool has_ltr_req_fix;234 u32 cdm_chk_int_en_bit;235 u32 gen4_preset_vec;236 u8 n_fts[2];237};238 239struct tegra_pcie_dw {240 struct device *dev;241 struct resource *appl_res;242 struct resource *dbi_res;243 struct resource *atu_dma_res;244 void __iomem *appl_base;245 struct clk *core_clk;246 struct reset_control *core_apb_rst;247 struct reset_control *core_rst;248 struct dw_pcie pci;249 struct tegra_bpmp *bpmp;250 251 struct tegra_pcie_dw_of_data *of_data;252 253 bool supports_clkreq;254 bool enable_cdm_check;255 bool enable_srns;256 bool link_state;257 bool update_fc_fixup;258 bool enable_ext_refclk;259 u8 init_link_width;260 u32 msi_ctrl_int;261 u32 num_lanes;262 u32 cid;263 u32 cfg_link_cap_l1sub;264 u32 ras_des_cap;265 u32 pcie_cap_base;266 u32 aspm_cmrt;267 u32 aspm_pwr_on_t;268 u32 aspm_l0s_enter_lat;269 270 struct regulator *pex_ctl_supply;271 struct regulator *slot_ctl_3v3;272 struct regulator *slot_ctl_12v;273 274 unsigned int phy_count;275 struct phy **phys;276 277 struct dentry *debugfs;278 279 /* Endpoint mode specific */280 struct gpio_desc *pex_rst_gpiod;281 struct gpio_desc *pex_refclk_sel_gpiod;282 unsigned int pex_rst_irq;283 int ep_state;284 long link_status;285 struct icc_path *icc_path;286};287 288static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)289{290 return container_of(pci, struct tegra_pcie_dw, pci);291}292 293static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,294 const u32 reg)295{296 writel_relaxed(value, pcie->appl_base + reg);297}298 299static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)300{301 return readl_relaxed(pcie->appl_base + reg);302}303 304static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie)305{306 struct dw_pcie *pci = &pcie->pci;307 u32 val, speed, width;308 309 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);310 311 speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val);312 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);313 314 val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]);315 316 if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0))317 dev_err(pcie->dev, "can't set bw[%u]\n", val);318 319 if (speed >= ARRAY_SIZE(pcie_gen_freq))320 speed = 0;321 322 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]);323}324 325static void apply_bad_link_workaround(struct dw_pcie_rp *pp)326{327 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);328 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);329 u32 current_link_width;330 u16 val;331 332 /*333 * NOTE:- Since this scenario is uncommon and link as such is not334 * stable anyway, not waiting to confirm if link is really335 * transitioning to Gen-2 speed336 */337 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);338 if (val & PCI_EXP_LNKSTA_LBMS) {339 current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val);340 if (pcie->init_link_width > current_link_width) {341 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");342 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +343 PCI_EXP_LNKCTL2);344 val &= ~PCI_EXP_LNKCTL2_TLS;345 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;346 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +347 PCI_EXP_LNKCTL2, val);348 349 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +350 PCI_EXP_LNKCTL);351 val |= PCI_EXP_LNKCTL_RL;352 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +353 PCI_EXP_LNKCTL, val);354 }355 }356}357 358static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)359{360 struct tegra_pcie_dw *pcie = arg;361 struct dw_pcie *pci = &pcie->pci;362 struct dw_pcie_rp *pp = &pci->pp;363 u32 val, status_l0, status_l1;364 u16 val_w;365 366 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);367 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {368 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);369 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);370 if (!pcie->of_data->has_sbr_reset_fix &&371 status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {372 /* SBR & Surprise Link Down WAR */373 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);374 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;375 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);376 udelay(1);377 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);378 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;379 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);380 381 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);382 val |= PORT_LOGIC_SPEED_CHANGE;383 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);384 }385 }386 387 if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) {388 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);389 if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {390 appl_writel(pcie,391 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,392 APPL_INTR_STATUS_L1_8_0);393 apply_bad_link_workaround(pp);394 }395 if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {396 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +397 PCI_EXP_LNKSTA);398 val_w |= PCI_EXP_LNKSTA_LBMS;399 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +400 PCI_EXP_LNKSTA, val_w);401 402 appl_writel(pcie,403 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,404 APPL_INTR_STATUS_L1_8_0);405 406 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +407 PCI_EXP_LNKSTA);408 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &409 PCI_EXP_LNKSTA_CLS);410 }411 }412 413 if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {414 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18);415 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);416 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {417 dev_info(pci->dev, "CDM check complete\n");418 val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;419 }420 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {421 dev_err(pci->dev, "CDM comparison mismatch\n");422 val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;423 }424 if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {425 dev_err(pci->dev, "CDM Logic error\n");426 val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;427 }428 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val);429 val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);430 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val);431 }432 433 return IRQ_HANDLED;434}435 436static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)437{438 u32 val;439 440 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);441 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);442 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);443 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);444 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);445 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);446 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);447 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);448 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);449 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);450 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);451 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);452 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);453 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);454 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);455 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);456 457 val = appl_readl(pcie, APPL_CTRL);458 val |= APPL_CTRL_LTSSM_EN;459 appl_writel(pcie, val, APPL_CTRL);460}461 462static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)463{464 struct tegra_pcie_dw *pcie = arg;465 struct dw_pcie_ep *ep = &pcie->pci.ep;466 struct dw_pcie *pci = &pcie->pci;467 u32 val;468 469 if (test_and_clear_bit(0, &pcie->link_status))470 dw_pcie_ep_linkup(ep);471 472 tegra_pcie_icc_set(pcie);473 474 if (pcie->of_data->has_ltr_req_fix)475 return IRQ_HANDLED;476 477 /* If EP doesn't advertise L1SS, just return */478 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);479 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))480 return IRQ_HANDLED;481 482 /* Check if BME is set to '1' */483 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);484 if (val & PCI_COMMAND_MASTER) {485 ktime_t timeout;486 487 /* 110us for both snoop and no-snoop */488 val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) |489 FIELD_PREP(PCI_LTR_SCALE_MASK, 2) |490 LTR_MSG_REQ |491 FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) |492 FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) |493 LTR_NOSNOOP_MSG_REQ;494 appl_writel(pcie, val, APPL_LTR_MSG_1);495 496 /* Send LTR upstream */497 val = appl_readl(pcie, APPL_LTR_MSG_2);498 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;499 appl_writel(pcie, val, APPL_LTR_MSG_2);500 501 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);502 for (;;) {503 val = appl_readl(pcie, APPL_LTR_MSG_2);504 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))505 break;506 if (ktime_after(ktime_get(), timeout))507 break;508 usleep_range(1000, 1100);509 }510 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)511 dev_err(pcie->dev, "Failed to send LTR message\n");512 }513 514 return IRQ_HANDLED;515}516 517static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)518{519 struct tegra_pcie_dw *pcie = arg;520 int spurious = 1;521 u32 status_l0, status_l1, link_status;522 523 status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0);524 if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) {525 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);526 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0);527 528 if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)529 pex_ep_event_hot_rst_done(pcie);530 531 if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {532 link_status = appl_readl(pcie, APPL_LINK_STATUS);533 if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) {534 dev_dbg(pcie->dev, "Link is up with Host\n");535 set_bit(0, &pcie->link_status);536 return IRQ_WAKE_THREAD;537 }538 }539 540 spurious = 0;541 }542 543 if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {544 status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15);545 appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15);546 547 if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)548 return IRQ_WAKE_THREAD;549 550 spurious = 0;551 }552 553 if (spurious) {554 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",555 status_l0);556 appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0);557 }558 559 return IRQ_HANDLED;560}561 562static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,563 int size, u32 *val)564{565 struct dw_pcie_rp *pp = bus->sysdata;566 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);567 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);568 569 /*570 * This is an endpoint mode specific register happen to appear even571 * when controller is operating in root port mode and system hangs572 * when it is accessed with link being in ASPM-L1 state.573 * So skip accessing it altogether574 */575 if (!pcie->of_data->has_msix_doorbell_access_fix &&576 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {577 *val = 0x00000000;578 return PCIBIOS_SUCCESSFUL;579 }580 581 return pci_generic_config_read(bus, devfn, where, size, val);582}583 584static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,585 int size, u32 val)586{587 struct dw_pcie_rp *pp = bus->sysdata;588 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);589 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);590 591 /*592 * This is an endpoint mode specific register happen to appear even593 * when controller is operating in root port mode and system hangs594 * when it is accessed with link being in ASPM-L1 state.595 * So skip accessing it altogether596 */597 if (!pcie->of_data->has_msix_doorbell_access_fix &&598 !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)599 return PCIBIOS_SUCCESSFUL;600 601 return pci_generic_config_write(bus, devfn, where, size, val);602}603 604static struct pci_ops tegra_pci_ops = {605 .map_bus = dw_pcie_own_conf_map_bus,606 .read = tegra_pcie_dw_rd_own_conf,607 .write = tegra_pcie_dw_wr_own_conf,608};609 610#if defined(CONFIG_PCIEASPM)611static void disable_aspm_l11(struct tegra_pcie_dw *pcie)612{613 u32 val;614 615 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);616 val &= ~PCI_L1SS_CAP_ASPM_L1_1;617 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);618}619 620static void disable_aspm_l12(struct tegra_pcie_dw *pcie)621{622 u32 val;623 624 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);625 val &= ~PCI_L1SS_CAP_ASPM_L1_2;626 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);627}628 629static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)630{631 u32 val;632 633 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +634 PCIE_RAS_DES_EVENT_COUNTER_CONTROL);635 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);636 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;637 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;638 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;639 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +640 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);641 val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap +642 PCIE_RAS_DES_EVENT_COUNTER_DATA);643 644 return val;645}646 647static int aspm_state_cnt(struct seq_file *s, void *data)648{649 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)650 dev_get_drvdata(s->private);651 u32 val;652 653 seq_printf(s, "Tx L0s entry count : %u\n",654 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));655 656 seq_printf(s, "Rx L0s entry count : %u\n",657 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));658 659 seq_printf(s, "Link L1 entry count : %u\n",660 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));661 662 seq_printf(s, "Link L1.1 entry count : %u\n",663 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));664 665 seq_printf(s, "Link L1.2 entry count : %u\n",666 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));667 668 /* Clear all counters */669 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +670 PCIE_RAS_DES_EVENT_COUNTER_CONTROL,671 EVENT_COUNTER_ALL_CLEAR);672 673 /* Re-enable counting */674 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;675 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;676 dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap +677 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);678 679 return 0;680}681 682static void init_host_aspm(struct tegra_pcie_dw *pcie)683{684 struct dw_pcie *pci = &pcie->pci;685 u32 val;686 687 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);688 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;689 690 pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci,691 PCI_EXT_CAP_ID_VNDR);692 693 /* Enable ASPM counters */694 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;695 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;696 dw_pcie_writel_dbi(pci, pcie->ras_des_cap +697 PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val);698 699 /* Program T_cmrt and T_pwr_on values */700 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);701 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);702 val |= (pcie->aspm_cmrt << 8);703 val |= (pcie->aspm_pwr_on_t << 19);704 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);705 706 /* Program L0s and L1 entrance latencies */707 val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);708 val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;709 val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);710 val |= PORT_AFR_ENTER_ASPM;711 dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);712}713 714static void init_debugfs(struct tegra_pcie_dw *pcie)715{716 debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", pcie->debugfs,717 aspm_state_cnt);718}719#else720static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }721static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }722static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }723static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; }724#endif725 726static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp)727{728 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);729 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);730 u32 val;731 u16 val_w;732 733 val = appl_readl(pcie, APPL_INTR_EN_L0_0);734 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;735 appl_writel(pcie, val, APPL_INTR_EN_L0_0);736 737 if (!pcie->of_data->has_sbr_reset_fix) {738 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);739 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;740 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);741 }742 743 if (pcie->enable_cdm_check) {744 val = appl_readl(pcie, APPL_INTR_EN_L0_0);745 val |= pcie->of_data->cdm_chk_int_en_bit;746 appl_writel(pcie, val, APPL_INTR_EN_L0_0);747 748 val = appl_readl(pcie, APPL_INTR_EN_L1_18);749 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;750 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;751 appl_writel(pcie, val, APPL_INTR_EN_L1_18);752 }753 754 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +755 PCI_EXP_LNKSTA);756 pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w);757 758 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +759 PCI_EXP_LNKCTL);760 val_w |= PCI_EXP_LNKCTL_LBMIE;761 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,762 val_w);763}764 765static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp)766{767 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);768 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);769 u32 val;770 771 /* Enable INTX interrupt generation */772 val = appl_readl(pcie, APPL_INTR_EN_L0_0);773 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;774 val |= APPL_INTR_EN_L0_0_INT_INT_EN;775 appl_writel(pcie, val, APPL_INTR_EN_L0_0);776 777 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);778 val |= APPL_INTR_EN_L1_8_INTX_EN;779 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;780 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;781 if (IS_ENABLED(CONFIG_PCIEAER))782 val |= APPL_INTR_EN_L1_8_AER_INT_EN;783 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);784}785 786static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp)787{788 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);789 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);790 u32 val;791 792 /* Enable MSI interrupt generation */793 val = appl_readl(pcie, APPL_INTR_EN_L0_0);794 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;795 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;796 appl_writel(pcie, val, APPL_INTR_EN_L0_0);797}798 799static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp)800{801 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);802 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);803 804 /* Clear interrupt statuses before enabling interrupts */805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);818 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);819 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);820 821 tegra_pcie_enable_system_interrupts(pp);822 tegra_pcie_enable_intx_interrupts(pp);823 if (IS_ENABLED(CONFIG_PCI_MSI))824 tegra_pcie_enable_msi_interrupts(pp);825}826 827static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)828{829 struct dw_pcie *pci = &pcie->pci;830 u32 val, offset, i;831 832 /* Program init preset */833 for (i = 0; i < pcie->num_lanes; i++) {834 val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2));835 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;836 val |= GEN3_GEN4_EQ_PRESET_INIT;837 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;838 val |= (GEN3_GEN4_EQ_PRESET_INIT <<839 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);840 dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val);841 842 offset = dw_pcie_find_ext_capability(pci,843 PCI_EXT_CAP_ID_PL_16GT) +844 PCI_PL_16GT_LE_CTRL;845 val = dw_pcie_readb_dbi(pci, offset + i);846 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;847 val |= GEN3_GEN4_EQ_PRESET_INIT;848 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;849 val |= (GEN3_GEN4_EQ_PRESET_INIT <<850 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);851 dw_pcie_writeb_dbi(pci, offset + i, val);852 }853 854 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);855 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;856 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);857 858 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);859 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;860 val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff);861 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;862 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);863 864 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);865 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;866 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);868 869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;871 val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC,872 pcie->of_data->gen4_preset_vec);873 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;874 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);875 876 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);877 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;878 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);879}880 881static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp)882{883 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);884 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);885 u32 val;886 u16 val_16;887 888 pp->bridge->ops = &tegra_pci_ops;889 890 if (!pcie->pcie_cap_base)891 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,892 PCI_CAP_ID_EXP);893 894 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);895 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);896 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);897 898 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);899 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;900 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;901 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);902 903 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);904 905 /* Enable as 0xFFFF0001 response for RRS */906 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);907 val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT);908 val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 <<909 AMBA_ERROR_RESPONSE_RRS_SHIFT);910 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);911 912 /* Clear Slot Clock Configuration bit if SRNS configuration */913 if (pcie->enable_srns) {914 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +915 PCI_EXP_LNKSTA);916 val_16 &= ~PCI_EXP_LNKSTA_SLC;917 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,918 val_16);919 }920 921 config_gen3_gen4_eq_presets(pcie);922 923 init_host_aspm(pcie);924 925 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */926 if (!pcie->supports_clkreq) {927 disable_aspm_l11(pcie);928 disable_aspm_l12(pcie);929 }930 931 if (!pcie->of_data->has_l1ss_exit_fix) {932 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);933 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;934 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);935 }936 937 if (pcie->update_fc_fixup) {938 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);939 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;940 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);941 }942 943 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);944 945 return 0;946}947 948static int tegra_pcie_dw_start_link(struct dw_pcie *pci)949{950 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);951 struct dw_pcie_rp *pp = &pci->pp;952 u32 val, offset, tmp;953 bool retry = true;954 955 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {956 enable_irq(pcie->pex_rst_irq);957 return 0;958 }959 960retry_link:961 /* Assert RST */962 val = appl_readl(pcie, APPL_PINMUX);963 val &= ~APPL_PINMUX_PEX_RST;964 appl_writel(pcie, val, APPL_PINMUX);965 966 usleep_range(100, 200);967 968 /* Enable LTSSM */969 val = appl_readl(pcie, APPL_CTRL);970 val |= APPL_CTRL_LTSSM_EN;971 appl_writel(pcie, val, APPL_CTRL);972 973 /* De-assert RST */974 val = appl_readl(pcie, APPL_PINMUX);975 val |= APPL_PINMUX_PEX_RST;976 appl_writel(pcie, val, APPL_PINMUX);977 978 msleep(100);979 980 if (dw_pcie_wait_for_link(pci)) {981 if (!retry)982 return 0;983 /*984 * There are some endpoints which can't get the link up if985 * root port has Data Link Feature (DLF) enabled.986 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info987 * on Scaled Flow Control and DLF.988 * So, need to confirm that is indeed the case here and attempt989 * link up once again with DLF disabled.990 */991 val = appl_readl(pcie, APPL_DEBUG);992 val &= APPL_DEBUG_LTSSM_STATE_MASK;993 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;994 tmp = appl_readl(pcie, APPL_LINK_STATUS);995 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;996 if (!(val == 0x11 && !tmp)) {997 /* Link is down for all good reasons */998 return 0;999 }1000 1001 dev_info(pci->dev, "Link is down in DLL");1002 dev_info(pci->dev, "Trying again with DLFE disabled\n");1003 /* Disable LTSSM */1004 val = appl_readl(pcie, APPL_CTRL);1005 val &= ~APPL_CTRL_LTSSM_EN;1006 appl_writel(pcie, val, APPL_CTRL);1007 1008 reset_control_assert(pcie->core_rst);1009 reset_control_deassert(pcie->core_rst);1010 1011 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);1012 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);1013 val &= ~PCI_DLF_EXCHANGE_ENABLE;1014 dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val);1015 1016 tegra_pcie_dw_host_init(pp);1017 dw_pcie_setup_rc(pp);1018 1019 retry = false;1020 goto retry_link;1021 }1022 1023 tegra_pcie_icc_set(pcie);1024 1025 tegra_pcie_enable_interrupts(pp);1026 1027 return 0;1028}1029 1030static int tegra_pcie_dw_link_up(struct dw_pcie *pci)1031{1032 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);1033 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);1034 1035 return !!(val & PCI_EXP_LNKSTA_DLLLA);1036}1037 1038static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)1039{1040 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);1041 1042 disable_irq(pcie->pex_rst_irq);1043}1044 1045static const struct dw_pcie_ops tegra_dw_pcie_ops = {1046 .link_up = tegra_pcie_dw_link_up,1047 .start_link = tegra_pcie_dw_start_link,1048 .stop_link = tegra_pcie_dw_stop_link,1049};1050 1051static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {1052 .init = tegra_pcie_dw_host_init,1053};1054 1055static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)1056{1057 unsigned int phy_count = pcie->phy_count;1058 1059 while (phy_count--) {1060 phy_power_off(pcie->phys[phy_count]);1061 phy_exit(pcie->phys[phy_count]);1062 }1063}1064 1065static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)1066{1067 unsigned int i;1068 int ret;1069 1070 for (i = 0; i < pcie->phy_count; i++) {1071 ret = phy_init(pcie->phys[i]);1072 if (ret < 0)1073 goto phy_power_off;1074 1075 ret = phy_power_on(pcie->phys[i]);1076 if (ret < 0)1077 goto phy_exit;1078 }1079 1080 return 0;1081 1082phy_power_off:1083 while (i--) {1084 phy_power_off(pcie->phys[i]);1085phy_exit:1086 phy_exit(pcie->phys[i]);1087 }1088 1089 return ret;1090}1091 1092static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)1093{1094 struct platform_device *pdev = to_platform_device(pcie->dev);1095 struct device_node *np = pcie->dev->of_node;1096 int ret;1097 1098 pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");1099 if (!pcie->dbi_res) {1100 dev_err(pcie->dev, "Failed to find \"dbi\" region\n");1101 return -ENODEV;1102 }1103 1104 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);1105 if (ret < 0) {1106 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);1107 return ret;1108 }1109 1110 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",1111 &pcie->aspm_pwr_on_t);1112 if (ret < 0)1113 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",1114 ret);1115 1116 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",1117 &pcie->aspm_l0s_enter_lat);1118 if (ret < 0)1119 dev_info(pcie->dev,1120 "Failed to read ASPM L0s Entrance latency: %d\n", ret);1121 1122 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);1123 if (ret < 0) {1124 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);1125 return ret;1126 }1127 1128 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);1129 if (ret) {1130 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);1131 return ret;1132 }1133 1134 ret = of_property_count_strings(np, "phy-names");1135 if (ret < 0) {1136 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",1137 ret);1138 return ret;1139 }1140 pcie->phy_count = ret;1141 1142 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))1143 pcie->update_fc_fixup = true;1144 1145 /* RP using an external REFCLK is supported only in Tegra234 */1146 if (pcie->of_data->version == TEGRA194_DWC_IP_VER) {1147 if (pcie->of_data->mode == DW_PCIE_EP_TYPE)1148 pcie->enable_ext_refclk = true;1149 } else {1150 pcie->enable_ext_refclk =1151 of_property_read_bool(pcie->dev->of_node,1152 "nvidia,enable-ext-refclk");1153 }1154 1155 pcie->supports_clkreq =1156 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");1157 1158 pcie->enable_cdm_check =1159 of_property_read_bool(np, "snps,enable-cdm-check");1160 1161 if (pcie->of_data->version == TEGRA234_DWC_IP_VER)1162 pcie->enable_srns =1163 of_property_read_bool(np, "nvidia,enable-srns");1164 1165 if (pcie->of_data->mode == DW_PCIE_RC_TYPE)1166 return 0;1167 1168 /* Endpoint mode specific DT entries */1169 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);1170 if (IS_ERR(pcie->pex_rst_gpiod)) {1171 int err = PTR_ERR(pcie->pex_rst_gpiod);1172 const char *level = KERN_ERR;1173 1174 if (err == -EPROBE_DEFER)1175 level = KERN_DEBUG;1176 1177 dev_printk(level, pcie->dev,1178 dev_fmt("Failed to get PERST GPIO: %d\n"),1179 err);1180 return err;1181 }1182 1183 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,1184 "nvidia,refclk-select",1185 GPIOD_OUT_HIGH);1186 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {1187 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);1188 const char *level = KERN_ERR;1189 1190 if (err == -EPROBE_DEFER)1191 level = KERN_DEBUG;1192 1193 dev_printk(level, pcie->dev,1194 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),1195 err);1196 pcie->pex_refclk_sel_gpiod = NULL;1197 }1198 1199 return 0;1200}1201 1202static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,1203 bool enable)1204{1205 struct mrq_uphy_response resp;1206 struct tegra_bpmp_message msg;1207 struct mrq_uphy_request req;1208 1209 /*1210 * Controller-5 doesn't need to have its state set by BPMP-FW in1211 * Tegra1941212 */1213 if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5)1214 return 0;1215 1216 memset(&req, 0, sizeof(req));1217 memset(&resp, 0, sizeof(resp));1218 1219 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;1220 req.controller_state.pcie_controller = pcie->cid;1221 req.controller_state.enable = enable;1222 1223 memset(&msg, 0, sizeof(msg));1224 msg.mrq = MRQ_UPHY;1225 msg.tx.data = &req;1226 msg.tx.size = sizeof(req);1227 msg.rx.data = &resp;1228 msg.rx.size = sizeof(resp);1229 1230 return tegra_bpmp_transfer(pcie->bpmp, &msg);1231}1232 1233static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,1234 bool enable)1235{1236 struct mrq_uphy_response resp;1237 struct tegra_bpmp_message msg;1238 struct mrq_uphy_request req;1239 1240 memset(&req, 0, sizeof(req));1241 memset(&resp, 0, sizeof(resp));1242 1243 if (enable) {1244 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;1245 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;1246 } else {1247 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;1248 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;1249 }1250 1251 memset(&msg, 0, sizeof(msg));1252 msg.mrq = MRQ_UPHY;1253 msg.tx.data = &req;1254 msg.tx.size = sizeof(req);1255 msg.rx.data = &resp;1256 msg.rx.size = sizeof(resp);1257 1258 return tegra_bpmp_transfer(pcie->bpmp, &msg);1259}1260 1261static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)1262{1263 struct dw_pcie_rp *pp = &pcie->pci.pp;1264 struct pci_bus *child, *root_bus = NULL;1265 struct pci_dev *pdev;1266 1267 /*1268 * link doesn't go into L2 state with some of the endpoints with Tegra1269 * if they are not in D0 state. So, need to make sure that immediate1270 * downstream devices are in D0 state before sending PME_TurnOff to put1271 * link into L2 state.1272 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,1273 * 5.2 Link State Power Management (Page #428).1274 */1275 1276 list_for_each_entry(child, &pp->bridge->bus->children, node) {1277 /* Bring downstream devices to D0 if they are not already in */1278 if (child->parent == pp->bridge->bus) {1279 root_bus = child;1280 break;1281 }1282 }1283 1284 if (!root_bus) {1285 dev_err(pcie->dev, "Failed to find downstream devices\n");1286 return;1287 }1288 1289 list_for_each_entry(pdev, &root_bus->devices, bus_list) {1290 if (PCI_SLOT(pdev->devfn) == 0) {1291 if (pci_set_power_state(pdev, PCI_D0))1292 dev_err(pcie->dev,1293 "Failed to transition %s to D0 state\n",1294 dev_name(&pdev->dev));1295 }1296 }1297}1298 1299static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)1300{1301 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");1302 if (IS_ERR(pcie->slot_ctl_3v3)) {1303 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)1304 return PTR_ERR(pcie->slot_ctl_3v3);1305 1306 pcie->slot_ctl_3v3 = NULL;1307 }1308 1309 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");1310 if (IS_ERR(pcie->slot_ctl_12v)) {1311 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)1312 return PTR_ERR(pcie->slot_ctl_12v);1313 1314 pcie->slot_ctl_12v = NULL;1315 }1316 1317 return 0;1318}1319 1320static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)1321{1322 int ret;1323 1324 if (pcie->slot_ctl_3v3) {1325 ret = regulator_enable(pcie->slot_ctl_3v3);1326 if (ret < 0) {1327 dev_err(pcie->dev,1328 "Failed to enable 3.3V slot supply: %d\n", ret);1329 return ret;1330 }1331 }1332 1333 if (pcie->slot_ctl_12v) {1334 ret = regulator_enable(pcie->slot_ctl_12v);1335 if (ret < 0) {1336 dev_err(pcie->dev,1337 "Failed to enable 12V slot supply: %d\n", ret);1338 goto fail_12v_enable;1339 }1340 }1341 1342 /*1343 * According to PCI Express Card Electromechanical Specification1344 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)1345 * should be a minimum of 100ms.1346 */1347 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)1348 msleep(100);1349 1350 return 0;1351 1352fail_12v_enable:1353 if (pcie->slot_ctl_3v3)1354 regulator_disable(pcie->slot_ctl_3v3);1355 return ret;1356}1357 1358static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)1359{1360 if (pcie->slot_ctl_12v)1361 regulator_disable(pcie->slot_ctl_12v);1362 if (pcie->slot_ctl_3v3)1363 regulator_disable(pcie->slot_ctl_3v3);1364}1365 1366static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,1367 bool en_hw_hot_rst)1368{1369 int ret;1370 u32 val;1371 1372 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);1373 if (ret) {1374 dev_err(pcie->dev,1375 "Failed to enable controller %u: %d\n", pcie->cid, ret);1376 return ret;1377 }1378 1379 if (pcie->enable_ext_refclk) {1380 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);1381 if (ret) {1382 dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret);1383 goto fail_pll_init;1384 }1385 }1386 1387 ret = tegra_pcie_enable_slot_regulators(pcie);1388 if (ret < 0)1389 goto fail_slot_reg_en;1390 1391 ret = regulator_enable(pcie->pex_ctl_supply);1392 if (ret < 0) {1393 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);1394 goto fail_reg_en;1395 }1396 1397 ret = clk_prepare_enable(pcie->core_clk);1398 if (ret) {1399 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);1400 goto fail_core_clk;1401 }1402 1403 ret = reset_control_deassert(pcie->core_apb_rst);1404 if (ret) {1405 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",1406 ret);1407 goto fail_core_apb_rst;1408 }1409 1410 if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) {1411 /* Enable HW_HOT_RST mode */1412 val = appl_readl(pcie, APPL_CTRL);1413 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<1414 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);1415 val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN <<1416 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);1417 val |= APPL_CTRL_HW_HOT_RST_EN;1418 appl_writel(pcie, val, APPL_CTRL);1419 }1420 1421 ret = tegra_pcie_enable_phy(pcie);1422 if (ret) {1423 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);1424 goto fail_phy;1425 }1426 1427 /* Update CFG base address */1428 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,1429 APPL_CFG_BASE_ADDR);1430 1431 /* Configure this core for RP mode operation */1432 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);1433 1434 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);1435 1436 val = appl_readl(pcie, APPL_CTRL);1437 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);1438 1439 val = appl_readl(pcie, APPL_CFG_MISC);1440 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);1441 appl_writel(pcie, val, APPL_CFG_MISC);1442 1443 if (pcie->enable_srns || pcie->enable_ext_refclk) {1444 /*1445 * When Tegra PCIe RP is using external clock, it cannot supply1446 * same clock to its downstream hierarchy. Hence, gate PCIe RP1447 * REFCLK out pads when RP & EP are using separate clocks or RP1448 * is using an external REFCLK.1449 */1450 val = appl_readl(pcie, APPL_PINMUX);1451 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;1452 val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;1453 appl_writel(pcie, val, APPL_PINMUX);1454 }1455 1456 if (!pcie->supports_clkreq) {1457 val = appl_readl(pcie, APPL_PINMUX);1458 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;1459 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;1460 appl_writel(pcie, val, APPL_PINMUX);1461 }1462 1463 /* Update iATU_DMA base address */1464 appl_writel(pcie,1465 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,1466 APPL_CFG_IATU_DMA_BASE_ADDR);1467 1468 reset_control_deassert(pcie->core_rst);1469 1470 return ret;1471 1472fail_phy:1473 reset_control_assert(pcie->core_apb_rst);1474fail_core_apb_rst:1475 clk_disable_unprepare(pcie->core_clk);1476fail_core_clk:1477 regulator_disable(pcie->pex_ctl_supply);1478fail_reg_en:1479 tegra_pcie_disable_slot_regulators(pcie);1480fail_slot_reg_en:1481 if (pcie->enable_ext_refclk)1482 tegra_pcie_bpmp_set_pll_state(pcie, false);1483fail_pll_init:1484 tegra_pcie_bpmp_set_ctrl_state(pcie, false);1485 1486 return ret;1487}1488 1489static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie)1490{1491 int ret;1492 1493 ret = reset_control_assert(pcie->core_rst);1494 if (ret)1495 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret);1496 1497 tegra_pcie_disable_phy(pcie);1498 1499 ret = reset_control_assert(pcie->core_apb_rst);1500 if (ret)1501 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);1502 1503 clk_disable_unprepare(pcie->core_clk);1504 1505 ret = regulator_disable(pcie->pex_ctl_supply);1506 if (ret)1507 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);1508 1509 tegra_pcie_disable_slot_regulators(pcie);1510 1511 if (pcie->enable_ext_refclk) {1512 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);1513 if (ret)1514 dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret);1515 }1516 1517 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);1518 if (ret)1519 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",1520 pcie->cid, ret);1521}1522 1523static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)1524{1525 struct dw_pcie *pci = &pcie->pci;1526 struct dw_pcie_rp *pp = &pci->pp;1527 int ret;1528 1529 ret = tegra_pcie_config_controller(pcie, false);1530 if (ret < 0)1531 return ret;1532 1533 pp->ops = &tegra_pcie_dw_host_ops;1534 1535 ret = dw_pcie_host_init(pp);1536 if (ret < 0) {1537 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);1538 goto fail_host_init;1539 }1540 1541 return 0;1542 1543fail_host_init:1544 tegra_pcie_unconfig_controller(pcie);1545 return ret;1546}1547 1548static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)1549{1550 u32 val;1551 1552 if (!tegra_pcie_dw_link_up(&pcie->pci))1553 return 0;1554 1555 val = appl_readl(pcie, APPL_RADM_STATUS);1556 val |= APPL_PM_XMT_TURNOFF_STATE;1557 appl_writel(pcie, val, APPL_RADM_STATUS);1558 1559 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,1560 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,1561 1, PME_ACK_TIMEOUT);1562}1563 1564static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)1565{1566 u32 data;1567 int err;1568 1569 if (!tegra_pcie_dw_link_up(&pcie->pci)) {1570 dev_dbg(pcie->dev, "PCIe link is not up...!\n");1571 return;1572 }1573 1574 /*1575 * PCIe controller exits from L2 only if reset is applied, so1576 * controller doesn't handle interrupts. But in cases where1577 * L2 entry fails, PERST# is asserted which can trigger surprise1578 * link down AER. However this function call happens in1579 * suspend_noirq(), so AER interrupt will not be processed.1580 * Disable all interrupts to avoid such a scenario.1581 */1582 appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0);1583 1584 if (tegra_pcie_try_link_l2(pcie)) {1585 dev_info(pcie->dev, "Link didn't transition to L2 state\n");1586 /*1587 * TX lane clock freq will reset to Gen1 only if link is in L21588 * or detect state.1589 * So apply pex_rst to end point to force RP to go into detect1590 * state1591 */1592 data = appl_readl(pcie, APPL_PINMUX);1593 data &= ~APPL_PINMUX_PEX_RST;1594 appl_writel(pcie, data, APPL_PINMUX);1595 1596 /*1597 * Some cards do not go to detect state even after de-asserting1598 * PERST#. So, de-assert LTSSM to bring link to detect state.1599 */1600 data = readl(pcie->appl_base + APPL_CTRL);1601 data &= ~APPL_CTRL_LTSSM_EN;1602 writel(data, pcie->appl_base + APPL_CTRL);1603 1604 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,1605 data,1606 ((data &1607 APPL_DEBUG_LTSSM_STATE_MASK) >>1608 APPL_DEBUG_LTSSM_STATE_SHIFT) ==1609 LTSSM_STATE_PRE_DETECT,1610 1, LTSSM_TIMEOUT);1611 if (err)1612 dev_info(pcie->dev, "Link didn't go to detect state\n");1613 }1614 /*1615 * DBI registers may not be accessible after this as PLL-E would be1616 * down depending on how CLKREQ is pulled by end point1617 */1618 data = appl_readl(pcie, APPL_PINMUX);1619 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);1620 /* Cut REFCLK to slot */1621 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;1622 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;1623 appl_writel(pcie, data, APPL_PINMUX);1624}1625 1626static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)1627{1628 tegra_pcie_downstream_dev_to_D0(pcie);1629 dw_pcie_host_deinit(&pcie->pci.pp);1630 tegra_pcie_dw_pme_turnoff(pcie);1631 tegra_pcie_unconfig_controller(pcie);1632}1633 1634static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)1635{1636 struct device *dev = pcie->dev;1637 char *name;1638 int ret;1639 1640 pm_runtime_enable(dev);1641 1642 ret = pm_runtime_get_sync(dev);1643 if (ret < 0) {1644 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",1645 ret);1646 goto fail_pm_get_sync;1647 }1648 1649 ret = pinctrl_pm_select_default_state(dev);1650 if (ret < 0) {1651 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);1652 goto fail_pm_get_sync;1653 }1654 1655 ret = tegra_pcie_init_controller(pcie);1656 if (ret < 0) {1657 dev_err(dev, "Failed to initialize controller: %d\n", ret);1658 goto fail_pm_get_sync;1659 }1660 1661 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);1662 if (!pcie->link_state) {1663 ret = -ENOMEDIUM;1664 goto fail_host_init;1665 }1666 1667 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);1668 if (!name) {1669 ret = -ENOMEM;1670 goto fail_host_init;1671 }1672 1673 pcie->debugfs = debugfs_create_dir(name, NULL);1674 init_debugfs(pcie);1675 1676 return ret;1677 1678fail_host_init:1679 tegra_pcie_deinit_controller(pcie);1680fail_pm_get_sync:1681 pm_runtime_put_sync(dev);1682 pm_runtime_disable(dev);1683 return ret;1684}1685 1686static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)1687{1688 u32 val;1689 int ret;1690 1691 if (pcie->ep_state == EP_STATE_DISABLED)1692 return;1693 1694 /* Disable LTSSM */1695 val = appl_readl(pcie, APPL_CTRL);1696 val &= ~APPL_CTRL_LTSSM_EN;1697 appl_writel(pcie, val, APPL_CTRL);1698 1699 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,1700 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>1701 APPL_DEBUG_LTSSM_STATE_SHIFT) ==1702 LTSSM_STATE_PRE_DETECT,1703 1, LTSSM_TIMEOUT);1704 if (ret)1705 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);1706 1707 pci_epc_deinit_notify(pcie->pci.ep.epc);1708 dw_pcie_ep_cleanup(&pcie->pci.ep);1709 1710 reset_control_assert(pcie->core_rst);1711 1712 tegra_pcie_disable_phy(pcie);1713 1714 reset_control_assert(pcie->core_apb_rst);1715 1716 clk_disable_unprepare(pcie->core_clk);1717 1718 pm_runtime_put_sync(pcie->dev);1719 1720 if (pcie->enable_ext_refclk) {1721 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);1722 if (ret)1723 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n",1724 ret);1725 }1726 1727 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);1728 if (ret)1729 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);1730 1731 pcie->ep_state = EP_STATE_DISABLED;1732 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");1733}1734 1735static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)1736{1737 struct dw_pcie *pci = &pcie->pci;1738 struct dw_pcie_ep *ep = &pci->ep;1739 struct device *dev = pcie->dev;1740 u32 val;1741 int ret;1742 u16 val_16;1743 1744 if (pcie->ep_state == EP_STATE_ENABLED)1745 return;1746 1747 ret = pm_runtime_resume_and_get(dev);1748 if (ret < 0) {1749 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",1750 ret);1751 return;1752 }1753 1754 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);1755 if (ret) {1756 dev_err(pcie->dev, "Failed to enable controller %u: %d\n",1757 pcie->cid, ret);1758 goto fail_set_ctrl_state;1759 }1760 1761 if (pcie->enable_ext_refclk) {1762 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);1763 if (ret) {1764 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n",1765 ret);1766 goto fail_pll_init;1767 }1768 }1769 1770 ret = clk_prepare_enable(pcie->core_clk);1771 if (ret) {1772 dev_err(dev, "Failed to enable core clock: %d\n", ret);1773 goto fail_core_clk_enable;1774 }1775 1776 ret = reset_control_deassert(pcie->core_apb_rst);1777 if (ret) {1778 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);1779 goto fail_core_apb_rst;1780 }1781 1782 ret = tegra_pcie_enable_phy(pcie);1783 if (ret) {1784 dev_err(dev, "Failed to enable PHY: %d\n", ret);1785 goto fail_phy;1786 }1787 1788 /* Clear any stale interrupt statuses */1789 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);1790 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);1791 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);1792 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);1793 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);1794 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);1795 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);1796 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);1797 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);1798 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);1799 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);1800 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);1801 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);1802 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);1803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);1804 1805 /* configure this core for EP mode operation */1806 val = appl_readl(pcie, APPL_DM_TYPE);1807 val &= ~APPL_DM_TYPE_MASK;1808 val |= APPL_DM_TYPE_EP;1809 appl_writel(pcie, val, APPL_DM_TYPE);1810 1811 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);1812 1813 val = appl_readl(pcie, APPL_CTRL);1814 val |= APPL_CTRL_SYS_PRE_DET_STATE;1815 val |= APPL_CTRL_HW_HOT_RST_EN;1816 appl_writel(pcie, val, APPL_CTRL);1817 1818 val = appl_readl(pcie, APPL_CFG_MISC);1819 val |= APPL_CFG_MISC_SLV_EP_MODE;1820 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);1821 appl_writel(pcie, val, APPL_CFG_MISC);1822 1823 val = appl_readl(pcie, APPL_PINMUX);1824 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;1825 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;1826 appl_writel(pcie, val, APPL_PINMUX);1827 1828 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,1829 APPL_CFG_BASE_ADDR);1830 1831 appl_writel(pcie, pcie->atu_dma_res->start &1832 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,1833 APPL_CFG_IATU_DMA_BASE_ADDR);1834 1835 val = appl_readl(pcie, APPL_INTR_EN_L0_0);1836 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;1837 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;1838 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;1839 appl_writel(pcie, val, APPL_INTR_EN_L0_0);1840 1841 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);1842 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;1843 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;1844 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);1845 1846 reset_control_deassert(pcie->core_rst);1847 1848 if (pcie->update_fc_fixup) {1849 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);1850 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;1851 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);1852 }1853 1854 config_gen3_gen4_eq_presets(pcie);1855 1856 init_host_aspm(pcie);1857 1858 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */1859 if (!pcie->supports_clkreq) {1860 disable_aspm_l11(pcie);1861 disable_aspm_l12(pcie);1862 }1863 1864 if (!pcie->of_data->has_l1ss_exit_fix) {1865 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);1866 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;1867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);1868 }1869 1870 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,1871 PCI_CAP_ID_EXP);1872 1873 /* Clear Slot Clock Configuration bit if SRNS configuration */1874 if (pcie->enable_srns) {1875 val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +1876 PCI_EXP_LNKSTA);1877 val_16 &= ~PCI_EXP_LNKSTA_SLC;1878 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA,1879 val_16);1880 }1881 1882 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);1883 1884 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);1885 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;1886 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);1887 val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);1888 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);1889 1890 ret = dw_pcie_ep_init_registers(ep);1891 if (ret) {1892 dev_err(dev, "Failed to complete initialization: %d\n", ret);1893 goto fail_init_complete;1894 }1895 1896 pci_epc_init_notify(ep->epc);1897 1898 /* Program the private control to allow sending LTR upstream */1899 if (pcie->of_data->has_ltr_req_fix) {1900 val = appl_readl(pcie, APPL_LTR_MSG_2);1901 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;1902 appl_writel(pcie, val, APPL_LTR_MSG_2);1903 }1904 1905 /* Enable LTSSM */1906 val = appl_readl(pcie, APPL_CTRL);1907 val |= APPL_CTRL_LTSSM_EN;1908 appl_writel(pcie, val, APPL_CTRL);1909 1910 pcie->ep_state = EP_STATE_ENABLED;1911 dev_dbg(dev, "Initialization of endpoint is completed\n");1912 1913 return;1914 1915fail_init_complete:1916 reset_control_assert(pcie->core_rst);1917 tegra_pcie_disable_phy(pcie);1918fail_phy:1919 reset_control_assert(pcie->core_apb_rst);1920fail_core_apb_rst:1921 clk_disable_unprepare(pcie->core_clk);1922fail_core_clk_enable:1923 tegra_pcie_bpmp_set_pll_state(pcie, false);1924fail_pll_init:1925 tegra_pcie_bpmp_set_ctrl_state(pcie, false);1926fail_set_ctrl_state:1927 pm_runtime_put_sync(dev);1928}1929 1930static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)1931{1932 struct tegra_pcie_dw *pcie = arg;1933 1934 if (gpiod_get_value(pcie->pex_rst_gpiod))1935 pex_ep_event_pex_rst_assert(pcie);1936 else1937 pex_ep_event_pex_rst_deassert(pcie);1938 1939 return IRQ_HANDLED;1940}1941 1942static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq)1943{1944 /* Tegra194 supports only INTA */1945 if (irq > 1)1946 return -EINVAL;1947 1948 appl_writel(pcie, 1, APPL_LEGACY_INTX);1949 usleep_range(1000, 2000);1950 appl_writel(pcie, 0, APPL_LEGACY_INTX);1951 return 0;1952}1953 1954static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)1955{1956 if (unlikely(irq > 31))1957 return -EINVAL;1958 1959 appl_writel(pcie, BIT(irq), APPL_MSI_CTRL_1);1960 1961 return 0;1962}1963 1964static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)1965{1966 struct dw_pcie_ep *ep = &pcie->pci.ep;1967 1968 writel(irq, ep->msi_mem);1969 1970 return 0;1971}1972 1973static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,1974 unsigned int type, u16 interrupt_num)1975{1976 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);1977 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);1978 1979 switch (type) {1980 case PCI_IRQ_INTX:1981 return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num);1982 1983 case PCI_IRQ_MSI:1984 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);1985 1986 case PCI_IRQ_MSIX:1987 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);1988 1989 default:1990 dev_err(pci->dev, "Unknown IRQ type\n");1991 return -EPERM;1992 }1993 1994 return 0;1995}1996 1997static const struct pci_epc_features tegra_pcie_epc_features = {1998 .linkup_notifier = true,1999 .msi_capable = false,2000 .msix_capable = false,2001 .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M,2002 .only_64bit = true, },2003 .bar[BAR_1] = { .type = BAR_RESERVED, },2004 .bar[BAR_2] = { .type = BAR_RESERVED, },2005 .bar[BAR_3] = { .type = BAR_RESERVED, },2006 .bar[BAR_4] = { .type = BAR_RESERVED, },2007 .bar[BAR_5] = { .type = BAR_RESERVED, },2008 .align = SZ_64K,2009};2010 2011static const struct pci_epc_features*2012tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)2013{2014 return &tegra_pcie_epc_features;2015}2016 2017static const struct dw_pcie_ep_ops pcie_ep_ops = {2018 .raise_irq = tegra_pcie_ep_raise_irq,2019 .get_features = tegra_pcie_ep_get_features,2020};2021 2022static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,2023 struct platform_device *pdev)2024{2025 struct dw_pcie *pci = &pcie->pci;2026 struct device *dev = pcie->dev;2027 struct dw_pcie_ep *ep;2028 char *name;2029 int ret;2030 2031 ep = &pci->ep;2032 ep->ops = &pcie_ep_ops;2033 2034 ep->page_size = SZ_64K;2035 2036 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);2037 if (ret < 0) {2038 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",2039 ret);2040 return ret;2041 }2042 2043 ret = gpiod_to_irq(pcie->pex_rst_gpiod);2044 if (ret < 0) {2045 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);2046 return ret;2047 }2048 pcie->pex_rst_irq = (unsigned int)ret;2049 2050 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",2051 pcie->cid);2052 if (!name) {2053 dev_err(dev, "Failed to create PERST IRQ string\n");2054 return -ENOMEM;2055 }2056 2057 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);2058 2059 pcie->ep_state = EP_STATE_DISABLED;2060 2061 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,2062 tegra_pcie_ep_pex_rst_irq,2063 IRQF_TRIGGER_RISING |2064 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,2065 name, (void *)pcie);2066 if (ret < 0) {2067 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);2068 return ret;2069 }2070 2071 pm_runtime_enable(dev);2072 2073 ret = dw_pcie_ep_init(ep);2074 if (ret) {2075 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",2076 ret);2077 pm_runtime_disable(dev);2078 return ret;2079 }2080 2081 return 0;2082}2083 2084static int tegra_pcie_dw_probe(struct platform_device *pdev)2085{2086 const struct tegra_pcie_dw_of_data *data;2087 struct device *dev = &pdev->dev;2088 struct resource *atu_dma_res;2089 struct tegra_pcie_dw *pcie;2090 struct dw_pcie_rp *pp;2091 struct dw_pcie *pci;2092 struct phy **phys;2093 char *name;2094 int ret;2095 u32 i;2096 2097 data = of_device_get_match_data(dev);2098 2099 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);2100 if (!pcie)2101 return -ENOMEM;2102 2103 pci = &pcie->pci;2104 pci->dev = &pdev->dev;2105 pci->ops = &tegra_dw_pcie_ops;2106 pcie->dev = &pdev->dev;2107 pcie->of_data = (struct tegra_pcie_dw_of_data *)data;2108 pci->n_fts[0] = pcie->of_data->n_fts[0];2109 pci->n_fts[1] = pcie->of_data->n_fts[1];2110 pp = &pci->pp;2111 pp->num_vectors = MAX_MSI_IRQS;2112 2113 ret = tegra_pcie_dw_parse_dt(pcie);2114 if (ret < 0) {2115 const char *level = KERN_ERR;2116 2117 if (ret == -EPROBE_DEFER)2118 level = KERN_DEBUG;2119 2120 dev_printk(level, dev,2121 dev_fmt("Failed to parse device tree: %d\n"),2122 ret);2123 return ret;2124 }2125 2126 ret = tegra_pcie_get_slot_regulators(pcie);2127 if (ret < 0) {2128 const char *level = KERN_ERR;2129 2130 if (ret == -EPROBE_DEFER)2131 level = KERN_DEBUG;2132 2133 dev_printk(level, dev,2134 dev_fmt("Failed to get slot regulators: %d\n"),2135 ret);2136 return ret;2137 }2138 2139 if (pcie->pex_refclk_sel_gpiod)2140 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);2141 2142 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");2143 if (IS_ERR(pcie->pex_ctl_supply)) {2144 ret = PTR_ERR(pcie->pex_ctl_supply);2145 if (ret != -EPROBE_DEFER)2146 dev_err(dev, "Failed to get regulator: %ld\n",2147 PTR_ERR(pcie->pex_ctl_supply));2148 return ret;2149 }2150 2151 pcie->core_clk = devm_clk_get(dev, "core");2152 if (IS_ERR(pcie->core_clk)) {2153 dev_err(dev, "Failed to get core clock: %ld\n",2154 PTR_ERR(pcie->core_clk));2155 return PTR_ERR(pcie->core_clk);2156 }2157 2158 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,2159 "appl");2160 if (!pcie->appl_res) {2161 dev_err(dev, "Failed to find \"appl\" region\n");2162 return -ENODEV;2163 }2164 2165 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);2166 if (IS_ERR(pcie->appl_base))2167 return PTR_ERR(pcie->appl_base);2168 2169 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");2170 if (IS_ERR(pcie->core_apb_rst)) {2171 dev_err(dev, "Failed to get APB reset: %ld\n",2172 PTR_ERR(pcie->core_apb_rst));2173 return PTR_ERR(pcie->core_apb_rst);2174 }2175 2176 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);2177 if (!phys)2178 return -ENOMEM;2179 2180 for (i = 0; i < pcie->phy_count; i++) {2181 name = kasprintf(GFP_KERNEL, "p2u-%u", i);2182 if (!name) {2183 dev_err(dev, "Failed to create P2U string\n");2184 return -ENOMEM;2185 }2186 phys[i] = devm_phy_get(dev, name);2187 kfree(name);2188 if (IS_ERR(phys[i])) {2189 ret = PTR_ERR(phys[i]);2190 if (ret != -EPROBE_DEFER)2191 dev_err(dev, "Failed to get PHY: %d\n", ret);2192 return ret;2193 }2194 }2195 2196 pcie->phys = phys;2197 2198 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,2199 "atu_dma");2200 if (!atu_dma_res) {2201 dev_err(dev, "Failed to find \"atu_dma\" region\n");2202 return -ENODEV;2203 }2204 pcie->atu_dma_res = atu_dma_res;2205 2206 pci->atu_size = resource_size(atu_dma_res);2207 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);2208 if (IS_ERR(pci->atu_base))2209 return PTR_ERR(pci->atu_base);2210 2211 pcie->core_rst = devm_reset_control_get(dev, "core");2212 if (IS_ERR(pcie->core_rst)) {2213 dev_err(dev, "Failed to get core reset: %ld\n",2214 PTR_ERR(pcie->core_rst));2215 return PTR_ERR(pcie->core_rst);2216 }2217 2218 pp->irq = platform_get_irq_byname(pdev, "intr");2219 if (pp->irq < 0)2220 return pp->irq;2221 2222 pcie->bpmp = tegra_bpmp_get(dev);2223 if (IS_ERR(pcie->bpmp))2224 return PTR_ERR(pcie->bpmp);2225 2226 platform_set_drvdata(pdev, pcie);2227 2228 pcie->icc_path = devm_of_icc_get(&pdev->dev, "write");2229 ret = PTR_ERR_OR_ZERO(pcie->icc_path);2230 if (ret) {2231 tegra_bpmp_put(pcie->bpmp);2232 dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n");2233 return ret;2234 }2235 2236 switch (pcie->of_data->mode) {2237 case DW_PCIE_RC_TYPE:2238 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,2239 IRQF_SHARED, "tegra-pcie-intr", pcie);2240 if (ret) {2241 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,2242 ret);2243 goto fail;2244 }2245 2246 ret = tegra_pcie_config_rp(pcie);2247 if (ret && ret != -ENOMEDIUM)2248 goto fail;2249 else2250 return 0;2251 break;2252 2253 case DW_PCIE_EP_TYPE:2254 ret = devm_request_threaded_irq(dev, pp->irq,2255 tegra_pcie_ep_hard_irq,2256 tegra_pcie_ep_irq_thread,2257 IRQF_SHARED | IRQF_ONESHOT,2258 "tegra-pcie-ep-intr", pcie);2259 if (ret) {2260 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,2261 ret);2262 goto fail;2263 }2264 2265 ret = tegra_pcie_config_ep(pcie, pdev);2266 if (ret < 0)2267 goto fail;2268 else2269 return 0;2270 break;2271 2272 default:2273 dev_err(dev, "Invalid PCIe device type %d\n",2274 pcie->of_data->mode);2275 ret = -EINVAL;2276 }2277 2278fail:2279 tegra_bpmp_put(pcie->bpmp);2280 return ret;2281}2282 2283static void tegra_pcie_dw_remove(struct platform_device *pdev)2284{2285 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);2286 2287 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {2288 if (!pcie->link_state)2289 return;2290 2291 debugfs_remove_recursive(pcie->debugfs);2292 tegra_pcie_deinit_controller(pcie);2293 pm_runtime_put_sync(pcie->dev);2294 } else {2295 disable_irq(pcie->pex_rst_irq);2296 pex_ep_event_pex_rst_assert(pcie);2297 }2298 2299 pm_runtime_disable(pcie->dev);2300 tegra_bpmp_put(pcie->bpmp);2301 if (pcie->pex_refclk_sel_gpiod)2302 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);2303}2304 2305static int tegra_pcie_dw_suspend_late(struct device *dev)2306{2307 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);2308 u32 val;2309 2310 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {2311 dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n");2312 return -EPERM;2313 }2314 2315 if (!pcie->link_state)2316 return 0;2317 2318 /* Enable HW_HOT_RST mode */2319 if (!pcie->of_data->has_sbr_reset_fix) {2320 val = appl_readl(pcie, APPL_CTRL);2321 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<2322 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);2323 val |= APPL_CTRL_HW_HOT_RST_EN;2324 appl_writel(pcie, val, APPL_CTRL);2325 }2326 2327 return 0;2328}2329 2330static int tegra_pcie_dw_suspend_noirq(struct device *dev)2331{2332 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);2333 2334 if (!pcie->link_state)2335 return 0;2336 2337 tegra_pcie_downstream_dev_to_D0(pcie);2338 tegra_pcie_dw_pme_turnoff(pcie);2339 tegra_pcie_unconfig_controller(pcie);2340 2341 return 0;2342}2343 2344static int tegra_pcie_dw_resume_noirq(struct device *dev)2345{2346 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);2347 int ret;2348 2349 if (!pcie->link_state)2350 return 0;2351 2352 ret = tegra_pcie_config_controller(pcie, true);2353 if (ret < 0)2354 return ret;2355 2356 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);2357 if (ret < 0) {2358 dev_err(dev, "Failed to init host: %d\n", ret);2359 goto fail_host_init;2360 }2361 2362 dw_pcie_setup_rc(&pcie->pci.pp);2363 2364 ret = tegra_pcie_dw_start_link(&pcie->pci);2365 if (ret < 0)2366 goto fail_host_init;2367 2368 return 0;2369 2370fail_host_init:2371 tegra_pcie_unconfig_controller(pcie);2372 return ret;2373}2374 2375static int tegra_pcie_dw_resume_early(struct device *dev)2376{2377 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);2378 u32 val;2379 2380 if (pcie->of_data->mode == DW_PCIE_EP_TYPE) {2381 dev_err(dev, "Suspend is not supported in EP mode");2382 return -ENOTSUPP;2383 }2384 2385 if (!pcie->link_state)2386 return 0;2387 2388 /* Disable HW_HOT_RST mode */2389 if (!pcie->of_data->has_sbr_reset_fix) {2390 val = appl_readl(pcie, APPL_CTRL);2391 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<2392 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);2393 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<2394 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;2395 val &= ~APPL_CTRL_HW_HOT_RST_EN;2396 appl_writel(pcie, val, APPL_CTRL);2397 }2398 2399 return 0;2400}2401 2402static void tegra_pcie_dw_shutdown(struct platform_device *pdev)2403{2404 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);2405 2406 if (pcie->of_data->mode == DW_PCIE_RC_TYPE) {2407 if (!pcie->link_state)2408 return;2409 2410 debugfs_remove_recursive(pcie->debugfs);2411 tegra_pcie_downstream_dev_to_D0(pcie);2412 2413 disable_irq(pcie->pci.pp.irq);2414 if (IS_ENABLED(CONFIG_PCI_MSI))2415 disable_irq(pcie->pci.pp.msi_irq[0]);2416 2417 tegra_pcie_dw_pme_turnoff(pcie);2418 tegra_pcie_unconfig_controller(pcie);2419 pm_runtime_put_sync(pcie->dev);2420 } else {2421 disable_irq(pcie->pex_rst_irq);2422 pex_ep_event_pex_rst_assert(pcie);2423 }2424}2425 2426static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = {2427 .version = TEGRA194_DWC_IP_VER,2428 .mode = DW_PCIE_RC_TYPE,2429 .cdm_chk_int_en_bit = BIT(19),2430 /* Gen4 - 5, 6, 8 and 9 presets enabled */2431 .gen4_preset_vec = 0x360,2432 .n_fts = { 52, 52 },2433};2434 2435static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = {2436 .version = TEGRA194_DWC_IP_VER,2437 .mode = DW_PCIE_EP_TYPE,2438 .cdm_chk_int_en_bit = BIT(19),2439 /* Gen4 - 5, 6, 8 and 9 presets enabled */2440 .gen4_preset_vec = 0x360,2441 .n_fts = { 52, 52 },2442};2443 2444static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = {2445 .version = TEGRA234_DWC_IP_VER,2446 .mode = DW_PCIE_RC_TYPE,2447 .has_msix_doorbell_access_fix = true,2448 .has_sbr_reset_fix = true,2449 .has_l1ss_exit_fix = true,2450 .cdm_chk_int_en_bit = BIT(18),2451 /* Gen4 - 6, 8 and 9 presets enabled */2452 .gen4_preset_vec = 0x340,2453 .n_fts = { 52, 80 },2454};2455 2456static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = {2457 .version = TEGRA234_DWC_IP_VER,2458 .mode = DW_PCIE_EP_TYPE,2459 .has_l1ss_exit_fix = true,2460 .has_ltr_req_fix = true,2461 .cdm_chk_int_en_bit = BIT(18),2462 /* Gen4 - 6, 8 and 9 presets enabled */2463 .gen4_preset_vec = 0x340,2464 .n_fts = { 52, 80 },2465};2466 2467static const struct of_device_id tegra_pcie_dw_of_match[] = {2468 {2469 .compatible = "nvidia,tegra194-pcie",2470 .data = &tegra194_pcie_dw_rc_of_data,2471 },2472 {2473 .compatible = "nvidia,tegra194-pcie-ep",2474 .data = &tegra194_pcie_dw_ep_of_data,2475 },2476 {2477 .compatible = "nvidia,tegra234-pcie",2478 .data = &tegra234_pcie_dw_rc_of_data,2479 },2480 {2481 .compatible = "nvidia,tegra234-pcie-ep",2482 .data = &tegra234_pcie_dw_ep_of_data,2483 },2484 {}2485};2486 2487static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {2488 .suspend_late = tegra_pcie_dw_suspend_late,2489 .suspend_noirq = tegra_pcie_dw_suspend_noirq,2490 .resume_noirq = tegra_pcie_dw_resume_noirq,2491 .resume_early = tegra_pcie_dw_resume_early,2492};2493 2494static struct platform_driver tegra_pcie_dw_driver = {2495 .probe = tegra_pcie_dw_probe,2496 .remove_new = tegra_pcie_dw_remove,2497 .shutdown = tegra_pcie_dw_shutdown,2498 .driver = {2499 .name = "tegra194-pcie",2500 .pm = &tegra_pcie_dw_pm_ops,2501 .of_match_table = tegra_pcie_dw_of_match,2502 },2503};2504module_platform_driver(tegra_pcie_dw_driver);2505 2506MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);2507 2508MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");2509MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");2510MODULE_LICENSE("GPL v2");2511