1712 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Enable PCIe link L0s/L1 state and Clock Power Management4 *5 * Copyright (C) 2007 Intel6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)8 */9 10#include <linux/bitfield.h>11#include <linux/bits.h>12#include <linux/build_bug.h>13#include <linux/kernel.h>14#include <linux/limits.h>15#include <linux/math.h>16#include <linux/module.h>17#include <linux/moduleparam.h>18#include <linux/pci.h>19#include <linux/pci_regs.h>20#include <linux/errno.h>21#include <linux/pm.h>22#include <linux/init.h>23#include <linux/printk.h>24#include <linux/slab.h>25#include <linux/time.h>26 27#include "../pci.h"28 29void pci_save_ltr_state(struct pci_dev *dev)30{31 int ltr;32 struct pci_cap_saved_state *save_state;33 u32 *cap;34 35 if (!pci_is_pcie(dev))36 return;37 38 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);39 if (!ltr)40 return;41 42 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);43 if (!save_state) {44 pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n");45 return;46 }47 48 /* Some broken devices only support dword access to LTR */49 cap = &save_state->cap.data[0];50 pci_read_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap);51}52 53void pci_restore_ltr_state(struct pci_dev *dev)54{55 struct pci_cap_saved_state *save_state;56 int ltr;57 u32 *cap;58 59 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);60 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);61 if (!save_state || !ltr)62 return;63 64 /* Some broken devices only support dword access to LTR */65 cap = &save_state->cap.data[0];66 pci_write_config_dword(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap);67}68 69void pci_configure_aspm_l1ss(struct pci_dev *pdev)70{71 int rc;72 73 pdev->l1ss = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);74 75 rc = pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_L1SS,76 2 * sizeof(u32));77 if (rc)78 pci_err(pdev, "unable to allocate ASPM L1SS save buffer (%pe)\n",79 ERR_PTR(rc));80}81 82void pci_save_aspm_l1ss_state(struct pci_dev *pdev)83{84 struct pci_cap_saved_state *save_state;85 u16 l1ss = pdev->l1ss;86 u32 *cap;87 88 /*89 * Save L1 substate configuration. The ASPM L0s/L1 configuration90 * in PCI_EXP_LNKCTL_ASPMC is saved by pci_save_pcie_state().91 */92 if (!l1ss)93 return;94 95 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_L1SS);96 if (!save_state)97 return;98 99 cap = &save_state->cap.data[0];100 pci_read_config_dword(pdev, l1ss + PCI_L1SS_CTL2, cap++);101 pci_read_config_dword(pdev, l1ss + PCI_L1SS_CTL1, cap++);102}103 104void pci_restore_aspm_l1ss_state(struct pci_dev *pdev)105{106 struct pci_cap_saved_state *pl_save_state, *cl_save_state;107 struct pci_dev *parent = pdev->bus->self;108 u32 *cap, pl_ctl1, pl_ctl2, pl_l1_2_enable;109 u32 cl_ctl1, cl_ctl2, cl_l1_2_enable;110 u16 clnkctl, plnkctl;111 112 /*113 * In case BIOS enabled L1.2 when resuming, we need to disable it first114 * on the downstream component before the upstream. So, don't attempt to115 * restore either until we are at the downstream component.116 */117 if (pcie_downstream_port(pdev) || !parent)118 return;119 120 if (!pdev->l1ss || !parent->l1ss)121 return;122 123 cl_save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_L1SS);124 pl_save_state = pci_find_saved_ext_cap(parent, PCI_EXT_CAP_ID_L1SS);125 if (!cl_save_state || !pl_save_state)126 return;127 128 cap = &cl_save_state->cap.data[0];129 cl_ctl2 = *cap++;130 cl_ctl1 = *cap;131 cap = &pl_save_state->cap.data[0];132 pl_ctl2 = *cap++;133 pl_ctl1 = *cap;134 135 /* Make sure L0s/L1 are disabled before updating L1SS config */136 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &clnkctl);137 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &plnkctl);138 if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, clnkctl) ||139 FIELD_GET(PCI_EXP_LNKCTL_ASPMC, plnkctl)) {140 pcie_capability_write_word(pdev, PCI_EXP_LNKCTL,141 clnkctl & ~PCI_EXP_LNKCTL_ASPMC);142 pcie_capability_write_word(parent, PCI_EXP_LNKCTL,143 plnkctl & ~PCI_EXP_LNKCTL_ASPMC);144 }145 146 /*147 * Disable L1.2 on this downstream endpoint device first, followed148 * by the upstream149 */150 pci_clear_and_set_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1,151 PCI_L1SS_CTL1_L1_2_MASK, 0);152 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,153 PCI_L1SS_CTL1_L1_2_MASK, 0);154 155 /*156 * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD157 * in PCI_L1SS_CTL1 must be programmed *before* setting the L1.2158 * enable bits, even though they're all in PCI_L1SS_CTL1.159 */160 pl_l1_2_enable = pl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK;161 pl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;162 cl_l1_2_enable = cl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK;163 cl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;164 165 /* Write back without enables first (above we cleared them in ctl1) */166 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, pl_ctl2);167 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL2, cl_ctl2);168 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, pl_ctl1);169 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1, cl_ctl1);170 171 /* Then write back the enables */172 if (pl_l1_2_enable || cl_l1_2_enable) {173 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,174 pl_ctl1 | pl_l1_2_enable);175 pci_write_config_dword(pdev, pdev->l1ss + PCI_L1SS_CTL1,176 cl_ctl1 | cl_l1_2_enable);177 }178 179 /* Restore L0s/L1 if they were enabled */180 if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, clnkctl) ||181 FIELD_GET(PCI_EXP_LNKCTL_ASPMC, plnkctl)) {182 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, plnkctl);183 pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, clnkctl);184 }185}186 187#ifdef CONFIG_PCIEASPM188 189#ifdef MODULE_PARAM_PREFIX190#undef MODULE_PARAM_PREFIX191#endif192#define MODULE_PARAM_PREFIX "pcie_aspm."193 194/* Note: these are not register definitions */195#define PCIE_LINK_STATE_L0S_UP BIT(0) /* Upstream direction L0s state */196#define PCIE_LINK_STATE_L0S_DW BIT(1) /* Downstream direction L0s state */197static_assert(PCIE_LINK_STATE_L0S == (PCIE_LINK_STATE_L0S_UP | PCIE_LINK_STATE_L0S_DW));198 199#define PCIE_LINK_STATE_L1_SS_PCIPM (PCIE_LINK_STATE_L1_1_PCIPM |\200 PCIE_LINK_STATE_L1_2_PCIPM)201#define PCIE_LINK_STATE_L1_2_MASK (PCIE_LINK_STATE_L1_2 |\202 PCIE_LINK_STATE_L1_2_PCIPM)203#define PCIE_LINK_STATE_L1SS (PCIE_LINK_STATE_L1_1 |\204 PCIE_LINK_STATE_L1_1_PCIPM |\205 PCIE_LINK_STATE_L1_2_MASK)206 207struct pcie_link_state {208 struct pci_dev *pdev; /* Upstream component of the Link */209 struct pci_dev *downstream; /* Downstream component, function 0 */210 struct pcie_link_state *root; /* pointer to the root port link */211 struct pcie_link_state *parent; /* pointer to the parent Link state */212 struct list_head sibling; /* node in link_list */213 214 /* ASPM state */215 u32 aspm_support:7; /* Supported ASPM state */216 u32 aspm_enabled:7; /* Enabled ASPM state */217 u32 aspm_capable:7; /* Capable ASPM state with latency */218 u32 aspm_default:7; /* Default ASPM state by BIOS */219 u32 aspm_disable:7; /* Disabled ASPM state */220 221 /* Clock PM state */222 u32 clkpm_capable:1; /* Clock PM capable? */223 u32 clkpm_enabled:1; /* Current Clock PM state */224 u32 clkpm_default:1; /* Default Clock PM state by BIOS */225 u32 clkpm_disable:1; /* Clock PM disabled */226};227 228static int aspm_disabled, aspm_force;229static bool aspm_support_enabled = true;230static DEFINE_MUTEX(aspm_lock);231static LIST_HEAD(link_list);232 233#define POLICY_DEFAULT 0 /* BIOS default setting */234#define POLICY_PERFORMANCE 1 /* high performance */235#define POLICY_POWERSAVE 2 /* high power saving */236#define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */237 238#ifdef CONFIG_PCIEASPM_PERFORMANCE239static int aspm_policy = POLICY_PERFORMANCE;240#elif defined CONFIG_PCIEASPM_POWERSAVE241static int aspm_policy = POLICY_POWERSAVE;242#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE243static int aspm_policy = POLICY_POWER_SUPERSAVE;244#else245static int aspm_policy;246#endif247 248static const char *policy_str[] = {249 [POLICY_DEFAULT] = "default",250 [POLICY_PERFORMANCE] = "performance",251 [POLICY_POWERSAVE] = "powersave",252 [POLICY_POWER_SUPERSAVE] = "powersupersave"253};254 255/*256 * The L1 PM substate capability is only implemented in function 0 in a257 * multi function device.258 */259static struct pci_dev *pci_function_0(struct pci_bus *linkbus)260{261 struct pci_dev *child;262 263 list_for_each_entry(child, &linkbus->devices, bus_list)264 if (PCI_FUNC(child->devfn) == 0)265 return child;266 return NULL;267}268 269static int policy_to_aspm_state(struct pcie_link_state *link)270{271 switch (aspm_policy) {272 case POLICY_PERFORMANCE:273 /* Disable ASPM and Clock PM */274 return 0;275 case POLICY_POWERSAVE:276 /* Enable ASPM L0s/L1 */277 return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;278 case POLICY_POWER_SUPERSAVE:279 /* Enable Everything */280 return PCIE_LINK_STATE_ASPM_ALL;281 case POLICY_DEFAULT:282 return link->aspm_default;283 }284 return 0;285}286 287static int policy_to_clkpm_state(struct pcie_link_state *link)288{289 switch (aspm_policy) {290 case POLICY_PERFORMANCE:291 /* Disable ASPM and Clock PM */292 return 0;293 case POLICY_POWERSAVE:294 case POLICY_POWER_SUPERSAVE:295 /* Enable Clock PM */296 return 1;297 case POLICY_DEFAULT:298 return link->clkpm_default;299 }300 return 0;301}302 303static void pci_update_aspm_saved_state(struct pci_dev *dev)304{305 struct pci_cap_saved_state *save_state;306 u16 *cap, lnkctl, aspm_ctl;307 308 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);309 if (!save_state)310 return;311 312 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnkctl);313 314 /*315 * Update ASPM and CLKREQ bits of LNKCTL in save_state. We only316 * write PCI_EXP_LNKCTL_CCC during enumeration, so it shouldn't317 * change after being captured in save_state.318 */319 aspm_ctl = lnkctl & (PCI_EXP_LNKCTL_ASPMC | PCI_EXP_LNKCTL_CLKREQ_EN);320 lnkctl &= ~(PCI_EXP_LNKCTL_ASPMC | PCI_EXP_LNKCTL_CLKREQ_EN);321 322 /* Depends on pci_save_pcie_state(): cap[1] is LNKCTL */323 cap = (u16 *)&save_state->cap.data[0];324 cap[1] = lnkctl | aspm_ctl;325}326 327static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)328{329 struct pci_dev *child;330 struct pci_bus *linkbus = link->pdev->subordinate;331 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;332 333 list_for_each_entry(child, &linkbus->devices, bus_list) {334 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,335 PCI_EXP_LNKCTL_CLKREQ_EN,336 val);337 pci_update_aspm_saved_state(child);338 }339 link->clkpm_enabled = !!enable;340}341 342static void pcie_set_clkpm(struct pcie_link_state *link, int enable)343{344 /*345 * Don't enable Clock PM if the link is not Clock PM capable346 * or Clock PM is disabled347 */348 if (!link->clkpm_capable || link->clkpm_disable)349 enable = 0;350 /* Need nothing if the specified equals to current state */351 if (link->clkpm_enabled == enable)352 return;353 pcie_set_clkpm_nocheck(link, enable);354}355 356static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)357{358 int capable = 1, enabled = 1;359 u32 reg32;360 u16 reg16;361 struct pci_dev *child;362 struct pci_bus *linkbus = link->pdev->subordinate;363 364 /* All functions should have the same cap and state, take the worst */365 list_for_each_entry(child, &linkbus->devices, bus_list) {366 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);367 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {368 capable = 0;369 enabled = 0;370 break;371 }372 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);373 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))374 enabled = 0;375 }376 link->clkpm_enabled = enabled;377 link->clkpm_default = enabled;378 link->clkpm_capable = capable;379 link->clkpm_disable = blacklist ? 1 : 0;380}381 382/*383 * pcie_aspm_configure_common_clock: check if the 2 ends of a link384 * could use common clock. If they are, configure them to use the385 * common clock. That will reduce the ASPM state exit latency.386 */387static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)388{389 int same_clock = 1;390 u16 reg16, ccc, parent_old_ccc, child_old_ccc[8];391 struct pci_dev *child, *parent = link->pdev;392 struct pci_bus *linkbus = parent->subordinate;393 /*394 * All functions of a slot should have the same Slot Clock395 * Configuration, so just check one function396 */397 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);398 BUG_ON(!pci_is_pcie(child));399 400 /* Check downstream component if bit Slot Clock Configuration is 1 */401 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);402 if (!(reg16 & PCI_EXP_LNKSTA_SLC))403 same_clock = 0;404 405 /* Check upstream component if bit Slot Clock Configuration is 1 */406 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);407 if (!(reg16 & PCI_EXP_LNKSTA_SLC))408 same_clock = 0;409 410 /* Port might be already in common clock mode */411 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);412 parent_old_ccc = reg16 & PCI_EXP_LNKCTL_CCC;413 if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {414 bool consistent = true;415 416 list_for_each_entry(child, &linkbus->devices, bus_list) {417 pcie_capability_read_word(child, PCI_EXP_LNKCTL,418 ®16);419 if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {420 consistent = false;421 break;422 }423 }424 if (consistent)425 return;426 pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");427 }428 429 ccc = same_clock ? PCI_EXP_LNKCTL_CCC : 0;430 /* Configure downstream component, all functions */431 list_for_each_entry(child, &linkbus->devices, bus_list) {432 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);433 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC;434 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,435 PCI_EXP_LNKCTL_CCC, ccc);436 }437 438 /* Configure upstream component */439 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,440 PCI_EXP_LNKCTL_CCC, ccc);441 442 if (pcie_retrain_link(link->pdev, true)) {443 444 /* Training failed. Restore common clock configurations */445 pci_err(parent, "ASPM: Could not configure common clock\n");446 list_for_each_entry(child, &linkbus->devices, bus_list)447 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,448 PCI_EXP_LNKCTL_CCC,449 child_old_ccc[PCI_FUNC(child->devfn)]);450 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,451 PCI_EXP_LNKCTL_CCC, parent_old_ccc);452 }453}454 455/* Convert L0s latency encoding to ns */456static u32 calc_l0s_latency(u32 lnkcap)457{458 u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L0SEL, lnkcap);459 460 if (encoding == 0x7)461 return 5 * NSEC_PER_USEC; /* > 4us */462 return (64 << encoding);463}464 465/* Convert L0s acceptable latency encoding to ns */466static u32 calc_l0s_acceptable(u32 encoding)467{468 if (encoding == 0x7)469 return U32_MAX;470 return (64 << encoding);471}472 473/* Convert L1 latency encoding to ns */474static u32 calc_l1_latency(u32 lnkcap)475{476 u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L1EL, lnkcap);477 478 if (encoding == 0x7)479 return 65 * NSEC_PER_USEC; /* > 64us */480 return NSEC_PER_USEC << encoding;481}482 483/* Convert L1 acceptable latency encoding to ns */484static u32 calc_l1_acceptable(u32 encoding)485{486 if (encoding == 0x7)487 return U32_MAX;488 return NSEC_PER_USEC << encoding;489}490 491/* Convert L1SS T_pwr encoding to usec */492static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val)493{494 switch (scale) {495 case 0:496 return val * 2;497 case 1:498 return val * 10;499 case 2:500 return val * 100;501 }502 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);503 return 0;504}505 506/*507 * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1508 * register. Ports enter L1.2 when the most recent LTR value is greater509 * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we510 * don't enter L1.2 too aggressively.511 *512 * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.513 */514static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)515{516 u64 threshold_ns = (u64)threshold_us * NSEC_PER_USEC;517 518 /*519 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max520 * value of 0x3ff.521 */522 if (threshold_ns <= 1 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {523 *scale = 0; /* Value times 1ns */524 *value = threshold_ns;525 } else if (threshold_ns <= 32 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {526 *scale = 1; /* Value times 32ns */527 *value = roundup(threshold_ns, 32) / 32;528 } else if (threshold_ns <= 1024 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {529 *scale = 2; /* Value times 1024ns */530 *value = roundup(threshold_ns, 1024) / 1024;531 } else if (threshold_ns <= 32768 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {532 *scale = 3; /* Value times 32768ns */533 *value = roundup(threshold_ns, 32768) / 32768;534 } else if (threshold_ns <= 1048576 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {535 *scale = 4; /* Value times 1048576ns */536 *value = roundup(threshold_ns, 1048576) / 1048576;537 } else if (threshold_ns <= (u64)33554432 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {538 *scale = 5; /* Value times 33554432ns */539 *value = roundup(threshold_ns, 33554432) / 33554432;540 } else {541 *scale = 5;542 *value = FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE);543 }544}545 546static void pcie_aspm_check_latency(struct pci_dev *endpoint)547{548 u32 latency, encoding, lnkcap_up, lnkcap_dw;549 u32 l1_switch_latency = 0, latency_up_l0s;550 u32 latency_up_l1, latency_dw_l0s, latency_dw_l1;551 u32 acceptable_l0s, acceptable_l1;552 struct pcie_link_state *link;553 554 /* Device not in D0 doesn't need latency check */555 if ((endpoint->current_state != PCI_D0) &&556 (endpoint->current_state != PCI_UNKNOWN))557 return;558 559 link = endpoint->bus->self->link_state;560 561 /* Calculate endpoint L0s acceptable latency */562 encoding = FIELD_GET(PCI_EXP_DEVCAP_L0S, endpoint->devcap);563 acceptable_l0s = calc_l0s_acceptable(encoding);564 565 /* Calculate endpoint L1 acceptable latency */566 encoding = FIELD_GET(PCI_EXP_DEVCAP_L1, endpoint->devcap);567 acceptable_l1 = calc_l1_acceptable(encoding);568 569 while (link) {570 struct pci_dev *dev = pci_function_0(link->pdev->subordinate);571 572 /* Read direction exit latencies */573 pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP,574 &lnkcap_up);575 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP,576 &lnkcap_dw);577 latency_up_l0s = calc_l0s_latency(lnkcap_up);578 latency_up_l1 = calc_l1_latency(lnkcap_up);579 latency_dw_l0s = calc_l0s_latency(lnkcap_dw);580 latency_dw_l1 = calc_l1_latency(lnkcap_dw);581 582 /* Check upstream direction L0s latency */583 if ((link->aspm_capable & PCIE_LINK_STATE_L0S_UP) &&584 (latency_up_l0s > acceptable_l0s))585 link->aspm_capable &= ~PCIE_LINK_STATE_L0S_UP;586 587 /* Check downstream direction L0s latency */588 if ((link->aspm_capable & PCIE_LINK_STATE_L0S_DW) &&589 (latency_dw_l0s > acceptable_l0s))590 link->aspm_capable &= ~PCIE_LINK_STATE_L0S_DW;591 /*592 * Check L1 latency.593 * Every switch on the path to root complex need 1594 * more microsecond for L1. Spec doesn't mention L0s.595 *596 * The exit latencies for L1 substates are not advertised597 * by a device. Since the spec also doesn't mention a way598 * to determine max latencies introduced by enabling L1599 * substates on the components, it is not clear how to do600 * a L1 substate exit latency check. We assume that the601 * L1 exit latencies advertised by a device include L1602 * substate latencies (and hence do not do any check).603 */604 latency = max_t(u32, latency_up_l1, latency_dw_l1);605 if ((link->aspm_capable & PCIE_LINK_STATE_L1) &&606 (latency + l1_switch_latency > acceptable_l1))607 link->aspm_capable &= ~PCIE_LINK_STATE_L1;608 l1_switch_latency += NSEC_PER_USEC;609 610 link = link->parent;611 }612}613 614/* Calculate L1.2 PM substate timing parameters */615static void aspm_calc_l12_info(struct pcie_link_state *link,616 u32 parent_l1ss_cap, u32 child_l1ss_cap)617{618 struct pci_dev *child = link->downstream, *parent = link->pdev;619 u32 val1, val2, scale1, scale2;620 u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;621 u32 ctl1 = 0, ctl2 = 0;622 u32 pctl1, pctl2, cctl1, cctl2;623 u32 pl1_2_enables, cl1_2_enables;624 625 /* Choose the greater of the two Port Common_Mode_Restore_Times */626 val1 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, parent_l1ss_cap);627 val2 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, child_l1ss_cap);628 t_common_mode = max(val1, val2);629 630 /* Choose the greater of the two Port T_POWER_ON times */631 val1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, parent_l1ss_cap);632 scale1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, parent_l1ss_cap);633 val2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, child_l1ss_cap);634 scale2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, child_l1ss_cap);635 636 if (calc_l12_pwron(parent, scale1, val1) >637 calc_l12_pwron(child, scale2, val2)) {638 ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale1) |639 FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val1);640 t_power_on = calc_l12_pwron(parent, scale1, val1);641 } else {642 ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale2) |643 FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val2);644 t_power_on = calc_l12_pwron(child, scale2, val2);645 }646 647 /*648 * Set LTR_L1.2_THRESHOLD to the time required to transition the649 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if650 * downstream devices report (via LTR) that they can tolerate at651 * least that much latency.652 *653 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and654 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at655 * least 4us.656 */657 l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;658 encode_l12_threshold(l1_2_threshold, &scale, &value);659 ctl1 |= FIELD_PREP(PCI_L1SS_CTL1_CM_RESTORE_TIME, t_common_mode) |660 FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_VALUE, value) |661 FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_SCALE, scale);662 663 /* Some broken devices only support dword access to L1 SS */664 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);665 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);666 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);667 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);668 669 if (ctl1 == pctl1 && ctl1 == cctl1 &&670 ctl2 == pctl2 && ctl2 == cctl2)671 return;672 673 /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */674 pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;675 cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;676 677 if (pl1_2_enables || cl1_2_enables) {678 pci_clear_and_set_config_dword(child,679 child->l1ss + PCI_L1SS_CTL1,680 PCI_L1SS_CTL1_L1_2_MASK, 0);681 pci_clear_and_set_config_dword(parent,682 parent->l1ss + PCI_L1SS_CTL1,683 PCI_L1SS_CTL1_L1_2_MASK, 0);684 }685 686 /* Program T_POWER_ON times in both ports */687 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);688 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);689 690 /* Program Common_Mode_Restore_Time in upstream device */691 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,692 PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);693 694 /* Program LTR_L1.2_THRESHOLD time in both ports */695 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,696 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |697 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,698 ctl1);699 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,700 PCI_L1SS_CTL1_LTR_L12_TH_VALUE |701 PCI_L1SS_CTL1_LTR_L12_TH_SCALE,702 ctl1);703 704 if (pl1_2_enables || cl1_2_enables) {705 pci_clear_and_set_config_dword(parent,706 parent->l1ss + PCI_L1SS_CTL1, 0,707 pl1_2_enables);708 pci_clear_and_set_config_dword(child,709 child->l1ss + PCI_L1SS_CTL1, 0,710 cl1_2_enables);711 }712}713 714static void aspm_l1ss_init(struct pcie_link_state *link)715{716 struct pci_dev *child = link->downstream, *parent = link->pdev;717 u32 parent_l1ss_cap, child_l1ss_cap;718 u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;719 720 if (!parent->l1ss || !child->l1ss)721 return;722 723 /* Setup L1 substate */724 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,725 &parent_l1ss_cap);726 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,727 &child_l1ss_cap);728 729 if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))730 parent_l1ss_cap = 0;731 if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))732 child_l1ss_cap = 0;733 734 /*735 * If we don't have LTR for the entire path from the Root Complex736 * to this device, we can't use ASPM L1.2 because it relies on the737 * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.738 */739 if (!child->ltr_path)740 child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;741 742 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)743 link->aspm_support |= PCIE_LINK_STATE_L1_1;744 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)745 link->aspm_support |= PCIE_LINK_STATE_L1_2;746 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)747 link->aspm_support |= PCIE_LINK_STATE_L1_1_PCIPM;748 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)749 link->aspm_support |= PCIE_LINK_STATE_L1_2_PCIPM;750 751 if (parent_l1ss_cap)752 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,753 &parent_l1ss_ctl1);754 if (child_l1ss_cap)755 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,756 &child_l1ss_ctl1);757 758 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)759 link->aspm_enabled |= PCIE_LINK_STATE_L1_1;760 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)761 link->aspm_enabled |= PCIE_LINK_STATE_L1_2;762 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)763 link->aspm_enabled |= PCIE_LINK_STATE_L1_1_PCIPM;764 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)765 link->aspm_enabled |= PCIE_LINK_STATE_L1_2_PCIPM;766 767 if (link->aspm_support & PCIE_LINK_STATE_L1_2_MASK)768 aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap);769}770 771static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)772{773 struct pci_dev *child = link->downstream, *parent = link->pdev;774 u32 parent_lnkcap, child_lnkcap;775 u16 parent_lnkctl, child_lnkctl;776 struct pci_bus *linkbus = parent->subordinate;777 778 if (blacklist) {779 /* Set enabled/disable so that we will disable ASPM later */780 link->aspm_enabled = PCIE_LINK_STATE_ASPM_ALL;781 link->aspm_disable = PCIE_LINK_STATE_ASPM_ALL;782 return;783 }784 785 /*786 * If ASPM not supported, don't mess with the clocks and link,787 * bail out now.788 */789 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);790 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);791 if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))792 return;793 794 /* Configure common clock before checking latencies */795 pcie_aspm_configure_common_clock(link);796 797 /*798 * Re-read upstream/downstream components' register state after799 * clock configuration. L0s & L1 exit latencies in the otherwise800 * read-only Link Capabilities may change depending on common clock801 * configuration (PCIe r5.0, sec 7.5.3.6).802 */803 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);804 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);805 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);806 pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);807 808 /*809 * Setup L0s state810 *811 * Note that we must not enable L0s in either direction on a812 * given link unless components on both sides of the link each813 * support L0s.814 */815 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)816 link->aspm_support |= PCIE_LINK_STATE_L0S;817 818 if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)819 link->aspm_enabled |= PCIE_LINK_STATE_L0S_UP;820 if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)821 link->aspm_enabled |= PCIE_LINK_STATE_L0S_DW;822 823 /* Setup L1 state */824 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)825 link->aspm_support |= PCIE_LINK_STATE_L1;826 827 if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)828 link->aspm_enabled |= PCIE_LINK_STATE_L1;829 830 aspm_l1ss_init(link);831 832 /* Save default state */833 link->aspm_default = link->aspm_enabled;834 835 /* Setup initial capable state. Will be updated later */836 link->aspm_capable = link->aspm_support;837 838 /* Get and check endpoint acceptable latencies */839 list_for_each_entry(child, &linkbus->devices, bus_list) {840 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&841 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)842 continue;843 844 pcie_aspm_check_latency(child);845 }846}847 848/* Configure the ASPM L1 substates */849static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)850{851 u32 val, enable_req;852 struct pci_dev *child = link->downstream, *parent = link->pdev;853 854 enable_req = (link->aspm_enabled ^ state) & state;855 856 /*857 * Here are the rules specified in the PCIe spec for enabling L1SS:858 * - When enabling L1.x, enable bit at parent first, then at child859 * - When disabling L1.x, disable bit at child first, then at parent860 * - When enabling ASPM L1.x, need to disable L1861 * (at child followed by parent).862 * - The ASPM/PCIPM L1.2 must be disabled while programming timing863 * parameters864 *865 * To keep it simple, disable all L1SS bits first, and later enable866 * what is needed.867 */868 869 /* Disable all L1 substates */870 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,871 PCI_L1SS_CTL1_L1SS_MASK, 0);872 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,873 PCI_L1SS_CTL1_L1SS_MASK, 0);874 /*875 * If needed, disable L1, and it gets enabled later876 * in pcie_config_aspm_link().877 */878 if (enable_req & (PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2)) {879 pcie_capability_clear_word(child, PCI_EXP_LNKCTL,880 PCI_EXP_LNKCTL_ASPM_L1);881 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL,882 PCI_EXP_LNKCTL_ASPM_L1);883 }884 885 val = 0;886 if (state & PCIE_LINK_STATE_L1_1)887 val |= PCI_L1SS_CTL1_ASPM_L1_1;888 if (state & PCIE_LINK_STATE_L1_2)889 val |= PCI_L1SS_CTL1_ASPM_L1_2;890 if (state & PCIE_LINK_STATE_L1_1_PCIPM)891 val |= PCI_L1SS_CTL1_PCIPM_L1_1;892 if (state & PCIE_LINK_STATE_L1_2_PCIPM)893 val |= PCI_L1SS_CTL1_PCIPM_L1_2;894 895 /* Enable what we need to enable */896 pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,897 PCI_L1SS_CTL1_L1SS_MASK, val);898 pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1,899 PCI_L1SS_CTL1_L1SS_MASK, val);900}901 902static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)903{904 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,905 PCI_EXP_LNKCTL_ASPMC, val);906}907 908static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)909{910 u32 upstream = 0, dwstream = 0;911 struct pci_dev *child = link->downstream, *parent = link->pdev;912 struct pci_bus *linkbus = parent->subordinate;913 914 /* Enable only the states that were not explicitly disabled */915 state &= (link->aspm_capable & ~link->aspm_disable);916 917 /* Can't enable any substates if L1 is not enabled */918 if (!(state & PCIE_LINK_STATE_L1))919 state &= ~PCIE_LINK_STATE_L1SS;920 921 /* Spec says both ports must be in D0 before enabling PCI PM substates*/922 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {923 state &= ~PCIE_LINK_STATE_L1_SS_PCIPM;924 state |= (link->aspm_enabled & PCIE_LINK_STATE_L1_SS_PCIPM);925 }926 927 /* Nothing to do if the link is already in the requested state */928 if (link->aspm_enabled == state)929 return;930 /* Convert ASPM state to upstream/downstream ASPM register state */931 if (state & PCIE_LINK_STATE_L0S_UP)932 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;933 if (state & PCIE_LINK_STATE_L0S_DW)934 upstream |= PCI_EXP_LNKCTL_ASPM_L0S;935 if (state & PCIE_LINK_STATE_L1) {936 upstream |= PCI_EXP_LNKCTL_ASPM_L1;937 dwstream |= PCI_EXP_LNKCTL_ASPM_L1;938 }939 940 if (link->aspm_capable & PCIE_LINK_STATE_L1SS)941 pcie_config_aspm_l1ss(link, state);942 943 /*944 * Spec 2.0 suggests all functions should be configured the945 * same setting for ASPM. Enabling ASPM L1 should be done in946 * upstream component first and then downstream, and vice947 * versa for disabling ASPM L1. Spec doesn't mention L0S.948 */949 if (state & PCIE_LINK_STATE_L1)950 pcie_config_aspm_dev(parent, upstream);951 list_for_each_entry(child, &linkbus->devices, bus_list)952 pcie_config_aspm_dev(child, dwstream);953 if (!(state & PCIE_LINK_STATE_L1))954 pcie_config_aspm_dev(parent, upstream);955 956 link->aspm_enabled = state;957 958 /* Update latest ASPM configuration in saved context */959 pci_save_aspm_l1ss_state(link->downstream);960 pci_update_aspm_saved_state(link->downstream);961 pci_save_aspm_l1ss_state(parent);962 pci_update_aspm_saved_state(parent);963}964 965static void pcie_config_aspm_path(struct pcie_link_state *link)966{967 while (link) {968 pcie_config_aspm_link(link, policy_to_aspm_state(link));969 link = link->parent;970 }971}972 973static void free_link_state(struct pcie_link_state *link)974{975 link->pdev->link_state = NULL;976 kfree(link);977}978 979static int pcie_aspm_sanity_check(struct pci_dev *pdev)980{981 struct pci_dev *child;982 u32 reg32;983 984 /*985 * Some functions in a slot might not all be PCIe functions,986 * very strange. Disable ASPM for the whole slot987 */988 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {989 if (!pci_is_pcie(child))990 return -EINVAL;991 992 /*993 * If ASPM is disabled then we're not going to change994 * the BIOS state. It's safe to continue even if it's a995 * pre-1.1 device996 */997 998 if (aspm_disabled)999 continue;1000 1001 /*1002 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use1003 * RBER bit to determine if a function is 1.1 version device1004 */1005 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);1006 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {1007 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");1008 return -EINVAL;1009 }1010 }1011 return 0;1012}1013 1014static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)1015{1016 struct pcie_link_state *link;1017 1018 link = kzalloc(sizeof(*link), GFP_KERNEL);1019 if (!link)1020 return NULL;1021 1022 INIT_LIST_HEAD(&link->sibling);1023 link->pdev = pdev;1024 link->downstream = pci_function_0(pdev->subordinate);1025 1026 /*1027 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe1028 * hierarchies. Note that some PCIe host implementations omit1029 * the root ports entirely, in which case a downstream port on1030 * a switch may become the root of the link state chain for all1031 * its subordinate endpoints.1032 */1033 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||1034 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||1035 !pdev->bus->parent->self) {1036 link->root = link;1037 } else {1038 struct pcie_link_state *parent;1039 1040 parent = pdev->bus->parent->self->link_state;1041 if (!parent) {1042 kfree(link);1043 return NULL;1044 }1045 1046 link->parent = parent;1047 link->root = link->parent->root;1048 }1049 1050 list_add(&link->sibling, &link_list);1051 pdev->link_state = link;1052 return link;1053}1054 1055static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)1056{1057 struct pci_dev *child;1058 1059 list_for_each_entry(child, &pdev->subordinate->devices, bus_list)1060 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);1061}1062 1063/*1064 * pcie_aspm_init_link_state: Initiate PCI express link state.1065 * It is called after the pcie and its children devices are scanned.1066 * @pdev: the root port or switch downstream port1067 */1068void pcie_aspm_init_link_state(struct pci_dev *pdev)1069{1070 struct pcie_link_state *link;1071 int blacklist = !!pcie_aspm_sanity_check(pdev);1072 1073 if (!aspm_support_enabled)1074 return;1075 1076 if (pdev->link_state)1077 return;1078 1079 /*1080 * We allocate pcie_link_state for the component on the upstream1081 * end of a Link, so there's nothing to do unless this device is1082 * downstream port.1083 */1084 if (!pcie_downstream_port(pdev))1085 return;1086 1087 /* VIA has a strange chipset, root port is under a bridge */1088 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&1089 pdev->bus->self)1090 return;1091 1092 down_read(&pci_bus_sem);1093 if (list_empty(&pdev->subordinate->devices))1094 goto out;1095 1096 mutex_lock(&aspm_lock);1097 link = alloc_pcie_link_state(pdev);1098 if (!link)1099 goto unlock;1100 /*1101 * Setup initial ASPM state. Note that we need to configure1102 * upstream links also because capable state of them can be1103 * update through pcie_aspm_cap_init().1104 */1105 pcie_aspm_cap_init(link, blacklist);1106 1107 /* Setup initial Clock PM state */1108 pcie_clkpm_cap_init(link, blacklist);1109 1110 /*1111 * At this stage drivers haven't had an opportunity to change the1112 * link policy setting. Enabling ASPM on broken hardware can cripple1113 * it even before the driver has had a chance to disable ASPM, so1114 * default to a safe level right now. If we're enabling ASPM beyond1115 * the BIOS's expectation, we'll do so once pci_enable_device() is1116 * called.1117 */1118 if (aspm_policy != POLICY_POWERSAVE &&1119 aspm_policy != POLICY_POWER_SUPERSAVE) {1120 pcie_config_aspm_path(link);1121 pcie_set_clkpm(link, policy_to_clkpm_state(link));1122 }1123 1124 pcie_aspm_update_sysfs_visibility(pdev);1125 1126unlock:1127 mutex_unlock(&aspm_lock);1128out:1129 up_read(&pci_bus_sem);1130}1131 1132void pci_bridge_reconfigure_ltr(struct pci_dev *pdev)1133{1134 struct pci_dev *bridge;1135 u32 ctl;1136 1137 bridge = pci_upstream_bridge(pdev);1138 if (bridge && bridge->ltr_path) {1139 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl);1140 if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) {1141 pci_dbg(bridge, "re-enabling LTR\n");1142 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,1143 PCI_EXP_DEVCTL2_LTR_EN);1144 }1145 }1146}1147 1148void pci_configure_ltr(struct pci_dev *pdev)1149{1150 struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);1151 struct pci_dev *bridge;1152 u32 cap, ctl;1153 1154 if (!pci_is_pcie(pdev))1155 return;1156 1157 pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &cap);1158 if (!(cap & PCI_EXP_DEVCAP2_LTR))1159 return;1160 1161 pcie_capability_read_dword(pdev, PCI_EXP_DEVCTL2, &ctl);1162 if (ctl & PCI_EXP_DEVCTL2_LTR_EN) {1163 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) {1164 pdev->ltr_path = 1;1165 return;1166 }1167 1168 bridge = pci_upstream_bridge(pdev);1169 if (bridge && bridge->ltr_path)1170 pdev->ltr_path = 1;1171 1172 return;1173 }1174 1175 if (!host->native_ltr)1176 return;1177 1178 /*1179 * Software must not enable LTR in an Endpoint unless the Root1180 * Complex and all intermediate Switches indicate support for LTR.1181 * PCIe r4.0, sec 6.18.1182 */1183 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) {1184 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,1185 PCI_EXP_DEVCTL2_LTR_EN);1186 pdev->ltr_path = 1;1187 return;1188 }1189 1190 /*1191 * If we're configuring a hot-added device, LTR was likely1192 * disabled in the upstream bridge, so re-enable it before enabling1193 * it in the new device.1194 */1195 bridge = pci_upstream_bridge(pdev);1196 if (bridge && bridge->ltr_path) {1197 pci_bridge_reconfigure_ltr(pdev);1198 pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,1199 PCI_EXP_DEVCTL2_LTR_EN);1200 pdev->ltr_path = 1;1201 }1202}1203 1204/* Recheck latencies and update aspm_capable for links under the root */1205static void pcie_update_aspm_capable(struct pcie_link_state *root)1206{1207 struct pcie_link_state *link;1208 BUG_ON(root->parent);1209 list_for_each_entry(link, &link_list, sibling) {1210 if (link->root != root)1211 continue;1212 link->aspm_capable = link->aspm_support;1213 }1214 list_for_each_entry(link, &link_list, sibling) {1215 struct pci_dev *child;1216 struct pci_bus *linkbus = link->pdev->subordinate;1217 if (link->root != root)1218 continue;1219 list_for_each_entry(child, &linkbus->devices, bus_list) {1220 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&1221 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))1222 continue;1223 pcie_aspm_check_latency(child);1224 }1225 }1226}1227 1228/* @pdev: the endpoint device */1229void pcie_aspm_exit_link_state(struct pci_dev *pdev)1230{1231 struct pci_dev *parent = pdev->bus->self;1232 struct pcie_link_state *link, *root, *parent_link;1233 1234 if (!parent || !parent->link_state)1235 return;1236 1237 down_read(&pci_bus_sem);1238 mutex_lock(&aspm_lock);1239 1240 link = parent->link_state;1241 root = link->root;1242 parent_link = link->parent;1243 1244 /*1245 * link->downstream is a pointer to the pci_dev of function 0. If1246 * we remove that function, the pci_dev is about to be deallocated,1247 * so we can't use link->downstream again. Free the link state to1248 * avoid this.1249 *1250 * If we're removing a non-0 function, it's possible we could1251 * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends1252 * programming the same ASPM Control value for all functions of1253 * multi-function devices, so disable ASPM for all of them.1254 */1255 pcie_config_aspm_link(link, 0);1256 list_del(&link->sibling);1257 free_link_state(link);1258 1259 /* Recheck latencies and configure upstream links */1260 if (parent_link) {1261 pcie_update_aspm_capable(root);1262 pcie_config_aspm_path(parent_link);1263 }1264 1265 mutex_unlock(&aspm_lock);1266 up_read(&pci_bus_sem);1267}1268 1269/*1270 * @pdev: the root port or switch downstream port1271 * @locked: whether pci_bus_sem is held1272 */1273void pcie_aspm_pm_state_change(struct pci_dev *pdev, bool locked)1274{1275 struct pcie_link_state *link = pdev->link_state;1276 1277 if (aspm_disabled || !link)1278 return;1279 /*1280 * Devices changed PM state, we should recheck if latency1281 * meets all functions' requirement1282 */1283 if (!locked)1284 down_read(&pci_bus_sem);1285 mutex_lock(&aspm_lock);1286 pcie_update_aspm_capable(link->root);1287 pcie_config_aspm_path(link);1288 mutex_unlock(&aspm_lock);1289 if (!locked)1290 up_read(&pci_bus_sem);1291}1292 1293void pcie_aspm_powersave_config_link(struct pci_dev *pdev)1294{1295 struct pcie_link_state *link = pdev->link_state;1296 1297 if (aspm_disabled || !link)1298 return;1299 1300 if (aspm_policy != POLICY_POWERSAVE &&1301 aspm_policy != POLICY_POWER_SUPERSAVE)1302 return;1303 1304 down_read(&pci_bus_sem);1305 mutex_lock(&aspm_lock);1306 pcie_config_aspm_path(link);1307 pcie_set_clkpm(link, policy_to_clkpm_state(link));1308 mutex_unlock(&aspm_lock);1309 up_read(&pci_bus_sem);1310}1311 1312static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)1313{1314 struct pci_dev *bridge;1315 1316 if (!pci_is_pcie(pdev))1317 return NULL;1318 1319 bridge = pci_upstream_bridge(pdev);1320 if (!bridge || !pci_is_pcie(bridge))1321 return NULL;1322 1323 return bridge->link_state;1324}1325 1326static u8 pci_calc_aspm_disable_mask(int state)1327{1328 state &= ~PCIE_LINK_STATE_CLKPM;1329 1330 /* L1 PM substates require L1 */1331 if (state & PCIE_LINK_STATE_L1)1332 state |= PCIE_LINK_STATE_L1SS;1333 1334 return state;1335}1336 1337static u8 pci_calc_aspm_enable_mask(int state)1338{1339 state &= ~PCIE_LINK_STATE_CLKPM;1340 1341 /* L1 PM substates require L1 */1342 if (state & PCIE_LINK_STATE_L1SS)1343 state |= PCIE_LINK_STATE_L1;1344 1345 return state;1346}1347 1348static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked)1349{1350 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1351 1352 if (!link)1353 return -EINVAL;1354 /*1355 * A driver requested that ASPM be disabled on this device, but1356 * if we don't have permission to manage ASPM (e.g., on ACPI1357 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and1358 * the _OSC method), we can't honor that request. Windows has1359 * a similar mechanism using "PciASPMOptOut", which is also1360 * ignored in this situation.1361 */1362 if (aspm_disabled) {1363 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");1364 return -EPERM;1365 }1366 1367 if (!locked)1368 down_read(&pci_bus_sem);1369 mutex_lock(&aspm_lock);1370 link->aspm_disable |= pci_calc_aspm_disable_mask(state);1371 pcie_config_aspm_link(link, policy_to_aspm_state(link));1372 1373 if (state & PCIE_LINK_STATE_CLKPM)1374 link->clkpm_disable = 1;1375 pcie_set_clkpm(link, policy_to_clkpm_state(link));1376 mutex_unlock(&aspm_lock);1377 if (!locked)1378 up_read(&pci_bus_sem);1379 1380 return 0;1381}1382 1383int pci_disable_link_state_locked(struct pci_dev *pdev, int state)1384{1385 lockdep_assert_held_read(&pci_bus_sem);1386 1387 return __pci_disable_link_state(pdev, state, true);1388}1389EXPORT_SYMBOL(pci_disable_link_state_locked);1390 1391/**1392 * pci_disable_link_state - Disable device's link state, so the link will1393 * never enter specific states. Note that if the BIOS didn't grant ASPM1394 * control to the OS, this does nothing because we can't touch the LNKCTL1395 * register. Returns 0 or a negative errno.1396 *1397 * @pdev: PCI device1398 * @state: ASPM link state to disable1399 */1400int pci_disable_link_state(struct pci_dev *pdev, int state)1401{1402 return __pci_disable_link_state(pdev, state, false);1403}1404EXPORT_SYMBOL(pci_disable_link_state);1405 1406static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)1407{1408 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1409 1410 if (!link)1411 return -EINVAL;1412 /*1413 * A driver requested that ASPM be enabled on this device, but1414 * if we don't have permission to manage ASPM (e.g., on ACPI1415 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and1416 * the _OSC method), we can't honor that request.1417 */1418 if (aspm_disabled) {1419 pci_warn(pdev, "can't override BIOS ASPM; OS doesn't have ASPM control\n");1420 return -EPERM;1421 }1422 1423 if (!locked)1424 down_read(&pci_bus_sem);1425 mutex_lock(&aspm_lock);1426 link->aspm_default = pci_calc_aspm_enable_mask(state);1427 pcie_config_aspm_link(link, policy_to_aspm_state(link));1428 1429 link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;1430 pcie_set_clkpm(link, policy_to_clkpm_state(link));1431 mutex_unlock(&aspm_lock);1432 if (!locked)1433 up_read(&pci_bus_sem);1434 1435 return 0;1436}1437 1438/**1439 * pci_enable_link_state - Clear and set the default device link state so that1440 * the link may be allowed to enter the specified states. Note that if the1441 * BIOS didn't grant ASPM control to the OS, this does nothing because we can't1442 * touch the LNKCTL register. Also note that this does not enable states1443 * disabled by pci_disable_link_state(). Return 0 or a negative errno.1444 *1445 * @pdev: PCI device1446 * @state: Mask of ASPM link states to enable1447 */1448int pci_enable_link_state(struct pci_dev *pdev, int state)1449{1450 return __pci_enable_link_state(pdev, state, false);1451}1452EXPORT_SYMBOL(pci_enable_link_state);1453 1454/**1455 * pci_enable_link_state_locked - Clear and set the default device link state1456 * so that the link may be allowed to enter the specified states. Note that if1457 * the BIOS didn't grant ASPM control to the OS, this does nothing because we1458 * can't touch the LNKCTL register. Also note that this does not enable states1459 * disabled by pci_disable_link_state(). Return 0 or a negative errno.1460 *1461 * @pdev: PCI device1462 * @state: Mask of ASPM link states to enable1463 *1464 * Context: Caller holds pci_bus_sem read lock.1465 */1466int pci_enable_link_state_locked(struct pci_dev *pdev, int state)1467{1468 lockdep_assert_held_read(&pci_bus_sem);1469 1470 return __pci_enable_link_state(pdev, state, true);1471}1472EXPORT_SYMBOL(pci_enable_link_state_locked);1473 1474static int pcie_aspm_set_policy(const char *val,1475 const struct kernel_param *kp)1476{1477 int i;1478 struct pcie_link_state *link;1479 1480 if (aspm_disabled)1481 return -EPERM;1482 i = sysfs_match_string(policy_str, val);1483 if (i < 0)1484 return i;1485 if (i == aspm_policy)1486 return 0;1487 1488 down_read(&pci_bus_sem);1489 mutex_lock(&aspm_lock);1490 aspm_policy = i;1491 list_for_each_entry(link, &link_list, sibling) {1492 pcie_config_aspm_link(link, policy_to_aspm_state(link));1493 pcie_set_clkpm(link, policy_to_clkpm_state(link));1494 }1495 mutex_unlock(&aspm_lock);1496 up_read(&pci_bus_sem);1497 return 0;1498}1499 1500static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)1501{1502 int i, cnt = 0;1503 for (i = 0; i < ARRAY_SIZE(policy_str); i++)1504 if (i == aspm_policy)1505 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);1506 else1507 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);1508 cnt += sprintf(buffer + cnt, "\n");1509 return cnt;1510}1511 1512module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,1513 NULL, 0644);1514 1515/**1516 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.1517 * @pdev: Target device.1518 *1519 * Relies on the upstream bridge's link_state being valid. The link_state1520 * is deallocated only when the last child of the bridge (i.e., @pdev or a1521 * sibling) is removed, and the caller should be holding a reference to1522 * @pdev, so this should be safe.1523 */1524bool pcie_aspm_enabled(struct pci_dev *pdev)1525{1526 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1527 1528 if (!link)1529 return false;1530 1531 return link->aspm_enabled;1532}1533EXPORT_SYMBOL_GPL(pcie_aspm_enabled);1534 1535static ssize_t aspm_attr_show_common(struct device *dev,1536 struct device_attribute *attr,1537 char *buf, u8 state)1538{1539 struct pci_dev *pdev = to_pci_dev(dev);1540 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1541 1542 return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);1543}1544 1545static ssize_t aspm_attr_store_common(struct device *dev,1546 struct device_attribute *attr,1547 const char *buf, size_t len, u8 state)1548{1549 struct pci_dev *pdev = to_pci_dev(dev);1550 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1551 bool state_enable;1552 1553 if (kstrtobool(buf, &state_enable) < 0)1554 return -EINVAL;1555 1556 down_read(&pci_bus_sem);1557 mutex_lock(&aspm_lock);1558 1559 if (state_enable) {1560 link->aspm_disable &= ~state;1561 /* need to enable L1 for substates */1562 if (state & PCIE_LINK_STATE_L1SS)1563 link->aspm_disable &= ~PCIE_LINK_STATE_L1;1564 } else {1565 link->aspm_disable |= state;1566 if (state & PCIE_LINK_STATE_L1)1567 link->aspm_disable |= PCIE_LINK_STATE_L1SS;1568 }1569 1570 pcie_config_aspm_link(link, policy_to_aspm_state(link));1571 1572 mutex_unlock(&aspm_lock);1573 up_read(&pci_bus_sem);1574 1575 return len;1576}1577 1578#define ASPM_ATTR(_f, _s) \1579static ssize_t _f##_show(struct device *dev, \1580 struct device_attribute *attr, char *buf) \1581{ return aspm_attr_show_common(dev, attr, buf, PCIE_LINK_STATE_##_s); } \1582 \1583static ssize_t _f##_store(struct device *dev, \1584 struct device_attribute *attr, \1585 const char *buf, size_t len) \1586{ return aspm_attr_store_common(dev, attr, buf, len, PCIE_LINK_STATE_##_s); }1587 1588ASPM_ATTR(l0s_aspm, L0S)1589ASPM_ATTR(l1_aspm, L1)1590ASPM_ATTR(l1_1_aspm, L1_1)1591ASPM_ATTR(l1_2_aspm, L1_2)1592ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)1593ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)1594 1595static ssize_t clkpm_show(struct device *dev,1596 struct device_attribute *attr, char *buf)1597{1598 struct pci_dev *pdev = to_pci_dev(dev);1599 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1600 1601 return sysfs_emit(buf, "%d\n", link->clkpm_enabled);1602}1603 1604static ssize_t clkpm_store(struct device *dev,1605 struct device_attribute *attr,1606 const char *buf, size_t len)1607{1608 struct pci_dev *pdev = to_pci_dev(dev);1609 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1610 bool state_enable;1611 1612 if (kstrtobool(buf, &state_enable) < 0)1613 return -EINVAL;1614 1615 down_read(&pci_bus_sem);1616 mutex_lock(&aspm_lock);1617 1618 link->clkpm_disable = !state_enable;1619 pcie_set_clkpm(link, policy_to_clkpm_state(link));1620 1621 mutex_unlock(&aspm_lock);1622 up_read(&pci_bus_sem);1623 1624 return len;1625}1626 1627static DEVICE_ATTR_RW(clkpm);1628static DEVICE_ATTR_RW(l0s_aspm);1629static DEVICE_ATTR_RW(l1_aspm);1630static DEVICE_ATTR_RW(l1_1_aspm);1631static DEVICE_ATTR_RW(l1_2_aspm);1632static DEVICE_ATTR_RW(l1_1_pcipm);1633static DEVICE_ATTR_RW(l1_2_pcipm);1634 1635static struct attribute *aspm_ctrl_attrs[] = {1636 &dev_attr_clkpm.attr,1637 &dev_attr_l0s_aspm.attr,1638 &dev_attr_l1_aspm.attr,1639 &dev_attr_l1_1_aspm.attr,1640 &dev_attr_l1_2_aspm.attr,1641 &dev_attr_l1_1_pcipm.attr,1642 &dev_attr_l1_2_pcipm.attr,1643 NULL1644};1645 1646static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,1647 struct attribute *a, int n)1648{1649 struct device *dev = kobj_to_dev(kobj);1650 struct pci_dev *pdev = to_pci_dev(dev);1651 struct pcie_link_state *link = pcie_aspm_get_link(pdev);1652 static const u8 aspm_state_map[] = {1653 PCIE_LINK_STATE_L0S,1654 PCIE_LINK_STATE_L1,1655 PCIE_LINK_STATE_L1_1,1656 PCIE_LINK_STATE_L1_2,1657 PCIE_LINK_STATE_L1_1_PCIPM,1658 PCIE_LINK_STATE_L1_2_PCIPM,1659 };1660 1661 if (aspm_disabled || !link)1662 return 0;1663 1664 if (n == 0)1665 return link->clkpm_capable ? a->mode : 0;1666 1667 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;1668}1669 1670const struct attribute_group aspm_ctrl_attr_group = {1671 .name = "link",1672 .attrs = aspm_ctrl_attrs,1673 .is_visible = aspm_ctrl_attrs_are_visible,1674};1675 1676static int __init pcie_aspm_disable(char *str)1677{1678 if (!strcmp(str, "off")) {1679 aspm_policy = POLICY_DEFAULT;1680 aspm_disabled = 1;1681 aspm_support_enabled = false;1682 pr_info("PCIe ASPM is disabled\n");1683 } else if (!strcmp(str, "force")) {1684 aspm_force = 1;1685 pr_info("PCIe ASPM is forcibly enabled\n");1686 }1687 return 1;1688}1689 1690__setup("pcie_aspm=", pcie_aspm_disable);1691 1692void pcie_no_aspm(void)1693{1694 /*1695 * Disabling ASPM is intended to prevent the kernel from modifying1696 * existing hardware state, not to clear existing state. To that end:1697 * (a) set policy to POLICY_DEFAULT in order to avoid changing state1698 * (b) prevent userspace from changing policy1699 */1700 if (!aspm_force) {1701 aspm_policy = POLICY_DEFAULT;1702 aspm_disabled = 1;1703 }1704}1705 1706bool pcie_aspm_support_enabled(void)1707{1708 return aspm_support_enabled;1709}1710 1711#endif /* CONFIG_PCIEASPM */1712