271 lines · c
1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)2/*3 * Copyright(c) 2019 Intel Corporation.4 *5 */6 7#include "aspm.h"8 9/* Time after which the timer interrupt will re-enable ASPM */10#define ASPM_TIMER_MS 100011/* Time for which interrupts are ignored after a timer has been scheduled */12#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)13/* Two interrupts within this time trigger ASPM disable */14#define ASPM_TRIGGER_MS 115#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)16#define ASPM_L1_SUPPORTED(reg) \17 ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)18 19uint aspm_mode = ASPM_MODE_DISABLED;20module_param_named(aspm, aspm_mode, uint, 0444);21MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");22 23static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)24{25 struct pci_dev *parent = dd->pcidev->bus->self;26 u32 up, dn;27 28 /*29 * If the driver does not have access to the upstream component,30 * it cannot support ASPM L1 at all.31 */32 if (!parent)33 return false;34 35 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);36 dn = ASPM_L1_SUPPORTED(dn);37 38 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);39 up = ASPM_L1_SUPPORTED(up);40 41 /* ASPM works on A-step but is reported as not supported */42 return (!!dn || is_ax(dd)) && !!up;43}44 45/* Set L1 entrance latency for slower entry to L1 */46static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)47{48 u32 l1_ent_lat = 0x4u;49 u32 reg32;50 51 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);52 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;53 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;54 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);55}56 57static void aspm_hw_enable_l1(struct hfi1_devdata *dd)58{59 struct pci_dev *parent = dd->pcidev->bus->self;60 61 /*62 * If the driver does not have access to the upstream component,63 * it cannot support ASPM L1 at all.64 */65 if (!parent)66 return;67 68 /* Enable ASPM L1 first in upstream component and then downstream */69 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,70 PCI_EXP_LNKCTL_ASPMC,71 PCI_EXP_LNKCTL_ASPM_L1);72 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,73 PCI_EXP_LNKCTL_ASPMC,74 PCI_EXP_LNKCTL_ASPM_L1);75}76 77void aspm_hw_disable_l1(struct hfi1_devdata *dd)78{79 struct pci_dev *parent = dd->pcidev->bus->self;80 81 /* Disable ASPM L1 first in downstream component and then upstream */82 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,83 PCI_EXP_LNKCTL_ASPMC, 0x0);84 if (parent)85 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,86 PCI_EXP_LNKCTL_ASPMC, 0x0);87}88 89static void aspm_enable(struct hfi1_devdata *dd)90{91 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||92 !dd->aspm_supported)93 return;94 95 aspm_hw_enable_l1(dd);96 dd->aspm_enabled = true;97}98 99static void aspm_disable(struct hfi1_devdata *dd)100{101 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)102 return;103 104 aspm_hw_disable_l1(dd);105 dd->aspm_enabled = false;106}107 108static void aspm_disable_inc(struct hfi1_devdata *dd)109{110 unsigned long flags;111 112 spin_lock_irqsave(&dd->aspm_lock, flags);113 aspm_disable(dd);114 atomic_inc(&dd->aspm_disabled_cnt);115 spin_unlock_irqrestore(&dd->aspm_lock, flags);116}117 118static void aspm_enable_dec(struct hfi1_devdata *dd)119{120 unsigned long flags;121 122 spin_lock_irqsave(&dd->aspm_lock, flags);123 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))124 aspm_enable(dd);125 spin_unlock_irqrestore(&dd->aspm_lock, flags);126}127 128/* ASPM processing for each receive context interrupt */129void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)130{131 bool restart_timer;132 bool close_interrupts;133 unsigned long flags;134 ktime_t now, prev;135 136 spin_lock_irqsave(&rcd->aspm_lock, flags);137 /* PSM contexts are open */138 if (!rcd->aspm_intr_enable)139 goto unlock;140 141 prev = rcd->aspm_ts_last_intr;142 now = ktime_get();143 rcd->aspm_ts_last_intr = now;144 145 /* An interrupt pair close together in time */146 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;147 148 /* Don't push out our timer till this much time has elapsed */149 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >150 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;151 restart_timer = restart_timer && close_interrupts;152 153 /* Disable ASPM and schedule timer */154 if (rcd->aspm_enabled && close_interrupts) {155 aspm_disable_inc(rcd->dd);156 rcd->aspm_enabled = false;157 restart_timer = true;158 }159 160 if (restart_timer) {161 mod_timer(&rcd->aspm_timer,162 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));163 rcd->aspm_ts_timer_sched = now;164 }165unlock:166 spin_unlock_irqrestore(&rcd->aspm_lock, flags);167}168 169/* Timer function for re-enabling ASPM in the absence of interrupt activity */170static void aspm_ctx_timer_function(struct timer_list *t)171{172 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);173 unsigned long flags;174 175 spin_lock_irqsave(&rcd->aspm_lock, flags);176 aspm_enable_dec(rcd->dd);177 rcd->aspm_enabled = true;178 spin_unlock_irqrestore(&rcd->aspm_lock, flags);179}180 181/*182 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts183 * are open.184 */185void aspm_disable_all(struct hfi1_devdata *dd)186{187 struct hfi1_ctxtdata *rcd;188 unsigned long flags;189 u16 i;190 191 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {192 rcd = hfi1_rcd_get_by_index(dd, i);193 if (rcd) {194 del_timer_sync(&rcd->aspm_timer);195 spin_lock_irqsave(&rcd->aspm_lock, flags);196 rcd->aspm_intr_enable = false;197 spin_unlock_irqrestore(&rcd->aspm_lock, flags);198 hfi1_rcd_put(rcd);199 }200 }201 202 aspm_disable(dd);203 atomic_set(&dd->aspm_disabled_cnt, 0);204}205 206/* Re-enable interrupt processing for verbs contexts */207void aspm_enable_all(struct hfi1_devdata *dd)208{209 struct hfi1_ctxtdata *rcd;210 unsigned long flags;211 u16 i;212 213 aspm_enable(dd);214 215 if (aspm_mode != ASPM_MODE_DYNAMIC)216 return;217 218 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {219 rcd = hfi1_rcd_get_by_index(dd, i);220 if (rcd) {221 spin_lock_irqsave(&rcd->aspm_lock, flags);222 rcd->aspm_intr_enable = true;223 rcd->aspm_enabled = true;224 spin_unlock_irqrestore(&rcd->aspm_lock, flags);225 hfi1_rcd_put(rcd);226 }227 }228}229 230static void aspm_ctx_init(struct hfi1_ctxtdata *rcd)231{232 spin_lock_init(&rcd->aspm_lock);233 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);234 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&235 aspm_mode == ASPM_MODE_DYNAMIC &&236 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;237}238 239void aspm_init(struct hfi1_devdata *dd)240{241 struct hfi1_ctxtdata *rcd;242 u16 i;243 244 spin_lock_init(&dd->aspm_lock);245 dd->aspm_supported = aspm_hw_l1_supported(dd);246 247 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {248 rcd = hfi1_rcd_get_by_index(dd, i);249 if (rcd)250 aspm_ctx_init(rcd);251 hfi1_rcd_put(rcd);252 }253 254 /* Start with ASPM disabled */255 aspm_hw_set_l1_ent_latency(dd);256 dd->aspm_enabled = false;257 aspm_hw_disable_l1(dd);258 259 /* Now turn on ASPM if configured */260 aspm_enable_all(dd);261}262 263void aspm_exit(struct hfi1_devdata *dd)264{265 aspm_disable_all(dd);266 267 /* Turn on ASPM on exit to conserve power */268 aspm_enable(dd);269}270 271