2374 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * intel_idle.c - native hardware idle loop for modern Intel processors4 *5 * Copyright (c) 2013 - 2020, Intel Corporation.6 * Len Brown <len.brown@intel.com>7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>8 */9 10/*11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT12 * in lieu of the legacy ACPI processor_idle driver. The intent is to13 * make Linux more efficient on these processors, as intel_idle knows14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.15 */16 17/*18 * Design Assumptions19 *20 * All CPUs have same idle states as boot CPU21 *22 * Chipset BM_STS (bus master status) bit is a NOP23 * for preventing entry into deep C-states24 *25 * CPU will flush caches as needed when entering a C-state via MWAIT26 * (in contrast to entering ACPI C3, in which case the WBINVD27 * instruction needs to be executed to flush the caches)28 */29 30/*31 * Known limitations32 *33 * ACPI has a .suspend hack to turn off deep c-statees during suspend34 * to avoid complications with the lapic timer workaround.35 * Have not seen issues with suspend, but may need same workaround here.36 *37 */38 39/* un-comment DEBUG to enable pr_debug() statements */40/* #define DEBUG */41 42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt43 44#include <linux/acpi.h>45#include <linux/kernel.h>46#include <linux/cpuidle.h>47#include <linux/tick.h>48#include <trace/events/power.h>49#include <linux/sched.h>50#include <linux/sched/smt.h>51#include <linux/notifier.h>52#include <linux/cpu.h>53#include <linux/moduleparam.h>54#include <asm/cpu_device_id.h>55#include <asm/intel-family.h>56#include <asm/mwait.h>57#include <asm/spec-ctrl.h>58#include <asm/fpu/api.h>59 60#define INTEL_IDLE_VERSION "0.5.1"61 62static struct cpuidle_driver intel_idle_driver = {63 .name = "intel_idle",64 .owner = THIS_MODULE,65};66/* intel_idle.max_cstate=0 disables driver */67static int max_cstate = CPUIDLE_STATE_MAX - 1;68static unsigned int disabled_states_mask __read_mostly;69static unsigned int preferred_states_mask __read_mostly;70static bool force_irq_on __read_mostly;71static bool ibrs_off __read_mostly;72 73static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;74 75static unsigned long auto_demotion_disable_flags;76 77static enum {78 C1E_PROMOTION_PRESERVE,79 C1E_PROMOTION_ENABLE,80 C1E_PROMOTION_DISABLE81} c1e_promotion = C1E_PROMOTION_PRESERVE;82 83struct idle_cpu {84 struct cpuidle_state *state_table;85 86 /*87 * Hardware C-state auto-demotion may not always be optimal.88 * Indicate which enable bits to clear here.89 */90 unsigned long auto_demotion_disable_flags;91 bool byt_auto_demotion_disable_flag;92 bool disable_promotion_to_c1e;93 bool use_acpi;94};95 96static const struct idle_cpu *icpu __initdata;97static struct cpuidle_state *cpuidle_state_table __initdata;98 99static unsigned int mwait_substates __initdata;100 101/*102 * Enable interrupts before entering the C-state. On some platforms and for103 * some C-states, this may measurably decrease interrupt latency.104 */105#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)106 107/*108 * Enable this state by default even if the ACPI _CST does not list it.109 */110#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)111 112/*113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE114 * above.115 */116#define CPUIDLE_FLAG_IBRS BIT(16)117 118/*119 * Initialize large xstate for the C6-state entrance.120 */121#define CPUIDLE_FLAG_INIT_XSTATE BIT(17)122 123/*124 * Ignore the sub-state when matching mwait hints between the ACPI _CST and125 * custom tables.126 */127#define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18)128 129/*130 * MWAIT takes an 8-bit "hint" in EAX "suggesting"131 * the C-state (top nibble) and sub-state (bottom nibble)132 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.133 *134 * We store the hint at the top of our "flags" for each state.135 */136#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)137#define MWAIT2flg(eax) ((eax & 0xFF) << 24)138 139static __always_inline int __intel_idle(struct cpuidle_device *dev,140 struct cpuidle_driver *drv,141 int index, bool irqoff)142{143 struct cpuidle_state *state = &drv->states[index];144 unsigned long eax = flg2MWAIT(state->flags);145 unsigned long ecx = 1*irqoff; /* break on interrupt flag */146 147 mwait_idle_with_hints(eax, ecx);148 149 return index;150}151 152/**153 * intel_idle - Ask the processor to enter the given idle state.154 * @dev: cpuidle device of the target CPU.155 * @drv: cpuidle driver (assumed to point to intel_idle_driver).156 * @index: Target idle state index.157 *158 * Use the MWAIT instruction to notify the processor that the CPU represented by159 * @dev is idle and it can try to enter the idle state corresponding to @index.160 *161 * If the local APIC timer is not known to be reliable in the target idle state,162 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.163 *164 * Must be called under local_irq_disable().165 */166static __cpuidle int intel_idle(struct cpuidle_device *dev,167 struct cpuidle_driver *drv, int index)168{169 return __intel_idle(dev, drv, index, true);170}171 172static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,173 struct cpuidle_driver *drv, int index)174{175 return __intel_idle(dev, drv, index, false);176}177 178static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,179 struct cpuidle_driver *drv, int index)180{181 bool smt_active = sched_smt_active();182 u64 spec_ctrl = spec_ctrl_current();183 int ret;184 185 if (smt_active)186 __update_spec_ctrl(0);187 188 ret = __intel_idle(dev, drv, index, true);189 190 if (smt_active)191 __update_spec_ctrl(spec_ctrl);192 193 return ret;194}195 196static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,197 struct cpuidle_driver *drv, int index)198{199 fpu_idle_fpregs();200 return __intel_idle(dev, drv, index, true);201}202 203/**204 * intel_idle_s2idle - Ask the processor to enter the given idle state.205 * @dev: cpuidle device of the target CPU.206 * @drv: cpuidle driver (assumed to point to intel_idle_driver).207 * @index: Target idle state index.208 *209 * Use the MWAIT instruction to notify the processor that the CPU represented by210 * @dev is idle and it can try to enter the idle state corresponding to @index.211 *212 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen213 * scheduler tick and suspended scheduler clock on the target CPU.214 */215static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,216 struct cpuidle_driver *drv, int index)217{218 unsigned long ecx = 1; /* break on interrupt flag */219 struct cpuidle_state *state = &drv->states[index];220 unsigned long eax = flg2MWAIT(state->flags);221 222 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)223 fpu_idle_fpregs();224 225 mwait_idle_with_hints(eax, ecx);226 227 return 0;228}229 230/*231 * States are indexed by the cstate number,232 * which is also the index into the MWAIT hint array.233 * Thus C0 is a dummy.234 */235static struct cpuidle_state nehalem_cstates[] __initdata = {236 {237 .name = "C1",238 .desc = "MWAIT 0x00",239 .flags = MWAIT2flg(0x00),240 .exit_latency = 3,241 .target_residency = 6,242 .enter = &intel_idle,243 .enter_s2idle = intel_idle_s2idle, },244 {245 .name = "C1E",246 .desc = "MWAIT 0x01",247 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,248 .exit_latency = 10,249 .target_residency = 20,250 .enter = &intel_idle,251 .enter_s2idle = intel_idle_s2idle, },252 {253 .name = "C3",254 .desc = "MWAIT 0x10",255 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,256 .exit_latency = 20,257 .target_residency = 80,258 .enter = &intel_idle,259 .enter_s2idle = intel_idle_s2idle, },260 {261 .name = "C6",262 .desc = "MWAIT 0x20",263 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,264 .exit_latency = 200,265 .target_residency = 800,266 .enter = &intel_idle,267 .enter_s2idle = intel_idle_s2idle, },268 {269 .enter = NULL }270};271 272static struct cpuidle_state snb_cstates[] __initdata = {273 {274 .name = "C1",275 .desc = "MWAIT 0x00",276 .flags = MWAIT2flg(0x00),277 .exit_latency = 2,278 .target_residency = 2,279 .enter = &intel_idle,280 .enter_s2idle = intel_idle_s2idle, },281 {282 .name = "C1E",283 .desc = "MWAIT 0x01",284 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,285 .exit_latency = 10,286 .target_residency = 20,287 .enter = &intel_idle,288 .enter_s2idle = intel_idle_s2idle, },289 {290 .name = "C3",291 .desc = "MWAIT 0x10",292 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,293 .exit_latency = 80,294 .target_residency = 211,295 .enter = &intel_idle,296 .enter_s2idle = intel_idle_s2idle, },297 {298 .name = "C6",299 .desc = "MWAIT 0x20",300 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,301 .exit_latency = 104,302 .target_residency = 345,303 .enter = &intel_idle,304 .enter_s2idle = intel_idle_s2idle, },305 {306 .name = "C7",307 .desc = "MWAIT 0x30",308 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,309 .exit_latency = 109,310 .target_residency = 345,311 .enter = &intel_idle,312 .enter_s2idle = intel_idle_s2idle, },313 {314 .enter = NULL }315};316 317static struct cpuidle_state byt_cstates[] __initdata = {318 {319 .name = "C1",320 .desc = "MWAIT 0x00",321 .flags = MWAIT2flg(0x00),322 .exit_latency = 1,323 .target_residency = 1,324 .enter = &intel_idle,325 .enter_s2idle = intel_idle_s2idle, },326 {327 .name = "C6N",328 .desc = "MWAIT 0x58",329 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,330 .exit_latency = 300,331 .target_residency = 275,332 .enter = &intel_idle,333 .enter_s2idle = intel_idle_s2idle, },334 {335 .name = "C6S",336 .desc = "MWAIT 0x52",337 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,338 .exit_latency = 500,339 .target_residency = 560,340 .enter = &intel_idle,341 .enter_s2idle = intel_idle_s2idle, },342 {343 .name = "C7",344 .desc = "MWAIT 0x60",345 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,346 .exit_latency = 1200,347 .target_residency = 4000,348 .enter = &intel_idle,349 .enter_s2idle = intel_idle_s2idle, },350 {351 .name = "C7S",352 .desc = "MWAIT 0x64",353 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,354 .exit_latency = 10000,355 .target_residency = 20000,356 .enter = &intel_idle,357 .enter_s2idle = intel_idle_s2idle, },358 {359 .enter = NULL }360};361 362static struct cpuidle_state cht_cstates[] __initdata = {363 {364 .name = "C1",365 .desc = "MWAIT 0x00",366 .flags = MWAIT2flg(0x00),367 .exit_latency = 1,368 .target_residency = 1,369 .enter = &intel_idle,370 .enter_s2idle = intel_idle_s2idle, },371 {372 .name = "C6N",373 .desc = "MWAIT 0x58",374 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,375 .exit_latency = 80,376 .target_residency = 275,377 .enter = &intel_idle,378 .enter_s2idle = intel_idle_s2idle, },379 {380 .name = "C6S",381 .desc = "MWAIT 0x52",382 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,383 .exit_latency = 200,384 .target_residency = 560,385 .enter = &intel_idle,386 .enter_s2idle = intel_idle_s2idle, },387 {388 .name = "C7",389 .desc = "MWAIT 0x60",390 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,391 .exit_latency = 1200,392 .target_residency = 4000,393 .enter = &intel_idle,394 .enter_s2idle = intel_idle_s2idle, },395 {396 .name = "C7S",397 .desc = "MWAIT 0x64",398 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,399 .exit_latency = 10000,400 .target_residency = 20000,401 .enter = &intel_idle,402 .enter_s2idle = intel_idle_s2idle, },403 {404 .enter = NULL }405};406 407static struct cpuidle_state ivb_cstates[] __initdata = {408 {409 .name = "C1",410 .desc = "MWAIT 0x00",411 .flags = MWAIT2flg(0x00),412 .exit_latency = 1,413 .target_residency = 1,414 .enter = &intel_idle,415 .enter_s2idle = intel_idle_s2idle, },416 {417 .name = "C1E",418 .desc = "MWAIT 0x01",419 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,420 .exit_latency = 10,421 .target_residency = 20,422 .enter = &intel_idle,423 .enter_s2idle = intel_idle_s2idle, },424 {425 .name = "C3",426 .desc = "MWAIT 0x10",427 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,428 .exit_latency = 59,429 .target_residency = 156,430 .enter = &intel_idle,431 .enter_s2idle = intel_idle_s2idle, },432 {433 .name = "C6",434 .desc = "MWAIT 0x20",435 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,436 .exit_latency = 80,437 .target_residency = 300,438 .enter = &intel_idle,439 .enter_s2idle = intel_idle_s2idle, },440 {441 .name = "C7",442 .desc = "MWAIT 0x30",443 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,444 .exit_latency = 87,445 .target_residency = 300,446 .enter = &intel_idle,447 .enter_s2idle = intel_idle_s2idle, },448 {449 .enter = NULL }450};451 452static struct cpuidle_state ivt_cstates[] __initdata = {453 {454 .name = "C1",455 .desc = "MWAIT 0x00",456 .flags = MWAIT2flg(0x00),457 .exit_latency = 1,458 .target_residency = 1,459 .enter = &intel_idle,460 .enter_s2idle = intel_idle_s2idle, },461 {462 .name = "C1E",463 .desc = "MWAIT 0x01",464 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,465 .exit_latency = 10,466 .target_residency = 80,467 .enter = &intel_idle,468 .enter_s2idle = intel_idle_s2idle, },469 {470 .name = "C3",471 .desc = "MWAIT 0x10",472 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,473 .exit_latency = 59,474 .target_residency = 156,475 .enter = &intel_idle,476 .enter_s2idle = intel_idle_s2idle, },477 {478 .name = "C6",479 .desc = "MWAIT 0x20",480 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,481 .exit_latency = 82,482 .target_residency = 300,483 .enter = &intel_idle,484 .enter_s2idle = intel_idle_s2idle, },485 {486 .enter = NULL }487};488 489static struct cpuidle_state ivt_cstates_4s[] __initdata = {490 {491 .name = "C1",492 .desc = "MWAIT 0x00",493 .flags = MWAIT2flg(0x00),494 .exit_latency = 1,495 .target_residency = 1,496 .enter = &intel_idle,497 .enter_s2idle = intel_idle_s2idle, },498 {499 .name = "C1E",500 .desc = "MWAIT 0x01",501 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,502 .exit_latency = 10,503 .target_residency = 250,504 .enter = &intel_idle,505 .enter_s2idle = intel_idle_s2idle, },506 {507 .name = "C3",508 .desc = "MWAIT 0x10",509 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,510 .exit_latency = 59,511 .target_residency = 300,512 .enter = &intel_idle,513 .enter_s2idle = intel_idle_s2idle, },514 {515 .name = "C6",516 .desc = "MWAIT 0x20",517 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,518 .exit_latency = 84,519 .target_residency = 400,520 .enter = &intel_idle,521 .enter_s2idle = intel_idle_s2idle, },522 {523 .enter = NULL }524};525 526static struct cpuidle_state ivt_cstates_8s[] __initdata = {527 {528 .name = "C1",529 .desc = "MWAIT 0x00",530 .flags = MWAIT2flg(0x00),531 .exit_latency = 1,532 .target_residency = 1,533 .enter = &intel_idle,534 .enter_s2idle = intel_idle_s2idle, },535 {536 .name = "C1E",537 .desc = "MWAIT 0x01",538 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,539 .exit_latency = 10,540 .target_residency = 500,541 .enter = &intel_idle,542 .enter_s2idle = intel_idle_s2idle, },543 {544 .name = "C3",545 .desc = "MWAIT 0x10",546 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,547 .exit_latency = 59,548 .target_residency = 600,549 .enter = &intel_idle,550 .enter_s2idle = intel_idle_s2idle, },551 {552 .name = "C6",553 .desc = "MWAIT 0x20",554 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,555 .exit_latency = 88,556 .target_residency = 700,557 .enter = &intel_idle,558 .enter_s2idle = intel_idle_s2idle, },559 {560 .enter = NULL }561};562 563static struct cpuidle_state hsw_cstates[] __initdata = {564 {565 .name = "C1",566 .desc = "MWAIT 0x00",567 .flags = MWAIT2flg(0x00),568 .exit_latency = 2,569 .target_residency = 2,570 .enter = &intel_idle,571 .enter_s2idle = intel_idle_s2idle, },572 {573 .name = "C1E",574 .desc = "MWAIT 0x01",575 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,576 .exit_latency = 10,577 .target_residency = 20,578 .enter = &intel_idle,579 .enter_s2idle = intel_idle_s2idle, },580 {581 .name = "C3",582 .desc = "MWAIT 0x10",583 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,584 .exit_latency = 33,585 .target_residency = 100,586 .enter = &intel_idle,587 .enter_s2idle = intel_idle_s2idle, },588 {589 .name = "C6",590 .desc = "MWAIT 0x20",591 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,592 .exit_latency = 133,593 .target_residency = 400,594 .enter = &intel_idle,595 .enter_s2idle = intel_idle_s2idle, },596 {597 .name = "C7s",598 .desc = "MWAIT 0x32",599 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,600 .exit_latency = 166,601 .target_residency = 500,602 .enter = &intel_idle,603 .enter_s2idle = intel_idle_s2idle, },604 {605 .name = "C8",606 .desc = "MWAIT 0x40",607 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,608 .exit_latency = 300,609 .target_residency = 900,610 .enter = &intel_idle,611 .enter_s2idle = intel_idle_s2idle, },612 {613 .name = "C9",614 .desc = "MWAIT 0x50",615 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,616 .exit_latency = 600,617 .target_residency = 1800,618 .enter = &intel_idle,619 .enter_s2idle = intel_idle_s2idle, },620 {621 .name = "C10",622 .desc = "MWAIT 0x60",623 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,624 .exit_latency = 2600,625 .target_residency = 7700,626 .enter = &intel_idle,627 .enter_s2idle = intel_idle_s2idle, },628 {629 .enter = NULL }630};631static struct cpuidle_state bdw_cstates[] __initdata = {632 {633 .name = "C1",634 .desc = "MWAIT 0x00",635 .flags = MWAIT2flg(0x00),636 .exit_latency = 2,637 .target_residency = 2,638 .enter = &intel_idle,639 .enter_s2idle = intel_idle_s2idle, },640 {641 .name = "C1E",642 .desc = "MWAIT 0x01",643 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,644 .exit_latency = 10,645 .target_residency = 20,646 .enter = &intel_idle,647 .enter_s2idle = intel_idle_s2idle, },648 {649 .name = "C3",650 .desc = "MWAIT 0x10",651 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,652 .exit_latency = 40,653 .target_residency = 100,654 .enter = &intel_idle,655 .enter_s2idle = intel_idle_s2idle, },656 {657 .name = "C6",658 .desc = "MWAIT 0x20",659 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,660 .exit_latency = 133,661 .target_residency = 400,662 .enter = &intel_idle,663 .enter_s2idle = intel_idle_s2idle, },664 {665 .name = "C7s",666 .desc = "MWAIT 0x32",667 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,668 .exit_latency = 166,669 .target_residency = 500,670 .enter = &intel_idle,671 .enter_s2idle = intel_idle_s2idle, },672 {673 .name = "C8",674 .desc = "MWAIT 0x40",675 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,676 .exit_latency = 300,677 .target_residency = 900,678 .enter = &intel_idle,679 .enter_s2idle = intel_idle_s2idle, },680 {681 .name = "C9",682 .desc = "MWAIT 0x50",683 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,684 .exit_latency = 600,685 .target_residency = 1800,686 .enter = &intel_idle,687 .enter_s2idle = intel_idle_s2idle, },688 {689 .name = "C10",690 .desc = "MWAIT 0x60",691 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,692 .exit_latency = 2600,693 .target_residency = 7700,694 .enter = &intel_idle,695 .enter_s2idle = intel_idle_s2idle, },696 {697 .enter = NULL }698};699 700static struct cpuidle_state skl_cstates[] __initdata = {701 {702 .name = "C1",703 .desc = "MWAIT 0x00",704 .flags = MWAIT2flg(0x00),705 .exit_latency = 2,706 .target_residency = 2,707 .enter = &intel_idle,708 .enter_s2idle = intel_idle_s2idle, },709 {710 .name = "C1E",711 .desc = "MWAIT 0x01",712 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,713 .exit_latency = 10,714 .target_residency = 20,715 .enter = &intel_idle,716 .enter_s2idle = intel_idle_s2idle, },717 {718 .name = "C3",719 .desc = "MWAIT 0x10",720 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,721 .exit_latency = 70,722 .target_residency = 100,723 .enter = &intel_idle,724 .enter_s2idle = intel_idle_s2idle, },725 {726 .name = "C6",727 .desc = "MWAIT 0x20",728 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,729 .exit_latency = 85,730 .target_residency = 200,731 .enter = &intel_idle,732 .enter_s2idle = intel_idle_s2idle, },733 {734 .name = "C7s",735 .desc = "MWAIT 0x33",736 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,737 .exit_latency = 124,738 .target_residency = 800,739 .enter = &intel_idle,740 .enter_s2idle = intel_idle_s2idle, },741 {742 .name = "C8",743 .desc = "MWAIT 0x40",744 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,745 .exit_latency = 200,746 .target_residency = 800,747 .enter = &intel_idle,748 .enter_s2idle = intel_idle_s2idle, },749 {750 .name = "C9",751 .desc = "MWAIT 0x50",752 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,753 .exit_latency = 480,754 .target_residency = 5000,755 .enter = &intel_idle,756 .enter_s2idle = intel_idle_s2idle, },757 {758 .name = "C10",759 .desc = "MWAIT 0x60",760 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,761 .exit_latency = 890,762 .target_residency = 5000,763 .enter = &intel_idle,764 .enter_s2idle = intel_idle_s2idle, },765 {766 .enter = NULL }767};768 769static struct cpuidle_state skx_cstates[] __initdata = {770 {771 .name = "C1",772 .desc = "MWAIT 0x00",773 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,774 .exit_latency = 2,775 .target_residency = 2,776 .enter = &intel_idle,777 .enter_s2idle = intel_idle_s2idle, },778 {779 .name = "C1E",780 .desc = "MWAIT 0x01",781 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,782 .exit_latency = 10,783 .target_residency = 20,784 .enter = &intel_idle,785 .enter_s2idle = intel_idle_s2idle, },786 {787 .name = "C6",788 .desc = "MWAIT 0x20",789 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,790 .exit_latency = 133,791 .target_residency = 600,792 .enter = &intel_idle,793 .enter_s2idle = intel_idle_s2idle, },794 {795 .enter = NULL }796};797 798static struct cpuidle_state icx_cstates[] __initdata = {799 {800 .name = "C1",801 .desc = "MWAIT 0x00",802 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,803 .exit_latency = 1,804 .target_residency = 1,805 .enter = &intel_idle,806 .enter_s2idle = intel_idle_s2idle, },807 {808 .name = "C1E",809 .desc = "MWAIT 0x01",810 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,811 .exit_latency = 4,812 .target_residency = 4,813 .enter = &intel_idle,814 .enter_s2idle = intel_idle_s2idle, },815 {816 .name = "C6",817 .desc = "MWAIT 0x20",818 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,819 .exit_latency = 170,820 .target_residency = 600,821 .enter = &intel_idle,822 .enter_s2idle = intel_idle_s2idle, },823 {824 .enter = NULL }825};826 827/*828 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.829 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.830 * But in this case there is effectively no C1, because C1 requests are831 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1832 * and C1E requests end up with C1, so there is effectively no C1E.833 *834 * By default we enable C1E and disable C1 by marking it with835 * 'CPUIDLE_FLAG_UNUSABLE'.836 */837static struct cpuidle_state adl_cstates[] __initdata = {838 {839 .name = "C1",840 .desc = "MWAIT 0x00",841 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,842 .exit_latency = 1,843 .target_residency = 1,844 .enter = &intel_idle,845 .enter_s2idle = intel_idle_s2idle, },846 {847 .name = "C1E",848 .desc = "MWAIT 0x01",849 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,850 .exit_latency = 2,851 .target_residency = 4,852 .enter = &intel_idle,853 .enter_s2idle = intel_idle_s2idle, },854 {855 .name = "C6",856 .desc = "MWAIT 0x20",857 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,858 .exit_latency = 220,859 .target_residency = 600,860 .enter = &intel_idle,861 .enter_s2idle = intel_idle_s2idle, },862 {863 .name = "C8",864 .desc = "MWAIT 0x40",865 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,866 .exit_latency = 280,867 .target_residency = 800,868 .enter = &intel_idle,869 .enter_s2idle = intel_idle_s2idle, },870 {871 .name = "C10",872 .desc = "MWAIT 0x60",873 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,874 .exit_latency = 680,875 .target_residency = 2000,876 .enter = &intel_idle,877 .enter_s2idle = intel_idle_s2idle, },878 {879 .enter = NULL }880};881 882static struct cpuidle_state adl_l_cstates[] __initdata = {883 {884 .name = "C1",885 .desc = "MWAIT 0x00",886 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,887 .exit_latency = 1,888 .target_residency = 1,889 .enter = &intel_idle,890 .enter_s2idle = intel_idle_s2idle, },891 {892 .name = "C1E",893 .desc = "MWAIT 0x01",894 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,895 .exit_latency = 2,896 .target_residency = 4,897 .enter = &intel_idle,898 .enter_s2idle = intel_idle_s2idle, },899 {900 .name = "C6",901 .desc = "MWAIT 0x20",902 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,903 .exit_latency = 170,904 .target_residency = 500,905 .enter = &intel_idle,906 .enter_s2idle = intel_idle_s2idle, },907 {908 .name = "C8",909 .desc = "MWAIT 0x40",910 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,911 .exit_latency = 200,912 .target_residency = 600,913 .enter = &intel_idle,914 .enter_s2idle = intel_idle_s2idle, },915 {916 .name = "C10",917 .desc = "MWAIT 0x60",918 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,919 .exit_latency = 230,920 .target_residency = 700,921 .enter = &intel_idle,922 .enter_s2idle = intel_idle_s2idle, },923 {924 .enter = NULL }925};926 927static struct cpuidle_state mtl_l_cstates[] __initdata = {928 {929 .name = "C1E",930 .desc = "MWAIT 0x01",931 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,932 .exit_latency = 1,933 .target_residency = 1,934 .enter = &intel_idle,935 .enter_s2idle = intel_idle_s2idle, },936 {937 .name = "C6",938 .desc = "MWAIT 0x20",939 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,940 .exit_latency = 140,941 .target_residency = 420,942 .enter = &intel_idle,943 .enter_s2idle = intel_idle_s2idle, },944 {945 .name = "C10",946 .desc = "MWAIT 0x60",947 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,948 .exit_latency = 310,949 .target_residency = 930,950 .enter = &intel_idle,951 .enter_s2idle = intel_idle_s2idle, },952 {953 .enter = NULL }954};955 956static struct cpuidle_state gmt_cstates[] __initdata = {957 {958 .name = "C1",959 .desc = "MWAIT 0x00",960 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,961 .exit_latency = 1,962 .target_residency = 1,963 .enter = &intel_idle,964 .enter_s2idle = intel_idle_s2idle, },965 {966 .name = "C1E",967 .desc = "MWAIT 0x01",968 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,969 .exit_latency = 2,970 .target_residency = 4,971 .enter = &intel_idle,972 .enter_s2idle = intel_idle_s2idle, },973 {974 .name = "C6",975 .desc = "MWAIT 0x20",976 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,977 .exit_latency = 195,978 .target_residency = 585,979 .enter = &intel_idle,980 .enter_s2idle = intel_idle_s2idle, },981 {982 .name = "C8",983 .desc = "MWAIT 0x40",984 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,985 .exit_latency = 260,986 .target_residency = 1040,987 .enter = &intel_idle,988 .enter_s2idle = intel_idle_s2idle, },989 {990 .name = "C10",991 .desc = "MWAIT 0x60",992 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,993 .exit_latency = 660,994 .target_residency = 1980,995 .enter = &intel_idle,996 .enter_s2idle = intel_idle_s2idle, },997 {998 .enter = NULL }999};1000 1001static struct cpuidle_state spr_cstates[] __initdata = {1002 {1003 .name = "C1",1004 .desc = "MWAIT 0x00",1005 .flags = MWAIT2flg(0x00),1006 .exit_latency = 1,1007 .target_residency = 1,1008 .enter = &intel_idle,1009 .enter_s2idle = intel_idle_s2idle, },1010 {1011 .name = "C1E",1012 .desc = "MWAIT 0x01",1013 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1014 .exit_latency = 2,1015 .target_residency = 4,1016 .enter = &intel_idle,1017 .enter_s2idle = intel_idle_s2idle, },1018 {1019 .name = "C6",1020 .desc = "MWAIT 0x20",1021 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |1022 CPUIDLE_FLAG_INIT_XSTATE,1023 .exit_latency = 290,1024 .target_residency = 800,1025 .enter = &intel_idle,1026 .enter_s2idle = intel_idle_s2idle, },1027 {1028 .enter = NULL }1029};1030 1031static struct cpuidle_state gnr_cstates[] __initdata = {1032 {1033 .name = "C1",1034 .desc = "MWAIT 0x00",1035 .flags = MWAIT2flg(0x00),1036 .exit_latency = 1,1037 .target_residency = 1,1038 .enter = &intel_idle,1039 .enter_s2idle = intel_idle_s2idle, },1040 {1041 .name = "C1E",1042 .desc = "MWAIT 0x01",1043 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1044 .exit_latency = 4,1045 .target_residency = 4,1046 .enter = &intel_idle,1047 .enter_s2idle = intel_idle_s2idle, },1048 {1049 .name = "C6",1050 .desc = "MWAIT 0x20",1051 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |1052 CPUIDLE_FLAG_INIT_XSTATE |1053 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,1054 .exit_latency = 170,1055 .target_residency = 650,1056 .enter = &intel_idle,1057 .enter_s2idle = intel_idle_s2idle, },1058 {1059 .name = "C6P",1060 .desc = "MWAIT 0x21",1061 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |1062 CPUIDLE_FLAG_INIT_XSTATE |1063 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,1064 .exit_latency = 210,1065 .target_residency = 1000,1066 .enter = &intel_idle,1067 .enter_s2idle = intel_idle_s2idle, },1068 {1069 .enter = NULL }1070};1071 1072static struct cpuidle_state atom_cstates[] __initdata = {1073 {1074 .name = "C1E",1075 .desc = "MWAIT 0x00",1076 .flags = MWAIT2flg(0x00),1077 .exit_latency = 10,1078 .target_residency = 20,1079 .enter = &intel_idle,1080 .enter_s2idle = intel_idle_s2idle, },1081 {1082 .name = "C2",1083 .desc = "MWAIT 0x10",1084 .flags = MWAIT2flg(0x10),1085 .exit_latency = 20,1086 .target_residency = 80,1087 .enter = &intel_idle,1088 .enter_s2idle = intel_idle_s2idle, },1089 {1090 .name = "C4",1091 .desc = "MWAIT 0x30",1092 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,1093 .exit_latency = 100,1094 .target_residency = 400,1095 .enter = &intel_idle,1096 .enter_s2idle = intel_idle_s2idle, },1097 {1098 .name = "C6",1099 .desc = "MWAIT 0x52",1100 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,1101 .exit_latency = 140,1102 .target_residency = 560,1103 .enter = &intel_idle,1104 .enter_s2idle = intel_idle_s2idle, },1105 {1106 .enter = NULL }1107};1108static struct cpuidle_state tangier_cstates[] __initdata = {1109 {1110 .name = "C1",1111 .desc = "MWAIT 0x00",1112 .flags = MWAIT2flg(0x00),1113 .exit_latency = 1,1114 .target_residency = 4,1115 .enter = &intel_idle,1116 .enter_s2idle = intel_idle_s2idle, },1117 {1118 .name = "C4",1119 .desc = "MWAIT 0x30",1120 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,1121 .exit_latency = 100,1122 .target_residency = 400,1123 .enter = &intel_idle,1124 .enter_s2idle = intel_idle_s2idle, },1125 {1126 .name = "C6",1127 .desc = "MWAIT 0x52",1128 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,1129 .exit_latency = 140,1130 .target_residency = 560,1131 .enter = &intel_idle,1132 .enter_s2idle = intel_idle_s2idle, },1133 {1134 .name = "C7",1135 .desc = "MWAIT 0x60",1136 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,1137 .exit_latency = 1200,1138 .target_residency = 4000,1139 .enter = &intel_idle,1140 .enter_s2idle = intel_idle_s2idle, },1141 {1142 .name = "C9",1143 .desc = "MWAIT 0x64",1144 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,1145 .exit_latency = 10000,1146 .target_residency = 20000,1147 .enter = &intel_idle,1148 .enter_s2idle = intel_idle_s2idle, },1149 {1150 .enter = NULL }1151};1152static struct cpuidle_state avn_cstates[] __initdata = {1153 {1154 .name = "C1",1155 .desc = "MWAIT 0x00",1156 .flags = MWAIT2flg(0x00),1157 .exit_latency = 2,1158 .target_residency = 2,1159 .enter = &intel_idle,1160 .enter_s2idle = intel_idle_s2idle, },1161 {1162 .name = "C6",1163 .desc = "MWAIT 0x51",1164 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,1165 .exit_latency = 15,1166 .target_residency = 45,1167 .enter = &intel_idle,1168 .enter_s2idle = intel_idle_s2idle, },1169 {1170 .enter = NULL }1171};1172static struct cpuidle_state knl_cstates[] __initdata = {1173 {1174 .name = "C1",1175 .desc = "MWAIT 0x00",1176 .flags = MWAIT2flg(0x00),1177 .exit_latency = 1,1178 .target_residency = 2,1179 .enter = &intel_idle,1180 .enter_s2idle = intel_idle_s2idle },1181 {1182 .name = "C6",1183 .desc = "MWAIT 0x10",1184 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,1185 .exit_latency = 120,1186 .target_residency = 500,1187 .enter = &intel_idle,1188 .enter_s2idle = intel_idle_s2idle },1189 {1190 .enter = NULL }1191};1192 1193static struct cpuidle_state bxt_cstates[] __initdata = {1194 {1195 .name = "C1",1196 .desc = "MWAIT 0x00",1197 .flags = MWAIT2flg(0x00),1198 .exit_latency = 2,1199 .target_residency = 2,1200 .enter = &intel_idle,1201 .enter_s2idle = intel_idle_s2idle, },1202 {1203 .name = "C1E",1204 .desc = "MWAIT 0x01",1205 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1206 .exit_latency = 10,1207 .target_residency = 20,1208 .enter = &intel_idle,1209 .enter_s2idle = intel_idle_s2idle, },1210 {1211 .name = "C6",1212 .desc = "MWAIT 0x20",1213 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,1214 .exit_latency = 133,1215 .target_residency = 133,1216 .enter = &intel_idle,1217 .enter_s2idle = intel_idle_s2idle, },1218 {1219 .name = "C7s",1220 .desc = "MWAIT 0x31",1221 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,1222 .exit_latency = 155,1223 .target_residency = 155,1224 .enter = &intel_idle,1225 .enter_s2idle = intel_idle_s2idle, },1226 {1227 .name = "C8",1228 .desc = "MWAIT 0x40",1229 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,1230 .exit_latency = 1000,1231 .target_residency = 1000,1232 .enter = &intel_idle,1233 .enter_s2idle = intel_idle_s2idle, },1234 {1235 .name = "C9",1236 .desc = "MWAIT 0x50",1237 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,1238 .exit_latency = 2000,1239 .target_residency = 2000,1240 .enter = &intel_idle,1241 .enter_s2idle = intel_idle_s2idle, },1242 {1243 .name = "C10",1244 .desc = "MWAIT 0x60",1245 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,1246 .exit_latency = 10000,1247 .target_residency = 10000,1248 .enter = &intel_idle,1249 .enter_s2idle = intel_idle_s2idle, },1250 {1251 .enter = NULL }1252};1253 1254static struct cpuidle_state dnv_cstates[] __initdata = {1255 {1256 .name = "C1",1257 .desc = "MWAIT 0x00",1258 .flags = MWAIT2flg(0x00),1259 .exit_latency = 2,1260 .target_residency = 2,1261 .enter = &intel_idle,1262 .enter_s2idle = intel_idle_s2idle, },1263 {1264 .name = "C1E",1265 .desc = "MWAIT 0x01",1266 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1267 .exit_latency = 10,1268 .target_residency = 20,1269 .enter = &intel_idle,1270 .enter_s2idle = intel_idle_s2idle, },1271 {1272 .name = "C6",1273 .desc = "MWAIT 0x20",1274 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,1275 .exit_latency = 50,1276 .target_residency = 500,1277 .enter = &intel_idle,1278 .enter_s2idle = intel_idle_s2idle, },1279 {1280 .enter = NULL }1281};1282 1283/*1284 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support1285 * C6, and this is indicated in the CPUID mwait leaf.1286 */1287static struct cpuidle_state snr_cstates[] __initdata = {1288 {1289 .name = "C1",1290 .desc = "MWAIT 0x00",1291 .flags = MWAIT2flg(0x00),1292 .exit_latency = 2,1293 .target_residency = 2,1294 .enter = &intel_idle,1295 .enter_s2idle = intel_idle_s2idle, },1296 {1297 .name = "C1E",1298 .desc = "MWAIT 0x01",1299 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1300 .exit_latency = 15,1301 .target_residency = 25,1302 .enter = &intel_idle,1303 .enter_s2idle = intel_idle_s2idle, },1304 {1305 .name = "C6",1306 .desc = "MWAIT 0x20",1307 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,1308 .exit_latency = 130,1309 .target_residency = 500,1310 .enter = &intel_idle,1311 .enter_s2idle = intel_idle_s2idle, },1312 {1313 .enter = NULL }1314};1315 1316static struct cpuidle_state grr_cstates[] __initdata = {1317 {1318 .name = "C1",1319 .desc = "MWAIT 0x00",1320 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,1321 .exit_latency = 1,1322 .target_residency = 1,1323 .enter = &intel_idle,1324 .enter_s2idle = intel_idle_s2idle, },1325 {1326 .name = "C1E",1327 .desc = "MWAIT 0x01",1328 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1329 .exit_latency = 2,1330 .target_residency = 10,1331 .enter = &intel_idle,1332 .enter_s2idle = intel_idle_s2idle, },1333 {1334 .name = "C6S",1335 .desc = "MWAIT 0x22",1336 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,1337 .exit_latency = 140,1338 .target_residency = 500,1339 .enter = &intel_idle,1340 .enter_s2idle = intel_idle_s2idle, },1341 {1342 .enter = NULL }1343};1344 1345static struct cpuidle_state srf_cstates[] __initdata = {1346 {1347 .name = "C1",1348 .desc = "MWAIT 0x00",1349 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,1350 .exit_latency = 1,1351 .target_residency = 1,1352 .enter = &intel_idle,1353 .enter_s2idle = intel_idle_s2idle, },1354 {1355 .name = "C1E",1356 .desc = "MWAIT 0x01",1357 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,1358 .exit_latency = 2,1359 .target_residency = 10,1360 .enter = &intel_idle,1361 .enter_s2idle = intel_idle_s2idle, },1362 {1363 .name = "C6S",1364 .desc = "MWAIT 0x22",1365 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |1366 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,1367 .exit_latency = 270,1368 .target_residency = 700,1369 .enter = &intel_idle,1370 .enter_s2idle = intel_idle_s2idle, },1371 {1372 .name = "C6SP",1373 .desc = "MWAIT 0x23",1374 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |1375 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,1376 .exit_latency = 310,1377 .target_residency = 900,1378 .enter = &intel_idle,1379 .enter_s2idle = intel_idle_s2idle, },1380 {1381 .enter = NULL }1382};1383 1384static const struct idle_cpu idle_cpu_nehalem __initconst = {1385 .state_table = nehalem_cstates,1386 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,1387 .disable_promotion_to_c1e = true,1388};1389 1390static const struct idle_cpu idle_cpu_nhx __initconst = {1391 .state_table = nehalem_cstates,1392 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,1393 .disable_promotion_to_c1e = true,1394 .use_acpi = true,1395};1396 1397static const struct idle_cpu idle_cpu_atom __initconst = {1398 .state_table = atom_cstates,1399};1400 1401static const struct idle_cpu idle_cpu_tangier __initconst = {1402 .state_table = tangier_cstates,1403};1404 1405static const struct idle_cpu idle_cpu_lincroft __initconst = {1406 .state_table = atom_cstates,1407 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,1408};1409 1410static const struct idle_cpu idle_cpu_snb __initconst = {1411 .state_table = snb_cstates,1412 .disable_promotion_to_c1e = true,1413};1414 1415static const struct idle_cpu idle_cpu_snx __initconst = {1416 .state_table = snb_cstates,1417 .disable_promotion_to_c1e = true,1418 .use_acpi = true,1419};1420 1421static const struct idle_cpu idle_cpu_byt __initconst = {1422 .state_table = byt_cstates,1423 .disable_promotion_to_c1e = true,1424 .byt_auto_demotion_disable_flag = true,1425};1426 1427static const struct idle_cpu idle_cpu_cht __initconst = {1428 .state_table = cht_cstates,1429 .disable_promotion_to_c1e = true,1430 .byt_auto_demotion_disable_flag = true,1431};1432 1433static const struct idle_cpu idle_cpu_ivb __initconst = {1434 .state_table = ivb_cstates,1435 .disable_promotion_to_c1e = true,1436};1437 1438static const struct idle_cpu idle_cpu_ivt __initconst = {1439 .state_table = ivt_cstates,1440 .disable_promotion_to_c1e = true,1441 .use_acpi = true,1442};1443 1444static const struct idle_cpu idle_cpu_hsw __initconst = {1445 .state_table = hsw_cstates,1446 .disable_promotion_to_c1e = true,1447};1448 1449static const struct idle_cpu idle_cpu_hsx __initconst = {1450 .state_table = hsw_cstates,1451 .disable_promotion_to_c1e = true,1452 .use_acpi = true,1453};1454 1455static const struct idle_cpu idle_cpu_bdw __initconst = {1456 .state_table = bdw_cstates,1457 .disable_promotion_to_c1e = true,1458};1459 1460static const struct idle_cpu idle_cpu_bdx __initconst = {1461 .state_table = bdw_cstates,1462 .disable_promotion_to_c1e = true,1463 .use_acpi = true,1464};1465 1466static const struct idle_cpu idle_cpu_skl __initconst = {1467 .state_table = skl_cstates,1468 .disable_promotion_to_c1e = true,1469};1470 1471static const struct idle_cpu idle_cpu_skx __initconst = {1472 .state_table = skx_cstates,1473 .disable_promotion_to_c1e = true,1474 .use_acpi = true,1475};1476 1477static const struct idle_cpu idle_cpu_icx __initconst = {1478 .state_table = icx_cstates,1479 .disable_promotion_to_c1e = true,1480 .use_acpi = true,1481};1482 1483static const struct idle_cpu idle_cpu_adl __initconst = {1484 .state_table = adl_cstates,1485};1486 1487static const struct idle_cpu idle_cpu_adl_l __initconst = {1488 .state_table = adl_l_cstates,1489};1490 1491static const struct idle_cpu idle_cpu_mtl_l __initconst = {1492 .state_table = mtl_l_cstates,1493};1494 1495static const struct idle_cpu idle_cpu_gmt __initconst = {1496 .state_table = gmt_cstates,1497};1498 1499static const struct idle_cpu idle_cpu_spr __initconst = {1500 .state_table = spr_cstates,1501 .disable_promotion_to_c1e = true,1502 .use_acpi = true,1503};1504 1505static const struct idle_cpu idle_cpu_gnr __initconst = {1506 .state_table = gnr_cstates,1507 .disable_promotion_to_c1e = true,1508 .use_acpi = true,1509};1510 1511static const struct idle_cpu idle_cpu_avn __initconst = {1512 .state_table = avn_cstates,1513 .disable_promotion_to_c1e = true,1514 .use_acpi = true,1515};1516 1517static const struct idle_cpu idle_cpu_knl __initconst = {1518 .state_table = knl_cstates,1519 .use_acpi = true,1520};1521 1522static const struct idle_cpu idle_cpu_bxt __initconst = {1523 .state_table = bxt_cstates,1524 .disable_promotion_to_c1e = true,1525};1526 1527static const struct idle_cpu idle_cpu_dnv __initconst = {1528 .state_table = dnv_cstates,1529 .disable_promotion_to_c1e = true,1530 .use_acpi = true,1531};1532 1533static const struct idle_cpu idle_cpu_tmt __initconst = {1534 .disable_promotion_to_c1e = true,1535};1536 1537static const struct idle_cpu idle_cpu_snr __initconst = {1538 .state_table = snr_cstates,1539 .disable_promotion_to_c1e = true,1540 .use_acpi = true,1541};1542 1543static const struct idle_cpu idle_cpu_grr __initconst = {1544 .state_table = grr_cstates,1545 .disable_promotion_to_c1e = true,1546 .use_acpi = true,1547};1548 1549static const struct idle_cpu idle_cpu_srf __initconst = {1550 .state_table = srf_cstates,1551 .disable_promotion_to_c1e = true,1552 .use_acpi = true,1553};1554 1555static const struct x86_cpu_id intel_idle_ids[] __initconst = {1556 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx),1557 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem),1558 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem),1559 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem),1560 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx),1561 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx),1562 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom),1563 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft),1564 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx),1565 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb),1566 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx),1567 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom),1568 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt),1569 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),1570 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht),1571 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb),1572 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt),1573 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw),1574 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx),1575 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw),1576 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw),1577 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn),1578 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw),1579 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw),1580 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx),1581 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx),1582 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl),1583 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl),1584 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl),1585 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl),1586 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx),1587 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx),1588 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx),1589 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl),1590 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l),1591 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l),1592 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt),1593 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr),1594 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr),1595 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr),1596 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl),1597 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl),1598 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt),1599 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),1600 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv),1601 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt),1602 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt),1603 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr),1604 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr),1605 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf),1606 {}1607};1608 1609static const struct x86_cpu_id intel_mwait_ids[] __initconst = {1610 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),1611 {}1612};1613 1614static bool __init intel_idle_max_cstate_reached(int cstate)1615{1616 if (cstate + 1 > max_cstate) {1617 pr_info("max_cstate %d reached\n", max_cstate);1618 return true;1619 }1620 return false;1621}1622 1623static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)1624{1625 unsigned long eax = flg2MWAIT(state->flags);1626 1627 if (boot_cpu_has(X86_FEATURE_ARAT))1628 return false;1629 1630 /*1631 * Switch over to one-shot tick broadcast if the target C-state1632 * is deeper than C1.1633 */1634 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);1635}1636 1637#ifdef CONFIG_ACPI_PROCESSOR_CSTATE1638#include <acpi/processor.h>1639 1640static bool no_acpi __read_mostly;1641module_param(no_acpi, bool, 0444);1642MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");1643 1644static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */1645module_param_named(use_acpi, force_use_acpi, bool, 0444);1646MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");1647 1648static struct acpi_processor_power acpi_state_table __initdata;1649 1650/**1651 * intel_idle_cst_usable - Check if the _CST information can be used.1652 *1653 * Check if all of the C-states listed by _CST in the max_cstate range are1654 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.1655 */1656static bool __init intel_idle_cst_usable(void)1657{1658 int cstate, limit;1659 1660 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),1661 acpi_state_table.count);1662 1663 for (cstate = 1; cstate < limit; cstate++) {1664 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];1665 1666 if (cx->entry_method != ACPI_CSTATE_FFH)1667 return false;1668 }1669 1670 return true;1671}1672 1673static bool __init intel_idle_acpi_cst_extract(void)1674{1675 unsigned int cpu;1676 1677 if (no_acpi) {1678 pr_debug("Not allowed to use ACPI _CST\n");1679 return false;1680 }1681 1682 for_each_possible_cpu(cpu) {1683 struct acpi_processor *pr = per_cpu(processors, cpu);1684 1685 if (!pr)1686 continue;1687 1688 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))1689 continue;1690 1691 acpi_state_table.count++;1692 1693 if (!intel_idle_cst_usable())1694 continue;1695 1696 if (!acpi_processor_claim_cst_control())1697 break;1698 1699 return true;1700 }1701 1702 acpi_state_table.count = 0;1703 pr_debug("ACPI _CST not found or not usable\n");1704 return false;1705}1706 1707static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)1708{1709 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);1710 1711 /*1712 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of1713 * the interesting states are ACPI_CSTATE_FFH.1714 */1715 for (cstate = 1; cstate < limit; cstate++) {1716 struct acpi_processor_cx *cx;1717 struct cpuidle_state *state;1718 1719 if (intel_idle_max_cstate_reached(cstate - 1))1720 break;1721 1722 cx = &acpi_state_table.states[cstate];1723 1724 state = &drv->states[drv->state_count++];1725 1726 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);1727 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);1728 state->exit_latency = cx->latency;1729 /*1730 * For C1-type C-states use the same number for both the exit1731 * latency and target residency, because that is the case for1732 * C1 in the majority of the static C-states tables above.1733 * For the other types of C-states, however, set the target1734 * residency to 3 times the exit latency which should lead to1735 * a reasonable balance between energy-efficiency and1736 * performance in the majority of interesting cases.1737 */1738 state->target_residency = cx->latency;1739 if (cx->type > ACPI_STATE_C1)1740 state->target_residency *= 3;1741 1742 state->flags = MWAIT2flg(cx->address);1743 if (cx->type > ACPI_STATE_C2)1744 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;1745 1746 if (disabled_states_mask & BIT(cstate))1747 state->flags |= CPUIDLE_FLAG_OFF;1748 1749 if (intel_idle_state_needs_timer_stop(state))1750 state->flags |= CPUIDLE_FLAG_TIMER_STOP;1751 1752 state->enter = intel_idle;1753 state->enter_s2idle = intel_idle_s2idle;1754 }1755}1756 1757static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)1758{1759 int cstate, limit;1760 1761 /*1762 * If there are no _CST C-states, do not disable any C-states by1763 * default.1764 */1765 if (!acpi_state_table.count)1766 return false;1767 1768 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);1769 /*1770 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of1771 * the interesting states are ACPI_CSTATE_FFH.1772 */1773 for (cstate = 1; cstate < limit; cstate++) {1774 u32 acpi_hint = acpi_state_table.states[cstate].address;1775 u32 table_hint = mwait_hint;1776 1777 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {1778 acpi_hint &= ~MWAIT_SUBSTATE_MASK;1779 table_hint &= ~MWAIT_SUBSTATE_MASK;1780 }1781 1782 if (acpi_hint == table_hint)1783 return false;1784 }1785 return true;1786}1787#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */1788#define force_use_acpi (false)1789 1790static inline bool intel_idle_acpi_cst_extract(void) { return false; }1791static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }1792static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)1793{1794 return false;1795}1796#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */1797 1798/**1799 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.1800 *1801 * Tune IVT multi-socket targets.1802 * Assumption: num_sockets == (max_package_num + 1).1803 */1804static void __init ivt_idle_state_table_update(void)1805{1806 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */1807 int cpu, package_num, num_sockets = 1;1808 1809 for_each_online_cpu(cpu) {1810 package_num = topology_physical_package_id(cpu);1811 if (package_num + 1 > num_sockets) {1812 num_sockets = package_num + 1;1813 1814 if (num_sockets > 4) {1815 cpuidle_state_table = ivt_cstates_8s;1816 return;1817 }1818 }1819 }1820 1821 if (num_sockets > 2)1822 cpuidle_state_table = ivt_cstates_4s;1823 1824 /* else, 1 and 2 socket systems use default ivt_cstates */1825}1826 1827/**1828 * irtl_2_usec - IRTL to microseconds conversion.1829 * @irtl: IRTL MSR value.1830 *1831 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.1832 */1833static unsigned long long __init irtl_2_usec(unsigned long long irtl)1834{1835 static const unsigned int irtl_ns_units[] __initconst = {1836 1, 32, 1024, 32768, 1048576, 33554432, 0, 01837 };1838 unsigned long long ns;1839 1840 if (!irtl)1841 return 0;1842 1843 ns = irtl_ns_units[(irtl >> 10) & 0x7];1844 1845 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);1846}1847 1848/**1849 * bxt_idle_state_table_update - Fix up the Broxton idle states table.1850 *1851 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the1852 * definitive maximum latency and use the same value for target_residency.1853 */1854static void __init bxt_idle_state_table_update(void)1855{1856 unsigned long long msr;1857 unsigned int usec;1858 1859 rdmsrl(MSR_PKGC6_IRTL, msr);1860 usec = irtl_2_usec(msr);1861 if (usec) {1862 bxt_cstates[2].exit_latency = usec;1863 bxt_cstates[2].target_residency = usec;1864 }1865 1866 rdmsrl(MSR_PKGC7_IRTL, msr);1867 usec = irtl_2_usec(msr);1868 if (usec) {1869 bxt_cstates[3].exit_latency = usec;1870 bxt_cstates[3].target_residency = usec;1871 }1872 1873 rdmsrl(MSR_PKGC8_IRTL, msr);1874 usec = irtl_2_usec(msr);1875 if (usec) {1876 bxt_cstates[4].exit_latency = usec;1877 bxt_cstates[4].target_residency = usec;1878 }1879 1880 rdmsrl(MSR_PKGC9_IRTL, msr);1881 usec = irtl_2_usec(msr);1882 if (usec) {1883 bxt_cstates[5].exit_latency = usec;1884 bxt_cstates[5].target_residency = usec;1885 }1886 1887 rdmsrl(MSR_PKGC10_IRTL, msr);1888 usec = irtl_2_usec(msr);1889 if (usec) {1890 bxt_cstates[6].exit_latency = usec;1891 bxt_cstates[6].target_residency = usec;1892 }1893 1894}1895 1896/**1897 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.1898 *1899 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.1900 */1901static void __init sklh_idle_state_table_update(void)1902{1903 unsigned long long msr;1904 unsigned int eax, ebx, ecx, edx;1905 1906 1907 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */1908 if (max_cstate <= 7)1909 return;1910 1911 /* if PC10 not present in CPUID.MWAIT.EDX */1912 if ((mwait_substates & (0xF << 28)) == 0)1913 return;1914 1915 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);1916 1917 /* PC10 is not enabled in PKG C-state limit */1918 if ((msr & 0xF) != 8)1919 return;1920 1921 ecx = 0;1922 cpuid(7, &eax, &ebx, &ecx, &edx);1923 1924 /* if SGX is present */1925 if (ebx & (1 << 2)) {1926 1927 rdmsrl(MSR_IA32_FEAT_CTL, msr);1928 1929 /* if SGX is enabled */1930 if (msr & (1 << 18))1931 return;1932 }1933 1934 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */1935 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */1936}1937 1938/**1939 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake1940 * idle states table.1941 */1942static void __init skx_idle_state_table_update(void)1943{1944 unsigned long long msr;1945 1946 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);1947 1948 /*1949 * 000b: C0/C1 (no package C-state support)1950 * 001b: C21951 * 010b: C6 (non-retention)1952 * 011b: C6 (retention)1953 * 111b: No Package C state limits.1954 */1955 if ((msr & 0x7) < 2) {1956 /*1957 * Uses the CC6 + PC0 latency and 3 times of1958 * latency for target_residency if the PC61959 * is disabled in BIOS. This is consistent1960 * with how intel_idle driver uses _CST1961 * to set the target_residency.1962 */1963 skx_cstates[2].exit_latency = 92;1964 skx_cstates[2].target_residency = 276;1965 }1966}1967 1968/**1969 * adl_idle_state_table_update - Adjust AlderLake idle states table.1970 */1971static void __init adl_idle_state_table_update(void)1972{1973 /* Check if user prefers C1 over C1E. */1974 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {1975 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;1976 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;1977 1978 /* Disable C1E by clearing the "C1E promotion" bit. */1979 c1e_promotion = C1E_PROMOTION_DISABLE;1980 return;1981 }1982 1983 /* Make sure C1E is enabled by default */1984 c1e_promotion = C1E_PROMOTION_ENABLE;1985}1986 1987/**1988 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.1989 */1990static void __init spr_idle_state_table_update(void)1991{1992 unsigned long long msr;1993 1994 /*1995 * By default, the C6 state assumes the worst-case scenario of package1996 * C6. However, if PC6 is disabled, we update the numbers to match1997 * core C6.1998 */1999 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);2000 2001 /* Limit value 2 and above allow for PC6. */2002 if ((msr & 0x7) < 2) {2003 spr_cstates[2].exit_latency = 190;2004 spr_cstates[2].target_residency = 600;2005 }2006}2007 2008static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)2009{2010 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &2011 MWAIT_CSTATE_MASK;2012 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &2013 MWAIT_SUBSTATE_MASK;2014 2015 /* Ignore the C-state if there are NO sub-states in CPUID for it. */2016 if (num_substates == 0)2017 return false;2018 2019 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))2020 mark_tsc_unstable("TSC halts in idle states deeper than C2");2021 2022 return true;2023}2024 2025static void state_update_enter_method(struct cpuidle_state *state, int cstate)2026{2027 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {2028 /*2029 * Combining with XSTATE with IBRS or IRQ_ENABLE flags2030 * is not currently supported but this driver.2031 */2032 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);2033 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);2034 state->enter = intel_idle_xstate;2035 return;2036 }2037 2038 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&2039 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {2040 /*2041 * IBRS mitigation requires that C-states are entered2042 * with interrupts disabled.2043 */2044 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))2045 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;2046 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);2047 state->enter = intel_idle_ibrs;2048 return;2049 }2050 2051 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {2052 state->enter = intel_idle_irq;2053 return;2054 }2055 2056 if (force_irq_on) {2057 pr_info("forced intel_idle_irq for state %d\n", cstate);2058 state->enter = intel_idle_irq;2059 }2060}2061 2062static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)2063{2064 int cstate;2065 2066 switch (boot_cpu_data.x86_vfm) {2067 case INTEL_IVYBRIDGE_X:2068 ivt_idle_state_table_update();2069 break;2070 case INTEL_ATOM_GOLDMONT:2071 case INTEL_ATOM_GOLDMONT_PLUS:2072 bxt_idle_state_table_update();2073 break;2074 case INTEL_SKYLAKE:2075 sklh_idle_state_table_update();2076 break;2077 case INTEL_SKYLAKE_X:2078 skx_idle_state_table_update();2079 break;2080 case INTEL_SAPPHIRERAPIDS_X:2081 case INTEL_EMERALDRAPIDS_X:2082 spr_idle_state_table_update();2083 break;2084 case INTEL_ALDERLAKE:2085 case INTEL_ALDERLAKE_L:2086 case INTEL_ATOM_GRACEMONT:2087 adl_idle_state_table_update();2088 break;2089 }2090 2091 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {2092 struct cpuidle_state *state;2093 unsigned int mwait_hint;2094 2095 if (intel_idle_max_cstate_reached(cstate))2096 break;2097 2098 if (!cpuidle_state_table[cstate].enter &&2099 !cpuidle_state_table[cstate].enter_s2idle)2100 break;2101 2102 /* If marked as unusable, skip this state. */2103 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {2104 pr_debug("state %s is disabled\n",2105 cpuidle_state_table[cstate].name);2106 continue;2107 }2108 2109 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);2110 if (!intel_idle_verify_cstate(mwait_hint))2111 continue;2112 2113 /* Structure copy. */2114 drv->states[drv->state_count] = cpuidle_state_table[cstate];2115 state = &drv->states[drv->state_count];2116 2117 state_update_enter_method(state, cstate);2118 2119 2120 if ((disabled_states_mask & BIT(drv->state_count)) ||2121 ((icpu->use_acpi || force_use_acpi) &&2122 intel_idle_off_by_default(state->flags, mwait_hint) &&2123 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))2124 state->flags |= CPUIDLE_FLAG_OFF;2125 2126 if (intel_idle_state_needs_timer_stop(state))2127 state->flags |= CPUIDLE_FLAG_TIMER_STOP;2128 2129 drv->state_count++;2130 }2131 2132 if (icpu->byt_auto_demotion_disable_flag) {2133 wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);2134 wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);2135 }2136}2137 2138/**2139 * intel_idle_cpuidle_driver_init - Create the list of available idle states.2140 * @drv: cpuidle driver structure to initialize.2141 */2142static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)2143{2144 cpuidle_poll_state_init(drv);2145 2146 if (disabled_states_mask & BIT(0))2147 drv->states[0].flags |= CPUIDLE_FLAG_OFF;2148 2149 drv->state_count = 1;2150 2151 if (icpu && icpu->state_table)2152 intel_idle_init_cstates_icpu(drv);2153 else2154 intel_idle_init_cstates_acpi(drv);2155}2156 2157static void auto_demotion_disable(void)2158{2159 unsigned long long msr_bits;2160 2161 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);2162 msr_bits &= ~auto_demotion_disable_flags;2163 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);2164}2165 2166static void c1e_promotion_enable(void)2167{2168 unsigned long long msr_bits;2169 2170 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);2171 msr_bits |= 0x2;2172 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);2173}2174 2175static void c1e_promotion_disable(void)2176{2177 unsigned long long msr_bits;2178 2179 rdmsrl(MSR_IA32_POWER_CTL, msr_bits);2180 msr_bits &= ~0x2;2181 wrmsrl(MSR_IA32_POWER_CTL, msr_bits);2182}2183 2184/**2185 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.2186 * @cpu: CPU to initialize.2187 *2188 * Register a cpuidle device object for @cpu and update its MSRs in accordance2189 * with the processor model flags.2190 */2191static int intel_idle_cpu_init(unsigned int cpu)2192{2193 struct cpuidle_device *dev;2194 2195 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);2196 dev->cpu = cpu;2197 2198 if (cpuidle_register_device(dev)) {2199 pr_debug("cpuidle_register_device %d failed!\n", cpu);2200 return -EIO;2201 }2202 2203 if (auto_demotion_disable_flags)2204 auto_demotion_disable();2205 2206 if (c1e_promotion == C1E_PROMOTION_ENABLE)2207 c1e_promotion_enable();2208 else if (c1e_promotion == C1E_PROMOTION_DISABLE)2209 c1e_promotion_disable();2210 2211 return 0;2212}2213 2214static int intel_idle_cpu_online(unsigned int cpu)2215{2216 struct cpuidle_device *dev;2217 2218 if (!boot_cpu_has(X86_FEATURE_ARAT))2219 tick_broadcast_enable();2220 2221 /*2222 * Some systems can hotplug a cpu at runtime after2223 * the kernel has booted, we have to initialize the2224 * driver in this case2225 */2226 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);2227 if (!dev->registered)2228 return intel_idle_cpu_init(cpu);2229 2230 return 0;2231}2232 2233/**2234 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.2235 */2236static void __init intel_idle_cpuidle_devices_uninit(void)2237{2238 int i;2239 2240 for_each_online_cpu(i)2241 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));2242}2243 2244static int __init intel_idle_init(void)2245{2246 const struct x86_cpu_id *id;2247 unsigned int eax, ebx, ecx;2248 int retval;2249 2250 /* Do not load intel_idle at all for now if idle= is passed */2251 if (boot_option_idle_override != IDLE_NO_OVERRIDE)2252 return -ENODEV;2253 2254 if (max_cstate == 0) {2255 pr_debug("disabled\n");2256 return -EPERM;2257 }2258 2259 id = x86_match_cpu(intel_idle_ids);2260 if (id) {2261 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {2262 pr_debug("Please enable MWAIT in BIOS SETUP\n");2263 return -ENODEV;2264 }2265 } else {2266 id = x86_match_cpu(intel_mwait_ids);2267 if (!id)2268 return -ENODEV;2269 }2270 2271 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)2272 return -ENODEV;2273 2274 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);2275 2276 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||2277 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||2278 !mwait_substates)2279 return -ENODEV;2280 2281 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);2282 2283 icpu = (const struct idle_cpu *)id->driver_data;2284 if (icpu) {2285 if (icpu->state_table)2286 cpuidle_state_table = icpu->state_table;2287 else if (!intel_idle_acpi_cst_extract())2288 return -ENODEV;2289 2290 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;2291 if (icpu->disable_promotion_to_c1e)2292 c1e_promotion = C1E_PROMOTION_DISABLE;2293 if (icpu->use_acpi || force_use_acpi)2294 intel_idle_acpi_cst_extract();2295 } else if (!intel_idle_acpi_cst_extract()) {2296 return -ENODEV;2297 }2298 2299 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",2300 boot_cpu_data.x86_model);2301 2302 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);2303 if (!intel_idle_cpuidle_devices)2304 return -ENOMEM;2305 2306 intel_idle_cpuidle_driver_init(&intel_idle_driver);2307 2308 retval = cpuidle_register_driver(&intel_idle_driver);2309 if (retval) {2310 struct cpuidle_driver *drv = cpuidle_get_driver();2311 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),2312 drv ? drv->name : "none");2313 goto init_driver_fail;2314 }2315 2316 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",2317 intel_idle_cpu_online, NULL);2318 if (retval < 0)2319 goto hp_setup_fail;2320 2321 pr_debug("Local APIC timer is reliable in %s\n",2322 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");2323 2324 return 0;2325 2326hp_setup_fail:2327 intel_idle_cpuidle_devices_uninit();2328 cpuidle_unregister_driver(&intel_idle_driver);2329init_driver_fail:2330 free_percpu(intel_idle_cpuidle_devices);2331 return retval;2332 2333}2334device_initcall(intel_idle_init);2335 2336/*2337 * We are not really modular, but we used to support that. Meaning we also2338 * support "intel_idle.max_cstate=..." at boot and also a read-only export of2339 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param2340 * is the easiest way (currently) to continue doing that.2341 */2342module_param(max_cstate, int, 0444);2343/*2344 * The positions of the bits that are set in this number are the indices of the2345 * idle states to be disabled by default (as reflected by the names of the2346 * corresponding idle state directories in sysfs, "state0", "state1" ...2347 * "state<i>" ..., where <i> is the index of the given state).2348 */2349module_param_named(states_off, disabled_states_mask, uint, 0444);2350MODULE_PARM_DESC(states_off, "Mask of disabled idle states");2351/*2352 * Some platforms come with mutually exclusive C-states, so that if one is2353 * enabled, the other C-states must not be used. Example: C1 and C1E on2354 * Sapphire Rapids platform. This parameter allows for selecting the2355 * preferred C-states among the groups of mutually exclusive C-states - the2356 * selected C-states will be registered, the other C-states from the mutually2357 * exclusive group won't be registered. If the platform has no mutually2358 * exclusive C-states, this parameter has no effect.2359 */2360module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);2361MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");2362/*2363 * Debugging option that forces the driver to enter all C-states with2364 * interrupts enabled. Does not apply to C-states with2365 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.2366 */2367module_param(force_irq_on, bool, 0444);2368/*2369 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and2370 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.2371 */2372module_param(ibrs_off, bool, 0444);2373MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");2374