2345 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Common code for Intel Running Average Power Limit (RAPL) support.4 * Copyright (c) 2019, Intel Corporation.5 */6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt7 8#include <linux/bitmap.h>9#include <linux/cleanup.h>10#include <linux/cpu.h>11#include <linux/delay.h>12#include <linux/device.h>13#include <linux/intel_rapl.h>14#include <linux/kernel.h>15#include <linux/list.h>16#include <linux/log2.h>17#include <linux/module.h>18#include <linux/nospec.h>19#include <linux/perf_event.h>20#include <linux/platform_device.h>21#include <linux/powercap.h>22#include <linux/processor.h>23#include <linux/slab.h>24#include <linux/suspend.h>25#include <linux/sysfs.h>26#include <linux/types.h>27 28#include <asm/cpu_device_id.h>29#include <asm/intel-family.h>30#include <asm/iosf_mbi.h>31 32/* bitmasks for RAPL MSRs, used by primitive access functions */33#define ENERGY_STATUS_MASK 0xffffffff34 35#define POWER_LIMIT1_MASK 0x7FFF36#define POWER_LIMIT1_ENABLE BIT(15)37#define POWER_LIMIT1_CLAMP BIT(16)38 39#define POWER_LIMIT2_MASK (0x7FFFULL<<32)40#define POWER_LIMIT2_ENABLE BIT_ULL(47)41#define POWER_LIMIT2_CLAMP BIT_ULL(48)42#define POWER_HIGH_LOCK BIT_ULL(63)43#define POWER_LOW_LOCK BIT(31)44 45#define POWER_LIMIT4_MASK 0x1FFF46 47#define TIME_WINDOW1_MASK (0x7FULL<<17)48#define TIME_WINDOW2_MASK (0x7FULL<<49)49 50#define POWER_UNIT_OFFSET 051#define POWER_UNIT_MASK 0x0F52 53#define ENERGY_UNIT_OFFSET 0x0854#define ENERGY_UNIT_MASK 0x1F0055 56#define TIME_UNIT_OFFSET 0x1057#define TIME_UNIT_MASK 0xF000058 59#define POWER_INFO_MAX_MASK (0x7fffULL<<32)60#define POWER_INFO_MIN_MASK (0x7fffULL<<16)61#define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)62#define POWER_INFO_THERMAL_SPEC_MASK 0x7fff63 64#define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff65#define PP_POLICY_MASK 0x1F66 67/*68 * SPR has different layout for Psys Domain PowerLimit registers.69 * There are 17 bits of PL1 and PL2 instead of 15 bits.70 * The Enable bits and TimeWindow bits are also shifted as a result.71 */72#define PSYS_POWER_LIMIT1_MASK 0x1FFFF73#define PSYS_POWER_LIMIT1_ENABLE BIT(17)74 75#define PSYS_POWER_LIMIT2_MASK (0x1FFFFULL<<32)76#define PSYS_POWER_LIMIT2_ENABLE BIT_ULL(49)77 78#define PSYS_TIME_WINDOW1_MASK (0x7FULL<<19)79#define PSYS_TIME_WINDOW2_MASK (0x7FULL<<51)80 81/* bitmasks for RAPL TPMI, used by primitive access functions */82#define TPMI_POWER_LIMIT_MASK 0x3FFFF83#define TPMI_POWER_LIMIT_ENABLE BIT_ULL(62)84#define TPMI_TIME_WINDOW_MASK (0x7FULL<<18)85#define TPMI_INFO_SPEC_MASK 0x3FFFF86#define TPMI_INFO_MIN_MASK (0x3FFFFULL << 18)87#define TPMI_INFO_MAX_MASK (0x3FFFFULL << 36)88#define TPMI_INFO_MAX_TIME_WIN_MASK (0x7FULL << 54)89 90/* Non HW constants */91#define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */92#define RAPL_PRIMITIVE_DUMMY BIT(2)93 94#define TIME_WINDOW_MAX_MSEC 4000095#define TIME_WINDOW_MIN_MSEC 25096#define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */97enum unit_type {98 ARBITRARY_UNIT, /* no translation */99 POWER_UNIT,100 ENERGY_UNIT,101 TIME_UNIT,102};103 104/* per domain data, some are optional */105#define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)106 107#define DOMAIN_STATE_INACTIVE BIT(0)108#define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)109 110static const char *pl_names[NR_POWER_LIMITS] = {111 [POWER_LIMIT1] = "long_term",112 [POWER_LIMIT2] = "short_term",113 [POWER_LIMIT4] = "peak_power",114};115 116enum pl_prims {117 PL_ENABLE,118 PL_CLAMP,119 PL_LIMIT,120 PL_TIME_WINDOW,121 PL_MAX_POWER,122 PL_LOCK,123};124 125static bool is_pl_valid(struct rapl_domain *rd, int pl)126{127 if (pl < POWER_LIMIT1 || pl > POWER_LIMIT4)128 return false;129 return rd->rpl[pl].name ? true : false;130}131 132static int get_pl_lock_prim(struct rapl_domain *rd, int pl)133{134 if (rd->rp->priv->type == RAPL_IF_TPMI) {135 if (pl == POWER_LIMIT1)136 return PL1_LOCK;137 if (pl == POWER_LIMIT2)138 return PL2_LOCK;139 if (pl == POWER_LIMIT4)140 return PL4_LOCK;141 }142 143 /* MSR/MMIO Interface doesn't have Lock bit for PL4 */144 if (pl == POWER_LIMIT4)145 return -EINVAL;146 147 /*148 * Power Limit register that supports two power limits has a different149 * bit position for the Lock bit.150 */151 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))152 return FW_HIGH_LOCK;153 return FW_LOCK;154}155 156static int get_pl_prim(struct rapl_domain *rd, int pl, enum pl_prims prim)157{158 switch (pl) {159 case POWER_LIMIT1:160 if (prim == PL_ENABLE)161 return PL1_ENABLE;162 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)163 return PL1_CLAMP;164 if (prim == PL_LIMIT)165 return POWER_LIMIT1;166 if (prim == PL_TIME_WINDOW)167 return TIME_WINDOW1;168 if (prim == PL_MAX_POWER)169 return THERMAL_SPEC_POWER;170 if (prim == PL_LOCK)171 return get_pl_lock_prim(rd, pl);172 return -EINVAL;173 case POWER_LIMIT2:174 if (prim == PL_ENABLE)175 return PL2_ENABLE;176 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)177 return PL2_CLAMP;178 if (prim == PL_LIMIT)179 return POWER_LIMIT2;180 if (prim == PL_TIME_WINDOW)181 return TIME_WINDOW2;182 if (prim == PL_MAX_POWER)183 return MAX_POWER;184 if (prim == PL_LOCK)185 return get_pl_lock_prim(rd, pl);186 return -EINVAL;187 case POWER_LIMIT4:188 if (prim == PL_LIMIT)189 return POWER_LIMIT4;190 if (prim == PL_ENABLE)191 return PL4_ENABLE;192 /* PL4 would be around two times PL2, use same prim as PL2. */193 if (prim == PL_MAX_POWER)194 return MAX_POWER;195 if (prim == PL_LOCK)196 return get_pl_lock_prim(rd, pl);197 return -EINVAL;198 default:199 return -EINVAL;200 }201}202 203#define power_zone_to_rapl_domain(_zone) \204 container_of(_zone, struct rapl_domain, power_zone)205 206struct rapl_defaults {207 u8 floor_freq_reg_addr;208 int (*check_unit)(struct rapl_domain *rd);209 void (*set_floor_freq)(struct rapl_domain *rd, bool mode);210 u64 (*compute_time_window)(struct rapl_domain *rd, u64 val,211 bool to_raw);212 unsigned int dram_domain_energy_unit;213 unsigned int psys_domain_energy_unit;214 bool spr_psys_bits;215};216static struct rapl_defaults *defaults_msr;217static const struct rapl_defaults defaults_tpmi;218 219static struct rapl_defaults *get_defaults(struct rapl_package *rp)220{221 return rp->priv->defaults;222}223 224/* Sideband MBI registers */225#define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)226#define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)227 228#define PACKAGE_PLN_INT_SAVED BIT(0)229#define MAX_PRIM_NAME (32)230 231/* per domain data. used to describe individual knobs such that access function232 * can be consolidated into one instead of many inline functions.233 */234struct rapl_primitive_info {235 const char *name;236 u64 mask;237 int shift;238 enum rapl_domain_reg_id id;239 enum unit_type unit;240 u32 flag;241};242 243#define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \244 .name = #p, \245 .mask = m, \246 .shift = s, \247 .id = i, \248 .unit = u, \249 .flag = f \250 }251 252static void rapl_init_domains(struct rapl_package *rp);253static int rapl_read_data_raw(struct rapl_domain *rd,254 enum rapl_primitives prim,255 bool xlate, u64 *data);256static int rapl_write_data_raw(struct rapl_domain *rd,257 enum rapl_primitives prim,258 unsigned long long value);259static int rapl_read_pl_data(struct rapl_domain *rd, int pl,260 enum pl_prims pl_prim,261 bool xlate, u64 *data);262static int rapl_write_pl_data(struct rapl_domain *rd, int pl,263 enum pl_prims pl_prim,264 unsigned long long value);265static u64 rapl_unit_xlate(struct rapl_domain *rd,266 enum unit_type type, u64 value, int to_raw);267static void package_power_limit_irq_save(struct rapl_package *rp);268 269static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */270 271static const char *const rapl_domain_names[] = {272 "package",273 "core",274 "uncore",275 "dram",276 "psys",277};278 279static int get_energy_counter(struct powercap_zone *power_zone,280 u64 *energy_raw)281{282 struct rapl_domain *rd;283 u64 energy_now;284 285 /* prevent CPU hotplug, make sure the RAPL domain does not go286 * away while reading the counter.287 */288 cpus_read_lock();289 rd = power_zone_to_rapl_domain(power_zone);290 291 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) {292 *energy_raw = energy_now;293 cpus_read_unlock();294 295 return 0;296 }297 cpus_read_unlock();298 299 return -EIO;300}301 302static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)303{304 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);305 306 *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);307 return 0;308}309 310static int release_zone(struct powercap_zone *power_zone)311{312 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);313 struct rapl_package *rp = rd->rp;314 315 /* package zone is the last zone of a package, we can free316 * memory here since all children has been unregistered.317 */318 if (rd->id == RAPL_DOMAIN_PACKAGE) {319 kfree(rd);320 rp->domains = NULL;321 }322 323 return 0;324 325}326 327static int find_nr_power_limit(struct rapl_domain *rd)328{329 int i, nr_pl = 0;330 331 for (i = 0; i < NR_POWER_LIMITS; i++) {332 if (is_pl_valid(rd, i))333 nr_pl++;334 }335 336 return nr_pl;337}338 339static int set_domain_enable(struct powercap_zone *power_zone, bool mode)340{341 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);342 struct rapl_defaults *defaults = get_defaults(rd->rp);343 int ret;344 345 cpus_read_lock();346 ret = rapl_write_pl_data(rd, POWER_LIMIT1, PL_ENABLE, mode);347 if (!ret && defaults->set_floor_freq)348 defaults->set_floor_freq(rd, mode);349 cpus_read_unlock();350 351 return ret;352}353 354static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)355{356 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);357 u64 val;358 int ret;359 360 if (rd->rpl[POWER_LIMIT1].locked) {361 *mode = false;362 return 0;363 }364 cpus_read_lock();365 ret = rapl_read_pl_data(rd, POWER_LIMIT1, PL_ENABLE, true, &val);366 if (!ret)367 *mode = val;368 cpus_read_unlock();369 370 return ret;371}372 373/* per RAPL domain ops, in the order of rapl_domain_type */374static const struct powercap_zone_ops zone_ops[] = {375 /* RAPL_DOMAIN_PACKAGE */376 {377 .get_energy_uj = get_energy_counter,378 .get_max_energy_range_uj = get_max_energy_counter,379 .release = release_zone,380 .set_enable = set_domain_enable,381 .get_enable = get_domain_enable,382 },383 /* RAPL_DOMAIN_PP0 */384 {385 .get_energy_uj = get_energy_counter,386 .get_max_energy_range_uj = get_max_energy_counter,387 .release = release_zone,388 .set_enable = set_domain_enable,389 .get_enable = get_domain_enable,390 },391 /* RAPL_DOMAIN_PP1 */392 {393 .get_energy_uj = get_energy_counter,394 .get_max_energy_range_uj = get_max_energy_counter,395 .release = release_zone,396 .set_enable = set_domain_enable,397 .get_enable = get_domain_enable,398 },399 /* RAPL_DOMAIN_DRAM */400 {401 .get_energy_uj = get_energy_counter,402 .get_max_energy_range_uj = get_max_energy_counter,403 .release = release_zone,404 .set_enable = set_domain_enable,405 .get_enable = get_domain_enable,406 },407 /* RAPL_DOMAIN_PLATFORM */408 {409 .get_energy_uj = get_energy_counter,410 .get_max_energy_range_uj = get_max_energy_counter,411 .release = release_zone,412 .set_enable = set_domain_enable,413 .get_enable = get_domain_enable,414 },415};416 417/*418 * Constraint index used by powercap can be different than power limit (PL)419 * index in that some PLs maybe missing due to non-existent MSRs. So we420 * need to convert here by finding the valid PLs only (name populated).421 */422static int contraint_to_pl(struct rapl_domain *rd, int cid)423{424 int i, j;425 426 for (i = POWER_LIMIT1, j = 0; i < NR_POWER_LIMITS; i++) {427 if (is_pl_valid(rd, i) && j++ == cid) {428 pr_debug("%s: index %d\n", __func__, i);429 return i;430 }431 }432 pr_err("Cannot find matching power limit for constraint %d\n", cid);433 434 return -EINVAL;435}436 437static int set_power_limit(struct powercap_zone *power_zone, int cid,438 u64 power_limit)439{440 struct rapl_domain *rd;441 struct rapl_package *rp;442 int ret = 0;443 int id;444 445 cpus_read_lock();446 rd = power_zone_to_rapl_domain(power_zone);447 id = contraint_to_pl(rd, cid);448 rp = rd->rp;449 450 ret = rapl_write_pl_data(rd, id, PL_LIMIT, power_limit);451 if (!ret)452 package_power_limit_irq_save(rp);453 cpus_read_unlock();454 return ret;455}456 457static int get_current_power_limit(struct powercap_zone *power_zone, int cid,458 u64 *data)459{460 struct rapl_domain *rd;461 u64 val;462 int ret = 0;463 int id;464 465 cpus_read_lock();466 rd = power_zone_to_rapl_domain(power_zone);467 id = contraint_to_pl(rd, cid);468 469 ret = rapl_read_pl_data(rd, id, PL_LIMIT, true, &val);470 if (!ret)471 *data = val;472 473 cpus_read_unlock();474 475 return ret;476}477 478static int set_time_window(struct powercap_zone *power_zone, int cid,479 u64 window)480{481 struct rapl_domain *rd;482 int ret = 0;483 int id;484 485 cpus_read_lock();486 rd = power_zone_to_rapl_domain(power_zone);487 id = contraint_to_pl(rd, cid);488 489 ret = rapl_write_pl_data(rd, id, PL_TIME_WINDOW, window);490 491 cpus_read_unlock();492 return ret;493}494 495static int get_time_window(struct powercap_zone *power_zone, int cid,496 u64 *data)497{498 struct rapl_domain *rd;499 u64 val;500 int ret = 0;501 int id;502 503 cpus_read_lock();504 rd = power_zone_to_rapl_domain(power_zone);505 id = contraint_to_pl(rd, cid);506 507 ret = rapl_read_pl_data(rd, id, PL_TIME_WINDOW, true, &val);508 if (!ret)509 *data = val;510 511 cpus_read_unlock();512 513 return ret;514}515 516static const char *get_constraint_name(struct powercap_zone *power_zone,517 int cid)518{519 struct rapl_domain *rd;520 int id;521 522 rd = power_zone_to_rapl_domain(power_zone);523 id = contraint_to_pl(rd, cid);524 if (id >= 0)525 return rd->rpl[id].name;526 527 return NULL;528}529 530static int get_max_power(struct powercap_zone *power_zone, int cid, u64 *data)531{532 struct rapl_domain *rd;533 u64 val;534 int ret = 0;535 int id;536 537 cpus_read_lock();538 rd = power_zone_to_rapl_domain(power_zone);539 id = contraint_to_pl(rd, cid);540 541 ret = rapl_read_pl_data(rd, id, PL_MAX_POWER, true, &val);542 if (!ret)543 *data = val;544 545 /* As a generalization rule, PL4 would be around two times PL2. */546 if (id == POWER_LIMIT4)547 *data = *data * 2;548 549 cpus_read_unlock();550 551 return ret;552}553 554static const struct powercap_zone_constraint_ops constraint_ops = {555 .set_power_limit_uw = set_power_limit,556 .get_power_limit_uw = get_current_power_limit,557 .set_time_window_us = set_time_window,558 .get_time_window_us = get_time_window,559 .get_max_power_uw = get_max_power,560 .get_name = get_constraint_name,561};562 563/* Return the id used for read_raw/write_raw callback */564static int get_rid(struct rapl_package *rp)565{566 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;567}568 569/* called after domain detection and package level data are set */570static void rapl_init_domains(struct rapl_package *rp)571{572 enum rapl_domain_type i;573 enum rapl_domain_reg_id j;574 struct rapl_domain *rd = rp->domains;575 576 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {577 unsigned int mask = rp->domain_map & (1 << i);578 int t;579 580 if (!mask)581 continue;582 583 rd->rp = rp;584 585 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {586 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d",587 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :588 rp->id);589 } else {590 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s",591 rapl_domain_names[i]);592 }593 594 rd->id = i;595 596 /* PL1 is supported by default */597 rp->priv->limits[i] |= BIT(POWER_LIMIT1);598 599 for (t = POWER_LIMIT1; t < NR_POWER_LIMITS; t++) {600 if (rp->priv->limits[i] & BIT(t))601 rd->rpl[t].name = pl_names[t];602 }603 604 for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++)605 rd->regs[j] = rp->priv->regs[i][j];606 607 rd++;608 }609}610 611static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,612 u64 value, int to_raw)613{614 u64 units = 1;615 struct rapl_defaults *defaults = get_defaults(rd->rp);616 u64 scale = 1;617 618 switch (type) {619 case POWER_UNIT:620 units = rd->power_unit;621 break;622 case ENERGY_UNIT:623 scale = ENERGY_UNIT_SCALE;624 units = rd->energy_unit;625 break;626 case TIME_UNIT:627 return defaults->compute_time_window(rd, value, to_raw);628 case ARBITRARY_UNIT:629 default:630 return value;631 }632 633 if (to_raw)634 return div64_u64(value, units) * scale;635 636 value *= units;637 638 return div64_u64(value, scale);639}640 641/* RAPL primitives for MSR and MMIO I/F */642static struct rapl_primitive_info rpi_msr[NR_RAPL_PRIMITIVES] = {643 /* name, mask, shift, msr index, unit divisor */644 [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,645 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),646 [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,647 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),648 [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0,649 RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),650 [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,651 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),652 [FW_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31,653 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),654 [FW_HIGH_LOCK] = PRIMITIVE_INFO_INIT(FW_LOCK, POWER_HIGH_LOCK, 63,655 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),656 [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,657 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),658 [PL1_CLAMP] = PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,659 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),660 [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,661 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),662 [PL2_CLAMP] = PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,663 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),664 [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,665 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),666 [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,667 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),668 [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,669 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),670 [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,671 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),672 [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,673 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),674 [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,675 RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),676 [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,677 RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),678 [PRIORITY_LEVEL] = PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,679 RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),680 [PSYS_POWER_LIMIT1] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT1, PSYS_POWER_LIMIT1_MASK, 0,681 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),682 [PSYS_POWER_LIMIT2] = PRIMITIVE_INFO_INIT(PSYS_POWER_LIMIT2, PSYS_POWER_LIMIT2_MASK, 32,683 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),684 [PSYS_PL1_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL1_ENABLE, PSYS_POWER_LIMIT1_ENABLE, 17,685 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),686 [PSYS_PL2_ENABLE] = PRIMITIVE_INFO_INIT(PSYS_PL2_ENABLE, PSYS_POWER_LIMIT2_ENABLE, 49,687 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),688 [PSYS_TIME_WINDOW1] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW1_MASK, 19,689 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),690 [PSYS_TIME_WINDOW2] = PRIMITIVE_INFO_INIT(PSYS_TIME_WINDOW2, PSYS_TIME_WINDOW2_MASK, 51,691 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),692 /* non-hardware */693 [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,694 RAPL_PRIMITIVE_DERIVED),695};696 697/* RAPL primitives for TPMI I/F */698static struct rapl_primitive_info rpi_tpmi[NR_RAPL_PRIMITIVES] = {699 /* name, mask, shift, msr index, unit divisor */700 [POWER_LIMIT1] = PRIMITIVE_INFO_INIT(POWER_LIMIT1, TPMI_POWER_LIMIT_MASK, 0,701 RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),702 [POWER_LIMIT2] = PRIMITIVE_INFO_INIT(POWER_LIMIT2, TPMI_POWER_LIMIT_MASK, 0,703 RAPL_DOMAIN_REG_PL2, POWER_UNIT, 0),704 [POWER_LIMIT4] = PRIMITIVE_INFO_INIT(POWER_LIMIT4, TPMI_POWER_LIMIT_MASK, 0,705 RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),706 [ENERGY_COUNTER] = PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,707 RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),708 [PL1_LOCK] = PRIMITIVE_INFO_INIT(PL1_LOCK, POWER_HIGH_LOCK, 63,709 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),710 [PL2_LOCK] = PRIMITIVE_INFO_INIT(PL2_LOCK, POWER_HIGH_LOCK, 63,711 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),712 [PL4_LOCK] = PRIMITIVE_INFO_INIT(PL4_LOCK, POWER_HIGH_LOCK, 63,713 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),714 [PL1_ENABLE] = PRIMITIVE_INFO_INIT(PL1_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,715 RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),716 [PL2_ENABLE] = PRIMITIVE_INFO_INIT(PL2_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,717 RAPL_DOMAIN_REG_PL2, ARBITRARY_UNIT, 0),718 [PL4_ENABLE] = PRIMITIVE_INFO_INIT(PL4_ENABLE, TPMI_POWER_LIMIT_ENABLE, 62,719 RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),720 [TIME_WINDOW1] = PRIMITIVE_INFO_INIT(TIME_WINDOW1, TPMI_TIME_WINDOW_MASK, 18,721 RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),722 [TIME_WINDOW2] = PRIMITIVE_INFO_INIT(TIME_WINDOW2, TPMI_TIME_WINDOW_MASK, 18,723 RAPL_DOMAIN_REG_PL2, TIME_UNIT, 0),724 [THERMAL_SPEC_POWER] = PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, TPMI_INFO_SPEC_MASK, 0,725 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),726 [MAX_POWER] = PRIMITIVE_INFO_INIT(MAX_POWER, TPMI_INFO_MAX_MASK, 36,727 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),728 [MIN_POWER] = PRIMITIVE_INFO_INIT(MIN_POWER, TPMI_INFO_MIN_MASK, 18,729 RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),730 [MAX_TIME_WINDOW] = PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, TPMI_INFO_MAX_TIME_WIN_MASK, 54,731 RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),732 [THROTTLED_TIME] = PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,733 RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),734 /* non-hardware */735 [AVERAGE_POWER] = PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0,736 POWER_UNIT, RAPL_PRIMITIVE_DERIVED),737};738 739static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)740{741 struct rapl_primitive_info *rpi = rp->priv->rpi;742 743 if (prim < 0 || prim >= NR_RAPL_PRIMITIVES || !rpi)744 return NULL;745 746 return &rpi[prim];747}748 749static int rapl_config(struct rapl_package *rp)750{751 switch (rp->priv->type) {752 /* MMIO I/F shares the same register layout as MSR registers */753 case RAPL_IF_MMIO:754 case RAPL_IF_MSR:755 rp->priv->defaults = (void *)defaults_msr;756 rp->priv->rpi = (void *)rpi_msr;757 break;758 case RAPL_IF_TPMI:759 rp->priv->defaults = (void *)&defaults_tpmi;760 rp->priv->rpi = (void *)rpi_tpmi;761 break;762 default:763 return -EINVAL;764 }765 766 /* defaults_msr can be NULL on unsupported platforms */767 if (!rp->priv->defaults || !rp->priv->rpi)768 return -ENODEV;769 770 return 0;771}772 773static enum rapl_primitives774prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim)775{776 struct rapl_defaults *defaults = get_defaults(rd->rp);777 778 if (!defaults->spr_psys_bits)779 return prim;780 781 if (rd->id != RAPL_DOMAIN_PLATFORM)782 return prim;783 784 switch (prim) {785 case POWER_LIMIT1:786 return PSYS_POWER_LIMIT1;787 case POWER_LIMIT2:788 return PSYS_POWER_LIMIT2;789 case PL1_ENABLE:790 return PSYS_PL1_ENABLE;791 case PL2_ENABLE:792 return PSYS_PL2_ENABLE;793 case TIME_WINDOW1:794 return PSYS_TIME_WINDOW1;795 case TIME_WINDOW2:796 return PSYS_TIME_WINDOW2;797 default:798 return prim;799 }800}801 802/* Read primitive data based on its related struct rapl_primitive_info.803 * if xlate flag is set, return translated data based on data units, i.e.804 * time, energy, and power.805 * RAPL MSRs are non-architectual and are laid out not consistently across806 * domains. Here we use primitive info to allow writing consolidated access807 * functions.808 * For a given primitive, it is processed by MSR mask and shift. Unit conversion809 * is pre-assigned based on RAPL unit MSRs read at init time.810 * 63-------------------------- 31--------------------------- 0811 * | xxxxx (mask) |812 * | |<- shift ----------------|813 * 63-------------------------- 31--------------------------- 0814 */815static int rapl_read_data_raw(struct rapl_domain *rd,816 enum rapl_primitives prim, bool xlate, u64 *data)817{818 u64 value;819 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);820 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);821 struct reg_action ra;822 823 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)824 return -EINVAL;825 826 ra.reg = rd->regs[rpi->id];827 if (!ra.reg.val)828 return -EINVAL;829 830 /* non-hardware data are collected by the polling thread */831 if (rpi->flag & RAPL_PRIMITIVE_DERIVED) {832 *data = rd->rdd.primitives[prim];833 return 0;834 }835 836 ra.mask = rpi->mask;837 838 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {839 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);840 return -EIO;841 }842 843 value = ra.value >> rpi->shift;844 845 if (xlate)846 *data = rapl_unit_xlate(rd, rpi->unit, value, 0);847 else848 *data = value;849 850 return 0;851}852 853/* Similar use of primitive info in the read counterpart */854static int rapl_write_data_raw(struct rapl_domain *rd,855 enum rapl_primitives prim,856 unsigned long long value)857{858 enum rapl_primitives prim_fixed = prim_fixups(rd, prim);859 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);860 u64 bits;861 struct reg_action ra;862 int ret;863 864 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY)865 return -EINVAL;866 867 bits = rapl_unit_xlate(rd, rpi->unit, value, 1);868 bits <<= rpi->shift;869 bits &= rpi->mask;870 871 memset(&ra, 0, sizeof(ra));872 873 ra.reg = rd->regs[rpi->id];874 ra.mask = rpi->mask;875 ra.value = bits;876 877 ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra);878 879 return ret;880}881 882static int rapl_read_pl_data(struct rapl_domain *rd, int pl,883 enum pl_prims pl_prim, bool xlate, u64 *data)884{885 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim);886 887 if (!is_pl_valid(rd, pl))888 return -EINVAL;889 890 return rapl_read_data_raw(rd, prim, xlate, data);891}892 893static int rapl_write_pl_data(struct rapl_domain *rd, int pl,894 enum pl_prims pl_prim,895 unsigned long long value)896{897 enum rapl_primitives prim = get_pl_prim(rd, pl, pl_prim);898 899 if (!is_pl_valid(rd, pl))900 return -EINVAL;901 902 if (rd->rpl[pl].locked) {903 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);904 return -EACCES;905 }906 907 return rapl_write_data_raw(rd, prim, value);908}909/*910 * Raw RAPL data stored in MSRs are in certain scales. We need to911 * convert them into standard units based on the units reported in912 * the RAPL unit MSRs. This is specific to CPUs as the method to913 * calculate units differ on different CPUs.914 * We convert the units to below format based on CPUs.915 * i.e.916 * energy unit: picoJoules : Represented in picoJoules by default917 * power unit : microWatts : Represented in milliWatts by default918 * time unit : microseconds: Represented in seconds by default919 */920static int rapl_check_unit_core(struct rapl_domain *rd)921{922 struct reg_action ra;923 u32 value;924 925 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];926 ra.mask = ~0;927 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {928 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",929 ra.reg.val, rd->rp->name, rd->name);930 return -ENODEV;931 }932 933 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;934 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);935 936 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;937 rd->power_unit = 1000000 / (1 << value);938 939 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;940 rd->time_unit = 1000000 / (1 << value);941 942 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",943 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);944 945 return 0;946}947 948static int rapl_check_unit_atom(struct rapl_domain *rd)949{950 struct reg_action ra;951 u32 value;952 953 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];954 ra.mask = ~0;955 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {956 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",957 ra.reg.val, rd->rp->name, rd->name);958 return -ENODEV;959 }960 961 value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;962 rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value;963 964 value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;965 rd->power_unit = (1 << value) * 1000;966 967 value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;968 rd->time_unit = 1000000 / (1 << value);969 970 pr_debug("Atom %s:%s energy=%dpJ, time=%dus, power=%duW\n",971 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);972 973 return 0;974}975 976static void power_limit_irq_save_cpu(void *info)977{978 u32 l, h = 0;979 struct rapl_package *rp = (struct rapl_package *)info;980 981 /* save the state of PLN irq mask bit before disabling it */982 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);983 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {984 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;985 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;986 }987 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;988 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);989}990 991/* REVISIT:992 * When package power limit is set artificially low by RAPL, LVT993 * thermal interrupt for package power limit should be ignored994 * since we are not really exceeding the real limit. The intention995 * is to avoid excessive interrupts while we are trying to save power.996 * A useful feature might be routing the package_power_limit interrupt997 * to userspace via eventfd. once we have a usecase, this is simple998 * to do by adding an atomic notifier.999 */1000 1001static void package_power_limit_irq_save(struct rapl_package *rp)1002{1003 if (rp->lead_cpu < 0)1004 return;1005 1006 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))1007 return;1008 1009 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);1010}1011 1012/*1013 * Restore per package power limit interrupt enable state. Called from cpu1014 * hotplug code on package removal.1015 */1016static void package_power_limit_irq_restore(struct rapl_package *rp)1017{1018 u32 l, h;1019 1020 if (rp->lead_cpu < 0)1021 return;1022 1023 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))1024 return;1025 1026 /* irq enable state not saved, nothing to restore */1027 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))1028 return;1029 1030 rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);1031 1032 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)1033 l |= PACKAGE_THERM_INT_PLN_ENABLE;1034 else1035 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;1036 1037 wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);1038}1039 1040static void set_floor_freq_default(struct rapl_domain *rd, bool mode)1041{1042 int i;1043 1044 /* always enable clamp such that p-state can go below OS requested1045 * range. power capping priority over guranteed frequency.1046 */1047 rapl_write_pl_data(rd, POWER_LIMIT1, PL_CLAMP, mode);1048 1049 for (i = POWER_LIMIT2; i < NR_POWER_LIMITS; i++) {1050 rapl_write_pl_data(rd, i, PL_ENABLE, mode);1051 rapl_write_pl_data(rd, i, PL_CLAMP, mode);1052 }1053}1054 1055static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)1056{1057 static u32 power_ctrl_orig_val;1058 struct rapl_defaults *defaults = get_defaults(rd->rp);1059 u32 mdata;1060 1061 if (!defaults->floor_freq_reg_addr) {1062 pr_err("Invalid floor frequency config register\n");1063 return;1064 }1065 1066 if (!power_ctrl_orig_val)1067 iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,1068 defaults->floor_freq_reg_addr,1069 &power_ctrl_orig_val);1070 mdata = power_ctrl_orig_val;1071 if (enable) {1072 mdata &= ~(0x7f << 8);1073 mdata |= 1 << 8;1074 }1075 iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,1076 defaults->floor_freq_reg_addr, mdata);1077}1078 1079static u64 rapl_compute_time_window_core(struct rapl_domain *rd, u64 value,1080 bool to_raw)1081{1082 u64 f, y; /* fraction and exp. used for time unit */1083 1084 /*1085 * Special processing based on 2^Y*(1+F/4), refer1086 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.1087 */1088 if (!to_raw) {1089 f = (value & 0x60) >> 5;1090 y = value & 0x1f;1091 value = (1 << y) * (4 + f) * rd->time_unit / 4;1092 } else {1093 if (value < rd->time_unit)1094 return 0;1095 1096 do_div(value, rd->time_unit);1097 y = ilog2(value);1098 1099 /*1100 * The target hardware field is 7 bits wide, so return all ones1101 * if the exponent is too large.1102 */1103 if (y > 0x1f)1104 return 0x7f;1105 1106 f = div64_u64(4 * (value - (1ULL << y)), 1ULL << y);1107 value = (y & 0x1f) | ((f & 0x3) << 5);1108 }1109 return value;1110}1111 1112static u64 rapl_compute_time_window_atom(struct rapl_domain *rd, u64 value,1113 bool to_raw)1114{1115 /*1116 * Atom time unit encoding is straight forward val * time_unit,1117 * where time_unit is default to 1 sec. Never 0.1118 */1119 if (!to_raw)1120 return (value) ? value * rd->time_unit : rd->time_unit;1121 1122 value = div64_u64(value, rd->time_unit);1123 1124 return value;1125}1126 1127/* TPMI Unit register has different layout */1128#define TPMI_POWER_UNIT_OFFSET POWER_UNIT_OFFSET1129#define TPMI_POWER_UNIT_MASK POWER_UNIT_MASK1130#define TPMI_ENERGY_UNIT_OFFSET 0x061131#define TPMI_ENERGY_UNIT_MASK 0x7C01132#define TPMI_TIME_UNIT_OFFSET 0x0C1133#define TPMI_TIME_UNIT_MASK 0xF0001134 1135static int rapl_check_unit_tpmi(struct rapl_domain *rd)1136{1137 struct reg_action ra;1138 u32 value;1139 1140 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT];1141 ra.mask = ~0;1142 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {1143 pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n",1144 ra.reg.val, rd->rp->name, rd->name);1145 return -ENODEV;1146 }1147 1148 value = (ra.value & TPMI_ENERGY_UNIT_MASK) >> TPMI_ENERGY_UNIT_OFFSET;1149 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);1150 1151 value = (ra.value & TPMI_POWER_UNIT_MASK) >> TPMI_POWER_UNIT_OFFSET;1152 rd->power_unit = 1000000 / (1 << value);1153 1154 value = (ra.value & TPMI_TIME_UNIT_MASK) >> TPMI_TIME_UNIT_OFFSET;1155 rd->time_unit = 1000000 / (1 << value);1156 1157 pr_debug("Core CPU %s:%s energy=%dpJ, time=%dus, power=%duW\n",1158 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);1159 1160 return 0;1161}1162 1163static const struct rapl_defaults defaults_tpmi = {1164 .check_unit = rapl_check_unit_tpmi,1165 /* Reuse existing logic, ignore the PL_CLAMP failures and enable all Power Limits */1166 .set_floor_freq = set_floor_freq_default,1167 .compute_time_window = rapl_compute_time_window_core,1168};1169 1170static const struct rapl_defaults rapl_defaults_core = {1171 .floor_freq_reg_addr = 0,1172 .check_unit = rapl_check_unit_core,1173 .set_floor_freq = set_floor_freq_default,1174 .compute_time_window = rapl_compute_time_window_core,1175};1176 1177static const struct rapl_defaults rapl_defaults_hsw_server = {1178 .check_unit = rapl_check_unit_core,1179 .set_floor_freq = set_floor_freq_default,1180 .compute_time_window = rapl_compute_time_window_core,1181 .dram_domain_energy_unit = 15300,1182};1183 1184static const struct rapl_defaults rapl_defaults_spr_server = {1185 .check_unit = rapl_check_unit_core,1186 .set_floor_freq = set_floor_freq_default,1187 .compute_time_window = rapl_compute_time_window_core,1188 .psys_domain_energy_unit = 1000000000,1189 .spr_psys_bits = true,1190};1191 1192static const struct rapl_defaults rapl_defaults_byt = {1193 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,1194 .check_unit = rapl_check_unit_atom,1195 .set_floor_freq = set_floor_freq_atom,1196 .compute_time_window = rapl_compute_time_window_atom,1197};1198 1199static const struct rapl_defaults rapl_defaults_tng = {1200 .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,1201 .check_unit = rapl_check_unit_atom,1202 .set_floor_freq = set_floor_freq_atom,1203 .compute_time_window = rapl_compute_time_window_atom,1204};1205 1206static const struct rapl_defaults rapl_defaults_ann = {1207 .floor_freq_reg_addr = 0,1208 .check_unit = rapl_check_unit_atom,1209 .set_floor_freq = NULL,1210 .compute_time_window = rapl_compute_time_window_atom,1211};1212 1213static const struct rapl_defaults rapl_defaults_cht = {1214 .floor_freq_reg_addr = 0,1215 .check_unit = rapl_check_unit_atom,1216 .set_floor_freq = NULL,1217 .compute_time_window = rapl_compute_time_window_atom,1218};1219 1220static const struct rapl_defaults rapl_defaults_amd = {1221 .check_unit = rapl_check_unit_core,1222};1223 1224static const struct x86_cpu_id rapl_ids[] __initconst = {1225 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &rapl_defaults_core),1226 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &rapl_defaults_core),1227 1228 X86_MATCH_VFM(INTEL_IVYBRIDGE, &rapl_defaults_core),1229 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &rapl_defaults_core),1230 1231 X86_MATCH_VFM(INTEL_HASWELL, &rapl_defaults_core),1232 X86_MATCH_VFM(INTEL_HASWELL_L, &rapl_defaults_core),1233 X86_MATCH_VFM(INTEL_HASWELL_G, &rapl_defaults_core),1234 X86_MATCH_VFM(INTEL_HASWELL_X, &rapl_defaults_hsw_server),1235 1236 X86_MATCH_VFM(INTEL_BROADWELL, &rapl_defaults_core),1237 X86_MATCH_VFM(INTEL_BROADWELL_G, &rapl_defaults_core),1238 X86_MATCH_VFM(INTEL_BROADWELL_D, &rapl_defaults_core),1239 X86_MATCH_VFM(INTEL_BROADWELL_X, &rapl_defaults_hsw_server),1240 1241 X86_MATCH_VFM(INTEL_SKYLAKE, &rapl_defaults_core),1242 X86_MATCH_VFM(INTEL_SKYLAKE_L, &rapl_defaults_core),1243 X86_MATCH_VFM(INTEL_SKYLAKE_X, &rapl_defaults_hsw_server),1244 X86_MATCH_VFM(INTEL_KABYLAKE_L, &rapl_defaults_core),1245 X86_MATCH_VFM(INTEL_KABYLAKE, &rapl_defaults_core),1246 X86_MATCH_VFM(INTEL_CANNONLAKE_L, &rapl_defaults_core),1247 X86_MATCH_VFM(INTEL_ICELAKE_L, &rapl_defaults_core),1248 X86_MATCH_VFM(INTEL_ICELAKE, &rapl_defaults_core),1249 X86_MATCH_VFM(INTEL_ICELAKE_NNPI, &rapl_defaults_core),1250 X86_MATCH_VFM(INTEL_ICELAKE_X, &rapl_defaults_hsw_server),1251 X86_MATCH_VFM(INTEL_ICELAKE_D, &rapl_defaults_hsw_server),1252 X86_MATCH_VFM(INTEL_COMETLAKE_L, &rapl_defaults_core),1253 X86_MATCH_VFM(INTEL_COMETLAKE, &rapl_defaults_core),1254 X86_MATCH_VFM(INTEL_TIGERLAKE_L, &rapl_defaults_core),1255 X86_MATCH_VFM(INTEL_TIGERLAKE, &rapl_defaults_core),1256 X86_MATCH_VFM(INTEL_ROCKETLAKE, &rapl_defaults_core),1257 X86_MATCH_VFM(INTEL_ALDERLAKE, &rapl_defaults_core),1258 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &rapl_defaults_core),1259 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &rapl_defaults_core),1260 X86_MATCH_VFM(INTEL_RAPTORLAKE, &rapl_defaults_core),1261 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, &rapl_defaults_core),1262 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &rapl_defaults_core),1263 X86_MATCH_VFM(INTEL_METEORLAKE, &rapl_defaults_core),1264 X86_MATCH_VFM(INTEL_METEORLAKE_L, &rapl_defaults_core),1265 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &rapl_defaults_spr_server),1266 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &rapl_defaults_spr_server),1267 X86_MATCH_VFM(INTEL_LUNARLAKE_M, &rapl_defaults_core),1268 X86_MATCH_VFM(INTEL_ARROWLAKE_H, &rapl_defaults_core),1269 X86_MATCH_VFM(INTEL_ARROWLAKE, &rapl_defaults_core),1270 X86_MATCH_VFM(INTEL_ARROWLAKE_U, &rapl_defaults_core),1271 X86_MATCH_VFM(INTEL_LAKEFIELD, &rapl_defaults_core),1272 1273 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &rapl_defaults_byt),1274 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &rapl_defaults_cht),1275 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &rapl_defaults_tng),1276 X86_MATCH_VFM(INTEL_ATOM_AIRMONT_MID, &rapl_defaults_ann),1277 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &rapl_defaults_core),1278 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &rapl_defaults_core),1279 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &rapl_defaults_core),1280 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &rapl_defaults_core),1281 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &rapl_defaults_core),1282 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &rapl_defaults_core),1283 1284 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &rapl_defaults_hsw_server),1285 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &rapl_defaults_hsw_server),1286 1287 X86_MATCH_VENDOR_FAM(AMD, 0x17, &rapl_defaults_amd),1288 X86_MATCH_VENDOR_FAM(AMD, 0x19, &rapl_defaults_amd),1289 X86_MATCH_VENDOR_FAM(AMD, 0x1A, &rapl_defaults_amd),1290 X86_MATCH_VENDOR_FAM(HYGON, 0x18, &rapl_defaults_amd),1291 {}1292};1293MODULE_DEVICE_TABLE(x86cpu, rapl_ids);1294 1295/* Read once for all raw primitive data for domains */1296static void rapl_update_domain_data(struct rapl_package *rp)1297{1298 int dmn, prim;1299 u64 val;1300 1301 for (dmn = 0; dmn < rp->nr_domains; dmn++) {1302 pr_debug("update %s domain %s data\n", rp->name,1303 rp->domains[dmn].name);1304 /* exclude non-raw primitives */1305 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {1306 struct rapl_primitive_info *rpi = get_rpi(rp, prim);1307 1308 if (!rapl_read_data_raw(&rp->domains[dmn], prim,1309 rpi->unit, &val))1310 rp->domains[dmn].rdd.primitives[prim] = val;1311 }1312 }1313 1314}1315 1316static int rapl_package_register_powercap(struct rapl_package *rp)1317{1318 struct rapl_domain *rd;1319 struct powercap_zone *power_zone = NULL;1320 int nr_pl, ret;1321 1322 /* Update the domain data of the new package */1323 rapl_update_domain_data(rp);1324 1325 /* first we register package domain as the parent zone */1326 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {1327 if (rd->id == RAPL_DOMAIN_PACKAGE) {1328 nr_pl = find_nr_power_limit(rd);1329 pr_debug("register package domain %s\n", rp->name);1330 power_zone = powercap_register_zone(&rd->power_zone,1331 rp->priv->control_type, rp->name,1332 NULL, &zone_ops[rd->id], nr_pl,1333 &constraint_ops);1334 if (IS_ERR(power_zone)) {1335 pr_debug("failed to register power zone %s\n",1336 rp->name);1337 return PTR_ERR(power_zone);1338 }1339 /* track parent zone in per package/socket data */1340 rp->power_zone = power_zone;1341 /* done, only one package domain per socket */1342 break;1343 }1344 }1345 if (!power_zone) {1346 pr_err("no package domain found, unknown topology!\n");1347 return -ENODEV;1348 }1349 /* now register domains as children of the socket/package */1350 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {1351 struct powercap_zone *parent = rp->power_zone;1352 1353 if (rd->id == RAPL_DOMAIN_PACKAGE)1354 continue;1355 if (rd->id == RAPL_DOMAIN_PLATFORM)1356 parent = NULL;1357 /* number of power limits per domain varies */1358 nr_pl = find_nr_power_limit(rd);1359 power_zone = powercap_register_zone(&rd->power_zone,1360 rp->priv->control_type,1361 rd->name, parent,1362 &zone_ops[rd->id], nr_pl,1363 &constraint_ops);1364 1365 if (IS_ERR(power_zone)) {1366 pr_debug("failed to register power_zone, %s:%s\n",1367 rp->name, rd->name);1368 ret = PTR_ERR(power_zone);1369 goto err_cleanup;1370 }1371 }1372 return 0;1373 1374err_cleanup:1375 /*1376 * Clean up previously initialized domains within the package if we1377 * failed after the first domain setup.1378 */1379 while (--rd >= rp->domains) {1380 pr_debug("unregister %s domain %s\n", rp->name, rd->name);1381 powercap_unregister_zone(rp->priv->control_type,1382 &rd->power_zone);1383 }1384 1385 return ret;1386}1387 1388static int rapl_check_domain(int domain, struct rapl_package *rp)1389{1390 struct reg_action ra;1391 1392 switch (domain) {1393 case RAPL_DOMAIN_PACKAGE:1394 case RAPL_DOMAIN_PP0:1395 case RAPL_DOMAIN_PP1:1396 case RAPL_DOMAIN_DRAM:1397 case RAPL_DOMAIN_PLATFORM:1398 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];1399 break;1400 default:1401 pr_err("invalid domain id %d\n", domain);1402 return -EINVAL;1403 }1404 /* make sure domain counters are available and contains non-zero1405 * values, otherwise skip it.1406 */1407 1408 ra.mask = ENERGY_STATUS_MASK;1409 if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value)1410 return -ENODEV;1411 1412 return 0;1413}1414 1415/*1416 * Get per domain energy/power/time unit.1417 * RAPL Interfaces without per domain unit register will use the package1418 * scope unit register to set per domain units.1419 */1420static int rapl_get_domain_unit(struct rapl_domain *rd)1421{1422 struct rapl_defaults *defaults = get_defaults(rd->rp);1423 int ret;1424 1425 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) {1426 if (!rd->rp->priv->reg_unit.val) {1427 pr_err("No valid Unit register found\n");1428 return -ENODEV;1429 }1430 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;1431 }1432 1433 if (!defaults->check_unit) {1434 pr_err("missing .check_unit() callback\n");1435 return -ENODEV;1436 }1437 1438 ret = defaults->check_unit(rd);1439 if (ret)1440 return ret;1441 1442 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit)1443 rd->energy_unit = defaults->dram_domain_energy_unit;1444 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit)1445 rd->energy_unit = defaults->psys_domain_energy_unit;1446 return 0;1447}1448 1449/*1450 * Check if power limits are available. Two cases when they are not available:1451 * 1. Locked by BIOS, in this case we still provide read-only access so that1452 * users can see what limit is set by the BIOS.1453 * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not1454 * exist at all. In this case, we do not show the constraints in powercap.1455 *1456 * Called after domains are detected and initialized.1457 */1458static void rapl_detect_powerlimit(struct rapl_domain *rd)1459{1460 u64 val64;1461 int i;1462 1463 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {1464 if (!rapl_read_pl_data(rd, i, PL_LOCK, false, &val64)) {1465 if (val64) {1466 rd->rpl[i].locked = true;1467 pr_info("%s:%s:%s locked by BIOS\n",1468 rd->rp->name, rd->name, pl_names[i]);1469 }1470 }1471 1472 if (rapl_read_pl_data(rd, i, PL_LIMIT, false, &val64))1473 rd->rpl[i].name = NULL;1474 }1475}1476 1477/* Detect active and valid domains for the given CPU, caller must1478 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.1479 */1480static int rapl_detect_domains(struct rapl_package *rp)1481{1482 struct rapl_domain *rd;1483 int i;1484 1485 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {1486 /* use physical package id to read counters */1487 if (!rapl_check_domain(i, rp)) {1488 rp->domain_map |= 1 << i;1489 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);1490 }1491 }1492 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);1493 if (!rp->nr_domains) {1494 pr_debug("no valid rapl domains found in %s\n", rp->name);1495 return -ENODEV;1496 }1497 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);1498 1499 rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain),1500 GFP_KERNEL);1501 if (!rp->domains)1502 return -ENOMEM;1503 1504 rapl_init_domains(rp);1505 1506 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {1507 rapl_get_domain_unit(rd);1508 rapl_detect_powerlimit(rd);1509 }1510 1511 return 0;1512}1513 1514#ifdef CONFIG_PERF_EVENTS1515 1516/*1517 * Support for RAPL PMU1518 *1519 * Register a PMU if any of the registered RAPL Packages have the requirement1520 * of exposing its energy counters via Perf PMU.1521 *1522 * PMU Name:1523 * power1524 *1525 * Events:1526 * Name Event id RAPL Domain1527 * energy_cores 0x01 RAPL_DOMAIN_PP01528 * energy_pkg 0x02 RAPL_DOMAIN_PACKAGE1529 * energy_ram 0x03 RAPL_DOMAIN_DRAM1530 * energy_gpu 0x04 RAPL_DOMAIN_PP11531 * energy_psys 0x05 RAPL_DOMAIN_PLATFORM1532 *1533 * Unit:1534 * Joules1535 *1536 * Scale:1537 * 2.3283064365386962890625e-101538 * The same RAPL domain in different RAPL Packages may have different1539 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as1540 * the fixed unit for all energy counters, and covert each hardware1541 * counter increase to N times of PMU event counter increases.1542 *1543 * This is fully compatible with the current MSR RAPL PMU. This means that1544 * userspace programs like turbostat can use the same code to handle RAPL Perf1545 * PMU, no matter what RAPL Interface driver (MSR/TPMI, etc) is running1546 * underlying on the platform.1547 *1548 * Note that RAPL Packages can be probed/removed dynamically, and the events1549 * supported by each TPMI RAPL device can be different. Thus the RAPL PMU1550 * support is done on demand, which means1551 * 1. PMU is registered only if it is needed by a RAPL Package. PMU events for1552 * unsupported counters are not exposed.1553 * 2. PMU is unregistered and registered when a new RAPL Package is probed and1554 * supports new counters that are not supported by current PMU.1555 * 3. PMU is unregistered when all registered RAPL Packages don't need PMU.1556 */1557 1558struct rapl_pmu {1559 struct pmu pmu; /* Perf PMU structure */1560 u64 timer_ms; /* Maximum expiration time to avoid counter overflow */1561 unsigned long domain_map; /* Events supported by current registered PMU */1562 bool registered; /* Whether the PMU has been registered or not */1563};1564 1565static struct rapl_pmu rapl_pmu;1566 1567/* PMU helpers */1568 1569static int get_pmu_cpu(struct rapl_package *rp)1570{1571 int cpu;1572 1573 if (!rp->has_pmu)1574 return nr_cpu_ids;1575 1576 /* Only TPMI RAPL is supported for now */1577 if (rp->priv->type != RAPL_IF_TPMI)1578 return nr_cpu_ids;1579 1580 /* TPMI RAPL uses any CPU in the package for PMU */1581 for_each_online_cpu(cpu)1582 if (topology_physical_package_id(cpu) == rp->id)1583 return cpu;1584 1585 return nr_cpu_ids;1586}1587 1588static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu)1589{1590 if (!rp->has_pmu)1591 return false;1592 1593 /* Only TPMI RAPL is supported for now */1594 if (rp->priv->type != RAPL_IF_TPMI)1595 return false;1596 1597 /* TPMI RAPL uses any CPU in the package for PMU */1598 return topology_physical_package_id(cpu) == rp->id;1599}1600 1601static struct rapl_package_pmu_data *event_to_pmu_data(struct perf_event *event)1602{1603 struct rapl_package *rp = event->pmu_private;1604 1605 return &rp->pmu_data;1606}1607 1608/* PMU event callbacks */1609 1610static u64 event_read_counter(struct perf_event *event)1611{1612 struct rapl_package *rp = event->pmu_private;1613 u64 val;1614 int ret;1615 1616 /* Return 0 for unsupported events */1617 if (event->hw.idx < 0)1618 return 0;1619 1620 ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val);1621 1622 /* Return 0 for failed read */1623 if (ret)1624 return 0;1625 1626 return val;1627}1628 1629static void __rapl_pmu_event_start(struct perf_event *event)1630{1631 struct rapl_package_pmu_data *data = event_to_pmu_data(event);1632 1633 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))1634 return;1635 1636 event->hw.state = 0;1637 1638 list_add_tail(&event->active_entry, &data->active_list);1639 1640 local64_set(&event->hw.prev_count, event_read_counter(event));1641 if (++data->n_active == 1)1642 hrtimer_start(&data->hrtimer, data->timer_interval,1643 HRTIMER_MODE_REL_PINNED);1644}1645 1646static void rapl_pmu_event_start(struct perf_event *event, int mode)1647{1648 struct rapl_package_pmu_data *data = event_to_pmu_data(event);1649 unsigned long flags;1650 1651 raw_spin_lock_irqsave(&data->lock, flags);1652 __rapl_pmu_event_start(event);1653 raw_spin_unlock_irqrestore(&data->lock, flags);1654}1655 1656static u64 rapl_event_update(struct perf_event *event)1657{1658 struct hw_perf_event *hwc = &event->hw;1659 struct rapl_package_pmu_data *data = event_to_pmu_data(event);1660 u64 prev_raw_count, new_raw_count;1661 s64 delta, sdelta;1662 1663 /*1664 * Follow the generic code to drain hwc->prev_count.1665 * The loop is not expected to run for multiple times.1666 */1667 prev_raw_count = local64_read(&hwc->prev_count);1668 do {1669 new_raw_count = event_read_counter(event);1670 } while (!local64_try_cmpxchg(&hwc->prev_count,1671 &prev_raw_count, new_raw_count));1672 1673 1674 /*1675 * Now we have the new raw value and have updated the prev1676 * timestamp already. We can now calculate the elapsed delta1677 * (event-)time and add that to the generic event.1678 */1679 delta = new_raw_count - prev_raw_count;1680 1681 /*1682 * Scale delta to smallest unit (2^-32)1683 * users must then scale back: count * 1/(1e9*2^32) to get Joules1684 * or use ldexp(count, -32).1685 * Watts = Joules/Time delta1686 */1687 sdelta = delta * data->scale[event->hw.flags];1688 1689 local64_add(sdelta, &event->count);1690 1691 return new_raw_count;1692}1693 1694static void rapl_pmu_event_stop(struct perf_event *event, int mode)1695{1696 struct rapl_package_pmu_data *data = event_to_pmu_data(event);1697 struct hw_perf_event *hwc = &event->hw;1698 unsigned long flags;1699 1700 raw_spin_lock_irqsave(&data->lock, flags);1701 1702 /* Mark event as deactivated and stopped */1703 if (!(hwc->state & PERF_HES_STOPPED)) {1704 WARN_ON_ONCE(data->n_active <= 0);1705 if (--data->n_active == 0)1706 hrtimer_cancel(&data->hrtimer);1707 1708 list_del(&event->active_entry);1709 1710 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);1711 hwc->state |= PERF_HES_STOPPED;1712 }1713 1714 /* Check if update of sw counter is necessary */1715 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {1716 /*1717 * Drain the remaining delta count out of a event1718 * that we are disabling:1719 */1720 rapl_event_update(event);1721 hwc->state |= PERF_HES_UPTODATE;1722 }1723 1724 raw_spin_unlock_irqrestore(&data->lock, flags);1725}1726 1727static int rapl_pmu_event_add(struct perf_event *event, int mode)1728{1729 struct rapl_package_pmu_data *data = event_to_pmu_data(event);1730 struct hw_perf_event *hwc = &event->hw;1731 unsigned long flags;1732 1733 raw_spin_lock_irqsave(&data->lock, flags);1734 1735 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;1736 1737 if (mode & PERF_EF_START)1738 __rapl_pmu_event_start(event);1739 1740 raw_spin_unlock_irqrestore(&data->lock, flags);1741 1742 return 0;1743}1744 1745static void rapl_pmu_event_del(struct perf_event *event, int flags)1746{1747 rapl_pmu_event_stop(event, PERF_EF_UPDATE);1748}1749 1750/* RAPL PMU event ids, same as shown in sysfs */1751enum perf_rapl_events {1752 PERF_RAPL_PP0 = 1, /* all cores */1753 PERF_RAPL_PKG, /* entire package */1754 PERF_RAPL_RAM, /* DRAM */1755 PERF_RAPL_PP1, /* gpu */1756 PERF_RAPL_PSYS, /* psys */1757 PERF_RAPL_MAX1758};1759#define RAPL_EVENT_MASK GENMASK(7, 0)1760 1761static const int event_to_domain[PERF_RAPL_MAX] = {1762 [PERF_RAPL_PP0] = RAPL_DOMAIN_PP0,1763 [PERF_RAPL_PKG] = RAPL_DOMAIN_PACKAGE,1764 [PERF_RAPL_RAM] = RAPL_DOMAIN_DRAM,1765 [PERF_RAPL_PP1] = RAPL_DOMAIN_PP1,1766 [PERF_RAPL_PSYS] = RAPL_DOMAIN_PLATFORM,1767};1768 1769static int rapl_pmu_event_init(struct perf_event *event)1770{1771 struct rapl_package *pos, *rp = NULL;1772 u64 cfg = event->attr.config & RAPL_EVENT_MASK;1773 int domain, idx;1774 1775 /* Only look at RAPL events */1776 if (event->attr.type != event->pmu->type)1777 return -ENOENT;1778 1779 /* Check for supported events only */1780 if (!cfg || cfg >= PERF_RAPL_MAX)1781 return -EINVAL;1782 1783 if (event->cpu < 0)1784 return -EINVAL;1785 1786 /* Find out which Package the event belongs to */1787 list_for_each_entry(pos, &rapl_packages, plist) {1788 if (is_rp_pmu_cpu(pos, event->cpu)) {1789 rp = pos;1790 break;1791 }1792 }1793 if (!rp)1794 return -ENODEV;1795 1796 /* Find out which RAPL Domain the event belongs to */1797 domain = event_to_domain[cfg];1798 1799 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;1800 event->pmu_private = rp; /* Which package */1801 event->hw.flags = domain; /* Which domain */1802 1803 event->hw.idx = -1;1804 /* Find out the index in rp->domains[] to get domain pointer */1805 for (idx = 0; idx < rp->nr_domains; idx++) {1806 if (rp->domains[idx].id == domain) {1807 event->hw.idx = idx;1808 break;1809 }1810 }1811 1812 return 0;1813}1814 1815static void rapl_pmu_event_read(struct perf_event *event)1816{1817 rapl_event_update(event);1818}1819 1820static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)1821{1822 struct rapl_package_pmu_data *data =1823 container_of(hrtimer, struct rapl_package_pmu_data, hrtimer);1824 struct perf_event *event;1825 unsigned long flags;1826 1827 if (!data->n_active)1828 return HRTIMER_NORESTART;1829 1830 raw_spin_lock_irqsave(&data->lock, flags);1831 1832 list_for_each_entry(event, &data->active_list, active_entry)1833 rapl_event_update(event);1834 1835 raw_spin_unlock_irqrestore(&data->lock, flags);1836 1837 hrtimer_forward_now(hrtimer, data->timer_interval);1838 1839 return HRTIMER_RESTART;1840}1841 1842/* PMU sysfs attributes */1843 1844/*1845 * There are no default events, but we need to create "events" group (with1846 * empty attrs) before updating it with detected events.1847 */1848static struct attribute *attrs_empty[] = {1849 NULL,1850};1851 1852static struct attribute_group pmu_events_group = {1853 .name = "events",1854 .attrs = attrs_empty,1855};1856 1857static ssize_t cpumask_show(struct device *dev,1858 struct device_attribute *attr, char *buf)1859{1860 struct rapl_package *rp;1861 cpumask_var_t cpu_mask;1862 int cpu;1863 int ret;1864 1865 if (!alloc_cpumask_var(&cpu_mask, GFP_KERNEL))1866 return -ENOMEM;1867 1868 cpus_read_lock();1869 1870 cpumask_clear(cpu_mask);1871 1872 /* Choose a cpu for each RAPL Package */1873 list_for_each_entry(rp, &rapl_packages, plist) {1874 cpu = get_pmu_cpu(rp);1875 if (cpu < nr_cpu_ids)1876 cpumask_set_cpu(cpu, cpu_mask);1877 }1878 cpus_read_unlock();1879 1880 ret = cpumap_print_to_pagebuf(true, buf, cpu_mask);1881 1882 free_cpumask_var(cpu_mask);1883 1884 return ret;1885}1886 1887static DEVICE_ATTR_RO(cpumask);1888 1889static struct attribute *pmu_cpumask_attrs[] = {1890 &dev_attr_cpumask.attr,1891 NULL1892};1893 1894static struct attribute_group pmu_cpumask_group = {1895 .attrs = pmu_cpumask_attrs,1896};1897 1898PMU_FORMAT_ATTR(event, "config:0-7");1899static struct attribute *pmu_format_attr[] = {1900 &format_attr_event.attr,1901 NULL1902};1903 1904static struct attribute_group pmu_format_group = {1905 .name = "format",1906 .attrs = pmu_format_attr,1907};1908 1909static const struct attribute_group *pmu_attr_groups[] = {1910 &pmu_events_group,1911 &pmu_cpumask_group,1912 &pmu_format_group,1913 NULL1914};1915 1916#define RAPL_EVENT_ATTR_STR(_name, v, str) \1917static struct perf_pmu_events_attr event_attr_##v = { \1918 .attr = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \1919 .event_str = str, \1920}1921 1922RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");1923RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02");1924RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03");1925RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04");1926RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");1927 1928RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules");1929RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules");1930RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules");1931RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules");1932RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules");1933 1934RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10");1935RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10");1936RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10");1937RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10");1938RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10");1939 1940#define RAPL_EVENT_GROUP(_name, domain) \1941static struct attribute *pmu_attr_##_name[] = { \1942 &event_attr_rapl_##_name.attr.attr, \1943 &event_attr_rapl_unit_##_name.attr.attr, \1944 &event_attr_rapl_scale_##_name.attr.attr, \1945 NULL \1946}; \1947static umode_t is_visible_##_name(struct kobject *kobj, struct attribute *attr, int event) \1948{ \1949 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \1950} \1951static struct attribute_group pmu_group_##_name = { \1952 .name = "events", \1953 .attrs = pmu_attr_##_name, \1954 .is_visible = is_visible_##_name, \1955}1956 1957RAPL_EVENT_GROUP(cores, RAPL_DOMAIN_PP0);1958RAPL_EVENT_GROUP(pkg, RAPL_DOMAIN_PACKAGE);1959RAPL_EVENT_GROUP(ram, RAPL_DOMAIN_DRAM);1960RAPL_EVENT_GROUP(gpu, RAPL_DOMAIN_PP1);1961RAPL_EVENT_GROUP(psys, RAPL_DOMAIN_PLATFORM);1962 1963static const struct attribute_group *pmu_attr_update[] = {1964 &pmu_group_cores,1965 &pmu_group_pkg,1966 &pmu_group_ram,1967 &pmu_group_gpu,1968 &pmu_group_psys,1969 NULL1970};1971 1972static int rapl_pmu_update(struct rapl_package *rp)1973{1974 int ret = 0;1975 1976 /* Return if PMU already covers all events supported by current RAPL Package */1977 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map)))1978 goto end;1979 1980 /* Unregister previous registered PMU */1981 if (rapl_pmu.registered)1982 perf_pmu_unregister(&rapl_pmu.pmu);1983 1984 rapl_pmu.registered = false;1985 rapl_pmu.domain_map |= rp->domain_map;1986 1987 memset(&rapl_pmu.pmu, 0, sizeof(struct pmu));1988 rapl_pmu.pmu.attr_groups = pmu_attr_groups;1989 rapl_pmu.pmu.attr_update = pmu_attr_update;1990 rapl_pmu.pmu.task_ctx_nr = perf_invalid_context;1991 rapl_pmu.pmu.event_init = rapl_pmu_event_init;1992 rapl_pmu.pmu.add = rapl_pmu_event_add;1993 rapl_pmu.pmu.del = rapl_pmu_event_del;1994 rapl_pmu.pmu.start = rapl_pmu_event_start;1995 rapl_pmu.pmu.stop = rapl_pmu_event_stop;1996 rapl_pmu.pmu.read = rapl_pmu_event_read;1997 rapl_pmu.pmu.module = THIS_MODULE;1998 rapl_pmu.pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT;1999 ret = perf_pmu_register(&rapl_pmu.pmu, "power", -1);2000 if (ret) {2001 pr_info("Failed to register PMU\n");2002 return ret;2003 }2004 2005 rapl_pmu.registered = true;2006end:2007 rp->has_pmu = true;2008 return ret;2009}2010 2011int rapl_package_add_pmu(struct rapl_package *rp)2012{2013 struct rapl_package_pmu_data *data = &rp->pmu_data;2014 int idx;2015 2016 if (rp->has_pmu)2017 return -EEXIST;2018 2019 guard(cpus_read_lock)();2020 2021 for (idx = 0; idx < rp->nr_domains; idx++) {2022 struct rapl_domain *rd = &rp->domains[idx];2023 int domain = rd->id;2024 u64 val;2025 2026 if (!test_bit(domain, &rp->domain_map))2027 continue;2028 2029 /*2030 * The RAPL PMU granularity is 2^-32 Joules2031 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase2032 */2033 val = rd->energy_unit * (1ULL << 32);2034 do_div(val, ENERGY_UNIT_SCALE * 1000000);2035 data->scale[domain] = val;2036 2037 if (!rapl_pmu.timer_ms) {2038 struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER);2039 2040 /*2041 * Calculate the timer rate:2042 * Use reference of 200W for scaling the timeout to avoid counter2043 * overflows.2044 *2045 * max_count = rpi->mask >> rpi->shift + 12046 * max_energy_pj = max_count * rd->energy_unit2047 * max_time_sec = (max_energy_pj / 1000000000) / 200w2048 *2049 * rapl_pmu.timer_ms = max_time_sec * 1000 / 22050 */2051 val = (rpi->mask >> rpi->shift) + 1;2052 val *= rd->energy_unit;2053 do_div(val, 1000000 * 200 * 2);2054 rapl_pmu.timer_ms = val;2055 2056 pr_debug("%llu ms overflow timer\n", rapl_pmu.timer_ms);2057 }2058 2059 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]);2060 }2061 2062 /* Initialize per package PMU data */2063 raw_spin_lock_init(&data->lock);2064 INIT_LIST_HEAD(&data->active_list);2065 data->timer_interval = ms_to_ktime(rapl_pmu.timer_ms);2066 hrtimer_init(&data->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);2067 data->hrtimer.function = rapl_hrtimer_handle;2068 2069 return rapl_pmu_update(rp);2070}2071EXPORT_SYMBOL_GPL(rapl_package_add_pmu);2072 2073void rapl_package_remove_pmu(struct rapl_package *rp)2074{2075 struct rapl_package *pos;2076 2077 if (!rp->has_pmu)2078 return;2079 2080 guard(cpus_read_lock)();2081 2082 list_for_each_entry(pos, &rapl_packages, plist) {2083 /* PMU is still needed */2084 if (pos->has_pmu && pos != rp)2085 return;2086 }2087 2088 perf_pmu_unregister(&rapl_pmu.pmu);2089 memset(&rapl_pmu, 0, sizeof(struct rapl_pmu));2090}2091EXPORT_SYMBOL_GPL(rapl_package_remove_pmu);2092#endif2093 2094/* called from CPU hotplug notifier, hotplug lock held */2095void rapl_remove_package_cpuslocked(struct rapl_package *rp)2096{2097 struct rapl_domain *rd, *rd_package = NULL;2098 2099 package_power_limit_irq_restore(rp);2100 2101 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {2102 int i;2103 2104 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {2105 rapl_write_pl_data(rd, i, PL_ENABLE, 0);2106 rapl_write_pl_data(rd, i, PL_CLAMP, 0);2107 }2108 2109 if (rd->id == RAPL_DOMAIN_PACKAGE) {2110 rd_package = rd;2111 continue;2112 }2113 pr_debug("remove package, undo power limit on %s: %s\n",2114 rp->name, rd->name);2115 powercap_unregister_zone(rp->priv->control_type,2116 &rd->power_zone);2117 }2118 /* do parent zone last */2119 powercap_unregister_zone(rp->priv->control_type,2120 &rd_package->power_zone);2121 list_del(&rp->plist);2122 kfree(rp);2123}2124EXPORT_SYMBOL_GPL(rapl_remove_package_cpuslocked);2125 2126void rapl_remove_package(struct rapl_package *rp)2127{2128 guard(cpus_read_lock)();2129 rapl_remove_package_cpuslocked(rp);2130}2131EXPORT_SYMBOL_GPL(rapl_remove_package);2132 2133/*2134 * RAPL Package energy counter scope:2135 * 1. AMD/HYGON platforms use per-PKG package energy counter2136 * 2. For Intel platforms2137 * 2.1 CLX-AP platform has per-DIE package energy counter2138 * 2.2 Other platforms that uses MSR RAPL are single die systems so the2139 * package energy counter can be considered as per-PKG/per-DIE,2140 * here it is considered as per-DIE.2141 * 2.3 New platforms that use TPMI RAPL doesn't care about the2142 * scope because they are not MSR/CPU based.2143 */2144#define rapl_msrs_are_pkg_scope() \2145 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || \2146 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)2147 2148/* caller to ensure CPU hotplug lock is held */2149struct rapl_package *rapl_find_package_domain_cpuslocked(int id, struct rapl_if_priv *priv,2150 bool id_is_cpu)2151{2152 struct rapl_package *rp;2153 int uid;2154 2155 if (id_is_cpu) {2156 uid = rapl_msrs_are_pkg_scope() ?2157 topology_physical_package_id(id) : topology_logical_die_id(id);2158 if (uid < 0) {2159 pr_err("topology_logical_(package/die)_id() returned a negative value");2160 return NULL;2161 }2162 }2163 else2164 uid = id;2165 2166 list_for_each_entry(rp, &rapl_packages, plist) {2167 if (rp->id == uid2168 && rp->priv->control_type == priv->control_type)2169 return rp;2170 }2171 2172 return NULL;2173}2174EXPORT_SYMBOL_GPL(rapl_find_package_domain_cpuslocked);2175 2176struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu)2177{2178 guard(cpus_read_lock)();2179 return rapl_find_package_domain_cpuslocked(id, priv, id_is_cpu);2180}2181EXPORT_SYMBOL_GPL(rapl_find_package_domain);2182 2183/* called from CPU hotplug notifier, hotplug lock held */2184struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *priv, bool id_is_cpu)2185{2186 struct rapl_package *rp;2187 int ret;2188 2189 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);2190 if (!rp)2191 return ERR_PTR(-ENOMEM);2192 2193 if (id_is_cpu) {2194 rp->id = rapl_msrs_are_pkg_scope() ?2195 topology_physical_package_id(id) : topology_logical_die_id(id);2196 if ((int)(rp->id) < 0) {2197 pr_err("topology_logical_(package/die)_id() returned a negative value");2198 return ERR_PTR(-EINVAL);2199 }2200 rp->lead_cpu = id;2201 if (!rapl_msrs_are_pkg_scope() && topology_max_dies_per_package() > 1)2202 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d",2203 topology_physical_package_id(id), topology_die_id(id));2204 else2205 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",2206 topology_physical_package_id(id));2207 } else {2208 rp->id = id;2209 rp->lead_cpu = -1;2210 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id);2211 }2212 2213 rp->priv = priv;2214 ret = rapl_config(rp);2215 if (ret)2216 goto err_free_package;2217 2218 /* check if the package contains valid domains */2219 if (rapl_detect_domains(rp)) {2220 ret = -ENODEV;2221 goto err_free_package;2222 }2223 ret = rapl_package_register_powercap(rp);2224 if (!ret) {2225 INIT_LIST_HEAD(&rp->plist);2226 list_add(&rp->plist, &rapl_packages);2227 return rp;2228 }2229 2230err_free_package:2231 kfree(rp->domains);2232 kfree(rp);2233 return ERR_PTR(ret);2234}2235EXPORT_SYMBOL_GPL(rapl_add_package_cpuslocked);2236 2237struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu)2238{2239 guard(cpus_read_lock)();2240 return rapl_add_package_cpuslocked(id, priv, id_is_cpu);2241}2242EXPORT_SYMBOL_GPL(rapl_add_package);2243 2244static void power_limit_state_save(void)2245{2246 struct rapl_package *rp;2247 struct rapl_domain *rd;2248 int ret, i;2249 2250 cpus_read_lock();2251 list_for_each_entry(rp, &rapl_packages, plist) {2252 if (!rp->power_zone)2253 continue;2254 rd = power_zone_to_rapl_domain(rp->power_zone);2255 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++) {2256 ret = rapl_read_pl_data(rd, i, PL_LIMIT, true,2257 &rd->rpl[i].last_power_limit);2258 if (ret)2259 rd->rpl[i].last_power_limit = 0;2260 }2261 }2262 cpus_read_unlock();2263}2264 2265static void power_limit_state_restore(void)2266{2267 struct rapl_package *rp;2268 struct rapl_domain *rd;2269 int i;2270 2271 cpus_read_lock();2272 list_for_each_entry(rp, &rapl_packages, plist) {2273 if (!rp->power_zone)2274 continue;2275 rd = power_zone_to_rapl_domain(rp->power_zone);2276 for (i = POWER_LIMIT1; i < NR_POWER_LIMITS; i++)2277 if (rd->rpl[i].last_power_limit)2278 rapl_write_pl_data(rd, i, PL_LIMIT,2279 rd->rpl[i].last_power_limit);2280 }2281 cpus_read_unlock();2282}2283 2284static int rapl_pm_callback(struct notifier_block *nb,2285 unsigned long mode, void *_unused)2286{2287 switch (mode) {2288 case PM_SUSPEND_PREPARE:2289 power_limit_state_save();2290 break;2291 case PM_POST_SUSPEND:2292 power_limit_state_restore();2293 break;2294 }2295 return NOTIFY_OK;2296}2297 2298static struct notifier_block rapl_pm_notifier = {2299 .notifier_call = rapl_pm_callback,2300};2301 2302static struct platform_device *rapl_msr_platdev;2303 2304static int __init rapl_init(void)2305{2306 const struct x86_cpu_id *id;2307 int ret;2308 2309 id = x86_match_cpu(rapl_ids);2310 if (id) {2311 defaults_msr = (struct rapl_defaults *)id->driver_data;2312 2313 rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0);2314 if (!rapl_msr_platdev)2315 return -ENOMEM;2316 2317 ret = platform_device_add(rapl_msr_platdev);2318 if (ret) {2319 platform_device_put(rapl_msr_platdev);2320 return ret;2321 }2322 }2323 2324 ret = register_pm_notifier(&rapl_pm_notifier);2325 if (ret && rapl_msr_platdev) {2326 platform_device_del(rapl_msr_platdev);2327 platform_device_put(rapl_msr_platdev);2328 }2329 2330 return ret;2331}2332 2333static void __exit rapl_exit(void)2334{2335 platform_device_unregister(rapl_msr_platdev);2336 unregister_pm_notifier(&rapl_pm_notifier);2337}2338 2339fs_initcall(rapl_init);2340module_exit(rapl_exit);2341 2342MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");2343MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");2344MODULE_LICENSE("GPL v2");2345