2906 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2019 Intel Corporation4 */5 6#include <linux/string_helpers.h>7 8#include <drm/intel/i915_drm.h>9 10#include "display/intel_display.h"11#include "display/intel_display_irq.h"12#include "i915_drv.h"13#include "i915_irq.h"14#include "i915_reg.h"15#include "intel_breadcrumbs.h"16#include "intel_gt.h"17#include "intel_gt_clock_utils.h"18#include "intel_gt_irq.h"19#include "intel_gt_pm.h"20#include "intel_gt_pm_irq.h"21#include "intel_gt_print.h"22#include "intel_gt_regs.h"23#include "intel_mchbar_regs.h"24#include "intel_pcode.h"25#include "intel_rps.h"26#include "vlv_sideband.h"27#include "../../../platform/x86/intel_ips.h"28 29#define BUSY_MAX_EI 20u /* ms */30 31/*32 * Lock protecting IPS related data structures33 */34static DEFINE_SPINLOCK(mchdev_lock);35 36static struct intel_gt *rps_to_gt(struct intel_rps *rps)37{38 return container_of(rps, struct intel_gt, rps);39}40 41static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)42{43 return rps_to_gt(rps)->i915;44}45 46static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)47{48 return rps_to_gt(rps)->uncore;49}50 51static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)52{53 struct intel_gt *gt = rps_to_gt(rps);54 55 return >_to_guc(gt)->slpc;56}57 58static bool rps_uses_slpc(struct intel_rps *rps)59{60 struct intel_gt *gt = rps_to_gt(rps);61 62 return intel_uc_uses_guc_slpc(>->uc);63}64 65static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)66{67 return mask & ~rps->pm_intrmsk_mbz;68}69 70static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)71{72 intel_uncore_write_fw(uncore, reg, val);73}74 75static void rps_timer(struct timer_list *t)76{77 struct intel_rps *rps = from_timer(rps, t, timer);78 struct intel_gt *gt = rps_to_gt(rps);79 struct intel_engine_cs *engine;80 ktime_t dt, last, timestamp;81 enum intel_engine_id id;82 s64 max_busy[3] = {};83 84 timestamp = 0;85 for_each_engine(engine, gt, id) {86 s64 busy;87 int i;88 89 dt = intel_engine_get_busy_time(engine, ×tamp);90 last = engine->stats.rps;91 engine->stats.rps = dt;92 93 busy = ktime_to_ns(ktime_sub(dt, last));94 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {95 if (busy > max_busy[i])96 swap(busy, max_busy[i]);97 }98 }99 last = rps->pm_timestamp;100 rps->pm_timestamp = timestamp;101 102 if (intel_rps_is_active(rps)) {103 s64 busy;104 int i;105 106 dt = ktime_sub(timestamp, last);107 108 /*109 * Our goal is to evaluate each engine independently, so we run110 * at the lowest clocks required to sustain the heaviest111 * workload. However, a task may be split into sequential112 * dependent operations across a set of engines, such that113 * the independent contributions do not account for high load,114 * but overall the task is GPU bound. For example, consider115 * video decode on vcs followed by colour post-processing116 * on vecs, followed by general post-processing on rcs.117 * Since multi-engines being active does imply a single118 * continuous workload across all engines, we hedge our119 * bets by only contributing a factor of the distributed120 * load into our busyness calculation.121 */122 busy = max_busy[0];123 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {124 if (!max_busy[i])125 break;126 127 busy += div_u64(max_busy[i], 1 << i);128 }129 GT_TRACE(gt,130 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",131 busy, (int)div64_u64(100 * busy, dt),132 max_busy[0], max_busy[1], max_busy[2],133 rps->pm_interval);134 135 if (100 * busy > rps->power.up_threshold * dt &&136 rps->cur_freq < rps->max_freq_softlimit) {137 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;138 rps->pm_interval = 1;139 queue_work(gt->i915->unordered_wq, &rps->work);140 } else if (100 * busy < rps->power.down_threshold * dt &&141 rps->cur_freq > rps->min_freq_softlimit) {142 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;143 rps->pm_interval = 1;144 queue_work(gt->i915->unordered_wq, &rps->work);145 } else {146 rps->last_adj = 0;147 }148 149 mod_timer(&rps->timer,150 jiffies + msecs_to_jiffies(rps->pm_interval));151 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);152 }153}154 155static void rps_start_timer(struct intel_rps *rps)156{157 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);158 rps->pm_interval = 1;159 mod_timer(&rps->timer, jiffies + 1);160}161 162static void rps_stop_timer(struct intel_rps *rps)163{164 del_timer_sync(&rps->timer);165 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);166 cancel_work_sync(&rps->work);167}168 169static u32 rps_pm_mask(struct intel_rps *rps, u8 val)170{171 u32 mask = 0;172 173 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */174 if (val > rps->min_freq_softlimit)175 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |176 GEN6_PM_RP_DOWN_THRESHOLD |177 GEN6_PM_RP_DOWN_TIMEOUT);178 179 if (val < rps->max_freq_softlimit)180 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;181 182 mask &= rps->pm_events;183 184 return rps_pm_sanitize_mask(rps, ~mask);185}186 187static void rps_reset_ei(struct intel_rps *rps)188{189 memset(&rps->ei, 0, sizeof(rps->ei));190}191 192static void rps_enable_interrupts(struct intel_rps *rps)193{194 struct intel_gt *gt = rps_to_gt(rps);195 196 GEM_BUG_ON(rps_uses_slpc(rps));197 198 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",199 rps->pm_events, rps_pm_mask(rps, rps->last_freq));200 201 rps_reset_ei(rps);202 203 spin_lock_irq(gt->irq_lock);204 gen6_gt_pm_enable_irq(gt, rps->pm_events);205 spin_unlock_irq(gt->irq_lock);206 207 intel_uncore_write(gt->uncore,208 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));209}210 211static void gen6_rps_reset_interrupts(struct intel_rps *rps)212{213 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);214}215 216static void gen11_rps_reset_interrupts(struct intel_rps *rps)217{218 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))219 ;220}221 222static void rps_reset_interrupts(struct intel_rps *rps)223{224 struct intel_gt *gt = rps_to_gt(rps);225 226 spin_lock_irq(gt->irq_lock);227 if (GRAPHICS_VER(gt->i915) >= 11)228 gen11_rps_reset_interrupts(rps);229 else230 gen6_rps_reset_interrupts(rps);231 232 rps->pm_iir = 0;233 spin_unlock_irq(gt->irq_lock);234}235 236static void rps_disable_interrupts(struct intel_rps *rps)237{238 struct intel_gt *gt = rps_to_gt(rps);239 240 intel_uncore_write(gt->uncore,241 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));242 243 spin_lock_irq(gt->irq_lock);244 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);245 spin_unlock_irq(gt->irq_lock);246 247 intel_synchronize_irq(gt->i915);248 249 /*250 * Now that we will not be generating any more work, flush any251 * outstanding tasks. As we are called on the RPS idle path,252 * we will reset the GPU to minimum frequencies, so the current253 * state of the worker can be discarded.254 */255 cancel_work_sync(&rps->work);256 257 rps_reset_interrupts(rps);258 GT_TRACE(gt, "interrupts:off\n");259}260 261static const struct cparams {262 u16 i;263 u16 t;264 u16 m;265 u16 c;266} cparams[] = {267 { 1, 1333, 301, 28664 },268 { 1, 1067, 294, 24460 },269 { 1, 800, 294, 25192 },270 { 0, 1333, 276, 27605 },271 { 0, 1067, 276, 27605 },272 { 0, 800, 231, 23784 },273};274 275static void gen5_rps_init(struct intel_rps *rps)276{277 struct drm_i915_private *i915 = rps_to_i915(rps);278 struct intel_uncore *uncore = rps_to_uncore(rps);279 u8 fmax, fmin, fstart;280 u32 rgvmodectl;281 int c_m, i;282 283 if (i915->fsb_freq <= 3200000)284 c_m = 0;285 else if (i915->fsb_freq <= 4800000)286 c_m = 1;287 else288 c_m = 2;289 290 for (i = 0; i < ARRAY_SIZE(cparams); i++) {291 if (cparams[i].i == c_m &&292 cparams[i].t == DIV_ROUND_CLOSEST(i915->mem_freq, 1000)) {293 rps->ips.m = cparams[i].m;294 rps->ips.c = cparams[i].c;295 break;296 }297 }298 299 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);300 301 /* Set up min, max, and cur for interrupt handling */302 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;303 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);304 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>305 MEMMODE_FSTART_SHIFT;306 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",307 fmax, fmin, fstart);308 309 rps->min_freq = fmax;310 rps->efficient_freq = fstart;311 rps->max_freq = fmin;312}313 314static unsigned long315__ips_chipset_val(struct intel_ips *ips)316{317 struct intel_uncore *uncore =318 rps_to_uncore(container_of(ips, struct intel_rps, ips));319 unsigned long now = jiffies_to_msecs(jiffies), dt;320 unsigned long result;321 u64 total, delta;322 323 lockdep_assert_held(&mchdev_lock);324 325 /*326 * Prevent division-by-zero if we are asking too fast.327 * Also, we don't get interesting results if we are polling328 * faster than once in 10ms, so just return the saved value329 * in such cases.330 */331 dt = now - ips->last_time1;332 if (dt <= 10)333 return ips->chipset_power;334 335 /* FIXME: handle per-counter overflow */336 total = intel_uncore_read(uncore, DMIEC);337 total += intel_uncore_read(uncore, DDREC);338 total += intel_uncore_read(uncore, CSIEC);339 340 delta = total - ips->last_count1;341 342 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);343 344 ips->last_count1 = total;345 ips->last_time1 = now;346 347 ips->chipset_power = result;348 349 return result;350}351 352static unsigned long ips_mch_val(struct intel_uncore *uncore)353{354 unsigned int m, x, b;355 u32 tsfs;356 357 tsfs = intel_uncore_read(uncore, TSFS);358 x = intel_uncore_read8(uncore, TR1);359 360 b = tsfs & TSFS_INTR_MASK;361 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;362 363 return m * x / 127 - b;364}365 366static int _pxvid_to_vd(u8 pxvid)367{368 if (pxvid == 0)369 return 0;370 371 if (pxvid >= 8 && pxvid < 31)372 pxvid = 31;373 374 return (pxvid + 2) * 125;375}376 377static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)378{379 const int vd = _pxvid_to_vd(pxvid);380 381 if (INTEL_INFO(i915)->is_mobile)382 return max(vd - 1125, 0);383 384 return vd;385}386 387static void __gen5_ips_update(struct intel_ips *ips)388{389 struct intel_uncore *uncore =390 rps_to_uncore(container_of(ips, struct intel_rps, ips));391 u64 now, delta, dt;392 u32 count;393 394 lockdep_assert_held(&mchdev_lock);395 396 now = ktime_get_raw_ns();397 dt = now - ips->last_time2;398 do_div(dt, NSEC_PER_MSEC);399 400 /* Don't divide by 0 */401 if (dt <= 10)402 return;403 404 count = intel_uncore_read(uncore, GFXEC);405 delta = count - ips->last_count2;406 407 ips->last_count2 = count;408 ips->last_time2 = now;409 410 /* More magic constants... */411 ips->gfx_power = div_u64(delta * 1181, dt * 10);412}413 414static void gen5_rps_update(struct intel_rps *rps)415{416 spin_lock_irq(&mchdev_lock);417 __gen5_ips_update(&rps->ips);418 spin_unlock_irq(&mchdev_lock);419}420 421static unsigned int gen5_invert_freq(struct intel_rps *rps,422 unsigned int val)423{424 /* Invert the frequency bin into an ips delay */425 val = rps->max_freq - val;426 val = rps->min_freq + val;427 428 return val;429}430 431static int __gen5_rps_set(struct intel_rps *rps, u8 val)432{433 struct intel_uncore *uncore = rps_to_uncore(rps);434 u16 rgvswctl;435 436 lockdep_assert_held(&mchdev_lock);437 438 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);439 if (rgvswctl & MEMCTL_CMD_STS) {440 drm_dbg(&rps_to_i915(rps)->drm,441 "gpu busy, RCS change rejected\n");442 return -EBUSY; /* still busy with another command */443 }444 445 /* Invert the frequency bin into an ips delay */446 val = gen5_invert_freq(rps, val);447 448 rgvswctl =449 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |450 (val << MEMCTL_FREQ_SHIFT) |451 MEMCTL_SFCAVM;452 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);453 intel_uncore_posting_read16(uncore, MEMSWCTL);454 455 rgvswctl |= MEMCTL_CMD_STS;456 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);457 458 return 0;459}460 461static int gen5_rps_set(struct intel_rps *rps, u8 val)462{463 int err;464 465 spin_lock_irq(&mchdev_lock);466 err = __gen5_rps_set(rps, val);467 spin_unlock_irq(&mchdev_lock);468 469 return err;470}471 472static unsigned long intel_pxfreq(u32 vidfreq)473{474 int div = (vidfreq & 0x3f0000) >> 16;475 int post = (vidfreq & 0x3000) >> 12;476 int pre = (vidfreq & 0x7);477 478 if (!pre)479 return 0;480 481 return div * 133333 / (pre << post);482}483 484static unsigned int init_emon(struct intel_uncore *uncore)485{486 u8 pxw[16];487 int i;488 489 /* Disable to program */490 intel_uncore_write(uncore, ECR, 0);491 intel_uncore_posting_read(uncore, ECR);492 493 /* Program energy weights for various events */494 intel_uncore_write(uncore, SDEW, 0x15040d00);495 intel_uncore_write(uncore, CSIEW0, 0x007f0000);496 intel_uncore_write(uncore, CSIEW1, 0x1e220004);497 intel_uncore_write(uncore, CSIEW2, 0x04000004);498 499 for (i = 0; i < 5; i++)500 intel_uncore_write(uncore, PEW(i), 0);501 for (i = 0; i < 3; i++)502 intel_uncore_write(uncore, DEW(i), 0);503 504 /* Program P-state weights to account for frequency power adjustment */505 for (i = 0; i < 16; i++) {506 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));507 unsigned int freq = intel_pxfreq(pxvidfreq);508 unsigned int vid =509 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;510 unsigned int val;511 512 val = vid * vid * freq / 1000 * 255;513 val /= 127 * 127 * 900;514 515 pxw[i] = val;516 }517 /* Render standby states get 0 weight */518 pxw[14] = 0;519 pxw[15] = 0;520 521 for (i = 0; i < 4; i++) {522 intel_uncore_write(uncore, PXW(i),523 pxw[i * 4 + 0] << 24 |524 pxw[i * 4 + 1] << 16 |525 pxw[i * 4 + 2] << 8 |526 pxw[i * 4 + 3] << 0);527 }528 529 /* Adjust magic regs to magic values (more experimental results) */530 intel_uncore_write(uncore, OGW0, 0);531 intel_uncore_write(uncore, OGW1, 0);532 intel_uncore_write(uncore, EG0, 0x00007f00);533 intel_uncore_write(uncore, EG1, 0x0000000e);534 intel_uncore_write(uncore, EG2, 0x000e0000);535 intel_uncore_write(uncore, EG3, 0x68000300);536 intel_uncore_write(uncore, EG4, 0x42000000);537 intel_uncore_write(uncore, EG5, 0x00140031);538 intel_uncore_write(uncore, EG6, 0);539 intel_uncore_write(uncore, EG7, 0);540 541 for (i = 0; i < 8; i++)542 intel_uncore_write(uncore, PXWL(i), 0);543 544 /* Enable PMON + select events */545 intel_uncore_write(uncore, ECR, 0x80000019);546 547 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;548}549 550static bool gen5_rps_enable(struct intel_rps *rps)551{552 struct drm_i915_private *i915 = rps_to_i915(rps);553 struct intel_uncore *uncore = rps_to_uncore(rps);554 u8 fstart, vstart;555 u32 rgvmodectl;556 557 spin_lock_irq(&mchdev_lock);558 559 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);560 561 /* Enable temp reporting */562 intel_uncore_write16(uncore, PMMISC,563 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);564 intel_uncore_write16(uncore, TSC1,565 intel_uncore_read16(uncore, TSC1) | TSE);566 567 /* 100ms RC evaluation intervals */568 intel_uncore_write(uncore, RCUPEI, 100000);569 intel_uncore_write(uncore, RCDNEI, 100000);570 571 /* Set max/min thresholds to 90ms and 80ms respectively */572 intel_uncore_write(uncore, RCBMAXAVG, 90000);573 intel_uncore_write(uncore, RCBMINAVG, 80000);574 575 intel_uncore_write(uncore, MEMIHYST, 1);576 577 /* Set up min, max, and cur for interrupt handling */578 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>579 MEMMODE_FSTART_SHIFT;580 581 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &582 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;583 584 intel_uncore_write(uncore,585 MEMINTREN,586 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);587 588 intel_uncore_write(uncore, VIDSTART, vstart);589 intel_uncore_posting_read(uncore, VIDSTART);590 591 rgvmodectl |= MEMMODE_SWMODE_EN;592 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);593 594 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &595 MEMCTL_CMD_STS) == 0, 10))596 drm_err(&uncore->i915->drm,597 "stuck trying to change perf mode\n");598 mdelay(1);599 600 __gen5_rps_set(rps, rps->cur_freq);601 602 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);603 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);604 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);605 rps->ips.last_time1 = jiffies_to_msecs(jiffies);606 607 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);608 rps->ips.last_time2 = ktime_get_raw_ns();609 610 spin_lock(&i915->irq_lock);611 ilk_enable_display_irq(i915, DE_PCU_EVENT);612 spin_unlock(&i915->irq_lock);613 614 spin_unlock_irq(&mchdev_lock);615 616 rps->ips.corr = init_emon(uncore);617 618 return true;619}620 621static void gen5_rps_disable(struct intel_rps *rps)622{623 struct drm_i915_private *i915 = rps_to_i915(rps);624 struct intel_uncore *uncore = rps_to_uncore(rps);625 u16 rgvswctl;626 627 spin_lock_irq(&mchdev_lock);628 629 spin_lock(&i915->irq_lock);630 ilk_disable_display_irq(i915, DE_PCU_EVENT);631 spin_unlock(&i915->irq_lock);632 633 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);634 635 /* Ack interrupts, disable EFC interrupt */636 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);637 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);638 639 /* Go back to the starting frequency */640 __gen5_rps_set(rps, rps->idle_freq);641 mdelay(1);642 rgvswctl |= MEMCTL_CMD_STS;643 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);644 mdelay(1);645 646 spin_unlock_irq(&mchdev_lock);647}648 649static u32 rps_limits(struct intel_rps *rps, u8 val)650{651 u32 limits;652 653 /*654 * Only set the down limit when we've reached the lowest level to avoid655 * getting more interrupts, otherwise leave this clear. This prevents a656 * race in the hw when coming out of rc6: There's a tiny window where657 * the hw runs at the minimal clock before selecting the desired658 * frequency, if the down threshold expires in that window we will not659 * receive a down interrupt.660 */661 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {662 limits = rps->max_freq_softlimit << 23;663 if (val <= rps->min_freq_softlimit)664 limits |= rps->min_freq_softlimit << 14;665 } else {666 limits = rps->max_freq_softlimit << 24;667 if (val <= rps->min_freq_softlimit)668 limits |= rps->min_freq_softlimit << 16;669 }670 671 return limits;672}673 674static void rps_set_power(struct intel_rps *rps, int new_power)675{676 struct intel_gt *gt = rps_to_gt(rps);677 struct intel_uncore *uncore = gt->uncore;678 u32 ei_up = 0, ei_down = 0;679 680 lockdep_assert_held(&rps->power.mutex);681 682 if (new_power == rps->power.mode)683 return;684 685 /* Note the units here are not exactly 1us, but 1280ns. */686 switch (new_power) {687 case LOW_POWER:688 ei_up = 16000;689 ei_down = 32000;690 break;691 692 case BETWEEN:693 ei_up = 13000;694 ei_down = 32000;695 break;696 697 case HIGH_POWER:698 ei_up = 10000;699 ei_down = 32000;700 break;701 }702 703 /* When byt can survive without system hang with dynamic704 * sw freq adjustments, this restriction can be lifted.705 */706 if (IS_VALLEYVIEW(gt->i915))707 goto skip_hw_write;708 709 GT_TRACE(gt,710 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",711 new_power,712 rps->power.up_threshold, ei_up,713 rps->power.down_threshold, ei_down);714 715 set(uncore, GEN6_RP_UP_EI,716 intel_gt_ns_to_pm_interval(gt, ei_up * 1000));717 set(uncore, GEN6_RP_UP_THRESHOLD,718 intel_gt_ns_to_pm_interval(gt,719 ei_up * rps->power.up_threshold * 10));720 721 set(uncore, GEN6_RP_DOWN_EI,722 intel_gt_ns_to_pm_interval(gt, ei_down * 1000));723 set(uncore, GEN6_RP_DOWN_THRESHOLD,724 intel_gt_ns_to_pm_interval(gt,725 ei_down *726 rps->power.down_threshold * 10));727 728 set(uncore, GEN6_RP_CONTROL,729 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |730 GEN6_RP_MEDIA_HW_NORMAL_MODE |731 GEN6_RP_MEDIA_IS_GFX |732 GEN6_RP_ENABLE |733 GEN6_RP_UP_BUSY_AVG |734 GEN6_RP_DOWN_IDLE_AVG);735 736skip_hw_write:737 rps->power.mode = new_power;738}739 740static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)741{742 int new_power;743 744 new_power = rps->power.mode;745 switch (rps->power.mode) {746 case LOW_POWER:747 if (val > rps->efficient_freq + 1 &&748 val > rps->cur_freq)749 new_power = BETWEEN;750 break;751 752 case BETWEEN:753 if (val <= rps->efficient_freq &&754 val < rps->cur_freq)755 new_power = LOW_POWER;756 else if (val >= rps->rp0_freq &&757 val > rps->cur_freq)758 new_power = HIGH_POWER;759 break;760 761 case HIGH_POWER:762 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&763 val < rps->cur_freq)764 new_power = BETWEEN;765 break;766 }767 /* Max/min bins are special */768 if (val <= rps->min_freq_softlimit)769 new_power = LOW_POWER;770 if (val >= rps->max_freq_softlimit)771 new_power = HIGH_POWER;772 773 mutex_lock(&rps->power.mutex);774 if (rps->power.interactive)775 new_power = HIGH_POWER;776 rps_set_power(rps, new_power);777 mutex_unlock(&rps->power.mutex);778}779 780void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)781{782 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",783 str_yes_no(interactive));784 785 mutex_lock(&rps->power.mutex);786 if (interactive) {787 if (!rps->power.interactive++ && intel_rps_is_active(rps))788 rps_set_power(rps, HIGH_POWER);789 } else {790 GEM_BUG_ON(!rps->power.interactive);791 rps->power.interactive--;792 }793 mutex_unlock(&rps->power.mutex);794}795 796static int gen6_rps_set(struct intel_rps *rps, u8 val)797{798 struct intel_uncore *uncore = rps_to_uncore(rps);799 struct drm_i915_private *i915 = rps_to_i915(rps);800 u32 swreq;801 802 GEM_BUG_ON(rps_uses_slpc(rps));803 804 if (GRAPHICS_VER(i915) >= 9)805 swreq = GEN9_FREQUENCY(val);806 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))807 swreq = HSW_FREQUENCY(val);808 else809 swreq = (GEN6_FREQUENCY(val) |810 GEN6_OFFSET(0) |811 GEN6_AGGRESSIVE_TURBO);812 set(uncore, GEN6_RPNSWREQ, swreq);813 814 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",815 val, intel_gpu_freq(rps, val), swreq);816 817 return 0;818}819 820static int vlv_rps_set(struct intel_rps *rps, u8 val)821{822 struct drm_i915_private *i915 = rps_to_i915(rps);823 int err;824 825 vlv_punit_get(i915);826 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);827 vlv_punit_put(i915);828 829 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",830 val, intel_gpu_freq(rps, val));831 832 return err;833}834 835static int rps_set(struct intel_rps *rps, u8 val, bool update)836{837 struct drm_i915_private *i915 = rps_to_i915(rps);838 int err;839 840 if (val == rps->last_freq)841 return 0;842 843 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))844 err = vlv_rps_set(rps, val);845 else if (GRAPHICS_VER(i915) >= 6)846 err = gen6_rps_set(rps, val);847 else848 err = gen5_rps_set(rps, val);849 if (err)850 return err;851 852 if (update && GRAPHICS_VER(i915) >= 6)853 gen6_rps_set_thresholds(rps, val);854 rps->last_freq = val;855 856 return 0;857}858 859void intel_rps_unpark(struct intel_rps *rps)860{861 if (!intel_rps_is_enabled(rps))862 return;863 864 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);865 866 /*867 * Use the user's desired frequency as a guide, but for better868 * performance, jump directly to RPe as our starting frequency.869 */870 mutex_lock(&rps->lock);871 872 intel_rps_set_active(rps);873 intel_rps_set(rps,874 clamp(rps->cur_freq,875 rps->min_freq_softlimit,876 rps->max_freq_softlimit));877 878 mutex_unlock(&rps->lock);879 880 rps->pm_iir = 0;881 if (intel_rps_has_interrupts(rps))882 rps_enable_interrupts(rps);883 if (intel_rps_uses_timer(rps))884 rps_start_timer(rps);885 886 if (GRAPHICS_VER(rps_to_i915(rps)) == 5)887 gen5_rps_update(rps);888}889 890void intel_rps_park(struct intel_rps *rps)891{892 int adj;893 894 if (!intel_rps_is_enabled(rps))895 return;896 897 if (!intel_rps_clear_active(rps))898 return;899 900 if (intel_rps_uses_timer(rps))901 rps_stop_timer(rps);902 if (intel_rps_has_interrupts(rps))903 rps_disable_interrupts(rps);904 905 if (rps->last_freq <= rps->idle_freq)906 return;907 908 /*909 * The punit delays the write of the frequency and voltage until it910 * determines the GPU is awake. During normal usage we don't want to911 * waste power changing the frequency if the GPU is sleeping (rc6).912 * However, the GPU and driver is now idle and we do not want to delay913 * switching to minimum voltage (reducing power whilst idle) as we do914 * not expect to be woken in the near future and so must flush the915 * change by waking the device.916 *917 * We choose to take the media powerwell (either would do to trick the918 * punit into committing the voltage change) as that takes a lot less919 * power than the render powerwell.920 */921 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);922 rps_set(rps, rps->idle_freq, false);923 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);924 925 /*926 * Since we will try and restart from the previously requested927 * frequency on unparking, treat this idle point as a downclock928 * interrupt and reduce the frequency for resume. If we park/unpark929 * more frequently than the rps worker can run, we will not respond930 * to any EI and never see a change in frequency.931 *932 * (Note we accommodate Cherryview's limitation of only using an933 * even bin by applying it to all.)934 */935 adj = rps->last_adj;936 if (adj < 0)937 adj *= 2;938 else /* CHV needs even encode values */939 adj = -2;940 rps->last_adj = adj;941 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);942 if (rps->cur_freq < rps->efficient_freq) {943 rps->cur_freq = rps->efficient_freq;944 rps->last_adj = 0;945 }946 947 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);948}949 950u32 intel_rps_get_boost_frequency(struct intel_rps *rps)951{952 struct intel_guc_slpc *slpc;953 954 if (rps_uses_slpc(rps)) {955 slpc = rps_to_slpc(rps);956 957 return slpc->boost_freq;958 } else {959 return intel_gpu_freq(rps, rps->boost_freq);960 }961}962 963static int rps_set_boost_freq(struct intel_rps *rps, u32 val)964{965 bool boost = false;966 967 /* Validate against (static) hardware limits */968 val = intel_freq_opcode(rps, val);969 if (val < rps->min_freq || val > rps->max_freq)970 return -EINVAL;971 972 mutex_lock(&rps->lock);973 if (val != rps->boost_freq) {974 rps->boost_freq = val;975 boost = atomic_read(&rps->num_waiters);976 }977 mutex_unlock(&rps->lock);978 if (boost)979 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);980 981 return 0;982}983 984int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)985{986 struct intel_guc_slpc *slpc;987 988 if (rps_uses_slpc(rps)) {989 slpc = rps_to_slpc(rps);990 991 return intel_guc_slpc_set_boost_freq(slpc, freq);992 } else {993 return rps_set_boost_freq(rps, freq);994 }995}996 997void intel_rps_dec_waiters(struct intel_rps *rps)998{999 struct intel_guc_slpc *slpc;1000 1001 if (rps_uses_slpc(rps)) {1002 slpc = rps_to_slpc(rps);1003 1004 intel_guc_slpc_dec_waiters(slpc);1005 } else {1006 atomic_dec(&rps->num_waiters);1007 }1008}1009 1010void intel_rps_boost(struct i915_request *rq)1011{1012 struct intel_guc_slpc *slpc;1013 1014 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))1015 return;1016 1017 /* Waitboost is not needed for contexts marked with a Freq hint */1018 if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))1019 return;1020 1021 /* Serializes with i915_request_retire() */1022 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {1023 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;1024 1025 if (rps_uses_slpc(rps)) {1026 slpc = rps_to_slpc(rps);1027 1028 if (slpc->min_freq_softlimit >= slpc->boost_freq)1029 return;1030 1031 /* Return if old value is non zero */1032 if (!atomic_fetch_inc(&slpc->num_waiters)) {1033 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",1034 rq->fence.context, rq->fence.seqno);1035 queue_work(rps_to_gt(rps)->i915->unordered_wq,1036 &slpc->boost_work);1037 }1038 1039 return;1040 }1041 1042 if (atomic_fetch_inc(&rps->num_waiters))1043 return;1044 1045 if (!intel_rps_is_active(rps))1046 return;1047 1048 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",1049 rq->fence.context, rq->fence.seqno);1050 1051 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)1052 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);1053 1054 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */1055 }1056}1057 1058int intel_rps_set(struct intel_rps *rps, u8 val)1059{1060 int err;1061 1062 lockdep_assert_held(&rps->lock);1063 GEM_BUG_ON(val > rps->max_freq);1064 GEM_BUG_ON(val < rps->min_freq);1065 1066 if (intel_rps_is_active(rps)) {1067 err = rps_set(rps, val, true);1068 if (err)1069 return err;1070 1071 /*1072 * Make sure we continue to get interrupts1073 * until we hit the minimum or maximum frequencies.1074 */1075 if (intel_rps_has_interrupts(rps)) {1076 struct intel_uncore *uncore = rps_to_uncore(rps);1077 1078 set(uncore,1079 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));1080 1081 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));1082 }1083 }1084 1085 rps->cur_freq = val;1086 return 0;1087}1088 1089static u32 intel_rps_read_state_cap(struct intel_rps *rps)1090{1091 struct drm_i915_private *i915 = rps_to_i915(rps);1092 struct intel_uncore *uncore = rps_to_uncore(rps);1093 1094 if (IS_GEN9_LP(i915))1095 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);1096 else1097 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);1098}1099 1100static void1101mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)1102{1103 struct intel_uncore *uncore = rps_to_uncore(rps);1104 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?1105 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :1106 intel_uncore_read(uncore, MTL_RP_STATE_CAP);1107 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?1108 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :1109 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);1110 1111 /* MTL values are in units of 16.67 MHz */1112 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);1113 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);1114 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);1115}1116 1117static void1118__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)1119{1120 struct drm_i915_private *i915 = rps_to_i915(rps);1121 u32 rp_state_cap;1122 1123 rp_state_cap = intel_rps_read_state_cap(rps);1124 1125 /* static values from HW: RP0 > RP1 > RPn (min_freq) */1126 if (IS_GEN9_LP(i915)) {1127 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;1128 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;1129 caps->min_freq = (rp_state_cap >> 0) & 0xff;1130 } else {1131 caps->rp0_freq = (rp_state_cap >> 0) & 0xff;1132 if (GRAPHICS_VER(i915) >= 10)1133 caps->rp1_freq = REG_FIELD_GET(RPE_MASK,1134 intel_uncore_read(to_gt(i915)->uncore,1135 GEN10_FREQ_INFO_REC));1136 else1137 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;1138 caps->min_freq = (rp_state_cap >> 16) & 0xff;1139 }1140 1141 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {1142 /*1143 * In this case rp_state_cap register reports frequencies in1144 * units of 50 MHz. Convert these to the actual "hw unit", i.e.1145 * units of 16.67 MHz1146 */1147 caps->rp0_freq *= GEN9_FREQ_SCALER;1148 caps->rp1_freq *= GEN9_FREQ_SCALER;1149 caps->min_freq *= GEN9_FREQ_SCALER;1150 }1151}1152 1153/**1154 * gen6_rps_get_freq_caps - Get freq caps exposed by HW1155 * @rps: the intel_rps structure1156 * @caps: returned freq caps1157 *1158 * Returned "caps" frequencies should be converted to MHz using1159 * intel_gpu_freq()1160 */1161void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)1162{1163 struct drm_i915_private *i915 = rps_to_i915(rps);1164 1165 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))1166 return mtl_get_freq_caps(rps, caps);1167 else1168 return __gen6_rps_get_freq_caps(rps, caps);1169}1170 1171static void gen6_rps_init(struct intel_rps *rps)1172{1173 struct drm_i915_private *i915 = rps_to_i915(rps);1174 struct intel_rps_freq_caps caps;1175 1176 gen6_rps_get_freq_caps(rps, &caps);1177 rps->rp0_freq = caps.rp0_freq;1178 rps->rp1_freq = caps.rp1_freq;1179 rps->min_freq = caps.min_freq;1180 1181 /* hw_max = RP0 until we check for overclocking */1182 rps->max_freq = rps->rp0_freq;1183 1184 rps->efficient_freq = rps->rp1_freq;1185 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||1186 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {1187 u32 ddcc_status = 0;1188 u32 mult = 1;1189 1190 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)1191 mult = GEN9_FREQ_SCALER;1192 if (snb_pcode_read(rps_to_gt(rps)->uncore,1193 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,1194 &ddcc_status, NULL) == 0)1195 rps->efficient_freq =1196 clamp_t(u32,1197 ((ddcc_status >> 8) & 0xff) * mult,1198 rps->min_freq,1199 rps->max_freq);1200 }1201}1202 1203static bool rps_reset(struct intel_rps *rps)1204{1205 struct drm_i915_private *i915 = rps_to_i915(rps);1206 1207 /* force a reset */1208 rps->power.mode = -1;1209 rps->last_freq = -1;1210 1211 if (rps_set(rps, rps->min_freq, true)) {1212 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");1213 return false;1214 }1215 1216 rps->cur_freq = rps->min_freq;1217 return true;1218}1219 1220/* See the Gen9_GT_PM_Programming_Guide doc for the below */1221static bool gen9_rps_enable(struct intel_rps *rps)1222{1223 struct intel_gt *gt = rps_to_gt(rps);1224 struct intel_uncore *uncore = gt->uncore;1225 1226 /* Program defaults and thresholds for RPS */1227 if (GRAPHICS_VER(gt->i915) == 9)1228 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,1229 GEN9_FREQUENCY(rps->rp1_freq));1230 1231 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);1232 1233 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;1234 1235 return rps_reset(rps);1236}1237 1238static bool gen8_rps_enable(struct intel_rps *rps)1239{1240 struct intel_uncore *uncore = rps_to_uncore(rps);1241 1242 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,1243 HSW_FREQUENCY(rps->rp1_freq));1244 1245 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);1246 1247 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;1248 1249 return rps_reset(rps);1250}1251 1252static bool gen6_rps_enable(struct intel_rps *rps)1253{1254 struct intel_uncore *uncore = rps_to_uncore(rps);1255 1256 /* Power down if completely idle for over 50ms */1257 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);1258 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);1259 1260 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |1261 GEN6_PM_RP_DOWN_THRESHOLD |1262 GEN6_PM_RP_DOWN_TIMEOUT);1263 1264 return rps_reset(rps);1265}1266 1267static int chv_rps_max_freq(struct intel_rps *rps)1268{1269 struct drm_i915_private *i915 = rps_to_i915(rps);1270 struct intel_gt *gt = rps_to_gt(rps);1271 u32 val;1272 1273 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);1274 1275 switch (gt->info.sseu.eu_total) {1276 case 8:1277 /* (2 * 4) config */1278 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;1279 break;1280 case 12:1281 /* (2 * 6) config */1282 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;1283 break;1284 case 16:1285 /* (2 * 8) config */1286 default:1287 /* Setting (2 * 8) Min RP0 for any other combination */1288 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;1289 break;1290 }1291 1292 return val & FB_GFX_FREQ_FUSE_MASK;1293}1294 1295static int chv_rps_rpe_freq(struct intel_rps *rps)1296{1297 struct drm_i915_private *i915 = rps_to_i915(rps);1298 u32 val;1299 1300 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);1301 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;1302 1303 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;1304}1305 1306static int chv_rps_guar_freq(struct intel_rps *rps)1307{1308 struct drm_i915_private *i915 = rps_to_i915(rps);1309 u32 val;1310 1311 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);1312 1313 return val & FB_GFX_FREQ_FUSE_MASK;1314}1315 1316static u32 chv_rps_min_freq(struct intel_rps *rps)1317{1318 struct drm_i915_private *i915 = rps_to_i915(rps);1319 u32 val;1320 1321 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);1322 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;1323 1324 return val & FB_GFX_FREQ_FUSE_MASK;1325}1326 1327static bool chv_rps_enable(struct intel_rps *rps)1328{1329 struct intel_uncore *uncore = rps_to_uncore(rps);1330 struct drm_i915_private *i915 = rps_to_i915(rps);1331 u32 val;1332 1333 /* 1: Program defaults and thresholds for RPS*/1334 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);1335 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);1336 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);1337 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);1338 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);1339 1340 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);1341 1342 /* 2: Enable RPS */1343 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,1344 GEN6_RP_MEDIA_HW_NORMAL_MODE |1345 GEN6_RP_MEDIA_IS_GFX |1346 GEN6_RP_ENABLE |1347 GEN6_RP_UP_BUSY_AVG |1348 GEN6_RP_DOWN_IDLE_AVG);1349 1350 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |1351 GEN6_PM_RP_DOWN_THRESHOLD |1352 GEN6_PM_RP_DOWN_TIMEOUT);1353 1354 /* Setting Fixed Bias */1355 vlv_punit_get(i915);1356 1357 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;1358 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);1359 1360 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);1361 1362 vlv_punit_put(i915);1363 1364 /* RPS code assumes GPLL is used */1365 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,1366 "GPLL not enabled\n");1367 1368 drm_dbg(&i915->drm, "GPLL enabled? %s\n",1369 str_yes_no(val & GPLLENABLE));1370 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);1371 1372 return rps_reset(rps);1373}1374 1375static int vlv_rps_guar_freq(struct intel_rps *rps)1376{1377 struct drm_i915_private *i915 = rps_to_i915(rps);1378 u32 val, rp1;1379 1380 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);1381 1382 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;1383 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;1384 1385 return rp1;1386}1387 1388static int vlv_rps_max_freq(struct intel_rps *rps)1389{1390 struct drm_i915_private *i915 = rps_to_i915(rps);1391 u32 val, rp0;1392 1393 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);1394 1395 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;1396 /* Clamp to max */1397 rp0 = min_t(u32, rp0, 0xea);1398 1399 return rp0;1400}1401 1402static int vlv_rps_rpe_freq(struct intel_rps *rps)1403{1404 struct drm_i915_private *i915 = rps_to_i915(rps);1405 u32 val, rpe;1406 1407 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);1408 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;1409 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);1410 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;1411 1412 return rpe;1413}1414 1415static int vlv_rps_min_freq(struct intel_rps *rps)1416{1417 struct drm_i915_private *i915 = rps_to_i915(rps);1418 u32 val;1419 1420 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;1421 /*1422 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value1423 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on1424 * a BYT-M B0 the above register contains 0xbf. Moreover when setting1425 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc01426 * to make sure it matches what Punit accepts.1427 */1428 return max_t(u32, val, 0xc0);1429}1430 1431static bool vlv_rps_enable(struct intel_rps *rps)1432{1433 struct intel_uncore *uncore = rps_to_uncore(rps);1434 struct drm_i915_private *i915 = rps_to_i915(rps);1435 u32 val;1436 1437 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);1438 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);1439 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);1440 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);1441 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);1442 1443 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);1444 1445 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,1446 GEN6_RP_MEDIA_TURBO |1447 GEN6_RP_MEDIA_HW_NORMAL_MODE |1448 GEN6_RP_MEDIA_IS_GFX |1449 GEN6_RP_ENABLE |1450 GEN6_RP_UP_BUSY_AVG |1451 GEN6_RP_DOWN_IDLE_CONT);1452 1453 /* WaGsvRC0ResidencyMethod:vlv */1454 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;1455 1456 vlv_punit_get(i915);1457 1458 /* Setting Fixed Bias */1459 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;1460 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);1461 1462 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);1463 1464 vlv_punit_put(i915);1465 1466 /* RPS code assumes GPLL is used */1467 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,1468 "GPLL not enabled\n");1469 1470 drm_dbg(&i915->drm, "GPLL enabled? %s\n",1471 str_yes_no(val & GPLLENABLE));1472 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);1473 1474 return rps_reset(rps);1475}1476 1477static unsigned long __ips_gfx_val(struct intel_ips *ips)1478{1479 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);1480 struct intel_uncore *uncore = rps_to_uncore(rps);1481 unsigned int t, state1, state2;1482 u32 pxvid, ext_v;1483 u64 corr, corr2;1484 1485 lockdep_assert_held(&mchdev_lock);1486 1487 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));1488 pxvid = (pxvid >> 24) & 0x7f;1489 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);1490 1491 state1 = ext_v;1492 1493 /* Revel in the empirically derived constants */1494 1495 /* Correction factor in 1/100000 units */1496 t = ips_mch_val(uncore);1497 if (t > 80)1498 corr = t * 2349 + 135940;1499 else if (t >= 50)1500 corr = t * 964 + 29317;1501 else /* < 50 */1502 corr = t * 301 + 1004;1503 1504 corr = div_u64(corr * 150142 * state1, 10000) - 78642;1505 corr2 = div_u64(corr, 100000) * ips->corr;1506 1507 state2 = div_u64(corr2 * state1, 10000);1508 state2 /= 100; /* convert to mW */1509 1510 __gen5_ips_update(ips);1511 1512 return ips->gfx_power + state2;1513}1514 1515static bool has_busy_stats(struct intel_rps *rps)1516{1517 struct intel_engine_cs *engine;1518 enum intel_engine_id id;1519 1520 for_each_engine(engine, rps_to_gt(rps), id) {1521 if (!intel_engine_supports_stats(engine))1522 return false;1523 }1524 1525 return true;1526}1527 1528void intel_rps_enable(struct intel_rps *rps)1529{1530 struct drm_i915_private *i915 = rps_to_i915(rps);1531 struct intel_uncore *uncore = rps_to_uncore(rps);1532 bool enabled = false;1533 1534 if (!HAS_RPS(i915))1535 return;1536 1537 if (rps_uses_slpc(rps))1538 return;1539 1540 intel_gt_check_clock_frequency(rps_to_gt(rps));1541 1542 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);1543 if (rps->max_freq <= rps->min_freq)1544 /* leave disabled, no room for dynamic reclocking */;1545 else if (IS_CHERRYVIEW(i915))1546 enabled = chv_rps_enable(rps);1547 else if (IS_VALLEYVIEW(i915))1548 enabled = vlv_rps_enable(rps);1549 else if (GRAPHICS_VER(i915) >= 9)1550 enabled = gen9_rps_enable(rps);1551 else if (GRAPHICS_VER(i915) >= 8)1552 enabled = gen8_rps_enable(rps);1553 else if (GRAPHICS_VER(i915) >= 6)1554 enabled = gen6_rps_enable(rps);1555 else if (IS_IRONLAKE_M(i915))1556 enabled = gen5_rps_enable(rps);1557 else1558 MISSING_CASE(GRAPHICS_VER(i915));1559 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);1560 if (!enabled)1561 return;1562 1563 GT_TRACE(rps_to_gt(rps),1564 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",1565 rps->min_freq, rps->max_freq,1566 intel_gpu_freq(rps, rps->min_freq),1567 intel_gpu_freq(rps, rps->max_freq),1568 rps->power.up_threshold,1569 rps->power.down_threshold);1570 1571 GEM_BUG_ON(rps->max_freq < rps->min_freq);1572 GEM_BUG_ON(rps->idle_freq > rps->max_freq);1573 1574 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);1575 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);1576 1577 if (has_busy_stats(rps))1578 intel_rps_set_timer(rps);1579 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)1580 intel_rps_set_interrupts(rps);1581 else1582 /* Ironlake currently uses intel_ips.ko */ {}1583 1584 intel_rps_set_enabled(rps);1585}1586 1587static void gen6_rps_disable(struct intel_rps *rps)1588{1589 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);1590}1591 1592void intel_rps_disable(struct intel_rps *rps)1593{1594 struct drm_i915_private *i915 = rps_to_i915(rps);1595 1596 if (!intel_rps_is_enabled(rps))1597 return;1598 1599 intel_rps_clear_enabled(rps);1600 intel_rps_clear_interrupts(rps);1601 intel_rps_clear_timer(rps);1602 1603 if (GRAPHICS_VER(i915) >= 6)1604 gen6_rps_disable(rps);1605 else if (IS_IRONLAKE_M(i915))1606 gen5_rps_disable(rps);1607}1608 1609static int byt_gpu_freq(struct intel_rps *rps, int val)1610{1611 /*1612 * N = val - 0xb71613 * Slow = Fast = GPLL ref * N1614 */1615 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);1616}1617 1618static int byt_freq_opcode(struct intel_rps *rps, int val)1619{1620 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;1621}1622 1623static int chv_gpu_freq(struct intel_rps *rps, int val)1624{1625 /*1626 * N = val / 21627 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 21628 */1629 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);1630}1631 1632static int chv_freq_opcode(struct intel_rps *rps, int val)1633{1634 /* CHV needs even values */1635 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;1636}1637 1638int intel_gpu_freq(struct intel_rps *rps, int val)1639{1640 struct drm_i915_private *i915 = rps_to_i915(rps);1641 1642 if (GRAPHICS_VER(i915) >= 9)1643 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,1644 GEN9_FREQ_SCALER);1645 else if (IS_CHERRYVIEW(i915))1646 return chv_gpu_freq(rps, val);1647 else if (IS_VALLEYVIEW(i915))1648 return byt_gpu_freq(rps, val);1649 else if (GRAPHICS_VER(i915) >= 6)1650 return val * GT_FREQUENCY_MULTIPLIER;1651 else1652 return val;1653}1654 1655int intel_freq_opcode(struct intel_rps *rps, int val)1656{1657 struct drm_i915_private *i915 = rps_to_i915(rps);1658 1659 if (GRAPHICS_VER(i915) >= 9)1660 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,1661 GT_FREQUENCY_MULTIPLIER);1662 else if (IS_CHERRYVIEW(i915))1663 return chv_freq_opcode(rps, val);1664 else if (IS_VALLEYVIEW(i915))1665 return byt_freq_opcode(rps, val);1666 else if (GRAPHICS_VER(i915) >= 6)1667 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);1668 else1669 return val;1670}1671 1672static void vlv_init_gpll_ref_freq(struct intel_rps *rps)1673{1674 struct drm_i915_private *i915 = rps_to_i915(rps);1675 1676 rps->gpll_ref_freq =1677 vlv_get_cck_clock(i915, "GPLL ref",1678 CCK_GPLL_CLOCK_CONTROL,1679 i915->czclk_freq);1680 1681 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",1682 rps->gpll_ref_freq);1683}1684 1685static void vlv_rps_init(struct intel_rps *rps)1686{1687 struct drm_i915_private *i915 = rps_to_i915(rps);1688 1689 vlv_iosf_sb_get(i915,1690 BIT(VLV_IOSF_SB_PUNIT) |1691 BIT(VLV_IOSF_SB_NC) |1692 BIT(VLV_IOSF_SB_CCK));1693 1694 vlv_init_gpll_ref_freq(rps);1695 1696 rps->max_freq = vlv_rps_max_freq(rps);1697 rps->rp0_freq = rps->max_freq;1698 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",1699 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);1700 1701 rps->efficient_freq = vlv_rps_rpe_freq(rps);1702 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",1703 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);1704 1705 rps->rp1_freq = vlv_rps_guar_freq(rps);1706 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",1707 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);1708 1709 rps->min_freq = vlv_rps_min_freq(rps);1710 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",1711 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);1712 1713 vlv_iosf_sb_put(i915,1714 BIT(VLV_IOSF_SB_PUNIT) |1715 BIT(VLV_IOSF_SB_NC) |1716 BIT(VLV_IOSF_SB_CCK));1717}1718 1719static void chv_rps_init(struct intel_rps *rps)1720{1721 struct drm_i915_private *i915 = rps_to_i915(rps);1722 1723 vlv_iosf_sb_get(i915,1724 BIT(VLV_IOSF_SB_PUNIT) |1725 BIT(VLV_IOSF_SB_NC) |1726 BIT(VLV_IOSF_SB_CCK));1727 1728 vlv_init_gpll_ref_freq(rps);1729 1730 rps->max_freq = chv_rps_max_freq(rps);1731 rps->rp0_freq = rps->max_freq;1732 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",1733 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);1734 1735 rps->efficient_freq = chv_rps_rpe_freq(rps);1736 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",1737 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);1738 1739 rps->rp1_freq = chv_rps_guar_freq(rps);1740 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",1741 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);1742 1743 rps->min_freq = chv_rps_min_freq(rps);1744 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",1745 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);1746 1747 vlv_iosf_sb_put(i915,1748 BIT(VLV_IOSF_SB_PUNIT) |1749 BIT(VLV_IOSF_SB_NC) |1750 BIT(VLV_IOSF_SB_CCK));1751 1752 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |1753 rps->rp1_freq | rps->min_freq) & 1,1754 "Odd GPU freq values\n");1755}1756 1757static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)1758{1759 ei->ktime = ktime_get_raw();1760 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);1761 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);1762}1763 1764static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)1765{1766 struct intel_uncore *uncore = rps_to_uncore(rps);1767 const struct intel_rps_ei *prev = &rps->ei;1768 struct intel_rps_ei now;1769 u32 events = 0;1770 1771 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)1772 return 0;1773 1774 vlv_c0_read(uncore, &now);1775 1776 if (prev->ktime) {1777 u64 time, c0;1778 u32 render, media;1779 1780 time = ktime_us_delta(now.ktime, prev->ktime);1781 1782 time *= rps_to_i915(rps)->czclk_freq;1783 1784 /* Workload can be split between render + media,1785 * e.g. SwapBuffers being blitted in X after being rendered in1786 * mesa. To account for this we need to combine both engines1787 * into our activity counter.1788 */1789 render = now.render_c0 - prev->render_c0;1790 media = now.media_c0 - prev->media_c0;1791 c0 = max(render, media);1792 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */1793 1794 if (c0 > time * rps->power.up_threshold)1795 events = GEN6_PM_RP_UP_THRESHOLD;1796 else if (c0 < time * rps->power.down_threshold)1797 events = GEN6_PM_RP_DOWN_THRESHOLD;1798 }1799 1800 rps->ei = now;1801 return events;1802}1803 1804static void rps_work(struct work_struct *work)1805{1806 struct intel_rps *rps = container_of(work, typeof(*rps), work);1807 struct intel_gt *gt = rps_to_gt(rps);1808 struct drm_i915_private *i915 = rps_to_i915(rps);1809 bool client_boost = false;1810 int new_freq, adj, min, max;1811 u32 pm_iir = 0;1812 1813 spin_lock_irq(gt->irq_lock);1814 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;1815 client_boost = atomic_read(&rps->num_waiters);1816 spin_unlock_irq(gt->irq_lock);1817 1818 /* Make sure we didn't queue anything we're not going to process. */1819 if (!pm_iir && !client_boost)1820 goto out;1821 1822 mutex_lock(&rps->lock);1823 if (!intel_rps_is_active(rps)) {1824 mutex_unlock(&rps->lock);1825 return;1826 }1827 1828 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);1829 1830 adj = rps->last_adj;1831 new_freq = rps->cur_freq;1832 min = rps->min_freq_softlimit;1833 max = rps->max_freq_softlimit;1834 if (client_boost)1835 max = rps->max_freq;1836 1837 GT_TRACE(gt,1838 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",1839 pm_iir, str_yes_no(client_boost),1840 adj, new_freq, min, max);1841 1842 if (client_boost && new_freq < rps->boost_freq) {1843 new_freq = rps->boost_freq;1844 adj = 0;1845 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {1846 if (adj > 0)1847 adj *= 2;1848 else /* CHV needs even encode values */1849 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;1850 1851 if (new_freq >= rps->max_freq_softlimit)1852 adj = 0;1853 } else if (client_boost) {1854 adj = 0;1855 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {1856 if (rps->cur_freq > rps->efficient_freq)1857 new_freq = rps->efficient_freq;1858 else if (rps->cur_freq > rps->min_freq_softlimit)1859 new_freq = rps->min_freq_softlimit;1860 adj = 0;1861 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {1862 if (adj < 0)1863 adj *= 2;1864 else /* CHV needs even encode values */1865 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;1866 1867 if (new_freq <= rps->min_freq_softlimit)1868 adj = 0;1869 } else { /* unknown event */1870 adj = 0;1871 }1872 1873 /*1874 * sysfs frequency limits may have snuck in while1875 * servicing the interrupt1876 */1877 new_freq += adj;1878 new_freq = clamp_t(int, new_freq, min, max);1879 1880 if (intel_rps_set(rps, new_freq)) {1881 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");1882 adj = 0;1883 }1884 rps->last_adj = adj;1885 1886 mutex_unlock(&rps->lock);1887 1888out:1889 spin_lock_irq(gt->irq_lock);1890 gen6_gt_pm_unmask_irq(gt, rps->pm_events);1891 spin_unlock_irq(gt->irq_lock);1892}1893 1894void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)1895{1896 struct intel_gt *gt = rps_to_gt(rps);1897 const u32 events = rps->pm_events & pm_iir;1898 1899 lockdep_assert_held(gt->irq_lock);1900 1901 if (unlikely(!events))1902 return;1903 1904 GT_TRACE(gt, "irq events:%x\n", events);1905 1906 gen6_gt_pm_mask_irq(gt, events);1907 1908 rps->pm_iir |= events;1909 queue_work(gt->i915->unordered_wq, &rps->work);1910}1911 1912void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)1913{1914 struct intel_gt *gt = rps_to_gt(rps);1915 u32 events;1916 1917 events = pm_iir & rps->pm_events;1918 if (events) {1919 spin_lock(gt->irq_lock);1920 1921 GT_TRACE(gt, "irq events:%x\n", events);1922 1923 gen6_gt_pm_mask_irq(gt, events);1924 rps->pm_iir |= events;1925 1926 queue_work(gt->i915->unordered_wq, &rps->work);1927 spin_unlock(gt->irq_lock);1928 }1929 1930 if (GRAPHICS_VER(gt->i915) >= 8)1931 return;1932 1933 if (pm_iir & PM_VEBOX_USER_INTERRUPT)1934 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);1935 1936 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)1937 drm_dbg(&rps_to_i915(rps)->drm,1938 "Command parser error, pm_iir 0x%08x\n", pm_iir);1939}1940 1941void gen5_rps_irq_handler(struct intel_rps *rps)1942{1943 struct intel_uncore *uncore = rps_to_uncore(rps);1944 u32 busy_up, busy_down, max_avg, min_avg;1945 u8 new_freq;1946 1947 spin_lock(&mchdev_lock);1948 1949 intel_uncore_write16(uncore,1950 MEMINTRSTS,1951 intel_uncore_read(uncore, MEMINTRSTS));1952 1953 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);1954 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);1955 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);1956 max_avg = intel_uncore_read(uncore, RCBMAXAVG);1957 min_avg = intel_uncore_read(uncore, RCBMINAVG);1958 1959 /* Handle RCS change request from hw */1960 new_freq = rps->cur_freq;1961 if (busy_up > max_avg)1962 new_freq++;1963 else if (busy_down < min_avg)1964 new_freq--;1965 new_freq = clamp(new_freq,1966 rps->min_freq_softlimit,1967 rps->max_freq_softlimit);1968 1969 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))1970 rps->cur_freq = new_freq;1971 1972 spin_unlock(&mchdev_lock);1973}1974 1975void intel_rps_init_early(struct intel_rps *rps)1976{1977 mutex_init(&rps->lock);1978 mutex_init(&rps->power.mutex);1979 1980 INIT_WORK(&rps->work, rps_work);1981 timer_setup(&rps->timer, rps_timer, 0);1982 1983 atomic_set(&rps->num_waiters, 0);1984}1985 1986void intel_rps_init(struct intel_rps *rps)1987{1988 struct drm_i915_private *i915 = rps_to_i915(rps);1989 1990 if (rps_uses_slpc(rps))1991 return;1992 1993 if (IS_CHERRYVIEW(i915))1994 chv_rps_init(rps);1995 else if (IS_VALLEYVIEW(i915))1996 vlv_rps_init(rps);1997 else if (GRAPHICS_VER(i915) >= 6)1998 gen6_rps_init(rps);1999 else if (IS_IRONLAKE_M(i915))2000 gen5_rps_init(rps);2001 2002 /* Derive initial user preferences/limits from the hardware limits */2003 rps->max_freq_softlimit = rps->max_freq;2004 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;2005 rps->min_freq_softlimit = rps->min_freq;2006 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;2007 2008 /* After setting max-softlimit, find the overclock max freq */2009 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {2010 u32 params = 0;2011 2012 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL);2013 if (params & BIT(31)) { /* OC supported */2014 drm_dbg(&i915->drm,2015 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",2016 (rps->max_freq & 0xff) * 50,2017 (params & 0xff) * 50);2018 rps->max_freq = params & 0xff;2019 }2020 }2021 2022 /* Set default thresholds in % */2023 rps->power.up_threshold = 95;2024 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;2025 rps->power.down_threshold = 85;2026 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;2027 2028 /* Finally allow us to boost to max by default */2029 rps->boost_freq = rps->max_freq;2030 rps->idle_freq = rps->min_freq;2031 2032 /* Start in the middle, from here we will autotune based on workload */2033 rps->cur_freq = rps->efficient_freq;2034 2035 rps->pm_intrmsk_mbz = 0;2036 2037 /*2038 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer2039 * if GEN6_PM_UP_EI_EXPIRED is masked.2040 *2041 * TODO: verify if this can be reproduced on VLV,CHV.2042 */2043 if (GRAPHICS_VER(i915) <= 7)2044 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;2045 2046 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)2047 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;2048 2049 /* GuC needs ARAT expired interrupt unmasked */2050 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))2051 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;2052}2053 2054void intel_rps_sanitize(struct intel_rps *rps)2055{2056 if (rps_uses_slpc(rps))2057 return;2058 2059 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)2060 rps_disable_interrupts(rps);2061}2062 2063u32 intel_rps_read_rpstat(struct intel_rps *rps)2064{2065 struct drm_i915_private *i915 = rps_to_i915(rps);2066 i915_reg_t rpstat;2067 2068 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;2069 2070 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);2071}2072 2073static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)2074{2075 struct drm_i915_private *i915 = rps_to_i915(rps);2076 u32 cagf;2077 2078 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))2079 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);2080 else if (GRAPHICS_VER(i915) >= 12)2081 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);2082 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))2083 cagf = REG_FIELD_GET(RPE_MASK, rpstat);2084 else if (GRAPHICS_VER(i915) >= 9)2085 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);2086 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))2087 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);2088 else if (GRAPHICS_VER(i915) >= 6)2089 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);2090 else2091 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));2092 2093 return cagf;2094}2095 2096static u32 __read_cagf(struct intel_rps *rps, bool take_fw)2097{2098 struct drm_i915_private *i915 = rps_to_i915(rps);2099 struct intel_uncore *uncore = rps_to_uncore(rps);2100 i915_reg_t r = INVALID_MMIO_REG;2101 u32 freq;2102 2103 /*2104 * For Gen12+ reading freq from HW does not need a forcewake and2105 * registers will return 0 freq when GT is in RC62106 */2107 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {2108 r = MTL_MIRROR_TARGET_WP1;2109 } else if (GRAPHICS_VER(i915) >= 12) {2110 r = GEN12_RPSTAT1;2111 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {2112 vlv_punit_get(i915);2113 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);2114 vlv_punit_put(i915);2115 } else if (GRAPHICS_VER(i915) >= 6) {2116 r = GEN6_RPSTAT1;2117 } else {2118 r = MEMSTAT_ILK;2119 }2120 2121 if (i915_mmio_reg_valid(r))2122 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);2123 2124 return intel_rps_get_cagf(rps, freq);2125}2126 2127static u32 read_cagf(struct intel_rps *rps)2128{2129 return __read_cagf(rps, true);2130}2131 2132u32 intel_rps_read_actual_frequency(struct intel_rps *rps)2133{2134 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;2135 intel_wakeref_t wakeref;2136 u32 freq = 0;2137 2138 with_intel_runtime_pm_if_in_use(rpm, wakeref)2139 freq = intel_gpu_freq(rps, read_cagf(rps));2140 2141 return freq;2142}2143 2144u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)2145{2146 return intel_gpu_freq(rps, __read_cagf(rps, false));2147}2148 2149static u32 intel_rps_read_punit_req(struct intel_rps *rps)2150{2151 struct intel_uncore *uncore = rps_to_uncore(rps);2152 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;2153 intel_wakeref_t wakeref;2154 u32 freq = 0;2155 2156 with_intel_runtime_pm_if_in_use(rpm, wakeref)2157 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);2158 2159 return freq;2160}2161 2162static u32 intel_rps_get_req(u32 pureq)2163{2164 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;2165 2166 return req;2167}2168 2169u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)2170{2171 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));2172 2173 return intel_gpu_freq(rps, freq);2174}2175 2176u32 intel_rps_get_requested_frequency(struct intel_rps *rps)2177{2178 if (rps_uses_slpc(rps))2179 return intel_rps_read_punit_req_frequency(rps);2180 else2181 return intel_gpu_freq(rps, rps->cur_freq);2182}2183 2184u32 intel_rps_get_max_frequency(struct intel_rps *rps)2185{2186 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2187 2188 if (rps_uses_slpc(rps))2189 return slpc->max_freq_softlimit;2190 else2191 return intel_gpu_freq(rps, rps->max_freq_softlimit);2192}2193 2194/**2195 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.2196 * @rps: the intel_rps structure2197 *2198 * Returns the max frequency in a raw format. In newer platforms raw is in2199 * units of 50 MHz.2200 */2201u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)2202{2203 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2204 u32 freq;2205 2206 if (rps_uses_slpc(rps)) {2207 return DIV_ROUND_CLOSEST(slpc->rp0_freq,2208 GT_FREQUENCY_MULTIPLIER);2209 } else {2210 freq = rps->max_freq;2211 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {2212 /* Convert GT frequency to 50 MHz units */2213 freq /= GEN9_FREQ_SCALER;2214 }2215 return freq;2216 }2217}2218 2219u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)2220{2221 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2222 2223 if (rps_uses_slpc(rps))2224 return slpc->rp0_freq;2225 else2226 return intel_gpu_freq(rps, rps->rp0_freq);2227}2228 2229u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)2230{2231 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2232 2233 if (rps_uses_slpc(rps))2234 return slpc->rp1_freq;2235 else2236 return intel_gpu_freq(rps, rps->rp1_freq);2237}2238 2239u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)2240{2241 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2242 2243 if (rps_uses_slpc(rps))2244 return slpc->min_freq;2245 else2246 return intel_gpu_freq(rps, rps->min_freq);2247}2248 2249static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)2250{2251 struct intel_gt *gt = rps_to_gt(rps);2252 struct drm_i915_private *i915 = gt->i915;2253 struct intel_uncore *uncore = gt->uncore;2254 struct intel_rps_freq_caps caps;2255 u32 rp_state_limits;2256 u32 gt_perf_status;2257 u32 rpmodectl, rpinclimit, rpdeclimit;2258 u32 rpstat, cagf, reqf;2259 u32 rpcurupei, rpcurup, rpprevup;2260 u32 rpcurdownei, rpcurdown, rpprevdown;2261 u32 rpupei, rpupt, rpdownei, rpdownt;2262 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;2263 2264 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);2265 gen6_rps_get_freq_caps(rps, &caps);2266 if (IS_GEN9_LP(i915))2267 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);2268 else2269 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);2270 2271 /* RPSTAT1 is in the GT power well */2272 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);2273 2274 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);2275 if (GRAPHICS_VER(i915) >= 9) {2276 reqf >>= 23;2277 } else {2278 reqf &= ~GEN6_TURBO_DISABLE;2279 if (IS_HASWELL(i915) || IS_BROADWELL(i915))2280 reqf >>= 24;2281 else2282 reqf >>= 25;2283 }2284 reqf = intel_gpu_freq(rps, reqf);2285 2286 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);2287 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);2288 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);2289 2290 rpstat = intel_rps_read_rpstat(rps);2291 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;2292 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;2293 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;2294 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;2295 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;2296 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;2297 2298 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);2299 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);2300 2301 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);2302 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);2303 2304 cagf = intel_rps_read_actual_frequency(rps);2305 2306 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);2307 2308 if (GRAPHICS_VER(i915) >= 11) {2309 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);2310 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);2311 /*2312 * The equivalent to the PM ISR & IIR cannot be read2313 * without affecting the current state of the system2314 */2315 pm_isr = 0;2316 pm_iir = 0;2317 } else if (GRAPHICS_VER(i915) >= 8) {2318 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));2319 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));2320 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));2321 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));2322 } else {2323 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);2324 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);2325 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);2326 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);2327 }2328 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);2329 2330 drm_printf(p, "Video Turbo Mode: %s\n",2331 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));2332 drm_printf(p, "HW control enabled: %s\n",2333 str_yes_no(rpmodectl & GEN6_RP_ENABLE));2334 drm_printf(p, "SW control enabled: %s\n",2335 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));2336 2337 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",2338 pm_ier, pm_imr, pm_mask);2339 if (GRAPHICS_VER(i915) <= 10)2340 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",2341 pm_isr, pm_iir);2342 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",2343 rps->pm_intrmsk_mbz);2344 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);2345 drm_printf(p, "Render p-state ratio: %d\n",2346 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);2347 drm_printf(p, "Render p-state VID: %d\n",2348 gt_perf_status & 0xff);2349 drm_printf(p, "Render p-state limit: %d\n",2350 rp_state_limits & 0xff);2351 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);2352 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);2353 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);2354 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);2355 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);2356 drm_printf(p, "CAGF: %dMHz\n", cagf);2357 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",2358 rpcurupei,2359 intel_gt_pm_interval_to_ns(gt, rpcurupei));2360 drm_printf(p, "RP CUR UP: %d (%lldns)\n",2361 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));2362 drm_printf(p, "RP PREV UP: %d (%lldns)\n",2363 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));2364 drm_printf(p, "Up threshold: %d%%\n",2365 rps->power.up_threshold);2366 drm_printf(p, "RP UP EI: %d (%lldns)\n",2367 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));2368 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",2369 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));2370 2371 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",2372 rpcurdownei,2373 intel_gt_pm_interval_to_ns(gt, rpcurdownei));2374 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",2375 rpcurdown,2376 intel_gt_pm_interval_to_ns(gt, rpcurdown));2377 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",2378 rpprevdown,2379 intel_gt_pm_interval_to_ns(gt, rpprevdown));2380 drm_printf(p, "Down threshold: %d%%\n",2381 rps->power.down_threshold);2382 drm_printf(p, "RP DOWN EI: %d (%lldns)\n",2383 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));2384 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",2385 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));2386 2387 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",2388 intel_gpu_freq(rps, caps.min_freq));2389 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",2390 intel_gpu_freq(rps, caps.rp1_freq));2391 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",2392 intel_gpu_freq(rps, caps.rp0_freq));2393 drm_printf(p, "Max overclocked frequency: %dMHz\n",2394 intel_gpu_freq(rps, rps->max_freq));2395 2396 drm_printf(p, "Current freq: %d MHz\n",2397 intel_gpu_freq(rps, rps->cur_freq));2398 drm_printf(p, "Actual freq: %d MHz\n", cagf);2399 drm_printf(p, "Idle freq: %d MHz\n",2400 intel_gpu_freq(rps, rps->idle_freq));2401 drm_printf(p, "Min freq: %d MHz\n",2402 intel_gpu_freq(rps, rps->min_freq));2403 drm_printf(p, "Boost freq: %d MHz\n",2404 intel_gpu_freq(rps, rps->boost_freq));2405 drm_printf(p, "Max freq: %d MHz\n",2406 intel_gpu_freq(rps, rps->max_freq));2407 drm_printf(p,2408 "efficient (RPe) frequency: %d MHz\n",2409 intel_gpu_freq(rps, rps->efficient_freq));2410}2411 2412static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)2413{2414 struct intel_gt *gt = rps_to_gt(rps);2415 struct intel_uncore *uncore = gt->uncore;2416 struct intel_rps_freq_caps caps;2417 u32 pm_mask;2418 2419 gen6_rps_get_freq_caps(rps, &caps);2420 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);2421 2422 drm_printf(p, "PM MASK=0x%08x\n", pm_mask);2423 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",2424 rps->pm_intrmsk_mbz);2425 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));2426 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));2427 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",2428 intel_gpu_freq(rps, caps.min_freq));2429 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",2430 intel_gpu_freq(rps, caps.rp1_freq));2431 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",2432 intel_gpu_freq(rps, caps.rp0_freq));2433 drm_printf(p, "Current freq: %d MHz\n",2434 intel_rps_get_requested_frequency(rps));2435 drm_printf(p, "Actual freq: %d MHz\n",2436 intel_rps_read_actual_frequency(rps));2437 drm_printf(p, "Min freq: %d MHz\n",2438 intel_rps_get_min_frequency(rps));2439 drm_printf(p, "Boost freq: %d MHz\n",2440 intel_rps_get_boost_frequency(rps));2441 drm_printf(p, "Max freq: %d MHz\n",2442 intel_rps_get_max_frequency(rps));2443 drm_printf(p,2444 "efficient (RPe) frequency: %d MHz\n",2445 intel_gpu_freq(rps, caps.rp1_freq));2446}2447 2448void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)2449{2450 if (rps_uses_slpc(rps))2451 return slpc_frequency_dump(rps, p);2452 else2453 return rps_frequency_dump(rps, p);2454}2455 2456static int set_max_freq(struct intel_rps *rps, u32 val)2457{2458 struct drm_i915_private *i915 = rps_to_i915(rps);2459 int ret = 0;2460 2461 mutex_lock(&rps->lock);2462 2463 val = intel_freq_opcode(rps, val);2464 if (val < rps->min_freq ||2465 val > rps->max_freq ||2466 val < rps->min_freq_softlimit) {2467 ret = -EINVAL;2468 goto unlock;2469 }2470 2471 if (val > rps->rp0_freq)2472 drm_dbg(&i915->drm, "User requested overclocking to %d\n",2473 intel_gpu_freq(rps, val));2474 2475 rps->max_freq_softlimit = val;2476 2477 val = clamp_t(int, rps->cur_freq,2478 rps->min_freq_softlimit,2479 rps->max_freq_softlimit);2480 2481 /*2482 * We still need *_set_rps to process the new max_delay and2483 * update the interrupt limits and PMINTRMSK even though2484 * frequency request may be unchanged.2485 */2486 intel_rps_set(rps, val);2487 2488unlock:2489 mutex_unlock(&rps->lock);2490 2491 return ret;2492}2493 2494int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)2495{2496 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2497 2498 if (rps_uses_slpc(rps))2499 return intel_guc_slpc_set_max_freq(slpc, val);2500 else2501 return set_max_freq(rps, val);2502}2503 2504u32 intel_rps_get_min_frequency(struct intel_rps *rps)2505{2506 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2507 2508 if (rps_uses_slpc(rps))2509 return slpc->min_freq_softlimit;2510 else2511 return intel_gpu_freq(rps, rps->min_freq_softlimit);2512}2513 2514/**2515 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.2516 * @rps: the intel_rps structure2517 *2518 * Returns the min frequency in a raw format. In newer platforms raw is in2519 * units of 50 MHz.2520 */2521u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)2522{2523 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2524 u32 freq;2525 2526 if (rps_uses_slpc(rps)) {2527 return DIV_ROUND_CLOSEST(slpc->min_freq,2528 GT_FREQUENCY_MULTIPLIER);2529 } else {2530 freq = rps->min_freq;2531 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {2532 /* Convert GT frequency to 50 MHz units */2533 freq /= GEN9_FREQ_SCALER;2534 }2535 return freq;2536 }2537}2538 2539static int set_min_freq(struct intel_rps *rps, u32 val)2540{2541 int ret = 0;2542 2543 mutex_lock(&rps->lock);2544 2545 val = intel_freq_opcode(rps, val);2546 if (val < rps->min_freq ||2547 val > rps->max_freq ||2548 val > rps->max_freq_softlimit) {2549 ret = -EINVAL;2550 goto unlock;2551 }2552 2553 rps->min_freq_softlimit = val;2554 2555 val = clamp_t(int, rps->cur_freq,2556 rps->min_freq_softlimit,2557 rps->max_freq_softlimit);2558 2559 /*2560 * We still need *_set_rps to process the new min_delay and2561 * update the interrupt limits and PMINTRMSK even though2562 * frequency request may be unchanged.2563 */2564 intel_rps_set(rps, val);2565 2566unlock:2567 mutex_unlock(&rps->lock);2568 2569 return ret;2570}2571 2572int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)2573{2574 struct intel_guc_slpc *slpc = rps_to_slpc(rps);2575 2576 if (rps_uses_slpc(rps))2577 return intel_guc_slpc_set_min_freq(slpc, val);2578 else2579 return set_min_freq(rps, val);2580}2581 2582u8 intel_rps_get_up_threshold(struct intel_rps *rps)2583{2584 return rps->power.up_threshold;2585}2586 2587static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)2588{2589 int ret;2590 2591 if (val > 100)2592 return -EINVAL;2593 2594 ret = mutex_lock_interruptible(&rps->lock);2595 if (ret)2596 return ret;2597 2598 if (*threshold == val)2599 goto out_unlock;2600 2601 *threshold = val;2602 2603 /* Force reset. */2604 rps->last_freq = -1;2605 mutex_lock(&rps->power.mutex);2606 rps->power.mode = -1;2607 mutex_unlock(&rps->power.mutex);2608 2609 intel_rps_set(rps, clamp(rps->cur_freq,2610 rps->min_freq_softlimit,2611 rps->max_freq_softlimit));2612 2613out_unlock:2614 mutex_unlock(&rps->lock);2615 2616 return ret;2617}2618 2619int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)2620{2621 return rps_set_threshold(rps, &rps->power.up_threshold, threshold);2622}2623 2624u8 intel_rps_get_down_threshold(struct intel_rps *rps)2625{2626 return rps->power.down_threshold;2627}2628 2629int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)2630{2631 return rps_set_threshold(rps, &rps->power.down_threshold, threshold);2632}2633 2634static void intel_rps_set_manual(struct intel_rps *rps, bool enable)2635{2636 struct intel_uncore *uncore = rps_to_uncore(rps);2637 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;2638 2639 /* Allow punit to process software requests */2640 intel_uncore_write(uncore, GEN6_RP_CONTROL, state);2641}2642 2643void intel_rps_raise_unslice(struct intel_rps *rps)2644{2645 struct intel_uncore *uncore = rps_to_uncore(rps);2646 2647 mutex_lock(&rps->lock);2648 2649 if (rps_uses_slpc(rps)) {2650 /* RP limits have not been initialized yet for SLPC path */2651 struct intel_rps_freq_caps caps;2652 2653 gen6_rps_get_freq_caps(rps, &caps);2654 2655 intel_rps_set_manual(rps, true);2656 intel_uncore_write(uncore, GEN6_RPNSWREQ,2657 ((caps.rp0_freq <<2658 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |2659 GEN9_IGNORE_SLICE_RATIO));2660 intel_rps_set_manual(rps, false);2661 } else {2662 intel_rps_set(rps, rps->rp0_freq);2663 }2664 2665 mutex_unlock(&rps->lock);2666}2667 2668void intel_rps_lower_unslice(struct intel_rps *rps)2669{2670 struct intel_uncore *uncore = rps_to_uncore(rps);2671 2672 mutex_lock(&rps->lock);2673 2674 if (rps_uses_slpc(rps)) {2675 /* RP limits have not been initialized yet for SLPC path */2676 struct intel_rps_freq_caps caps;2677 2678 gen6_rps_get_freq_caps(rps, &caps);2679 2680 intel_rps_set_manual(rps, true);2681 intel_uncore_write(uncore, GEN6_RPNSWREQ,2682 ((caps.min_freq <<2683 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |2684 GEN9_IGNORE_SLICE_RATIO));2685 intel_rps_set_manual(rps, false);2686 } else {2687 intel_rps_set(rps, rps->min_freq);2688 }2689 2690 mutex_unlock(&rps->lock);2691}2692 2693static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)2694{2695 struct intel_gt *gt = rps_to_gt(rps);2696 intel_wakeref_t wakeref;2697 u32 val;2698 2699 with_intel_runtime_pm(gt->uncore->rpm, wakeref)2700 val = intel_uncore_read(gt->uncore, reg32);2701 2702 return val;2703}2704 2705bool rps_read_mask_mmio(struct intel_rps *rps,2706 i915_reg_t reg32, u32 mask)2707{2708 return rps_read_mmio(rps, reg32) & mask;2709}2710 2711/* External interface for intel_ips.ko */2712 2713static struct drm_i915_private __rcu *ips_mchdev;2714 2715/*2716 * Tells the intel_ips driver that the i915 driver is now loaded, if2717 * IPS got loaded first.2718 *2719 * This awkward dance is so that neither module has to depend on the2720 * other in order for IPS to do the appropriate communication of2721 * GPU turbo limits to i915.2722 */2723static void2724ips_ping_for_i915_load(void)2725{2726 void (*link)(void);2727 2728 link = symbol_get(ips_link_to_i915_driver);2729 if (link) {2730 link();2731 symbol_put(ips_link_to_i915_driver);2732 }2733}2734 2735void intel_rps_driver_register(struct intel_rps *rps)2736{2737 struct intel_gt *gt = rps_to_gt(rps);2738 2739 /*2740 * We only register the i915 ips part with intel-ips once everything is2741 * set up, to avoid intel-ips sneaking in and reading bogus values.2742 */2743 if (GRAPHICS_VER(gt->i915) == 5) {2744 GEM_BUG_ON(ips_mchdev);2745 rcu_assign_pointer(ips_mchdev, gt->i915);2746 ips_ping_for_i915_load();2747 }2748}2749 2750void intel_rps_driver_unregister(struct intel_rps *rps)2751{2752 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))2753 rcu_assign_pointer(ips_mchdev, NULL);2754}2755 2756static struct drm_i915_private *mchdev_get(void)2757{2758 struct drm_i915_private *i915;2759 2760 rcu_read_lock();2761 i915 = rcu_dereference(ips_mchdev);2762 if (i915 && !kref_get_unless_zero(&i915->drm.ref))2763 i915 = NULL;2764 rcu_read_unlock();2765 2766 return i915;2767}2768 2769/**2770 * i915_read_mch_val - return value for IPS use2771 *2772 * Calculate and return a value for the IPS driver to use when deciding whether2773 * we have thermal and power headroom to increase CPU or GPU power budget.2774 */2775unsigned long i915_read_mch_val(void)2776{2777 struct drm_i915_private *i915;2778 unsigned long chipset_val = 0;2779 unsigned long graphics_val = 0;2780 intel_wakeref_t wakeref;2781 2782 i915 = mchdev_get();2783 if (!i915)2784 return 0;2785 2786 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {2787 struct intel_ips *ips = &to_gt(i915)->rps.ips;2788 2789 spin_lock_irq(&mchdev_lock);2790 chipset_val = __ips_chipset_val(ips);2791 graphics_val = __ips_gfx_val(ips);2792 spin_unlock_irq(&mchdev_lock);2793 }2794 2795 drm_dev_put(&i915->drm);2796 return chipset_val + graphics_val;2797}2798EXPORT_SYMBOL_GPL(i915_read_mch_val);2799 2800/**2801 * i915_gpu_raise - raise GPU frequency limit2802 *2803 * Raise the limit; IPS indicates we have thermal headroom.2804 */2805bool i915_gpu_raise(void)2806{2807 struct drm_i915_private *i915;2808 struct intel_rps *rps;2809 2810 i915 = mchdev_get();2811 if (!i915)2812 return false;2813 2814 rps = &to_gt(i915)->rps;2815 2816 spin_lock_irq(&mchdev_lock);2817 if (rps->max_freq_softlimit < rps->max_freq)2818 rps->max_freq_softlimit++;2819 spin_unlock_irq(&mchdev_lock);2820 2821 drm_dev_put(&i915->drm);2822 return true;2823}2824EXPORT_SYMBOL_GPL(i915_gpu_raise);2825 2826/**2827 * i915_gpu_lower - lower GPU frequency limit2828 *2829 * IPS indicates we're close to a thermal limit, so throttle back the GPU2830 * frequency maximum.2831 */2832bool i915_gpu_lower(void)2833{2834 struct drm_i915_private *i915;2835 struct intel_rps *rps;2836 2837 i915 = mchdev_get();2838 if (!i915)2839 return false;2840 2841 rps = &to_gt(i915)->rps;2842 2843 spin_lock_irq(&mchdev_lock);2844 if (rps->max_freq_softlimit > rps->min_freq)2845 rps->max_freq_softlimit--;2846 spin_unlock_irq(&mchdev_lock);2847 2848 drm_dev_put(&i915->drm);2849 return true;2850}2851EXPORT_SYMBOL_GPL(i915_gpu_lower);2852 2853/**2854 * i915_gpu_busy - indicate GPU business to IPS2855 *2856 * Tell the IPS driver whether or not the GPU is busy.2857 */2858bool i915_gpu_busy(void)2859{2860 struct drm_i915_private *i915;2861 bool ret;2862 2863 i915 = mchdev_get();2864 if (!i915)2865 return false;2866 2867 ret = to_gt(i915)->awake;2868 2869 drm_dev_put(&i915->drm);2870 return ret;2871}2872EXPORT_SYMBOL_GPL(i915_gpu_busy);2873 2874/**2875 * i915_gpu_turbo_disable - disable graphics turbo2876 *2877 * Disable graphics turbo by resetting the max frequency and setting the2878 * current frequency to the default.2879 */2880bool i915_gpu_turbo_disable(void)2881{2882 struct drm_i915_private *i915;2883 struct intel_rps *rps;2884 bool ret;2885 2886 i915 = mchdev_get();2887 if (!i915)2888 return false;2889 2890 rps = &to_gt(i915)->rps;2891 2892 spin_lock_irq(&mchdev_lock);2893 rps->max_freq_softlimit = rps->min_freq;2894 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);2895 spin_unlock_irq(&mchdev_lock);2896 2897 drm_dev_put(&i915->drm);2898 return ret;2899}2900EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);2901 2902#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)2903#include "selftest_rps.c"2904#include "selftest_slpc.c"2905#endif2906