481 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2020 Intel Corporation4 */5 6#include <linux/string_helpers.h>7#include <linux/kernel.h>8 9#include <drm/drm_print.h>10 11#include "i915_drv.h"12#include "i915_reg.h"13#include "i915_trace.h"14#include "i915_utils.h"15#include "intel_clock_gating.h"16#include "vlv_suspend.h"17 18#include "gt/intel_gt_regs.h"19 20struct vlv_s0ix_state {21 /* GAM */22 u32 wr_watermark;23 u32 gfx_prio_ctrl;24 u32 arb_mode;25 u32 gfx_pend_tlb0;26 u32 gfx_pend_tlb1;27 u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];28 u32 media_max_req_count;29 u32 gfx_max_req_count;30 u32 render_hwsp;31 u32 ecochk;32 u32 bsd_hwsp;33 u32 blt_hwsp;34 u32 tlb_rd_addr;35 36 /* MBC */37 u32 g3dctl;38 u32 gsckgctl;39 u32 mbctl;40 41 /* GCP */42 u32 ucgctl1;43 u32 ucgctl3;44 u32 rcgctl1;45 u32 rcgctl2;46 u32 rstctl;47 u32 misccpctl;48 49 /* GPM */50 u32 gfxpause;51 u32 rpdeuhwtc;52 u32 rpdeuc;53 u32 ecobus;54 u32 pwrdwnupctl;55 u32 rp_down_timeout;56 u32 rp_deucsw;57 u32 rcubmabdtmr;58 u32 rcedata;59 u32 spare2gh;60 61 /* Display 1 CZ domain */62 u32 gt_imr;63 u32 gt_ier;64 u32 pm_imr;65 u32 pm_ier;66 u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];67 68 /* GT SA CZ domain */69 u32 tilectl;70 u32 gt_fifoctl;71 u32 gtlc_wake_ctrl;72 u32 gtlc_survive;73 u32 pmwgicz;74 75 /* Display 2 CZ domain */76 u32 gu_ctl0;77 u32 gu_ctl1;78 u32 pcbr;79 u32 clock_gate_dis2;80};81 82/*83 * Save all Gunit registers that may be lost after a D3 and a subsequent84 * S0i[R123] transition. The list of registers needing a save/restore is85 * defined in the VLV2_S0IXRegs document. This documents marks all Gunit86 * registers in the following way:87 * - Driver: saved/restored by the driver88 * - Punit : saved/restored by the Punit firmware89 * - No, w/o marking: no need to save/restore, since the register is R/O or90 * used internally by the HW in a way that doesn't depend91 * keeping the content across a suspend/resume.92 * - Debug : used for debugging93 *94 * We save/restore all registers marked with 'Driver', with the following95 * exceptions:96 * - Registers out of use, including also registers marked with 'Debug'.97 * These have no effect on the driver's operation, so we don't save/restore98 * them to reduce the overhead.99 * - Registers that are fully setup by an initialization function called from100 * the resume path. For example many clock gating and RPS/RC6 registers.101 * - Registers that provide the right functionality with their reset defaults.102 *103 * TODO: Except for registers that based on the above 3 criteria can be safely104 * ignored, we save/restore all others, practically treating the HW context as105 * a black-box for the driver. Further investigation is needed to reduce the106 * saved/restored registers even further, by following the same 3 criteria.107 */108static void vlv_save_gunit_s0ix_state(struct drm_i915_private *i915)109{110 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;111 struct intel_uncore *uncore = &i915->uncore;112 int i;113 114 if (!s)115 return;116 117 /* GAM 0x4000-0x4770 */118 s->wr_watermark = intel_uncore_read(uncore, GEN7_WR_WATERMARK);119 s->gfx_prio_ctrl = intel_uncore_read(uncore, GEN7_GFX_PRIO_CTRL);120 s->arb_mode = intel_uncore_read(uncore, ARB_MODE);121 s->gfx_pend_tlb0 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB0);122 s->gfx_pend_tlb1 = intel_uncore_read(uncore, GEN7_GFX_PEND_TLB1);123 124 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)125 s->lra_limits[i] = intel_uncore_read(uncore, GEN7_LRA_LIMITS(i));126 127 s->media_max_req_count = intel_uncore_read(uncore, GEN7_MEDIA_MAX_REQ_COUNT);128 s->gfx_max_req_count = intel_uncore_read(uncore, GEN7_GFX_MAX_REQ_COUNT);129 130 s->render_hwsp = intel_uncore_read(uncore, RENDER_HWS_PGA_GEN7);131 s->ecochk = intel_uncore_read(uncore, GAM_ECOCHK);132 s->bsd_hwsp = intel_uncore_read(uncore, BSD_HWS_PGA_GEN7);133 s->blt_hwsp = intel_uncore_read(uncore, BLT_HWS_PGA_GEN7);134 135 s->tlb_rd_addr = intel_uncore_read(uncore, GEN7_TLB_RD_ADDR);136 137 /* MBC 0x9024-0x91D0, 0x8500 */138 s->g3dctl = intel_uncore_read(uncore, VLV_G3DCTL);139 s->gsckgctl = intel_uncore_read(uncore, VLV_GSCKGCTL);140 s->mbctl = intel_uncore_read(uncore, GEN6_MBCTL);141 142 /* GCP 0x9400-0x9424, 0x8100-0x810C */143 s->ucgctl1 = intel_uncore_read(uncore, GEN6_UCGCTL1);144 s->ucgctl3 = intel_uncore_read(uncore, GEN6_UCGCTL3);145 s->rcgctl1 = intel_uncore_read(uncore, GEN6_RCGCTL1);146 s->rcgctl2 = intel_uncore_read(uncore, GEN6_RCGCTL2);147 s->rstctl = intel_uncore_read(uncore, GEN6_RSTCTL);148 s->misccpctl = intel_uncore_read(uncore, GEN7_MISCCPCTL);149 150 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */151 s->gfxpause = intel_uncore_read(uncore, GEN6_GFXPAUSE);152 s->rpdeuhwtc = intel_uncore_read(uncore, GEN6_RPDEUHWTC);153 s->rpdeuc = intel_uncore_read(uncore, GEN6_RPDEUC);154 s->ecobus = intel_uncore_read(uncore, ECOBUS);155 s->pwrdwnupctl = intel_uncore_read(uncore, VLV_PWRDWNUPCTL);156 s->rp_down_timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_TIMEOUT);157 s->rp_deucsw = intel_uncore_read(uncore, GEN6_RPDEUCSW);158 s->rcubmabdtmr = intel_uncore_read(uncore, GEN6_RCUBMABDTMR);159 s->rcedata = intel_uncore_read(uncore, VLV_RCEDATA);160 s->spare2gh = intel_uncore_read(uncore, VLV_SPAREG2H);161 162 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */163 s->gt_imr = intel_uncore_read(uncore, GTIMR);164 s->gt_ier = intel_uncore_read(uncore, GTIER);165 s->pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);166 s->pm_ier = intel_uncore_read(uncore, GEN6_PMIER);167 168 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)169 s->gt_scratch[i] = intel_uncore_read(uncore, GEN7_GT_SCRATCH(i));170 171 /* GT SA CZ domain, 0x100000-0x138124 */172 s->tilectl = intel_uncore_read(uncore, TILECTL);173 s->gt_fifoctl = intel_uncore_read(uncore, GTFIFOCTL);174 s->gtlc_wake_ctrl = intel_uncore_read(uncore, VLV_GTLC_WAKE_CTRL);175 s->gtlc_survive = intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG);176 s->pmwgicz = intel_uncore_read(uncore, VLV_PMWGICZ);177 178 /* Gunit-Display CZ domain, 0x182028-0x1821CF */179 s->gu_ctl0 = intel_uncore_read(uncore, VLV_GU_CTL0);180 s->gu_ctl1 = intel_uncore_read(uncore, VLV_GU_CTL1);181 s->pcbr = intel_uncore_read(uncore, VLV_PCBR);182 s->clock_gate_dis2 = intel_uncore_read(uncore, VLV_GUNIT_CLOCK_GATE2);183 184 /*185 * Not saving any of:186 * DFT, 0x9800-0x9EC0187 * SARB, 0xB000-0xB1FC188 * GAC, 0x5208-0x524C, 0x14000-0x14C000189 * PCI CFG190 */191}192 193static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *i915)194{195 struct vlv_s0ix_state *s = i915->vlv_s0ix_state;196 struct intel_uncore *uncore = &i915->uncore;197 int i;198 199 if (!s)200 return;201 202 /* GAM 0x4000-0x4770 */203 intel_uncore_write(uncore, GEN7_WR_WATERMARK, s->wr_watermark);204 intel_uncore_write(uncore, GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl);205 intel_uncore_write(uncore, ARB_MODE, s->arb_mode | (0xffff << 16));206 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0);207 intel_uncore_write(uncore, GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1);208 209 for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)210 intel_uncore_write(uncore, GEN7_LRA_LIMITS(i), s->lra_limits[i]);211 212 intel_uncore_write(uncore, GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);213 intel_uncore_write(uncore, GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);214 215 intel_uncore_write(uncore, RENDER_HWS_PGA_GEN7, s->render_hwsp);216 intel_uncore_write(uncore, GAM_ECOCHK, s->ecochk);217 intel_uncore_write(uncore, BSD_HWS_PGA_GEN7, s->bsd_hwsp);218 intel_uncore_write(uncore, BLT_HWS_PGA_GEN7, s->blt_hwsp);219 220 intel_uncore_write(uncore, GEN7_TLB_RD_ADDR, s->tlb_rd_addr);221 222 /* MBC 0x9024-0x91D0, 0x8500 */223 intel_uncore_write(uncore, VLV_G3DCTL, s->g3dctl);224 intel_uncore_write(uncore, VLV_GSCKGCTL, s->gsckgctl);225 intel_uncore_write(uncore, GEN6_MBCTL, s->mbctl);226 227 /* GCP 0x9400-0x9424, 0x8100-0x810C */228 intel_uncore_write(uncore, GEN6_UCGCTL1, s->ucgctl1);229 intel_uncore_write(uncore, GEN6_UCGCTL3, s->ucgctl3);230 intel_uncore_write(uncore, GEN6_RCGCTL1, s->rcgctl1);231 intel_uncore_write(uncore, GEN6_RCGCTL2, s->rcgctl2);232 intel_uncore_write(uncore, GEN6_RSTCTL, s->rstctl);233 intel_uncore_write(uncore, GEN7_MISCCPCTL, s->misccpctl);234 235 /* GPM 0xA000-0xAA84, 0x8000-0x80FC */236 intel_uncore_write(uncore, GEN6_GFXPAUSE, s->gfxpause);237 intel_uncore_write(uncore, GEN6_RPDEUHWTC, s->rpdeuhwtc);238 intel_uncore_write(uncore, GEN6_RPDEUC, s->rpdeuc);239 intel_uncore_write(uncore, ECOBUS, s->ecobus);240 intel_uncore_write(uncore, VLV_PWRDWNUPCTL, s->pwrdwnupctl);241 intel_uncore_write(uncore, GEN6_RP_DOWN_TIMEOUT, s->rp_down_timeout);242 intel_uncore_write(uncore, GEN6_RPDEUCSW, s->rp_deucsw);243 intel_uncore_write(uncore, GEN6_RCUBMABDTMR, s->rcubmabdtmr);244 intel_uncore_write(uncore, VLV_RCEDATA, s->rcedata);245 intel_uncore_write(uncore, VLV_SPAREG2H, s->spare2gh);246 247 /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */248 intel_uncore_write(uncore, GTIMR, s->gt_imr);249 intel_uncore_write(uncore, GTIER, s->gt_ier);250 intel_uncore_write(uncore, GEN6_PMIMR, s->pm_imr);251 intel_uncore_write(uncore, GEN6_PMIER, s->pm_ier);252 253 for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)254 intel_uncore_write(uncore, GEN7_GT_SCRATCH(i), s->gt_scratch[i]);255 256 /* GT SA CZ domain, 0x100000-0x138124 */257 intel_uncore_write(uncore, TILECTL, s->tilectl);258 intel_uncore_write(uncore, GTFIFOCTL, s->gt_fifoctl);259 /*260 * Preserve the GT allow wake and GFX force clock bit, they are not261 * be restored, as they are used to control the s0ix suspend/resume262 * sequence by the caller.263 */264 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, ~VLV_GTLC_ALLOWWAKEREQ,265 s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ);266 267 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, ~VLV_GFX_CLK_FORCE_ON_BIT,268 s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT);269 270 intel_uncore_write(uncore, VLV_PMWGICZ, s->pmwgicz);271 272 /* Gunit-Display CZ domain, 0x182028-0x1821CF */273 intel_uncore_write(uncore, VLV_GU_CTL0, s->gu_ctl0);274 intel_uncore_write(uncore, VLV_GU_CTL1, s->gu_ctl1);275 intel_uncore_write(uncore, VLV_PCBR, s->pcbr);276 intel_uncore_write(uncore, VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2);277}278 279static int vlv_wait_for_pw_status(struct drm_i915_private *i915,280 u32 mask, u32 val)281{282 i915_reg_t reg = VLV_GTLC_PW_STATUS;283 u32 reg_value;284 int ret;285 286 /* The HW does not like us polling for PW_STATUS frequently, so287 * use the sleeping loop rather than risk the busy spin within288 * intel_wait_for_register().289 *290 * Transitioning between RC6 states should be at most 2ms (see291 * valleyview_enable_rps) so use a 3ms timeout.292 */293 ret = wait_for(((reg_value =294 intel_uncore_read_notrace(&i915->uncore, reg)) & mask)295 == val, 3);296 297 /* just trace the final value */298 trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);299 300 return ret;301}302 303static int vlv_force_gfx_clock(struct drm_i915_private *i915, bool force_on)304{305 struct intel_uncore *uncore = &i915->uncore;306 int err;307 308 intel_uncore_rmw(uncore, VLV_GTLC_SURVIVABILITY_REG, VLV_GFX_CLK_FORCE_ON_BIT,309 force_on ? VLV_GFX_CLK_FORCE_ON_BIT : 0);310 311 if (!force_on)312 return 0;313 314 err = intel_wait_for_register(uncore,315 VLV_GTLC_SURVIVABILITY_REG,316 VLV_GFX_CLK_STATUS_BIT,317 VLV_GFX_CLK_STATUS_BIT,318 20);319 if (err)320 drm_err(&i915->drm,321 "timeout waiting for GFX clock force-on (%08x)\n",322 intel_uncore_read(uncore, VLV_GTLC_SURVIVABILITY_REG));323 324 return err;325}326 327static int vlv_allow_gt_wake(struct drm_i915_private *i915, bool allow)328{329 struct intel_uncore *uncore = &i915->uncore;330 u32 mask;331 u32 val;332 int err;333 334 intel_uncore_rmw(uncore, VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ,335 allow ? VLV_GTLC_ALLOWWAKEREQ : 0);336 intel_uncore_posting_read(uncore, VLV_GTLC_WAKE_CTRL);337 338 mask = VLV_GTLC_ALLOWWAKEACK;339 val = allow ? mask : 0;340 341 err = vlv_wait_for_pw_status(i915, mask, val);342 if (err)343 drm_err(&i915->drm, "timeout disabling GT waking\n");344 345 return err;346}347 348static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,349 bool wait_for_on)350{351 u32 mask;352 u32 val;353 354 mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;355 val = wait_for_on ? mask : 0;356 357 /*358 * RC6 transitioning can be delayed up to 2 msec (see359 * valleyview_enable_rps), use 3 msec for safety.360 *361 * This can fail to turn off the rc6 if the GPU is stuck after a failed362 * reset and we are trying to force the machine to sleep.363 */364 if (vlv_wait_for_pw_status(dev_priv, mask, val))365 drm_dbg(&dev_priv->drm,366 "timeout waiting for GT wells to go %s\n",367 str_on_off(wait_for_on));368}369 370static void vlv_check_no_gt_access(struct drm_i915_private *i915)371{372 struct intel_uncore *uncore = &i915->uncore;373 374 if (!(intel_uncore_read(uncore, VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))375 return;376 377 drm_dbg(&i915->drm, "GT register access while GT waking disabled\n");378 intel_uncore_write(uncore, VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);379}380 381int vlv_suspend_complete(struct drm_i915_private *dev_priv)382{383 u32 mask;384 int err;385 386 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))387 return 0;388 389 /*390 * Bspec defines the following GT well on flags as debug only, so391 * don't treat them as hard failures.392 */393 vlv_wait_for_gt_wells(dev_priv, false);394 395 mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;396 drm_WARN_ON(&dev_priv->drm,397 (intel_uncore_read(&dev_priv->uncore, VLV_GTLC_WAKE_CTRL) & mask) != mask);398 399 vlv_check_no_gt_access(dev_priv);400 401 err = vlv_force_gfx_clock(dev_priv, true);402 if (err)403 goto err1;404 405 err = vlv_allow_gt_wake(dev_priv, false);406 if (err)407 goto err2;408 409 vlv_save_gunit_s0ix_state(dev_priv);410 411 err = vlv_force_gfx_clock(dev_priv, false);412 if (err)413 goto err2;414 415 return 0;416 417err2:418 /* For safety always re-enable waking and disable gfx clock forcing */419 vlv_allow_gt_wake(dev_priv, true);420err1:421 vlv_force_gfx_clock(dev_priv, false);422 423 return err;424}425 426int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume)427{428 int err;429 int ret;430 431 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))432 return 0;433 434 /*435 * If any of the steps fail just try to continue, that's the best we436 * can do at this point. Return the first error code (which will also437 * leave RPM permanently disabled).438 */439 ret = vlv_force_gfx_clock(dev_priv, true);440 441 vlv_restore_gunit_s0ix_state(dev_priv);442 443 err = vlv_allow_gt_wake(dev_priv, true);444 if (!ret)445 ret = err;446 447 err = vlv_force_gfx_clock(dev_priv, false);448 if (!ret)449 ret = err;450 451 vlv_check_no_gt_access(dev_priv);452 453 if (rpm_resume)454 intel_clock_gating_init(dev_priv);455 456 return ret;457}458 459int vlv_suspend_init(struct drm_i915_private *i915)460{461 if (!IS_VALLEYVIEW(i915))462 return 0;463 464 /* we write all the values in the struct, so no need to zero it out */465 i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state),466 GFP_KERNEL);467 if (!i915->vlv_s0ix_state)468 return -ENOMEM;469 470 return 0;471}472 473void vlv_suspend_cleanup(struct drm_i915_private *i915)474{475 if (!i915->vlv_s0ix_state)476 return;477 478 kfree(i915->vlv_s0ix_state);479 i915->vlv_s0ix_state = NULL;480}481