brintos / linux-shallow public Read only

1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include "i915_drv.h"7#include "i915_reg.h"8#include "i9xx_wm.h"9#include "intel_atomic.h"10#include "intel_display.h"11#include "intel_display_trace.h"12#include "intel_mchbar_regs.h"13#include "intel_wm.h"14#include "skl_watermark.h"15#include "vlv_sideband.h"16 17/* used in computing the new watermarks state */18struct intel_wm_config {19	unsigned int num_pipes_active;20	bool sprites_enabled;21	bool sprites_scaled;22};23 24struct cxsr_latency {25	bool is_desktop : 1;26	bool is_ddr3 : 1;27	u16 fsb_freq;28	u16 mem_freq;29	u16 display_sr;30	u16 display_hpll_disable;31	u16 cursor_sr;32	u16 cursor_hpll_disable;33};34 35static const struct cxsr_latency cxsr_latency_table[] = {36	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */37	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */38	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */39	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */40	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */41 42	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */43	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */44	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */45	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */46	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */47 48	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */49	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */50	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */51	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */52	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */53 54	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */55	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */56	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */57	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */58	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */59 60	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */61	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */62	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */63	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */64	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */65 66	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */67	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */68	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */69	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */70	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */71};72 73static const struct cxsr_latency *pnv_get_cxsr_latency(struct drm_i915_private *i915)74{75	int i;76 77	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {78		const struct cxsr_latency *latency = &cxsr_latency_table[i];79		bool is_desktop = !IS_MOBILE(i915);80 81		if (is_desktop == latency->is_desktop &&82		    i915->is_ddr3 == latency->is_ddr3 &&83		    DIV_ROUND_CLOSEST(i915->fsb_freq, 1000) == latency->fsb_freq &&84		    DIV_ROUND_CLOSEST(i915->mem_freq, 1000) == latency->mem_freq)85			return latency;86	}87 88	drm_dbg_kms(&i915->drm,89		    "Could not find CxSR latency for DDR%s, FSB %u kHz, MEM %u kHz\n",90		    i915->is_ddr3 ? "3" : "2", i915->fsb_freq, i915->mem_freq);91 92	return NULL;93}94 95static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)96{97	u32 val;98 99	vlv_punit_get(dev_priv);100 101	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);102	if (enable)103		val &= ~FORCE_DDR_HIGH_FREQ;104	else105		val |= FORCE_DDR_HIGH_FREQ;106	val &= ~FORCE_DDR_LOW_FREQ;107	val |= FORCE_DDR_FREQ_REQ_ACK;108	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);109 110	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &111		      FORCE_DDR_FREQ_REQ_ACK) == 0, 3))112		drm_err(&dev_priv->drm,113			"timed out waiting for Punit DDR DVFS request\n");114 115	vlv_punit_put(dev_priv);116}117 118static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)119{120	u32 val;121 122	vlv_punit_get(dev_priv);123 124	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);125	if (enable)126		val |= DSP_MAXFIFO_PM5_ENABLE;127	else128		val &= ~DSP_MAXFIFO_PM5_ENABLE;129	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);130 131	vlv_punit_put(dev_priv);132}133 134#define FW_WM(value, plane) \135	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)136 137static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)138{139	bool was_enabled;140	u32 val;141 142	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {143		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;144		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);145		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);146	} else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {147		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;148		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);149		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);150	} else if (IS_PINEVIEW(dev_priv)) {151		val = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));152		was_enabled = val & PINEVIEW_SELF_REFRESH_EN;153		if (enable)154			val |= PINEVIEW_SELF_REFRESH_EN;155		else156			val &= ~PINEVIEW_SELF_REFRESH_EN;157		intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), val);158		intel_uncore_posting_read(&dev_priv->uncore, DSPFW3(dev_priv));159	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {160		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;161		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :162			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);163		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);164		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);165	} else if (IS_I915GM(dev_priv)) {166		/*167		 * FIXME can't find a bit like this for 915G, and168		 * yet it does have the related watermark in169		 * FW_BLC_SELF. What's going on?170		 */171		was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;172		val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :173			       _MASKED_BIT_DISABLE(INSTPM_SELF_EN);174		intel_uncore_write(&dev_priv->uncore, INSTPM, val);175		intel_uncore_posting_read(&dev_priv->uncore, INSTPM);176	} else {177		return false;178	}179 180	trace_intel_memory_cxsr(dev_priv, was_enabled, enable);181 182	drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",183		    str_enabled_disabled(enable),184		    str_enabled_disabled(was_enabled));185 186	return was_enabled;187}188 189/**190 * intel_set_memory_cxsr - Configure CxSR state191 * @dev_priv: i915 device192 * @enable: Allow vs. disallow CxSR193 *194 * Allow or disallow the system to enter a special CxSR195 * (C-state self refresh) state. What typically happens in CxSR mode196 * is that several display FIFOs may get combined into a single larger197 * FIFO for a particular plane (so called max FIFO mode) to allow the198 * system to defer memory fetches longer, and the memory will enter199 * self refresh.200 *201 * Note that enabling CxSR does not guarantee that the system enter202 * this special mode, nor does it guarantee that the system stays203 * in that mode once entered. So this just allows/disallows the system204 * to autonomously utilize the CxSR mode. Other factors such as core205 * C-states will affect when/if the system actually enters/exits the206 * CxSR mode.207 *208 * Note that on VLV/CHV this actually only controls the max FIFO mode,209 * and the system is free to enter/exit memory self refresh at any time210 * even when the use of CxSR has been disallowed.211 *212 * While the system is actually in the CxSR/max FIFO mode, some plane213 * control registers will not get latched on vblank. Thus in order to214 * guarantee the system will respond to changes in the plane registers215 * we must always disallow CxSR prior to making changes to those registers.216 * Unfortunately the system will re-evaluate the CxSR conditions at217 * frame start which happens after vblank start (which is when the plane218 * registers would get latched), so we can't proceed with the plane update219 * during the same frame where we disallowed CxSR.220 *221 * Certain platforms also have a deeper HPLL SR mode. Fortunately the222 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold223 * the hardware w.r.t. HPLL SR when writing to plane registers.224 * Disallowing just CxSR is sufficient.225 */226bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)227{228	bool ret;229 230	mutex_lock(&dev_priv->display.wm.wm_mutex);231	ret = _intel_set_memory_cxsr(dev_priv, enable);232	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))233		dev_priv->display.wm.vlv.cxsr = enable;234	else if (IS_G4X(dev_priv))235		dev_priv->display.wm.g4x.cxsr = enable;236	mutex_unlock(&dev_priv->display.wm.wm_mutex);237 238	return ret;239}240 241/*242 * Latency for FIFO fetches is dependent on several factors:243 *   - memory configuration (speed, channels)244 *   - chipset245 *   - current MCH state246 * It can be fairly high in some situations, so here we assume a fairly247 * pessimal value.  It's a tradeoff between extra memory fetches (if we248 * set this value too high, the FIFO will fetch frequently to stay full)249 * and power consumption (set it too low to save power and we might see250 * FIFO underruns and display "flicker").251 *252 * A value of 5us seems to be a good balance; safe for very low end253 * platforms but not overly aggressive on lower latency configs.254 */255static const int pessimal_latency_ns = 5000;256 257#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \258	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))259 260static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)261{262	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);263	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);264	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;265	enum pipe pipe = crtc->pipe;266	int sprite0_start, sprite1_start;267	u32 dsparb, dsparb2, dsparb3;268 269	switch (pipe) {270	case PIPE_A:271		dsparb = intel_uncore_read(&dev_priv->uncore,272					   DSPARB(dev_priv));273		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);274		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);275		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);276		break;277	case PIPE_B:278		dsparb = intel_uncore_read(&dev_priv->uncore,279					   DSPARB(dev_priv));280		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);281		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);282		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);283		break;284	case PIPE_C:285		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);286		dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);287		sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);288		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);289		break;290	default:291		MISSING_CASE(pipe);292		return;293	}294 295	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;296	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;297	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;298	fifo_state->plane[PLANE_CURSOR] = 63;299}300 301static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,302			      enum i9xx_plane_id i9xx_plane)303{304	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));305	int size;306 307	size = dsparb & 0x7f;308	if (i9xx_plane == PLANE_B)309		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;310 311	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",312		    dsparb, plane_name(i9xx_plane), size);313 314	return size;315}316 317static int i830_get_fifo_size(struct drm_i915_private *dev_priv,318			      enum i9xx_plane_id i9xx_plane)319{320	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));321	int size;322 323	size = dsparb & 0x1ff;324	if (i9xx_plane == PLANE_B)325		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;326	size >>= 1; /* Convert to cachelines */327 328	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",329		    dsparb, plane_name(i9xx_plane), size);330 331	return size;332}333 334static int i845_get_fifo_size(struct drm_i915_private *dev_priv,335			      enum i9xx_plane_id i9xx_plane)336{337	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));338	int size;339 340	size = dsparb & 0x7f;341	size >>= 2; /* Convert to cachelines */342 343	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",344		    dsparb, plane_name(i9xx_plane), size);345 346	return size;347}348 349/* Pineview has different values for various configs */350static const struct intel_watermark_params pnv_display_wm = {351	.fifo_size = PINEVIEW_DISPLAY_FIFO,352	.max_wm = PINEVIEW_MAX_WM,353	.default_wm = PINEVIEW_DFT_WM,354	.guard_size = PINEVIEW_GUARD_WM,355	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,356};357 358static const struct intel_watermark_params pnv_display_hplloff_wm = {359	.fifo_size = PINEVIEW_DISPLAY_FIFO,360	.max_wm = PINEVIEW_MAX_WM,361	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,362	.guard_size = PINEVIEW_GUARD_WM,363	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,364};365 366static const struct intel_watermark_params pnv_cursor_wm = {367	.fifo_size = PINEVIEW_CURSOR_FIFO,368	.max_wm = PINEVIEW_CURSOR_MAX_WM,369	.default_wm = PINEVIEW_CURSOR_DFT_WM,370	.guard_size = PINEVIEW_CURSOR_GUARD_WM,371	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,372};373 374static const struct intel_watermark_params pnv_cursor_hplloff_wm = {375	.fifo_size = PINEVIEW_CURSOR_FIFO,376	.max_wm = PINEVIEW_CURSOR_MAX_WM,377	.default_wm = PINEVIEW_CURSOR_DFT_WM,378	.guard_size = PINEVIEW_CURSOR_GUARD_WM,379	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,380};381 382static const struct intel_watermark_params i965_cursor_wm_info = {383	.fifo_size = I965_CURSOR_FIFO,384	.max_wm = I965_CURSOR_MAX_WM,385	.default_wm = I965_CURSOR_DFT_WM,386	.guard_size = 2,387	.cacheline_size = I915_FIFO_LINE_SIZE,388};389 390static const struct intel_watermark_params i945_wm_info = {391	.fifo_size = I945_FIFO_SIZE,392	.max_wm = I915_MAX_WM,393	.default_wm = 1,394	.guard_size = 2,395	.cacheline_size = I915_FIFO_LINE_SIZE,396};397 398static const struct intel_watermark_params i915_wm_info = {399	.fifo_size = I915_FIFO_SIZE,400	.max_wm = I915_MAX_WM,401	.default_wm = 1,402	.guard_size = 2,403	.cacheline_size = I915_FIFO_LINE_SIZE,404};405 406static const struct intel_watermark_params i830_a_wm_info = {407	.fifo_size = I855GM_FIFO_SIZE,408	.max_wm = I915_MAX_WM,409	.default_wm = 1,410	.guard_size = 2,411	.cacheline_size = I830_FIFO_LINE_SIZE,412};413 414static const struct intel_watermark_params i830_bc_wm_info = {415	.fifo_size = I855GM_FIFO_SIZE,416	.max_wm = I915_MAX_WM / 2,417	.default_wm = 1,418	.guard_size = 2,419	.cacheline_size = I830_FIFO_LINE_SIZE,420};421 422static const struct intel_watermark_params i845_wm_info = {423	.fifo_size = I830_FIFO_SIZE,424	.max_wm = I915_MAX_WM,425	.default_wm = 1,426	.guard_size = 2,427	.cacheline_size = I830_FIFO_LINE_SIZE,428};429 430/**431 * intel_wm_method1 - Method 1 / "small buffer" watermark formula432 * @pixel_rate: Pipe pixel rate in kHz433 * @cpp: Plane bytes per pixel434 * @latency: Memory wakeup latency in 0.1us units435 *436 * Compute the watermark using the method 1 or "small buffer"437 * formula. The caller may additonally add extra cachelines438 * to account for TLB misses and clock crossings.439 *440 * This method is concerned with the short term drain rate441 * of the FIFO, ie. it does not account for blanking periods442 * which would effectively reduce the average drain rate across443 * a longer period. The name "small" refers to the fact the444 * FIFO is relatively small compared to the amount of data445 * fetched.446 *447 * The FIFO level vs. time graph might look something like:448 *449 *   |\   |\450 *   | \  | \451 * __---__---__ (- plane active, _ blanking)452 * -> time453 *454 * or perhaps like this:455 *456 *   |\|\  |\|\457 * __----__----__ (- plane active, _ blanking)458 * -> time459 *460 * Returns:461 * The watermark in bytes462 */463static unsigned int intel_wm_method1(unsigned int pixel_rate,464				     unsigned int cpp,465				     unsigned int latency)466{467	u64 ret;468 469	ret = mul_u32_u32(pixel_rate, cpp * latency);470	ret = DIV_ROUND_UP_ULL(ret, 10000);471 472	return ret;473}474 475/**476 * intel_wm_method2 - Method 2 / "large buffer" watermark formula477 * @pixel_rate: Pipe pixel rate in kHz478 * @htotal: Pipe horizontal total479 * @width: Plane width in pixels480 * @cpp: Plane bytes per pixel481 * @latency: Memory wakeup latency in 0.1us units482 *483 * Compute the watermark using the method 2 or "large buffer"484 * formula. The caller may additonally add extra cachelines485 * to account for TLB misses and clock crossings.486 *487 * This method is concerned with the long term drain rate488 * of the FIFO, ie. it does account for blanking periods489 * which effectively reduce the average drain rate across490 * a longer period. The name "large" refers to the fact the491 * FIFO is relatively large compared to the amount of data492 * fetched.493 *494 * The FIFO level vs. time graph might look something like:495 *496 *    |\___       |\___497 *    |    \___   |    \___498 *    |        \  |        \499 * __ --__--__--__--__--__--__ (- plane active, _ blanking)500 * -> time501 *502 * Returns:503 * The watermark in bytes504 */505static unsigned int intel_wm_method2(unsigned int pixel_rate,506				     unsigned int htotal,507				     unsigned int width,508				     unsigned int cpp,509				     unsigned int latency)510{511	unsigned int ret;512 513	/*514	 * FIXME remove once all users are computing515	 * watermarks in the correct place.516	 */517	if (WARN_ON_ONCE(htotal == 0))518		htotal = 1;519 520	ret = (latency * pixel_rate) / (htotal * 10000);521	ret = (ret + 1) * width * cpp;522 523	return ret;524}525 526/**527 * intel_calculate_wm - calculate watermark level528 * @i915: the device529 * @pixel_rate: pixel clock530 * @wm: chip FIFO params531 * @fifo_size: size of the FIFO buffer532 * @cpp: bytes per pixel533 * @latency_ns: memory latency for the platform534 *535 * Calculate the watermark level (the level at which the display plane will536 * start fetching from memory again).  Each chip has a different display537 * FIFO size and allocation, so the caller needs to figure that out and pass538 * in the correct intel_watermark_params structure.539 *540 * As the pixel clock runs, the FIFO will be drained at a rate that depends541 * on the pixel size.  When it reaches the watermark level, it'll start542 * fetching FIFO line sized based chunks from memory until the FIFO fills543 * past the watermark point.  If the FIFO drains completely, a FIFO underrun544 * will occur, and a display engine hang could result.545 */546static unsigned int intel_calculate_wm(struct drm_i915_private *i915,547				       int pixel_rate,548				       const struct intel_watermark_params *wm,549				       int fifo_size, int cpp,550				       unsigned int latency_ns)551{552	int entries, wm_size;553 554	/*555	 * Note: we need to make sure we don't overflow for various clock &556	 * latency values.557	 * clocks go from a few thousand to several hundred thousand.558	 * latency is usually a few thousand559	 */560	entries = intel_wm_method1(pixel_rate, cpp,561				   latency_ns / 100);562	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +563		wm->guard_size;564	drm_dbg_kms(&i915->drm, "FIFO entries required for mode: %d\n", entries);565 566	wm_size = fifo_size - entries;567	drm_dbg_kms(&i915->drm, "FIFO watermark level: %d\n", wm_size);568 569	/* Don't promote wm_size to unsigned... */570	if (wm_size > wm->max_wm)571		wm_size = wm->max_wm;572	if (wm_size <= 0)573		wm_size = wm->default_wm;574 575	/*576	 * Bspec seems to indicate that the value shouldn't be lower than577	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.578	 * Lets go for 8 which is the burst size since certain platforms579	 * already use a hardcoded 8 (which is what the spec says should be580	 * done).581	 */582	if (wm_size <= 8)583		wm_size = 8;584 585	return wm_size;586}587 588static bool is_disabling(int old, int new, int threshold)589{590	return old >= threshold && new < threshold;591}592 593static bool is_enabling(int old, int new, int threshold)594{595	return old < threshold && new >= threshold;596}597 598static bool intel_crtc_active(struct intel_crtc *crtc)599{600	/* Be paranoid as we can arrive here with only partial601	 * state retrieved from the hardware during setup.602	 *603	 * We can ditch the adjusted_mode.crtc_clock check as soon604	 * as Haswell has gained clock readout/fastboot support.605	 *606	 * We can ditch the crtc->primary->state->fb check as soon as we can607	 * properly reconstruct framebuffers.608	 *609	 * FIXME: The intel_crtc->active here should be switched to610	 * crtc->state->active once we have proper CRTC states wired up611	 * for atomic.612	 */613	return crtc->active && crtc->base.primary->state->fb &&614		crtc->config->hw.adjusted_mode.crtc_clock;615}616 617static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)618{619	struct intel_crtc *crtc, *enabled = NULL;620 621	for_each_intel_crtc(&dev_priv->drm, crtc) {622		if (intel_crtc_active(crtc)) {623			if (enabled)624				return NULL;625			enabled = crtc;626		}627	}628 629	return enabled;630}631 632static void pnv_update_wm(struct drm_i915_private *dev_priv)633{634	struct intel_crtc *crtc;635	const struct cxsr_latency *latency;636	u32 reg;637	unsigned int wm;638 639	latency = pnv_get_cxsr_latency(dev_priv);640	if (!latency) {641		drm_dbg_kms(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");642		intel_set_memory_cxsr(dev_priv, false);643		return;644	}645 646	crtc = single_enabled_crtc(dev_priv);647	if (crtc) {648		const struct drm_framebuffer *fb =649			crtc->base.primary->state->fb;650		int pixel_rate = crtc->config->pixel_rate;651		int cpp = fb->format->cpp[0];652 653		/* Display SR */654		wm = intel_calculate_wm(dev_priv, pixel_rate,655					&pnv_display_wm,656					pnv_display_wm.fifo_size,657					cpp, latency->display_sr);658		reg = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));659		reg &= ~DSPFW_SR_MASK;660		reg |= FW_WM(wm, SR);661		intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv), reg);662		drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);663 664		/* cursor SR */665		wm = intel_calculate_wm(dev_priv, pixel_rate,666					&pnv_cursor_wm,667					pnv_display_wm.fifo_size,668					4, latency->cursor_sr);669		intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),670				 DSPFW_CURSOR_SR_MASK,671				 FW_WM(wm, CURSOR_SR));672 673		/* Display HPLL off SR */674		wm = intel_calculate_wm(dev_priv, pixel_rate,675					&pnv_display_hplloff_wm,676					pnv_display_hplloff_wm.fifo_size,677					cpp, latency->display_hpll_disable);678		intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),679				 DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));680 681		/* cursor HPLL off SR */682		wm = intel_calculate_wm(dev_priv, pixel_rate,683					&pnv_cursor_hplloff_wm,684					pnv_display_hplloff_wm.fifo_size,685					4, latency->cursor_hpll_disable);686		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));687		reg &= ~DSPFW_HPLL_CURSOR_MASK;688		reg |= FW_WM(wm, HPLL_CURSOR);689		intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), reg);690		drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);691 692		intel_set_memory_cxsr(dev_priv, true);693	} else {694		intel_set_memory_cxsr(dev_priv, false);695	}696}697 698/*699 * Documentation says:700 * "If the line size is small, the TLB fetches can get in the way of the701 *  data fetches, causing some lag in the pixel data return which is not702 *  accounted for in the above formulas. The following adjustment only703 *  needs to be applied if eight whole lines fit in the buffer at once.704 *  The WM is adjusted upwards by the difference between the FIFO size705 *  and the size of 8 whole lines. This adjustment is always performed706 *  in the actual pixel depth regardless of whether FBC is enabled or not."707 */708static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)709{710	int tlb_miss = fifo_size * 64 - width * cpp * 8;711 712	return max(0, tlb_miss);713}714 715static void g4x_write_wm_values(struct drm_i915_private *dev_priv,716				const struct g4x_wm_values *wm)717{718	enum pipe pipe;719 720	for_each_pipe(dev_priv, pipe)721		trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);722 723	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),724			   FW_WM(wm->sr.plane, SR) |725			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |726			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |727			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));728	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),729			   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |730			   FW_WM(wm->sr.fbc, FBC_SR) |731			   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |732			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |733			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |734			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));735	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),736			   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |737			   FW_WM(wm->sr.cursor, CURSOR_SR) |738			   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |739			   FW_WM(wm->hpll.plane, HPLL_SR));740 741	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));742}743 744#define FW_WM_VLV(value, plane) \745	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)746 747static void vlv_write_wm_values(struct drm_i915_private *dev_priv,748				const struct vlv_wm_values *wm)749{750	enum pipe pipe;751 752	for_each_pipe(dev_priv, pipe) {753		trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);754 755		intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),756				   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |757				   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |758				   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |759				   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));760	}761 762	/*763	 * Zero the (unused) WM1 watermarks, and also clear all the764	 * high order bits so that there are no out of bounds values765	 * present in the registers during the reprogramming.766	 */767	intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);768	intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);769	intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);770	intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);771	intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);772 773	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),774			   FW_WM(wm->sr.plane, SR) |775			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |776			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |777			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));778	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),779			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |780			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |781			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));782	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),783			   FW_WM(wm->sr.cursor, CURSOR_SR));784 785	if (IS_CHERRYVIEW(dev_priv)) {786		intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,787				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |788				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));789		intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,790				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |791				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));792		intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,793				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |794				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));795		intel_uncore_write(&dev_priv->uncore, DSPHOWM,796				   FW_WM(wm->sr.plane >> 9, SR_HI) |797				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |798				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |799				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |800				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |801				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |802				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |803				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |804				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |805				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));806	} else {807		intel_uncore_write(&dev_priv->uncore, DSPFW7,808				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |809				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));810		intel_uncore_write(&dev_priv->uncore, DSPHOWM,811				   FW_WM(wm->sr.plane >> 9, SR_HI) |812				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |813				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |814				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |815				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |816				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |817				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));818	}819 820	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));821}822 823#undef FW_WM_VLV824 825static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)826{827	/* all latencies in usec */828	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;829	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;830	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;831 832	dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;833}834 835static int g4x_plane_fifo_size(enum plane_id plane_id, int level)836{837	/*838	 * DSPCNTR[13] supposedly controls whether the839	 * primary plane can use the FIFO space otherwise840	 * reserved for the sprite plane. It's not 100% clear841	 * what the actual FIFO size is, but it looks like we842	 * can happily set both primary and sprite watermarks843	 * up to 127 cachelines. So that would seem to mean844	 * that either DSPCNTR[13] doesn't do anything, or that845	 * the total FIFO is >= 256 cachelines in size. Either846	 * way, we don't seem to have to worry about this847	 * repartitioning as the maximum watermark value the848	 * register can hold for each plane is lower than the849	 * minimum FIFO size.850	 */851	switch (plane_id) {852	case PLANE_CURSOR:853		return 63;854	case PLANE_PRIMARY:855		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;856	case PLANE_SPRITE0:857		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;858	default:859		MISSING_CASE(plane_id);860		return 0;861	}862}863 864static int g4x_fbc_fifo_size(int level)865{866	switch (level) {867	case G4X_WM_LEVEL_SR:868		return 7;869	case G4X_WM_LEVEL_HPLL:870		return 15;871	default:872		MISSING_CASE(level);873		return 0;874	}875}876 877static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,878			  const struct intel_plane_state *plane_state,879			  int level)880{881	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);882	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);883	const struct drm_display_mode *pipe_mode =884		&crtc_state->hw.pipe_mode;885	unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;886	unsigned int pixel_rate, htotal, cpp, width, wm;887 888	if (latency == 0)889		return USHRT_MAX;890 891	if (!intel_wm_plane_visible(crtc_state, plane_state))892		return 0;893 894	cpp = plane_state->hw.fb->format->cpp[0];895 896	/*897	 * WaUse32BppForSRWM:ctg,elk898	 *899	 * The spec fails to list this restriction for the900	 * HPLL watermark, which seems a little strange.901	 * Let's use 32bpp for the HPLL watermark as well.902	 */903	if (plane->id == PLANE_PRIMARY &&904	    level != G4X_WM_LEVEL_NORMAL)905		cpp = max(cpp, 4u);906 907	pixel_rate = crtc_state->pixel_rate;908	htotal = pipe_mode->crtc_htotal;909	width = drm_rect_width(&plane_state->uapi.src) >> 16;910 911	if (plane->id == PLANE_CURSOR) {912		wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);913	} else if (plane->id == PLANE_PRIMARY &&914		   level == G4X_WM_LEVEL_NORMAL) {915		wm = intel_wm_method1(pixel_rate, cpp, latency);916	} else {917		unsigned int small, large;918 919		small = intel_wm_method1(pixel_rate, cpp, latency);920		large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);921 922		wm = min(small, large);923	}924 925	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),926			      width, cpp);927 928	wm = DIV_ROUND_UP(wm, 64) + 2;929 930	return min_t(unsigned int, wm, USHRT_MAX);931}932 933static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,934				 int level, enum plane_id plane_id, u16 value)935{936	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);937	bool dirty = false;938 939	for (; level < dev_priv->display.wm.num_levels; level++) {940		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];941 942		dirty |= raw->plane[plane_id] != value;943		raw->plane[plane_id] = value;944	}945 946	return dirty;947}948 949static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,950			       int level, u16 value)951{952	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);953	bool dirty = false;954 955	/* NORMAL level doesn't have an FBC watermark */956	level = max(level, G4X_WM_LEVEL_SR);957 958	for (; level < dev_priv->display.wm.num_levels; level++) {959		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];960 961		dirty |= raw->fbc != value;962		raw->fbc = value;963	}964 965	return dirty;966}967 968static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,969			      const struct intel_plane_state *plane_state,970			      u32 pri_val);971 972static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,973				     const struct intel_plane_state *plane_state)974{975	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);976	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);977	enum plane_id plane_id = plane->id;978	bool dirty = false;979	int level;980 981	if (!intel_wm_plane_visible(crtc_state, plane_state)) {982		dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);983		if (plane_id == PLANE_PRIMARY)984			dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);985		goto out;986	}987 988	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {989		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];990		int wm, max_wm;991 992		wm = g4x_compute_wm(crtc_state, plane_state, level);993		max_wm = g4x_plane_fifo_size(plane_id, level);994 995		if (wm > max_wm)996			break;997 998		dirty |= raw->plane[plane_id] != wm;999		raw->plane[plane_id] = wm;1000 1001		if (plane_id != PLANE_PRIMARY ||1002		    level == G4X_WM_LEVEL_NORMAL)1003			continue;1004 1005		wm = ilk_compute_fbc_wm(crtc_state, plane_state,1006					raw->plane[plane_id]);1007		max_wm = g4x_fbc_fifo_size(level);1008 1009		/*1010		 * FBC wm is not mandatory as we1011		 * can always just disable its use.1012		 */1013		if (wm > max_wm)1014			wm = USHRT_MAX;1015 1016		dirty |= raw->fbc != wm;1017		raw->fbc = wm;1018	}1019 1020	/* mark watermarks as invalid */1021	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);1022 1023	if (plane_id == PLANE_PRIMARY)1024		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);1025 1026 out:1027	if (dirty) {1028		drm_dbg_kms(&dev_priv->drm,1029			    "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",1030			    plane->base.name,1031			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],1032			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],1033			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);1034 1035		if (plane_id == PLANE_PRIMARY)1036			drm_dbg_kms(&dev_priv->drm,1037				    "FBC watermarks: SR=%d, HPLL=%d\n",1038				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,1039				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);1040	}1041 1042	return dirty;1043}1044 1045static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,1046				      enum plane_id plane_id, int level)1047{1048	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];1049 1050	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);1051}1052 1053static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,1054				     int level)1055{1056	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);1057 1058	if (level >= dev_priv->display.wm.num_levels)1059		return false;1060 1061	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&1062		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&1063		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);1064}1065 1066/* mark all levels starting from 'level' as invalid */1067static void g4x_invalidate_wms(struct intel_crtc *crtc,1068			       struct g4x_wm_state *wm_state, int level)1069{1070	if (level <= G4X_WM_LEVEL_NORMAL) {1071		enum plane_id plane_id;1072 1073		for_each_plane_id_on_crtc(crtc, plane_id)1074			wm_state->wm.plane[plane_id] = USHRT_MAX;1075	}1076 1077	if (level <= G4X_WM_LEVEL_SR) {1078		wm_state->cxsr = false;1079		wm_state->sr.cursor = USHRT_MAX;1080		wm_state->sr.plane = USHRT_MAX;1081		wm_state->sr.fbc = USHRT_MAX;1082	}1083 1084	if (level <= G4X_WM_LEVEL_HPLL) {1085		wm_state->hpll_en = false;1086		wm_state->hpll.cursor = USHRT_MAX;1087		wm_state->hpll.plane = USHRT_MAX;1088		wm_state->hpll.fbc = USHRT_MAX;1089	}1090}1091 1092static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,1093			       int level)1094{1095	if (level < G4X_WM_LEVEL_SR)1096		return false;1097 1098	if (level >= G4X_WM_LEVEL_SR &&1099	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))1100		return false;1101 1102	if (level >= G4X_WM_LEVEL_HPLL &&1103	    wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))1104		return false;1105 1106	return true;1107}1108 1109static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)1110{1111	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);1112	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;1113	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);1114	const struct g4x_pipe_wm *raw;1115	enum plane_id plane_id;1116	int level;1117 1118	level = G4X_WM_LEVEL_NORMAL;1119	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))1120		goto out;1121 1122	raw = &crtc_state->wm.g4x.raw[level];1123	for_each_plane_id_on_crtc(crtc, plane_id)1124		wm_state->wm.plane[plane_id] = raw->plane[plane_id];1125 1126	level = G4X_WM_LEVEL_SR;1127	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))1128		goto out;1129 1130	raw = &crtc_state->wm.g4x.raw[level];1131	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];1132	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];1133	wm_state->sr.fbc = raw->fbc;1134 1135	wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);1136 1137	level = G4X_WM_LEVEL_HPLL;1138	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))1139		goto out;1140 1141	raw = &crtc_state->wm.g4x.raw[level];1142	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];1143	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];1144	wm_state->hpll.fbc = raw->fbc;1145 1146	wm_state->hpll_en = wm_state->cxsr;1147 1148	level++;1149 1150 out:1151	if (level == G4X_WM_LEVEL_NORMAL)1152		return -EINVAL;1153 1154	/* invalidate the higher levels */1155	g4x_invalidate_wms(crtc, wm_state, level);1156 1157	/*1158	 * Determine if the FBC watermark(s) can be used. IF1159	 * this isn't the case we prefer to disable the FBC1160	 * watermark(s) rather than disable the SR/HPLL1161	 * level(s) entirely. 'level-1' is the highest valid1162	 * level here.1163	 */1164	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);1165 1166	return 0;1167}1168 1169static int g4x_compute_pipe_wm(struct intel_atomic_state *state,1170			       struct intel_crtc *crtc)1171{1172	struct intel_crtc_state *crtc_state =1173		intel_atomic_get_new_crtc_state(state, crtc);1174	const struct intel_plane_state *old_plane_state;1175	const struct intel_plane_state *new_plane_state;1176	struct intel_plane *plane;1177	unsigned int dirty = 0;1178	int i;1179 1180	for_each_oldnew_intel_plane_in_state(state, plane,1181					     old_plane_state,1182					     new_plane_state, i) {1183		if (new_plane_state->hw.crtc != &crtc->base &&1184		    old_plane_state->hw.crtc != &crtc->base)1185			continue;1186 1187		if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))1188			dirty |= BIT(plane->id);1189	}1190 1191	if (!dirty)1192		return 0;1193 1194	return _g4x_compute_pipe_wm(crtc_state);1195}1196 1197static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,1198				       struct intel_crtc *crtc)1199{1200	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1201	struct intel_crtc_state *new_crtc_state =1202		intel_atomic_get_new_crtc_state(state, crtc);1203	const struct intel_crtc_state *old_crtc_state =1204		intel_atomic_get_old_crtc_state(state, crtc);1205	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;1206	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;1207	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;1208	enum plane_id plane_id;1209 1210	if (!new_crtc_state->hw.active ||1211	    intel_crtc_needs_modeset(new_crtc_state)) {1212		*intermediate = *optimal;1213 1214		intermediate->cxsr = false;1215		intermediate->hpll_en = false;1216		goto out;1217	}1218 1219	intermediate->cxsr = optimal->cxsr && active->cxsr &&1220		!new_crtc_state->disable_cxsr;1221	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&1222		!new_crtc_state->disable_cxsr;1223	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;1224 1225	for_each_plane_id_on_crtc(crtc, plane_id) {1226		intermediate->wm.plane[plane_id] =1227			max(optimal->wm.plane[plane_id],1228			    active->wm.plane[plane_id]);1229 1230		drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >1231			    g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));1232	}1233 1234	intermediate->sr.plane = max(optimal->sr.plane,1235				     active->sr.plane);1236	intermediate->sr.cursor = max(optimal->sr.cursor,1237				      active->sr.cursor);1238	intermediate->sr.fbc = max(optimal->sr.fbc,1239				   active->sr.fbc);1240 1241	intermediate->hpll.plane = max(optimal->hpll.plane,1242				       active->hpll.plane);1243	intermediate->hpll.cursor = max(optimal->hpll.cursor,1244					active->hpll.cursor);1245	intermediate->hpll.fbc = max(optimal->hpll.fbc,1246				     active->hpll.fbc);1247 1248	drm_WARN_ON(&dev_priv->drm,1249		    (intermediate->sr.plane >1250		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||1251		     intermediate->sr.cursor >1252		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&1253		    intermediate->cxsr);1254	drm_WARN_ON(&dev_priv->drm,1255		    (intermediate->sr.plane >1256		     g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||1257		     intermediate->sr.cursor >1258		     g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&1259		    intermediate->hpll_en);1260 1261	drm_WARN_ON(&dev_priv->drm,1262		    intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&1263		    intermediate->fbc_en && intermediate->cxsr);1264	drm_WARN_ON(&dev_priv->drm,1265		    intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&1266		    intermediate->fbc_en && intermediate->hpll_en);1267 1268out:1269	/*1270	 * If our intermediate WM are identical to the final WM, then we can1271	 * omit the post-vblank programming; only update if it's different.1272	 */1273	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)1274		new_crtc_state->wm.need_postvbl_update = true;1275 1276	return 0;1277}1278 1279static void g4x_merge_wm(struct drm_i915_private *dev_priv,1280			 struct g4x_wm_values *wm)1281{1282	struct intel_crtc *crtc;1283	int num_active_pipes = 0;1284 1285	wm->cxsr = true;1286	wm->hpll_en = true;1287	wm->fbc_en = true;1288 1289	for_each_intel_crtc(&dev_priv->drm, crtc) {1290		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;1291 1292		if (!crtc->active)1293			continue;1294 1295		if (!wm_state->cxsr)1296			wm->cxsr = false;1297		if (!wm_state->hpll_en)1298			wm->hpll_en = false;1299		if (!wm_state->fbc_en)1300			wm->fbc_en = false;1301 1302		num_active_pipes++;1303	}1304 1305	if (num_active_pipes != 1) {1306		wm->cxsr = false;1307		wm->hpll_en = false;1308		wm->fbc_en = false;1309	}1310 1311	for_each_intel_crtc(&dev_priv->drm, crtc) {1312		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;1313		enum pipe pipe = crtc->pipe;1314 1315		wm->pipe[pipe] = wm_state->wm;1316		if (crtc->active && wm->cxsr)1317			wm->sr = wm_state->sr;1318		if (crtc->active && wm->hpll_en)1319			wm->hpll = wm_state->hpll;1320	}1321}1322 1323static void g4x_program_watermarks(struct drm_i915_private *dev_priv)1324{1325	struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;1326	struct g4x_wm_values new_wm = {};1327 1328	g4x_merge_wm(dev_priv, &new_wm);1329 1330	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)1331		return;1332 1333	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))1334		_intel_set_memory_cxsr(dev_priv, false);1335 1336	g4x_write_wm_values(dev_priv, &new_wm);1337 1338	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))1339		_intel_set_memory_cxsr(dev_priv, true);1340 1341	*old_wm = new_wm;1342}1343 1344static void g4x_initial_watermarks(struct intel_atomic_state *state,1345				   struct intel_crtc *crtc)1346{1347	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1348	const struct intel_crtc_state *crtc_state =1349		intel_atomic_get_new_crtc_state(state, crtc);1350 1351	mutex_lock(&dev_priv->display.wm.wm_mutex);1352	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;1353	g4x_program_watermarks(dev_priv);1354	mutex_unlock(&dev_priv->display.wm.wm_mutex);1355}1356 1357static void g4x_optimize_watermarks(struct intel_atomic_state *state,1358				    struct intel_crtc *crtc)1359{1360	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1361	const struct intel_crtc_state *crtc_state =1362		intel_atomic_get_new_crtc_state(state, crtc);1363 1364	if (!crtc_state->wm.need_postvbl_update)1365		return;1366 1367	mutex_lock(&dev_priv->display.wm.wm_mutex);1368	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;1369	g4x_program_watermarks(dev_priv);1370	mutex_unlock(&dev_priv->display.wm.wm_mutex);1371}1372 1373/* latency must be in 0.1us units. */1374static unsigned int vlv_wm_method2(unsigned int pixel_rate,1375				   unsigned int htotal,1376				   unsigned int width,1377				   unsigned int cpp,1378				   unsigned int latency)1379{1380	unsigned int ret;1381 1382	ret = intel_wm_method2(pixel_rate, htotal,1383			       width, cpp, latency);1384	ret = DIV_ROUND_UP(ret, 64);1385 1386	return ret;1387}1388 1389static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)1390{1391	/* all latencies in usec */1392	dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;1393 1394	dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;1395 1396	if (IS_CHERRYVIEW(dev_priv)) {1397		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;1398		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;1399 1400		dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;1401	}1402}1403 1404static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,1405				const struct intel_plane_state *plane_state,1406				int level)1407{1408	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);1409	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);1410	const struct drm_display_mode *pipe_mode =1411		&crtc_state->hw.pipe_mode;1412	unsigned int pixel_rate, htotal, cpp, width, wm;1413 1414	if (dev_priv->display.wm.pri_latency[level] == 0)1415		return USHRT_MAX;1416 1417	if (!intel_wm_plane_visible(crtc_state, plane_state))1418		return 0;1419 1420	cpp = plane_state->hw.fb->format->cpp[0];1421	pixel_rate = crtc_state->pixel_rate;1422	htotal = pipe_mode->crtc_htotal;1423	width = drm_rect_width(&plane_state->uapi.src) >> 16;1424 1425	if (plane->id == PLANE_CURSOR) {1426		/*1427		 * FIXME the formula gives values that are1428		 * too big for the cursor FIFO, and hence we1429		 * would never be able to use cursors. For1430		 * now just hardcode the watermark.1431		 */1432		wm = 63;1433	} else {1434		wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,1435				    dev_priv->display.wm.pri_latency[level] * 10);1436	}1437 1438	return min_t(unsigned int, wm, USHRT_MAX);1439}1440 1441static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)1442{1443	return (active_planes & (BIT(PLANE_SPRITE0) |1444				 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);1445}1446 1447static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)1448{1449	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);1450	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1451	const struct g4x_pipe_wm *raw =1452		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];1453	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;1454	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);1455	int num_active_planes = hweight8(active_planes);1456	const int fifo_size = 511;1457	int fifo_extra, fifo_left = fifo_size;1458	int sprite0_fifo_extra = 0;1459	unsigned int total_rate;1460	enum plane_id plane_id;1461 1462	/*1463	 * When enabling sprite0 after sprite1 has already been enabled1464	 * we tend to get an underrun unless sprite0 already has some1465	 * FIFO space allcoated. Hence we always allocate at least one1466	 * cacheline for sprite0 whenever sprite1 is enabled.1467	 *1468	 * All other plane enable sequences appear immune to this problem.1469	 */1470	if (vlv_need_sprite0_fifo_workaround(active_planes))1471		sprite0_fifo_extra = 1;1472 1473	total_rate = raw->plane[PLANE_PRIMARY] +1474		raw->plane[PLANE_SPRITE0] +1475		raw->plane[PLANE_SPRITE1] +1476		sprite0_fifo_extra;1477 1478	if (total_rate > fifo_size)1479		return -EINVAL;1480 1481	if (total_rate == 0)1482		total_rate = 1;1483 1484	for_each_plane_id_on_crtc(crtc, plane_id) {1485		unsigned int rate;1486 1487		if ((active_planes & BIT(plane_id)) == 0) {1488			fifo_state->plane[plane_id] = 0;1489			continue;1490		}1491 1492		rate = raw->plane[plane_id];1493		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;1494		fifo_left -= fifo_state->plane[plane_id];1495	}1496 1497	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;1498	fifo_left -= sprite0_fifo_extra;1499 1500	fifo_state->plane[PLANE_CURSOR] = 63;1501 1502	fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);1503 1504	/* spread the remainder evenly */1505	for_each_plane_id_on_crtc(crtc, plane_id) {1506		int plane_extra;1507 1508		if (fifo_left == 0)1509			break;1510 1511		if ((active_planes & BIT(plane_id)) == 0)1512			continue;1513 1514		plane_extra = min(fifo_extra, fifo_left);1515		fifo_state->plane[plane_id] += plane_extra;1516		fifo_left -= plane_extra;1517	}1518 1519	drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);1520 1521	/* give it all to the first plane if none are active */1522	if (active_planes == 0) {1523		drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);1524		fifo_state->plane[PLANE_PRIMARY] = fifo_left;1525	}1526 1527	return 0;1528}1529 1530/* mark all levels starting from 'level' as invalid */1531static void vlv_invalidate_wms(struct intel_crtc *crtc,1532			       struct vlv_wm_state *wm_state, int level)1533{1534	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1535 1536	for (; level < dev_priv->display.wm.num_levels; level++) {1537		enum plane_id plane_id;1538 1539		for_each_plane_id_on_crtc(crtc, plane_id)1540			wm_state->wm[level].plane[plane_id] = USHRT_MAX;1541 1542		wm_state->sr[level].cursor = USHRT_MAX;1543		wm_state->sr[level].plane = USHRT_MAX;1544	}1545}1546 1547static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)1548{1549	if (wm > fifo_size)1550		return USHRT_MAX;1551	else1552		return fifo_size - wm;1553}1554 1555/*1556 * Starting from 'level' set all higher1557 * levels to 'value' in the "raw" watermarks.1558 */1559static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,1560				 int level, enum plane_id plane_id, u16 value)1561{1562	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);1563	bool dirty = false;1564 1565	for (; level < dev_priv->display.wm.num_levels; level++) {1566		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];1567 1568		dirty |= raw->plane[plane_id] != value;1569		raw->plane[plane_id] = value;1570	}1571 1572	return dirty;1573}1574 1575static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,1576				     const struct intel_plane_state *plane_state)1577{1578	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);1579	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);1580	enum plane_id plane_id = plane->id;1581	int level;1582	bool dirty = false;1583 1584	if (!intel_wm_plane_visible(crtc_state, plane_state)) {1585		dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);1586		goto out;1587	}1588 1589	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {1590		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];1591		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);1592		int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;1593 1594		if (wm > max_wm)1595			break;1596 1597		dirty |= raw->plane[plane_id] != wm;1598		raw->plane[plane_id] = wm;1599	}1600 1601	/* mark all higher levels as invalid */1602	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);1603 1604out:1605	if (dirty)1606		drm_dbg_kms(&dev_priv->drm,1607			    "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",1608			    plane->base.name,1609			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],1610			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],1611			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);1612 1613	return dirty;1614}1615 1616static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,1617				      enum plane_id plane_id, int level)1618{1619	const struct g4x_pipe_wm *raw =1620		&crtc_state->wm.vlv.raw[level];1621	const struct vlv_fifo_state *fifo_state =1622		&crtc_state->wm.vlv.fifo_state;1623 1624	return raw->plane[plane_id] <= fifo_state->plane[plane_id];1625}1626 1627static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)1628{1629	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&1630		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&1631		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&1632		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);1633}1634 1635static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)1636{1637	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);1638	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1639	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;1640	const struct vlv_fifo_state *fifo_state =1641		&crtc_state->wm.vlv.fifo_state;1642	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);1643	int num_active_planes = hweight8(active_planes);1644	enum plane_id plane_id;1645	int level;1646 1647	/* initially allow all levels */1648	wm_state->num_levels = dev_priv->display.wm.num_levels;1649	/*1650	 * Note that enabling cxsr with no primary/sprite planes1651	 * enabled can wedge the pipe. Hence we only allow cxsr1652	 * with exactly one enabled primary/sprite plane.1653	 */1654	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;1655 1656	for (level = 0; level < wm_state->num_levels; level++) {1657		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];1658		const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;1659 1660		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))1661			break;1662 1663		for_each_plane_id_on_crtc(crtc, plane_id) {1664			wm_state->wm[level].plane[plane_id] =1665				vlv_invert_wm_value(raw->plane[plane_id],1666						    fifo_state->plane[plane_id]);1667		}1668 1669		wm_state->sr[level].plane =1670			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],1671						 raw->plane[PLANE_SPRITE0],1672						 raw->plane[PLANE_SPRITE1]),1673					    sr_fifo_size);1674 1675		wm_state->sr[level].cursor =1676			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],1677					    63);1678	}1679 1680	if (level == 0)1681		return -EINVAL;1682 1683	/* limit to only levels we can actually handle */1684	wm_state->num_levels = level;1685 1686	/* invalidate the higher levels */1687	vlv_invalidate_wms(crtc, wm_state, level);1688 1689	return 0;1690}1691 1692static int vlv_compute_pipe_wm(struct intel_atomic_state *state,1693			       struct intel_crtc *crtc)1694{1695	struct intel_crtc_state *crtc_state =1696		intel_atomic_get_new_crtc_state(state, crtc);1697	const struct intel_plane_state *old_plane_state;1698	const struct intel_plane_state *new_plane_state;1699	struct intel_plane *plane;1700	unsigned int dirty = 0;1701	int i;1702 1703	for_each_oldnew_intel_plane_in_state(state, plane,1704					     old_plane_state,1705					     new_plane_state, i) {1706		if (new_plane_state->hw.crtc != &crtc->base &&1707		    old_plane_state->hw.crtc != &crtc->base)1708			continue;1709 1710		if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))1711			dirty |= BIT(plane->id);1712	}1713 1714	/*1715	 * DSPARB registers may have been reset due to the1716	 * power well being turned off. Make sure we restore1717	 * them to a consistent state even if no primary/sprite1718	 * planes are initially active. We also force a FIFO1719	 * recomputation so that we are sure to sanitize the1720	 * FIFO setting we took over from the BIOS even if there1721	 * are no active planes on the crtc.1722	 */1723	if (intel_crtc_needs_modeset(crtc_state))1724		dirty = ~0;1725 1726	if (!dirty)1727		return 0;1728 1729	/* cursor changes don't warrant a FIFO recompute */1730	if (dirty & ~BIT(PLANE_CURSOR)) {1731		const struct intel_crtc_state *old_crtc_state =1732			intel_atomic_get_old_crtc_state(state, crtc);1733		const struct vlv_fifo_state *old_fifo_state =1734			&old_crtc_state->wm.vlv.fifo_state;1735		const struct vlv_fifo_state *new_fifo_state =1736			&crtc_state->wm.vlv.fifo_state;1737		int ret;1738 1739		ret = vlv_compute_fifo(crtc_state);1740		if (ret)1741			return ret;1742 1743		if (intel_crtc_needs_modeset(crtc_state) ||1744		    memcmp(old_fifo_state, new_fifo_state,1745			   sizeof(*new_fifo_state)) != 0)1746			crtc_state->fifo_changed = true;1747	}1748 1749	return _vlv_compute_pipe_wm(crtc_state);1750}1751 1752#define VLV_FIFO(plane, value) \1753	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)1754 1755static void vlv_atomic_update_fifo(struct intel_atomic_state *state,1756				   struct intel_crtc *crtc)1757{1758	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1759	struct intel_uncore *uncore = &dev_priv->uncore;1760	const struct intel_crtc_state *crtc_state =1761		intel_atomic_get_new_crtc_state(state, crtc);1762	const struct vlv_fifo_state *fifo_state =1763		&crtc_state->wm.vlv.fifo_state;1764	int sprite0_start, sprite1_start, fifo_size;1765	u32 dsparb, dsparb2, dsparb3;1766 1767	if (!crtc_state->fifo_changed)1768		return;1769 1770	sprite0_start = fifo_state->plane[PLANE_PRIMARY];1771	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;1772	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;1773 1774	drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);1775	drm_WARN_ON(&dev_priv->drm, fifo_size != 511);1776 1777	trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);1778 1779	/*1780	 * uncore.lock serves a double purpose here. It allows us to1781	 * use the less expensive I915_{READ,WRITE}_FW() functions, and1782	 * it protects the DSPARB registers from getting clobbered by1783	 * parallel updates from multiple pipes.1784	 *1785	 * intel_pipe_update_start() has already disabled interrupts1786	 * for us, so a plain spin_lock() is sufficient here.1787	 */1788	spin_lock(&uncore->lock);1789 1790	switch (crtc->pipe) {1791	case PIPE_A:1792		dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));1793		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);1794 1795		dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |1796			    VLV_FIFO(SPRITEB, 0xff));1797		dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |1798			   VLV_FIFO(SPRITEB, sprite1_start));1799 1800		dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |1801			     VLV_FIFO(SPRITEB_HI, 0x1));1802		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |1803			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));1804 1805		intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);1806		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);1807		break;1808	case PIPE_B:1809		dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));1810		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);1811 1812		dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |1813			    VLV_FIFO(SPRITED, 0xff));1814		dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |1815			   VLV_FIFO(SPRITED, sprite1_start));1816 1817		dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |1818			     VLV_FIFO(SPRITED_HI, 0xff));1819		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |1820			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));1821 1822		intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);1823		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);1824		break;1825	case PIPE_C:1826		dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);1827		dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);1828 1829		dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |1830			     VLV_FIFO(SPRITEF, 0xff));1831		dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |1832			    VLV_FIFO(SPRITEF, sprite1_start));1833 1834		dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |1835			     VLV_FIFO(SPRITEF_HI, 0xff));1836		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |1837			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));1838 1839		intel_uncore_write_fw(uncore, DSPARB3, dsparb3);1840		intel_uncore_write_fw(uncore, DSPARB2, dsparb2);1841		break;1842	default:1843		break;1844	}1845 1846	intel_uncore_posting_read_fw(uncore, DSPARB(dev_priv));1847 1848	spin_unlock(&uncore->lock);1849}1850 1851#undef VLV_FIFO1852 1853static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,1854				       struct intel_crtc *crtc)1855{1856	struct intel_crtc_state *new_crtc_state =1857		intel_atomic_get_new_crtc_state(state, crtc);1858	const struct intel_crtc_state *old_crtc_state =1859		intel_atomic_get_old_crtc_state(state, crtc);1860	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;1861	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;1862	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;1863	int level;1864 1865	if (!new_crtc_state->hw.active ||1866	    intel_crtc_needs_modeset(new_crtc_state)) {1867		*intermediate = *optimal;1868 1869		intermediate->cxsr = false;1870		goto out;1871	}1872 1873	intermediate->num_levels = min(optimal->num_levels, active->num_levels);1874	intermediate->cxsr = optimal->cxsr && active->cxsr &&1875		!new_crtc_state->disable_cxsr;1876 1877	for (level = 0; level < intermediate->num_levels; level++) {1878		enum plane_id plane_id;1879 1880		for_each_plane_id_on_crtc(crtc, plane_id) {1881			intermediate->wm[level].plane[plane_id] =1882				min(optimal->wm[level].plane[plane_id],1883				    active->wm[level].plane[plane_id]);1884		}1885 1886		intermediate->sr[level].plane = min(optimal->sr[level].plane,1887						    active->sr[level].plane);1888		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,1889						     active->sr[level].cursor);1890	}1891 1892	vlv_invalidate_wms(crtc, intermediate, level);1893 1894out:1895	/*1896	 * If our intermediate WM are identical to the final WM, then we can1897	 * omit the post-vblank programming; only update if it's different.1898	 */1899	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)1900		new_crtc_state->wm.need_postvbl_update = true;1901 1902	return 0;1903}1904 1905static void vlv_merge_wm(struct drm_i915_private *dev_priv,1906			 struct vlv_wm_values *wm)1907{1908	struct intel_crtc *crtc;1909	int num_active_pipes = 0;1910 1911	wm->level = dev_priv->display.wm.num_levels - 1;1912	wm->cxsr = true;1913 1914	for_each_intel_crtc(&dev_priv->drm, crtc) {1915		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;1916 1917		if (!crtc->active)1918			continue;1919 1920		if (!wm_state->cxsr)1921			wm->cxsr = false;1922 1923		num_active_pipes++;1924		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);1925	}1926 1927	if (num_active_pipes != 1)1928		wm->cxsr = false;1929 1930	if (num_active_pipes > 1)1931		wm->level = VLV_WM_LEVEL_PM2;1932 1933	for_each_intel_crtc(&dev_priv->drm, crtc) {1934		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;1935		enum pipe pipe = crtc->pipe;1936 1937		wm->pipe[pipe] = wm_state->wm[wm->level];1938		if (crtc->active && wm->cxsr)1939			wm->sr = wm_state->sr[wm->level];1940 1941		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;1942		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;1943		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;1944		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;1945	}1946}1947 1948static void vlv_program_watermarks(struct drm_i915_private *dev_priv)1949{1950	struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;1951	struct vlv_wm_values new_wm = {};1952 1953	vlv_merge_wm(dev_priv, &new_wm);1954 1955	if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)1956		return;1957 1958	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))1959		chv_set_memory_dvfs(dev_priv, false);1960 1961	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))1962		chv_set_memory_pm5(dev_priv, false);1963 1964	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))1965		_intel_set_memory_cxsr(dev_priv, false);1966 1967	vlv_write_wm_values(dev_priv, &new_wm);1968 1969	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))1970		_intel_set_memory_cxsr(dev_priv, true);1971 1972	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))1973		chv_set_memory_pm5(dev_priv, true);1974 1975	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))1976		chv_set_memory_dvfs(dev_priv, true);1977 1978	*old_wm = new_wm;1979}1980 1981static void vlv_initial_watermarks(struct intel_atomic_state *state,1982				   struct intel_crtc *crtc)1983{1984	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1985	const struct intel_crtc_state *crtc_state =1986		intel_atomic_get_new_crtc_state(state, crtc);1987 1988	mutex_lock(&dev_priv->display.wm.wm_mutex);1989	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;1990	vlv_program_watermarks(dev_priv);1991	mutex_unlock(&dev_priv->display.wm.wm_mutex);1992}1993 1994static void vlv_optimize_watermarks(struct intel_atomic_state *state,1995				    struct intel_crtc *crtc)1996{1997	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);1998	const struct intel_crtc_state *crtc_state =1999		intel_atomic_get_new_crtc_state(state, crtc);2000 2001	if (!crtc_state->wm.need_postvbl_update)2002		return;2003 2004	mutex_lock(&dev_priv->display.wm.wm_mutex);2005	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;2006	vlv_program_watermarks(dev_priv);2007	mutex_unlock(&dev_priv->display.wm.wm_mutex);2008}2009 2010static void i965_update_wm(struct drm_i915_private *dev_priv)2011{2012	struct intel_crtc *crtc;2013	int srwm = 1;2014	int cursor_sr = 16;2015	bool cxsr_enabled;2016 2017	/* Calc sr entries for one plane configs */2018	crtc = single_enabled_crtc(dev_priv);2019	if (crtc) {2020		/* self-refresh has much higher latency */2021		static const int sr_latency_ns = 12000;2022		const struct drm_display_mode *pipe_mode =2023			&crtc->config->hw.pipe_mode;2024		const struct drm_framebuffer *fb =2025			crtc->base.primary->state->fb;2026		int pixel_rate = crtc->config->pixel_rate;2027		int htotal = pipe_mode->crtc_htotal;2028		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;2029		int cpp = fb->format->cpp[0];2030		int entries;2031 2032		entries = intel_wm_method2(pixel_rate, htotal,2033					   width, cpp, sr_latency_ns / 100);2034		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);2035		srwm = I965_FIFO_SIZE - entries;2036		if (srwm < 0)2037			srwm = 1;2038		srwm &= 0x1ff;2039		drm_dbg_kms(&dev_priv->drm,2040			    "self-refresh entries: %d, wm: %d\n",2041			    entries, srwm);2042 2043		entries = intel_wm_method2(pixel_rate, htotal,2044					   crtc->base.cursor->state->crtc_w, 4,2045					   sr_latency_ns / 100);2046		entries = DIV_ROUND_UP(entries,2047				       i965_cursor_wm_info.cacheline_size) +2048			i965_cursor_wm_info.guard_size;2049 2050		cursor_sr = i965_cursor_wm_info.fifo_size - entries;2051		if (cursor_sr > i965_cursor_wm_info.max_wm)2052			cursor_sr = i965_cursor_wm_info.max_wm;2053 2054		drm_dbg_kms(&dev_priv->drm,2055			    "self-refresh watermark: display plane %d "2056			    "cursor %d\n", srwm, cursor_sr);2057 2058		cxsr_enabled = true;2059	} else {2060		cxsr_enabled = false;2061		/* Turn off self refresh if both pipes are enabled */2062		intel_set_memory_cxsr(dev_priv, false);2063	}2064 2065	drm_dbg_kms(&dev_priv->drm,2066		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",2067		    srwm);2068 2069	/* 965 has limitations... */2070	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),2071			   FW_WM(srwm, SR) |2072			   FW_WM(8, CURSORB) |2073			   FW_WM(8, PLANEB) |2074			   FW_WM(8, PLANEA));2075	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),2076			   FW_WM(8, CURSORA) |2077			   FW_WM(8, PLANEC_OLD));2078	/* update cursor SR watermark */2079	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),2080			   FW_WM(cursor_sr, CURSOR_SR));2081 2082	if (cxsr_enabled)2083		intel_set_memory_cxsr(dev_priv, true);2084}2085 2086#undef FW_WM2087 2088static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,2089					       enum i9xx_plane_id i9xx_plane)2090{2091	struct intel_plane *plane;2092 2093	for_each_intel_plane(&i915->drm, plane) {2094		if (plane->id == PLANE_PRIMARY &&2095		    plane->i9xx_plane == i9xx_plane)2096			return intel_crtc_for_pipe(i915, plane->pipe);2097	}2098 2099	return NULL;2100}2101 2102static void i9xx_update_wm(struct drm_i915_private *dev_priv)2103{2104	const struct intel_watermark_params *wm_info;2105	u32 fwater_lo;2106	u32 fwater_hi;2107	int cwm, srwm = 1;2108	int fifo_size;2109	int planea_wm, planeb_wm;2110	struct intel_crtc *crtc;2111 2112	if (IS_I945GM(dev_priv))2113		wm_info = &i945_wm_info;2114	else if (DISPLAY_VER(dev_priv) != 2)2115		wm_info = &i915_wm_info;2116	else2117		wm_info = &i830_a_wm_info;2118 2119	if (DISPLAY_VER(dev_priv) == 2)2120		fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);2121	else2122		fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);2123	crtc = intel_crtc_for_plane(dev_priv, PLANE_A);2124	if (intel_crtc_active(crtc)) {2125		const struct drm_framebuffer *fb =2126			crtc->base.primary->state->fb;2127		int cpp;2128 2129		if (DISPLAY_VER(dev_priv) == 2)2130			cpp = 4;2131		else2132			cpp = fb->format->cpp[0];2133 2134		planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,2135					       wm_info, fifo_size, cpp,2136					       pessimal_latency_ns);2137	} else {2138		planea_wm = fifo_size - wm_info->guard_size;2139		if (planea_wm > (long)wm_info->max_wm)2140			planea_wm = wm_info->max_wm;2141	}2142 2143	if (DISPLAY_VER(dev_priv) == 2)2144		wm_info = &i830_bc_wm_info;2145 2146	if (DISPLAY_VER(dev_priv) == 2)2147		fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);2148	else2149		fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);2150	crtc = intel_crtc_for_plane(dev_priv, PLANE_B);2151	if (intel_crtc_active(crtc)) {2152		const struct drm_framebuffer *fb =2153			crtc->base.primary->state->fb;2154		int cpp;2155 2156		if (DISPLAY_VER(dev_priv) == 2)2157			cpp = 4;2158		else2159			cpp = fb->format->cpp[0];2160 2161		planeb_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,2162					       wm_info, fifo_size, cpp,2163					       pessimal_latency_ns);2164	} else {2165		planeb_wm = fifo_size - wm_info->guard_size;2166		if (planeb_wm > (long)wm_info->max_wm)2167			planeb_wm = wm_info->max_wm;2168	}2169 2170	drm_dbg_kms(&dev_priv->drm,2171		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);2172 2173	crtc = single_enabled_crtc(dev_priv);2174	if (IS_I915GM(dev_priv) && crtc) {2175		struct drm_i915_gem_object *obj;2176 2177		obj = intel_fb_obj(crtc->base.primary->state->fb);2178 2179		/* self-refresh seems busted with untiled */2180		if (!i915_gem_object_is_tiled(obj))2181			crtc = NULL;2182	}2183 2184	/*2185	 * Overlay gets an aggressive default since video jitter is bad.2186	 */2187	cwm = 2;2188 2189	/* Play safe and disable self-refresh before adjusting watermarks. */2190	intel_set_memory_cxsr(dev_priv, false);2191 2192	/* Calc sr entries for one plane configs */2193	if (HAS_FW_BLC(dev_priv) && crtc) {2194		/* self-refresh has much higher latency */2195		static const int sr_latency_ns = 6000;2196		const struct drm_display_mode *pipe_mode =2197			&crtc->config->hw.pipe_mode;2198		const struct drm_framebuffer *fb =2199			crtc->base.primary->state->fb;2200		int pixel_rate = crtc->config->pixel_rate;2201		int htotal = pipe_mode->crtc_htotal;2202		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;2203		int cpp;2204		int entries;2205 2206		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))2207			cpp = 4;2208		else2209			cpp = fb->format->cpp[0];2210 2211		entries = intel_wm_method2(pixel_rate, htotal, width, cpp,2212					   sr_latency_ns / 100);2213		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);2214		drm_dbg_kms(&dev_priv->drm,2215			    "self-refresh entries: %d\n", entries);2216		srwm = wm_info->fifo_size - entries;2217		if (srwm < 0)2218			srwm = 1;2219 2220		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))2221			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,2222				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));2223		else2224			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);2225	}2226 2227	drm_dbg_kms(&dev_priv->drm,2228		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",2229		     planea_wm, planeb_wm, cwm, srwm);2230 2231	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);2232	fwater_hi = (cwm & 0x1f);2233 2234	/* Set request length to 8 cachelines per fetch */2235	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);2236	fwater_hi = fwater_hi | (1 << 8);2237 2238	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);2239	intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);2240 2241	if (crtc)2242		intel_set_memory_cxsr(dev_priv, true);2243}2244 2245static void i845_update_wm(struct drm_i915_private *dev_priv)2246{2247	struct intel_crtc *crtc;2248	u32 fwater_lo;2249	int planea_wm;2250 2251	crtc = single_enabled_crtc(dev_priv);2252	if (crtc == NULL)2253		return;2254 2255	planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,2256				       &i845_wm_info,2257				       i845_get_fifo_size(dev_priv, PLANE_A),2258				       4, pessimal_latency_ns);2259	fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;2260	fwater_lo |= (3<<8) | planea_wm;2261 2262	drm_dbg_kms(&dev_priv->drm,2263		    "Setting FIFO watermarks - A: %d\n", planea_wm);2264 2265	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);2266}2267 2268/* latency must be in 0.1us units. */2269static unsigned int ilk_wm_method1(unsigned int pixel_rate,2270				   unsigned int cpp,2271				   unsigned int latency)2272{2273	unsigned int ret;2274 2275	ret = intel_wm_method1(pixel_rate, cpp, latency);2276	ret = DIV_ROUND_UP(ret, 64) + 2;2277 2278	return ret;2279}2280 2281/* latency must be in 0.1us units. */2282static unsigned int ilk_wm_method2(unsigned int pixel_rate,2283				   unsigned int htotal,2284				   unsigned int width,2285				   unsigned int cpp,2286				   unsigned int latency)2287{2288	unsigned int ret;2289 2290	ret = intel_wm_method2(pixel_rate, htotal,2291			       width, cpp, latency);2292	ret = DIV_ROUND_UP(ret, 64) + 2;2293 2294	return ret;2295}2296 2297static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)2298{2299	/*2300	 * Neither of these should be possible since this function shouldn't be2301	 * called if the CRTC is off or the plane is invisible.  But let's be2302	 * extra paranoid to avoid a potential divide-by-zero if we screw up2303	 * elsewhere in the driver.2304	 */2305	if (WARN_ON(!cpp))2306		return 0;2307	if (WARN_ON(!horiz_pixels))2308		return 0;2309 2310	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;2311}2312 2313struct ilk_wm_maximums {2314	u16 pri;2315	u16 spr;2316	u16 cur;2317	u16 fbc;2318};2319 2320/*2321 * For both WM_PIPE and WM_LP.2322 * mem_value must be in 0.1us units.2323 */2324static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,2325			      const struct intel_plane_state *plane_state,2326			      u32 mem_value, bool is_lp)2327{2328	u32 method1, method2;2329	int cpp;2330 2331	if (mem_value == 0)2332		return U32_MAX;2333 2334	if (!intel_wm_plane_visible(crtc_state, plane_state))2335		return 0;2336 2337	cpp = plane_state->hw.fb->format->cpp[0];2338 2339	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);2340 2341	if (!is_lp)2342		return method1;2343 2344	method2 = ilk_wm_method2(crtc_state->pixel_rate,2345				 crtc_state->hw.pipe_mode.crtc_htotal,2346				 drm_rect_width(&plane_state->uapi.src) >> 16,2347				 cpp, mem_value);2348 2349	return min(method1, method2);2350}2351 2352/*2353 * For both WM_PIPE and WM_LP.2354 * mem_value must be in 0.1us units.2355 */2356static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,2357			      const struct intel_plane_state *plane_state,2358			      u32 mem_value)2359{2360	u32 method1, method2;2361	int cpp;2362 2363	if (mem_value == 0)2364		return U32_MAX;2365 2366	if (!intel_wm_plane_visible(crtc_state, plane_state))2367		return 0;2368 2369	cpp = plane_state->hw.fb->format->cpp[0];2370 2371	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);2372	method2 = ilk_wm_method2(crtc_state->pixel_rate,2373				 crtc_state->hw.pipe_mode.crtc_htotal,2374				 drm_rect_width(&plane_state->uapi.src) >> 16,2375				 cpp, mem_value);2376	return min(method1, method2);2377}2378 2379/*2380 * For both WM_PIPE and WM_LP.2381 * mem_value must be in 0.1us units.2382 */2383static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,2384			      const struct intel_plane_state *plane_state,2385			      u32 mem_value)2386{2387	int cpp;2388 2389	if (mem_value == 0)2390		return U32_MAX;2391 2392	if (!intel_wm_plane_visible(crtc_state, plane_state))2393		return 0;2394 2395	cpp = plane_state->hw.fb->format->cpp[0];2396 2397	return ilk_wm_method2(crtc_state->pixel_rate,2398			      crtc_state->hw.pipe_mode.crtc_htotal,2399			      drm_rect_width(&plane_state->uapi.src) >> 16,2400			      cpp, mem_value);2401}2402 2403/* Only for WM_LP. */2404static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,2405			      const struct intel_plane_state *plane_state,2406			      u32 pri_val)2407{2408	int cpp;2409 2410	if (!intel_wm_plane_visible(crtc_state, plane_state))2411		return 0;2412 2413	cpp = plane_state->hw.fb->format->cpp[0];2414 2415	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,2416			  cpp);2417}2418 2419static unsigned int2420ilk_display_fifo_size(const struct drm_i915_private *dev_priv)2421{2422	if (DISPLAY_VER(dev_priv) >= 8)2423		return 3072;2424	else if (DISPLAY_VER(dev_priv) >= 7)2425		return 768;2426	else2427		return 512;2428}2429 2430static unsigned int2431ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,2432		     int level, bool is_sprite)2433{2434	if (DISPLAY_VER(dev_priv) >= 8)2435		/* BDW primary/sprite plane watermarks */2436		return level == 0 ? 255 : 2047;2437	else if (DISPLAY_VER(dev_priv) >= 7)2438		/* IVB/HSW primary/sprite plane watermarks */2439		return level == 0 ? 127 : 1023;2440	else if (!is_sprite)2441		/* ILK/SNB primary plane watermarks */2442		return level == 0 ? 127 : 511;2443	else2444		/* ILK/SNB sprite plane watermarks */2445		return level == 0 ? 63 : 255;2446}2447 2448static unsigned int2449ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)2450{2451	if (DISPLAY_VER(dev_priv) >= 7)2452		return level == 0 ? 63 : 255;2453	else2454		return level == 0 ? 31 : 63;2455}2456 2457static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)2458{2459	if (DISPLAY_VER(dev_priv) >= 8)2460		return 31;2461	else2462		return 15;2463}2464 2465/* Calculate the maximum primary/sprite plane watermark */2466static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,2467				     int level,2468				     const struct intel_wm_config *config,2469				     enum intel_ddb_partitioning ddb_partitioning,2470				     bool is_sprite)2471{2472	unsigned int fifo_size = ilk_display_fifo_size(dev_priv);2473 2474	/* if sprites aren't enabled, sprites get nothing */2475	if (is_sprite && !config->sprites_enabled)2476		return 0;2477 2478	/* HSW allows LP1+ watermarks even with multiple pipes */2479	if (level == 0 || config->num_pipes_active > 1) {2480		fifo_size /= INTEL_NUM_PIPES(dev_priv);2481 2482		/*2483		 * For some reason the non self refresh2484		 * FIFO size is only half of the self2485		 * refresh FIFO size on ILK/SNB.2486		 */2487		if (DISPLAY_VER(dev_priv) < 7)2488			fifo_size /= 2;2489	}2490 2491	if (config->sprites_enabled) {2492		/* level 0 is always calculated with 1:1 split */2493		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {2494			if (is_sprite)2495				fifo_size *= 5;2496			fifo_size /= 6;2497		} else {2498			fifo_size /= 2;2499		}2500	}2501 2502	/* clamp to max that the registers can hold */2503	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));2504}2505 2506/* Calculate the maximum cursor plane watermark */2507static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,2508				      int level,2509				      const struct intel_wm_config *config)2510{2511	/* HSW LP1+ watermarks w/ multiple pipes */2512	if (level > 0 && config->num_pipes_active > 1)2513		return 64;2514 2515	/* otherwise just report max that registers can hold */2516	return ilk_cursor_wm_reg_max(dev_priv, level);2517}2518 2519static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,2520				    int level,2521				    const struct intel_wm_config *config,2522				    enum intel_ddb_partitioning ddb_partitioning,2523				    struct ilk_wm_maximums *max)2524{2525	max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);2526	max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);2527	max->cur = ilk_cursor_wm_max(dev_priv, level, config);2528	max->fbc = ilk_fbc_wm_reg_max(dev_priv);2529}2530 2531static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,2532					int level,2533					struct ilk_wm_maximums *max)2534{2535	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);2536	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);2537	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);2538	max->fbc = ilk_fbc_wm_reg_max(dev_priv);2539}2540 2541static bool ilk_validate_wm_level(struct drm_i915_private *i915,2542				  int level,2543				  const struct ilk_wm_maximums *max,2544				  struct intel_wm_level *result)2545{2546	bool ret;2547 2548	/* already determined to be invalid? */2549	if (!result->enable)2550		return false;2551 2552	result->enable = result->pri_val <= max->pri &&2553			 result->spr_val <= max->spr &&2554			 result->cur_val <= max->cur;2555 2556	ret = result->enable;2557 2558	/*2559	 * HACK until we can pre-compute everything,2560	 * and thus fail gracefully if LP0 watermarks2561	 * are exceeded...2562	 */2563	if (level == 0 && !result->enable) {2564		if (result->pri_val > max->pri)2565			drm_dbg_kms(&i915->drm,2566				    "Primary WM%d too large %u (max %u)\n",2567				    level, result->pri_val, max->pri);2568		if (result->spr_val > max->spr)2569			drm_dbg_kms(&i915->drm,2570				    "Sprite WM%d too large %u (max %u)\n",2571				    level, result->spr_val, max->spr);2572		if (result->cur_val > max->cur)2573			drm_dbg_kms(&i915->drm,2574				    "Cursor WM%d too large %u (max %u)\n",2575				    level, result->cur_val, max->cur);2576 2577		result->pri_val = min_t(u32, result->pri_val, max->pri);2578		result->spr_val = min_t(u32, result->spr_val, max->spr);2579		result->cur_val = min_t(u32, result->cur_val, max->cur);2580		result->enable = true;2581	}2582 2583	return ret;2584}2585 2586static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,2587				 const struct intel_crtc *crtc,2588				 int level,2589				 struct intel_crtc_state *crtc_state,2590				 const struct intel_plane_state *pristate,2591				 const struct intel_plane_state *sprstate,2592				 const struct intel_plane_state *curstate,2593				 struct intel_wm_level *result)2594{2595	u16 pri_latency = dev_priv->display.wm.pri_latency[level];2596	u16 spr_latency = dev_priv->display.wm.spr_latency[level];2597	u16 cur_latency = dev_priv->display.wm.cur_latency[level];2598 2599	/* WM1+ latency values stored in 0.5us units */2600	if (level > 0) {2601		pri_latency *= 5;2602		spr_latency *= 5;2603		cur_latency *= 5;2604	}2605 2606	if (pristate) {2607		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,2608						     pri_latency, level);2609		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);2610	}2611 2612	if (sprstate)2613		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);2614 2615	if (curstate)2616		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);2617 2618	result->enable = true;2619}2620 2621static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])2622{2623	u64 sskpd;2624 2625	i915->display.wm.num_levels = 5;2626 2627	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);2628 2629	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);2630	if (wm[0] == 0)2631		wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);2632	wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);2633	wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);2634	wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);2635	wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);2636}2637 2638static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])2639{2640	u32 sskpd;2641 2642	i915->display.wm.num_levels = 4;2643 2644	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);2645 2646	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);2647	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);2648	wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);2649	wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);2650}2651 2652static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])2653{2654	u32 mltr;2655 2656	i915->display.wm.num_levels = 3;2657 2658	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);2659 2660	/* ILK primary LP0 latency is 700 ns */2661	wm[0] = 7;2662	wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);2663	wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);2664}2665 2666static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,2667				       u16 wm[5])2668{2669	/* ILK sprite LP0 latency is 1300 ns */2670	if (DISPLAY_VER(dev_priv) == 5)2671		wm[0] = 13;2672}2673 2674static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,2675				       u16 wm[5])2676{2677	/* ILK cursor LP0 latency is 1300 ns */2678	if (DISPLAY_VER(dev_priv) == 5)2679		wm[0] = 13;2680}2681 2682static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,2683				    u16 wm[5], u16 min)2684{2685	int level;2686 2687	if (wm[0] >= min)2688		return false;2689 2690	wm[0] = max(wm[0], min);2691	for (level = 1; level < dev_priv->display.wm.num_levels; level++)2692		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));2693 2694	return true;2695}2696 2697static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)2698{2699	bool changed;2700 2701	/*2702	 * The BIOS provided WM memory latency values are often2703	 * inadequate for high resolution displays. Adjust them.2704	 */2705	changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);2706	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);2707	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);2708 2709	if (!changed)2710		return;2711 2712	drm_dbg_kms(&dev_priv->drm,2713		    "WM latency values increased to avoid potential underruns\n");2714	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);2715	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);2716	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);2717}2718 2719static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)2720{2721	/*2722	 * On some SNB machines (Thinkpad X220 Tablet at least)2723	 * LP3 usage can cause vblank interrupts to be lost.2724	 * The DEIIR bit will go high but it looks like the CPU2725	 * never gets interrupted.2726	 *2727	 * It's not clear whether other interrupt source could2728	 * be affected or if this is somehow limited to vblank2729	 * interrupts only. To play it safe we disable LP32730	 * watermarks entirely.2731	 */2732	if (dev_priv->display.wm.pri_latency[3] == 0 &&2733	    dev_priv->display.wm.spr_latency[3] == 0 &&2734	    dev_priv->display.wm.cur_latency[3] == 0)2735		return;2736 2737	dev_priv->display.wm.pri_latency[3] = 0;2738	dev_priv->display.wm.spr_latency[3] = 0;2739	dev_priv->display.wm.cur_latency[3] = 0;2740 2741	drm_dbg_kms(&dev_priv->drm,2742		    "LP3 watermarks disabled due to potential for lost interrupts\n");2743	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);2744	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);2745	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);2746}2747 2748static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)2749{2750	if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))2751		hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);2752	else if (DISPLAY_VER(dev_priv) >= 6)2753		snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);2754	else2755		ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);2756 2757	memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,2758	       sizeof(dev_priv->display.wm.pri_latency));2759	memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,2760	       sizeof(dev_priv->display.wm.pri_latency));2761 2762	intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);2763	intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);2764 2765	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);2766	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);2767	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);2768 2769	if (DISPLAY_VER(dev_priv) == 6) {2770		snb_wm_latency_quirk(dev_priv);2771		snb_wm_lp3_irq_quirk(dev_priv);2772	}2773}2774 2775static bool ilk_validate_pipe_wm(struct drm_i915_private *dev_priv,2776				 struct intel_pipe_wm *pipe_wm)2777{2778	/* LP0 watermark maximums depend on this pipe alone */2779	const struct intel_wm_config config = {2780		.num_pipes_active = 1,2781		.sprites_enabled = pipe_wm->sprites_enabled,2782		.sprites_scaled = pipe_wm->sprites_scaled,2783	};2784	struct ilk_wm_maximums max;2785 2786	/* LP0 watermarks always use 1/2 DDB partitioning */2787	ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);2788 2789	/* At least LP0 must be valid */2790	if (!ilk_validate_wm_level(dev_priv, 0, &max, &pipe_wm->wm[0])) {2791		drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");2792		return false;2793	}2794 2795	return true;2796}2797 2798/* Compute new watermarks for the pipe */2799static int ilk_compute_pipe_wm(struct intel_atomic_state *state,2800			       struct intel_crtc *crtc)2801{2802	struct drm_i915_private *dev_priv = to_i915(state->base.dev);2803	struct intel_crtc_state *crtc_state =2804		intel_atomic_get_new_crtc_state(state, crtc);2805	struct intel_pipe_wm *pipe_wm;2806	struct intel_plane *plane;2807	const struct intel_plane_state *plane_state;2808	const struct intel_plane_state *pristate = NULL;2809	const struct intel_plane_state *sprstate = NULL;2810	const struct intel_plane_state *curstate = NULL;2811	struct ilk_wm_maximums max;2812	int level, usable_level;2813 2814	pipe_wm = &crtc_state->wm.ilk.optimal;2815 2816	intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {2817		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)2818			pristate = plane_state;2819		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)2820			sprstate = plane_state;2821		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)2822			curstate = plane_state;2823	}2824 2825	pipe_wm->pipe_enabled = crtc_state->hw.active;2826	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);2827	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);2828 2829	usable_level = dev_priv->display.wm.num_levels - 1;2830 2831	/* ILK/SNB: LP2+ watermarks only w/o sprites */2832	if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)2833		usable_level = 1;2834 2835	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */2836	if (pipe_wm->sprites_scaled)2837		usable_level = 0;2838 2839	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));2840	ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,2841			     pristate, sprstate, curstate, &pipe_wm->wm[0]);2842 2843	if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))2844		return -EINVAL;2845 2846	ilk_compute_wm_reg_maximums(dev_priv, 1, &max);2847 2848	for (level = 1; level <= usable_level; level++) {2849		struct intel_wm_level *wm = &pipe_wm->wm[level];2850 2851		ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,2852				     pristate, sprstate, curstate, wm);2853 2854		/*2855		 * Disable any watermark level that exceeds the2856		 * register maximums since such watermarks are2857		 * always invalid.2858		 */2859		if (!ilk_validate_wm_level(dev_priv, level, &max, wm)) {2860			memset(wm, 0, sizeof(*wm));2861			break;2862		}2863	}2864 2865	return 0;2866}2867 2868/*2869 * Build a set of 'intermediate' watermark values that satisfy both the old2870 * state and the new state.  These can be programmed to the hardware2871 * immediately.2872 */2873static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,2874				       struct intel_crtc *crtc)2875{2876	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);2877	struct intel_crtc_state *new_crtc_state =2878		intel_atomic_get_new_crtc_state(state, crtc);2879	const struct intel_crtc_state *old_crtc_state =2880		intel_atomic_get_old_crtc_state(state, crtc);2881	struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;2882	const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;2883	int level;2884 2885	/*2886	 * Start with the final, target watermarks, then combine with the2887	 * currently active watermarks to get values that are safe both before2888	 * and after the vblank.2889	 */2890	*a = new_crtc_state->wm.ilk.optimal;2891	if (!new_crtc_state->hw.active ||2892	    intel_crtc_needs_modeset(new_crtc_state) ||2893	    state->skip_intermediate_wm)2894		return 0;2895 2896	a->pipe_enabled |= b->pipe_enabled;2897	a->sprites_enabled |= b->sprites_enabled;2898	a->sprites_scaled |= b->sprites_scaled;2899 2900	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {2901		struct intel_wm_level *a_wm = &a->wm[level];2902		const struct intel_wm_level *b_wm = &b->wm[level];2903 2904		a_wm->enable &= b_wm->enable;2905		a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);2906		a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);2907		a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);2908		a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);2909	}2910 2911	/*2912	 * We need to make sure that these merged watermark values are2913	 * actually a valid configuration themselves.  If they're not,2914	 * there's no safe way to transition from the old state to2915	 * the new state, so we need to fail the atomic transaction.2916	 */2917	if (!ilk_validate_pipe_wm(dev_priv, a))2918		return -EINVAL;2919 2920	/*2921	 * If our intermediate WM are identical to the final WM, then we can2922	 * omit the post-vblank programming; only update if it's different.2923	 */2924	if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)2925		new_crtc_state->wm.need_postvbl_update = true;2926 2927	return 0;2928}2929 2930/*2931 * Merge the watermarks from all active pipes for a specific level.2932 */2933static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,2934			       int level,2935			       struct intel_wm_level *ret_wm)2936{2937	const struct intel_crtc *crtc;2938 2939	ret_wm->enable = true;2940 2941	for_each_intel_crtc(&dev_priv->drm, crtc) {2942		const struct intel_pipe_wm *active = &crtc->wm.active.ilk;2943		const struct intel_wm_level *wm = &active->wm[level];2944 2945		if (!active->pipe_enabled)2946			continue;2947 2948		/*2949		 * The watermark values may have been used in the past,2950		 * so we must maintain them in the registers for some2951		 * time even if the level is now disabled.2952		 */2953		if (!wm->enable)2954			ret_wm->enable = false;2955 2956		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);2957		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);2958		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);2959		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);2960	}2961}2962 2963/*2964 * Merge all low power watermarks for all active pipes.2965 */2966static void ilk_wm_merge(struct drm_i915_private *dev_priv,2967			 const struct intel_wm_config *config,2968			 const struct ilk_wm_maximums *max,2969			 struct intel_pipe_wm *merged)2970{2971	int level, num_levels = dev_priv->display.wm.num_levels;2972	int last_enabled_level = num_levels - 1;2973 2974	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */2975	if ((DISPLAY_VER(dev_priv) < 7 || IS_IVYBRIDGE(dev_priv)) &&2976	    config->num_pipes_active > 1)2977		last_enabled_level = 0;2978 2979	/* ILK: FBC WM must be disabled always */2980	merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;2981 2982	/* merge each WM1+ level */2983	for (level = 1; level < num_levels; level++) {2984		struct intel_wm_level *wm = &merged->wm[level];2985 2986		ilk_merge_wm_level(dev_priv, level, wm);2987 2988		if (level > last_enabled_level)2989			wm->enable = false;2990		else if (!ilk_validate_wm_level(dev_priv, level, max, wm))2991			/* make sure all following levels get disabled */2992			last_enabled_level = level - 1;2993 2994		/*2995		 * The spec says it is preferred to disable2996		 * FBC WMs instead of disabling a WM level.2997		 */2998		if (wm->fbc_val > max->fbc) {2999			if (wm->enable)3000				merged->fbc_wm_enabled = false;3001			wm->fbc_val = 0;3002		}3003	}3004 3005	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */3006	if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&3007	    dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {3008		for (level = 2; level < num_levels; level++) {3009			struct intel_wm_level *wm = &merged->wm[level];3010 3011			wm->enable = false;3012		}3013	}3014}3015 3016static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)3017{3018	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */3019	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);3020}3021 3022/* The value we need to program into the WM_LPx latency field */3023static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,3024				      int level)3025{3026	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))3027		return 2 * level;3028	else3029		return dev_priv->display.wm.pri_latency[level];3030}3031 3032static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,3033				   const struct intel_pipe_wm *merged,3034				   enum intel_ddb_partitioning partitioning,3035				   struct ilk_wm_values *results)3036{3037	struct intel_crtc *crtc;3038	int level, wm_lp;3039 3040	results->enable_fbc_wm = merged->fbc_wm_enabled;3041	results->partitioning = partitioning;3042 3043	/* LP1+ register values */3044	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {3045		const struct intel_wm_level *r;3046 3047		level = ilk_wm_lp_to_level(wm_lp, merged);3048 3049		r = &merged->wm[level];3050 3051		/*3052		 * Maintain the watermark values even if the level is3053		 * disabled. Doing otherwise could cause underruns.3054		 */3055		results->wm_lp[wm_lp - 1] =3056			WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |3057			WM_LP_PRIMARY(r->pri_val) |3058			WM_LP_CURSOR(r->cur_val);3059 3060		if (r->enable)3061			results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;3062 3063		if (DISPLAY_VER(dev_priv) >= 8)3064			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);3065		else3066			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);3067 3068		results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);3069 3070		/*3071		 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the3072		 * level is disabled. Doing otherwise could cause underruns.3073		 */3074		if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {3075			drm_WARN_ON(&dev_priv->drm, wm_lp != 1);3076			results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;3077		}3078	}3079 3080	/* LP0 register values */3081	for_each_intel_crtc(&dev_priv->drm, crtc) {3082		enum pipe pipe = crtc->pipe;3083		const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;3084		const struct intel_wm_level *r = &pipe_wm->wm[0];3085 3086		if (drm_WARN_ON(&dev_priv->drm, !r->enable))3087			continue;3088 3089		results->wm_pipe[pipe] =3090			WM0_PIPE_PRIMARY(r->pri_val) |3091			WM0_PIPE_SPRITE(r->spr_val) |3092			WM0_PIPE_CURSOR(r->cur_val);3093	}3094}3095 3096/*3097 * Find the result with the highest level enabled. Check for enable_fbc_wm in3098 * case both are at the same level. Prefer r1 in case they're the same.3099 */3100static struct intel_pipe_wm *3101ilk_find_best_result(struct drm_i915_private *dev_priv,3102		     struct intel_pipe_wm *r1,3103		     struct intel_pipe_wm *r2)3104{3105	int level, level1 = 0, level2 = 0;3106 3107	for (level = 1; level < dev_priv->display.wm.num_levels; level++) {3108		if (r1->wm[level].enable)3109			level1 = level;3110		if (r2->wm[level].enable)3111			level2 = level;3112	}3113 3114	if (level1 == level2) {3115		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)3116			return r2;3117		else3118			return r1;3119	} else if (level1 > level2) {3120		return r1;3121	} else {3122		return r2;3123	}3124}3125 3126/* dirty bits used to track which watermarks need changes */3127#define WM_DIRTY_PIPE(pipe) (1 << (pipe))3128#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))3129#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))3130#define WM_DIRTY_FBC (1 << 24)3131#define WM_DIRTY_DDB (1 << 25)3132 3133static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,3134					 const struct ilk_wm_values *old,3135					 const struct ilk_wm_values *new)3136{3137	unsigned int dirty = 0;3138	enum pipe pipe;3139	int wm_lp;3140 3141	for_each_pipe(dev_priv, pipe) {3142		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {3143			dirty |= WM_DIRTY_PIPE(pipe);3144			/* Must disable LP1+ watermarks too */3145			dirty |= WM_DIRTY_LP_ALL;3146		}3147	}3148 3149	if (old->enable_fbc_wm != new->enable_fbc_wm) {3150		dirty |= WM_DIRTY_FBC;3151		/* Must disable LP1+ watermarks too */3152		dirty |= WM_DIRTY_LP_ALL;3153	}3154 3155	if (old->partitioning != new->partitioning) {3156		dirty |= WM_DIRTY_DDB;3157		/* Must disable LP1+ watermarks too */3158		dirty |= WM_DIRTY_LP_ALL;3159	}3160 3161	/* LP1+ watermarks already deemed dirty, no need to continue */3162	if (dirty & WM_DIRTY_LP_ALL)3163		return dirty;3164 3165	/* Find the lowest numbered LP1+ watermark in need of an update... */3166	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {3167		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||3168		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])3169			break;3170	}3171 3172	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */3173	for (; wm_lp <= 3; wm_lp++)3174		dirty |= WM_DIRTY_LP(wm_lp);3175 3176	return dirty;3177}3178 3179static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,3180			       unsigned int dirty)3181{3182	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;3183	bool changed = false;3184 3185	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {3186		previous->wm_lp[2] &= ~WM_LP_ENABLE;3187		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);3188		changed = true;3189	}3190	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {3191		previous->wm_lp[1] &= ~WM_LP_ENABLE;3192		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);3193		changed = true;3194	}3195	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {3196		previous->wm_lp[0] &= ~WM_LP_ENABLE;3197		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);3198		changed = true;3199	}3200 3201	/*3202	 * Don't touch WM_LP_SPRITE_ENABLE here.3203	 * Doing so could cause underruns.3204	 */3205 3206	return changed;3207}3208 3209/*3210 * The spec says we shouldn't write when we don't need, because every write3211 * causes WMs to be re-evaluated, expending some power.3212 */3213static void ilk_write_wm_values(struct drm_i915_private *dev_priv,3214				struct ilk_wm_values *results)3215{3216	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;3217	unsigned int dirty;3218 3219	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);3220	if (!dirty)3221		return;3222 3223	_ilk_disable_lp_wm(dev_priv, dirty);3224 3225	if (dirty & WM_DIRTY_PIPE(PIPE_A))3226		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);3227	if (dirty & WM_DIRTY_PIPE(PIPE_B))3228		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);3229	if (dirty & WM_DIRTY_PIPE(PIPE_C))3230		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);3231 3232	if (dirty & WM_DIRTY_DDB) {3233		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))3234			intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,3235					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :3236					 WM_MISC_DATA_PARTITION_5_6);3237		else3238			intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,3239					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :3240					 DISP_DATA_PARTITION_5_6);3241	}3242 3243	if (dirty & WM_DIRTY_FBC)3244		intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,3245				 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);3246 3247	if (dirty & WM_DIRTY_LP(1) &&3248	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])3249		intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);3250 3251	if (DISPLAY_VER(dev_priv) >= 7) {3252		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])3253			intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);3254		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])3255			intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);3256	}3257 3258	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])3259		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);3260	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])3261		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);3262	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])3263		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);3264 3265	dev_priv->display.wm.hw = *results;3266}3267 3268bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)3269{3270	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);3271}3272 3273static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,3274				  struct intel_wm_config *config)3275{3276	struct intel_crtc *crtc;3277 3278	/* Compute the currently _active_ config */3279	for_each_intel_crtc(&dev_priv->drm, crtc) {3280		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;3281 3282		if (!wm->pipe_enabled)3283			continue;3284 3285		config->sprites_enabled |= wm->sprites_enabled;3286		config->sprites_scaled |= wm->sprites_scaled;3287		config->num_pipes_active++;3288	}3289}3290 3291static void ilk_program_watermarks(struct drm_i915_private *dev_priv)3292{3293	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;3294	struct ilk_wm_maximums max;3295	struct intel_wm_config config = {};3296	struct ilk_wm_values results = {};3297	enum intel_ddb_partitioning partitioning;3298 3299	ilk_compute_wm_config(dev_priv, &config);3300 3301	ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);3302	ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);3303 3304	/* 5/6 split only in single pipe config on IVB+ */3305	if (DISPLAY_VER(dev_priv) >= 7 &&3306	    config.num_pipes_active == 1 && config.sprites_enabled) {3307		ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);3308		ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);3309 3310		best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);3311	} else {3312		best_lp_wm = &lp_wm_1_2;3313	}3314 3315	partitioning = (best_lp_wm == &lp_wm_1_2) ?3316		       INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;3317 3318	ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);3319 3320	ilk_write_wm_values(dev_priv, &results);3321}3322 3323static void ilk_initial_watermarks(struct intel_atomic_state *state,3324				   struct intel_crtc *crtc)3325{3326	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);3327	const struct intel_crtc_state *crtc_state =3328		intel_atomic_get_new_crtc_state(state, crtc);3329 3330	mutex_lock(&dev_priv->display.wm.wm_mutex);3331	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;3332	ilk_program_watermarks(dev_priv);3333	mutex_unlock(&dev_priv->display.wm.wm_mutex);3334}3335 3336static void ilk_optimize_watermarks(struct intel_atomic_state *state,3337				    struct intel_crtc *crtc)3338{3339	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);3340	const struct intel_crtc_state *crtc_state =3341		intel_atomic_get_new_crtc_state(state, crtc);3342 3343	if (!crtc_state->wm.need_postvbl_update)3344		return;3345 3346	mutex_lock(&dev_priv->display.wm.wm_mutex);3347	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;3348	ilk_program_watermarks(dev_priv);3349	mutex_unlock(&dev_priv->display.wm.wm_mutex);3350}3351 3352static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)3353{3354	struct drm_device *dev = crtc->base.dev;3355	struct drm_i915_private *dev_priv = to_i915(dev);3356	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;3357	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);3358	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;3359	enum pipe pipe = crtc->pipe;3360 3361	hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));3362 3363	memset(active, 0, sizeof(*active));3364 3365	active->pipe_enabled = crtc->active;3366 3367	if (active->pipe_enabled) {3368		u32 tmp = hw->wm_pipe[pipe];3369 3370		/*3371		 * For active pipes LP0 watermark is marked as3372		 * enabled, and LP1+ watermaks as disabled since3373		 * we can't really reverse compute them in case3374		 * multiple pipes are active.3375		 */3376		active->wm[0].enable = true;3377		active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);3378		active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);3379		active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);3380	} else {3381		int level;3382 3383		/*3384		 * For inactive pipes, all watermark levels3385		 * should be marked as enabled but zeroed,3386		 * which is what we'd compute them to.3387		 */3388		for (level = 0; level < dev_priv->display.wm.num_levels; level++)3389			active->wm[level].enable = true;3390	}3391 3392	crtc->wm.active.ilk = *active;3393}3394 3395static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)3396{3397	struct drm_plane *plane;3398	struct intel_crtc *crtc;3399 3400	for_each_intel_crtc(state->dev, crtc) {3401		struct intel_crtc_state *crtc_state;3402 3403		crtc_state = intel_atomic_get_crtc_state(state, crtc);3404		if (IS_ERR(crtc_state))3405			return PTR_ERR(crtc_state);3406 3407		if (crtc_state->hw.active) {3408			/*3409			 * Preserve the inherited flag to avoid3410			 * taking the full modeset path.3411			 */3412			crtc_state->inherited = true;3413		}3414	}3415 3416	drm_for_each_plane(plane, state->dev) {3417		struct drm_plane_state *plane_state;3418 3419		plane_state = drm_atomic_get_plane_state(state, plane);3420		if (IS_ERR(plane_state))3421			return PTR_ERR(plane_state);3422	}3423 3424	return 0;3425}3426 3427/*3428 * Calculate what we think the watermarks should be for the state we've read3429 * out of the hardware and then immediately program those watermarks so that3430 * we ensure the hardware settings match our internal state.3431 *3432 * We can calculate what we think WM's should be by creating a duplicate of the3433 * current state (which was constructed during hardware readout) and running it3434 * through the atomic check code to calculate new watermark values in the3435 * state object.3436 */3437void ilk_wm_sanitize(struct drm_i915_private *dev_priv)3438{3439	struct drm_atomic_state *state;3440	struct intel_atomic_state *intel_state;3441	struct intel_crtc *crtc;3442	struct intel_crtc_state *crtc_state;3443	struct drm_modeset_acquire_ctx ctx;3444	int ret;3445	int i;3446 3447	/* Only supported on platforms that use atomic watermark design */3448	if (!dev_priv->display.funcs.wm->optimize_watermarks)3449		return;3450 3451	if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))3452		return;3453 3454	state = drm_atomic_state_alloc(&dev_priv->drm);3455	if (drm_WARN_ON(&dev_priv->drm, !state))3456		return;3457 3458	intel_state = to_intel_atomic_state(state);3459 3460	drm_modeset_acquire_init(&ctx, 0);3461 3462	state->acquire_ctx = &ctx;3463	to_intel_atomic_state(state)->internal = true;3464 3465retry:3466	/*3467	 * Hardware readout is the only time we don't want to calculate3468	 * intermediate watermarks (since we don't trust the current3469	 * watermarks).3470	 */3471	if (!HAS_GMCH(dev_priv))3472		intel_state->skip_intermediate_wm = true;3473 3474	ret = ilk_sanitize_watermarks_add_affected(state);3475	if (ret)3476		goto fail;3477 3478	ret = intel_atomic_check(&dev_priv->drm, state);3479	if (ret)3480		goto fail;3481 3482	/* Write calculated watermark values back */3483	for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {3484		crtc_state->wm.need_postvbl_update = true;3485		intel_optimize_watermarks(intel_state, crtc);3486 3487		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;3488	}3489 3490fail:3491	if (ret == -EDEADLK) {3492		drm_atomic_state_clear(state);3493		drm_modeset_backoff(&ctx);3494		goto retry;3495	}3496 3497	/*3498	 * If we fail here, it means that the hardware appears to be3499	 * programmed in a way that shouldn't be possible, given our3500	 * understanding of watermark requirements.  This might mean a3501	 * mistake in the hardware readout code or a mistake in the3502	 * watermark calculations for a given platform.  Raise a WARN3503	 * so that this is noticeable.3504	 *3505	 * If this actually happens, we'll have to just leave the3506	 * BIOS-programmed watermarks untouched and hope for the best.3507	 */3508	drm_WARN(&dev_priv->drm, ret,3509		 "Could not determine valid watermarks for inherited state\n");3510 3511	drm_atomic_state_put(state);3512 3513	drm_modeset_drop_locks(&ctx);3514	drm_modeset_acquire_fini(&ctx);3515}3516 3517#define _FW_WM(value, plane) \3518	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)3519#define _FW_WM_VLV(value, plane) \3520	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)3521 3522static void g4x_read_wm_values(struct drm_i915_private *dev_priv,3523			       struct g4x_wm_values *wm)3524{3525	u32 tmp;3526 3527	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));3528	wm->sr.plane = _FW_WM(tmp, SR);3529	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);3530	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);3531	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);3532 3533	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));3534	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;3535	wm->sr.fbc = _FW_WM(tmp, FBC_SR);3536	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);3537	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);3538	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);3539	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);3540 3541	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));3542	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;3543	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);3544	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);3545	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);3546}3547 3548static void vlv_read_wm_values(struct drm_i915_private *dev_priv,3549			       struct vlv_wm_values *wm)3550{3551	enum pipe pipe;3552	u32 tmp;3553 3554	for_each_pipe(dev_priv, pipe) {3555		tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));3556 3557		wm->ddl[pipe].plane[PLANE_PRIMARY] =3558			(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);3559		wm->ddl[pipe].plane[PLANE_CURSOR] =3560			(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);3561		wm->ddl[pipe].plane[PLANE_SPRITE0] =3562			(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);3563		wm->ddl[pipe].plane[PLANE_SPRITE1] =3564			(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);3565	}3566 3567	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));3568	wm->sr.plane = _FW_WM(tmp, SR);3569	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);3570	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);3571	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);3572 3573	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));3574	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);3575	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);3576	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);3577 3578	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));3579	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);3580 3581	if (IS_CHERRYVIEW(dev_priv)) {3582		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);3583		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);3584		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);3585 3586		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);3587		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);3588		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);3589 3590		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);3591		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);3592		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);3593 3594		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);3595		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;3596		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;3597		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;3598		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;3599		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;3600		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;3601		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;3602		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;3603		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;3604		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;3605	} else {3606		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);3607		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);3608		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);3609 3610		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);3611		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;3612		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;3613		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;3614		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;3615		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;3616		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;3617		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;3618	}3619}3620 3621#undef _FW_WM3622#undef _FW_WM_VLV3623 3624static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)3625{3626	struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;3627	struct intel_crtc *crtc;3628 3629	g4x_read_wm_values(dev_priv, wm);3630 3631	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;3632 3633	for_each_intel_crtc(&dev_priv->drm, crtc) {3634		struct intel_crtc_state *crtc_state =3635			to_intel_crtc_state(crtc->base.state);3636		struct g4x_wm_state *active = &crtc->wm.active.g4x;3637		struct g4x_pipe_wm *raw;3638		enum pipe pipe = crtc->pipe;3639		enum plane_id plane_id;3640		int level, max_level;3641 3642		active->cxsr = wm->cxsr;3643		active->hpll_en = wm->hpll_en;3644		active->fbc_en = wm->fbc_en;3645 3646		active->sr = wm->sr;3647		active->hpll = wm->hpll;3648 3649		for_each_plane_id_on_crtc(crtc, plane_id) {3650			active->wm.plane[plane_id] =3651				wm->pipe[pipe].plane[plane_id];3652		}3653 3654		if (wm->cxsr && wm->hpll_en)3655			max_level = G4X_WM_LEVEL_HPLL;3656		else if (wm->cxsr)3657			max_level = G4X_WM_LEVEL_SR;3658		else3659			max_level = G4X_WM_LEVEL_NORMAL;3660 3661		level = G4X_WM_LEVEL_NORMAL;3662		raw = &crtc_state->wm.g4x.raw[level];3663		for_each_plane_id_on_crtc(crtc, plane_id)3664			raw->plane[plane_id] = active->wm.plane[plane_id];3665 3666		level = G4X_WM_LEVEL_SR;3667		if (level > max_level)3668			goto out;3669 3670		raw = &crtc_state->wm.g4x.raw[level];3671		raw->plane[PLANE_PRIMARY] = active->sr.plane;3672		raw->plane[PLANE_CURSOR] = active->sr.cursor;3673		raw->plane[PLANE_SPRITE0] = 0;3674		raw->fbc = active->sr.fbc;3675 3676		level = G4X_WM_LEVEL_HPLL;3677		if (level > max_level)3678			goto out;3679 3680		raw = &crtc_state->wm.g4x.raw[level];3681		raw->plane[PLANE_PRIMARY] = active->hpll.plane;3682		raw->plane[PLANE_CURSOR] = active->hpll.cursor;3683		raw->plane[PLANE_SPRITE0] = 0;3684		raw->fbc = active->hpll.fbc;3685 3686		level++;3687	out:3688		for_each_plane_id_on_crtc(crtc, plane_id)3689			g4x_raw_plane_wm_set(crtc_state, level,3690					     plane_id, USHRT_MAX);3691		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);3692 3693		g4x_invalidate_wms(crtc, active, level);3694 3695		crtc_state->wm.g4x.optimal = *active;3696		crtc_state->wm.g4x.intermediate = *active;3697 3698		drm_dbg_kms(&dev_priv->drm,3699			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",3700			    pipe_name(pipe),3701			    wm->pipe[pipe].plane[PLANE_PRIMARY],3702			    wm->pipe[pipe].plane[PLANE_CURSOR],3703			    wm->pipe[pipe].plane[PLANE_SPRITE0]);3704	}3705 3706	drm_dbg_kms(&dev_priv->drm,3707		    "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",3708		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);3709	drm_dbg_kms(&dev_priv->drm,3710		    "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",3711		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);3712	drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",3713		    str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),3714		    str_yes_no(wm->fbc_en));3715}3716 3717static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)3718{3719	struct intel_plane *plane;3720	struct intel_crtc *crtc;3721 3722	mutex_lock(&dev_priv->display.wm.wm_mutex);3723 3724	for_each_intel_plane(&dev_priv->drm, plane) {3725		struct intel_crtc *crtc =3726			intel_crtc_for_pipe(dev_priv, plane->pipe);3727		struct intel_crtc_state *crtc_state =3728			to_intel_crtc_state(crtc->base.state);3729		struct intel_plane_state *plane_state =3730			to_intel_plane_state(plane->base.state);3731		enum plane_id plane_id = plane->id;3732		int level;3733 3734		if (plane_state->uapi.visible)3735			continue;3736 3737		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {3738			struct g4x_pipe_wm *raw =3739				&crtc_state->wm.g4x.raw[level];3740 3741			raw->plane[plane_id] = 0;3742 3743			if (plane_id == PLANE_PRIMARY)3744				raw->fbc = 0;3745		}3746	}3747 3748	for_each_intel_crtc(&dev_priv->drm, crtc) {3749		struct intel_crtc_state *crtc_state =3750			to_intel_crtc_state(crtc->base.state);3751		int ret;3752 3753		ret = _g4x_compute_pipe_wm(crtc_state);3754		drm_WARN_ON(&dev_priv->drm, ret);3755 3756		crtc_state->wm.g4x.intermediate =3757			crtc_state->wm.g4x.optimal;3758		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;3759	}3760 3761	g4x_program_watermarks(dev_priv);3762 3763	mutex_unlock(&dev_priv->display.wm.wm_mutex);3764}3765 3766static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)3767{3768	g4x_wm_get_hw_state(i915);3769	g4x_wm_sanitize(i915);3770}3771 3772static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)3773{3774	struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;3775	struct intel_crtc *crtc;3776	u32 val;3777 3778	vlv_read_wm_values(dev_priv, wm);3779 3780	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;3781	wm->level = VLV_WM_LEVEL_PM2;3782 3783	if (IS_CHERRYVIEW(dev_priv)) {3784		vlv_punit_get(dev_priv);3785 3786		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);3787		if (val & DSP_MAXFIFO_PM5_ENABLE)3788			wm->level = VLV_WM_LEVEL_PM5;3789 3790		/*3791		 * If DDR DVFS is disabled in the BIOS, Punit3792		 * will never ack the request. So if that happens3793		 * assume we don't have to enable/disable DDR DVFS3794		 * dynamically. To test that just set the REQ_ACK3795		 * bit to poke the Punit, but don't change the3796		 * HIGH/LOW bits so that we don't actually change3797		 * the current state.3798		 */3799		val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);3800		val |= FORCE_DDR_FREQ_REQ_ACK;3801		vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);3802 3803		if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &3804			      FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {3805			drm_dbg_kms(&dev_priv->drm,3806				    "Punit not acking DDR DVFS request, "3807				    "assuming DDR DVFS is disabled\n");3808			dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;3809		} else {3810			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);3811			if ((val & FORCE_DDR_HIGH_FREQ) == 0)3812				wm->level = VLV_WM_LEVEL_DDR_DVFS;3813		}3814 3815		vlv_punit_put(dev_priv);3816	}3817 3818	for_each_intel_crtc(&dev_priv->drm, crtc) {3819		struct intel_crtc_state *crtc_state =3820			to_intel_crtc_state(crtc->base.state);3821		struct vlv_wm_state *active = &crtc->wm.active.vlv;3822		const struct vlv_fifo_state *fifo_state =3823			&crtc_state->wm.vlv.fifo_state;3824		enum pipe pipe = crtc->pipe;3825		enum plane_id plane_id;3826		int level;3827 3828		vlv_get_fifo_size(crtc_state);3829 3830		active->num_levels = wm->level + 1;3831		active->cxsr = wm->cxsr;3832 3833		for (level = 0; level < active->num_levels; level++) {3834			struct g4x_pipe_wm *raw =3835				&crtc_state->wm.vlv.raw[level];3836 3837			active->sr[level].plane = wm->sr.plane;3838			active->sr[level].cursor = wm->sr.cursor;3839 3840			for_each_plane_id_on_crtc(crtc, plane_id) {3841				active->wm[level].plane[plane_id] =3842					wm->pipe[pipe].plane[plane_id];3843 3844				raw->plane[plane_id] =3845					vlv_invert_wm_value(active->wm[level].plane[plane_id],3846							    fifo_state->plane[plane_id]);3847			}3848		}3849 3850		for_each_plane_id_on_crtc(crtc, plane_id)3851			vlv_raw_plane_wm_set(crtc_state, level,3852					     plane_id, USHRT_MAX);3853		vlv_invalidate_wms(crtc, active, level);3854 3855		crtc_state->wm.vlv.optimal = *active;3856		crtc_state->wm.vlv.intermediate = *active;3857 3858		drm_dbg_kms(&dev_priv->drm,3859			    "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",3860			    pipe_name(pipe),3861			    wm->pipe[pipe].plane[PLANE_PRIMARY],3862			    wm->pipe[pipe].plane[PLANE_CURSOR],3863			    wm->pipe[pipe].plane[PLANE_SPRITE0],3864			    wm->pipe[pipe].plane[PLANE_SPRITE1]);3865	}3866 3867	drm_dbg_kms(&dev_priv->drm,3868		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",3869		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);3870}3871 3872static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)3873{3874	struct intel_plane *plane;3875	struct intel_crtc *crtc;3876 3877	mutex_lock(&dev_priv->display.wm.wm_mutex);3878 3879	for_each_intel_plane(&dev_priv->drm, plane) {3880		struct intel_crtc *crtc =3881			intel_crtc_for_pipe(dev_priv, plane->pipe);3882		struct intel_crtc_state *crtc_state =3883			to_intel_crtc_state(crtc->base.state);3884		struct intel_plane_state *plane_state =3885			to_intel_plane_state(plane->base.state);3886		enum plane_id plane_id = plane->id;3887		int level;3888 3889		if (plane_state->uapi.visible)3890			continue;3891 3892		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {3893			struct g4x_pipe_wm *raw =3894				&crtc_state->wm.vlv.raw[level];3895 3896			raw->plane[plane_id] = 0;3897		}3898	}3899 3900	for_each_intel_crtc(&dev_priv->drm, crtc) {3901		struct intel_crtc_state *crtc_state =3902			to_intel_crtc_state(crtc->base.state);3903		int ret;3904 3905		ret = _vlv_compute_pipe_wm(crtc_state);3906		drm_WARN_ON(&dev_priv->drm, ret);3907 3908		crtc_state->wm.vlv.intermediate =3909			crtc_state->wm.vlv.optimal;3910		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;3911	}3912 3913	vlv_program_watermarks(dev_priv);3914 3915	mutex_unlock(&dev_priv->display.wm.wm_mutex);3916}3917 3918static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)3919{3920	vlv_wm_get_hw_state(i915);3921	vlv_wm_sanitize(i915);3922}3923 3924/*3925 * FIXME should probably kill this and improve3926 * the real watermark readout/sanitation instead3927 */3928static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)3929{3930	intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);3931	intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);3932	intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);3933 3934	/*3935	 * Don't touch WM_LP_SPRITE_ENABLE here.3936	 * Doing so could cause underruns.3937	 */3938}3939 3940static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)3941{3942	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;3943	struct intel_crtc *crtc;3944 3945	ilk_init_lp_watermarks(dev_priv);3946 3947	for_each_intel_crtc(&dev_priv->drm, crtc)3948		ilk_pipe_wm_get_hw_state(crtc);3949 3950	hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);3951	hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);3952	hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);3953 3954	hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);3955	if (DISPLAY_VER(dev_priv) >= 7) {3956		hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);3957		hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);3958	}3959 3960	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))3961		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &3962				    WM_MISC_DATA_PARTITION_5_6) ?3963			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;3964	else if (IS_IVYBRIDGE(dev_priv))3965		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &3966				    DISP_DATA_PARTITION_5_6) ?3967			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;3968 3969	hw->enable_fbc_wm =3970		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);3971}3972 3973static const struct intel_wm_funcs ilk_wm_funcs = {3974	.compute_pipe_wm = ilk_compute_pipe_wm,3975	.compute_intermediate_wm = ilk_compute_intermediate_wm,3976	.initial_watermarks = ilk_initial_watermarks,3977	.optimize_watermarks = ilk_optimize_watermarks,3978	.get_hw_state = ilk_wm_get_hw_state,3979};3980 3981static const struct intel_wm_funcs vlv_wm_funcs = {3982	.compute_pipe_wm = vlv_compute_pipe_wm,3983	.compute_intermediate_wm = vlv_compute_intermediate_wm,3984	.initial_watermarks = vlv_initial_watermarks,3985	.optimize_watermarks = vlv_optimize_watermarks,3986	.atomic_update_watermarks = vlv_atomic_update_fifo,3987	.get_hw_state = vlv_wm_get_hw_state_and_sanitize,3988};3989 3990static const struct intel_wm_funcs g4x_wm_funcs = {3991	.compute_pipe_wm = g4x_compute_pipe_wm,3992	.compute_intermediate_wm = g4x_compute_intermediate_wm,3993	.initial_watermarks = g4x_initial_watermarks,3994	.optimize_watermarks = g4x_optimize_watermarks,3995	.get_hw_state = g4x_wm_get_hw_state_and_sanitize,3996};3997 3998static const struct intel_wm_funcs pnv_wm_funcs = {3999	.update_wm = pnv_update_wm,4000};4001 4002static const struct intel_wm_funcs i965_wm_funcs = {4003	.update_wm = i965_update_wm,4004};4005 4006static const struct intel_wm_funcs i9xx_wm_funcs = {4007	.update_wm = i9xx_update_wm,4008};4009 4010static const struct intel_wm_funcs i845_wm_funcs = {4011	.update_wm = i845_update_wm,4012};4013 4014static const struct intel_wm_funcs nop_funcs = {4015};4016 4017void i9xx_wm_init(struct drm_i915_private *dev_priv)4018{4019	/* For FIFO watermark updates */4020	if (HAS_PCH_SPLIT(dev_priv)) {4021		ilk_setup_wm_latency(dev_priv);4022		dev_priv->display.funcs.wm = &ilk_wm_funcs;4023	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {4024		vlv_setup_wm_latency(dev_priv);4025		dev_priv->display.funcs.wm = &vlv_wm_funcs;4026	} else if (IS_G4X(dev_priv)) {4027		g4x_setup_wm_latency(dev_priv);4028		dev_priv->display.funcs.wm = &g4x_wm_funcs;4029	} else if (IS_PINEVIEW(dev_priv)) {4030		if (!pnv_get_cxsr_latency(dev_priv)) {4031			drm_info(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");4032			/* Disable CxSR and never update its watermark again */4033			intel_set_memory_cxsr(dev_priv, false);4034			dev_priv->display.funcs.wm = &nop_funcs;4035		} else {4036			dev_priv->display.funcs.wm = &pnv_wm_funcs;4037		}4038	} else if (DISPLAY_VER(dev_priv) == 4) {4039		dev_priv->display.funcs.wm = &i965_wm_funcs;4040	} else if (DISPLAY_VER(dev_priv) == 3) {4041		dev_priv->display.funcs.wm = &i9xx_wm_funcs;4042	} else if (DISPLAY_VER(dev_priv) == 2) {4043		if (INTEL_NUM_PIPES(dev_priv) == 1)4044			dev_priv->display.funcs.wm = &i845_wm_funcs;4045		else4046			dev_priv->display.funcs.wm = &i9xx_wm_funcs;4047	} else {4048		drm_err(&dev_priv->drm,4049			"unexpected fall-through in %s\n", __func__);4050		dev_priv->display.funcs.wm = &nop_funcs;4051	}4052}4053