brintos / linux-shallow public Read only

1/*2 * Copyright 2014 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/delay.h>25#include <linux/kernel.h>26#include <linux/firmware.h>27#include <linux/module.h>28#include <linux/pci.h>29 30#include "amdgpu.h"31#include "amdgpu_gfx.h"32#include "amdgpu_ring.h"33#include "vi.h"34#include "vi_structs.h"35#include "vid.h"36#include "amdgpu_ucode.h"37#include "amdgpu_atombios.h"38#include "atombios_i2c.h"39#include "clearstate_vi.h"40 41#include "gmc/gmc_8_2_d.h"42#include "gmc/gmc_8_2_sh_mask.h"43 44#include "oss/oss_3_0_d.h"45#include "oss/oss_3_0_sh_mask.h"46 47#include "bif/bif_5_0_d.h"48#include "bif/bif_5_0_sh_mask.h"49#include "gca/gfx_8_0_d.h"50#include "gca/gfx_8_0_enum.h"51#include "gca/gfx_8_0_sh_mask.h"52 53#include "dce/dce_10_0_d.h"54#include "dce/dce_10_0_sh_mask.h"55 56#include "smu/smu_7_1_3_d.h"57 58#include "ivsrcid/ivsrcid_vislands30.h"59 60#define GFX8_NUM_GFX_RINGS     161#define GFX8_MEC_HPD_SIZE 409662 63#define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x2201000164#define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x2201000165#define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x2201100266#define TONGA_GB_ADDR_CONFIG_GOLDEN 0x2201100367 68#define ARRAY_MODE(x)					((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)69#define PIPE_CONFIG(x)					((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)70#define TILE_SPLIT(x)					((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)71#define MICRO_TILE_MODE_NEW(x)				((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)72#define SAMPLE_SPLIT(x)					((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)73#define BANK_WIDTH(x)					((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)74#define BANK_HEIGHT(x)					((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)75#define MACRO_TILE_ASPECT(x)				((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)76#define NUM_BANKS(x)					((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)77 78#define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L79#define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L80#define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L81#define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L82#define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L83#define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L84 85/* BPM SERDES CMD */86#define SET_BPM_SERDES_CMD    187#define CLE_BPM_SERDES_CMD    088 89/* BPM Register Address*/90enum {91	BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */92	BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */93	BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */94	BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */95	BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */96	BPM_REG_FGCG_MAX97};98 99#define RLC_FormatDirectRegListLength        14100 101MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");102MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");103MODULE_FIRMWARE("amdgpu/carrizo_me.bin");104MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");105MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");106MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");107 108MODULE_FIRMWARE("amdgpu/stoney_ce.bin");109MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");110MODULE_FIRMWARE("amdgpu/stoney_me.bin");111MODULE_FIRMWARE("amdgpu/stoney_mec.bin");112MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");113 114MODULE_FIRMWARE("amdgpu/tonga_ce.bin");115MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");116MODULE_FIRMWARE("amdgpu/tonga_me.bin");117MODULE_FIRMWARE("amdgpu/tonga_mec.bin");118MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");119MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");120 121MODULE_FIRMWARE("amdgpu/topaz_ce.bin");122MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");123MODULE_FIRMWARE("amdgpu/topaz_me.bin");124MODULE_FIRMWARE("amdgpu/topaz_mec.bin");125MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");126 127MODULE_FIRMWARE("amdgpu/fiji_ce.bin");128MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");129MODULE_FIRMWARE("amdgpu/fiji_me.bin");130MODULE_FIRMWARE("amdgpu/fiji_mec.bin");131MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");132MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");133 134MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");135MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");136MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");137MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");138MODULE_FIRMWARE("amdgpu/polaris10_me.bin");139MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");140MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");141MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");142MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");143MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");144MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");145 146MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");147MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");148MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");149MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");150MODULE_FIRMWARE("amdgpu/polaris11_me.bin");151MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");152MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");153MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");154MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");155MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");156MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");157 158MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");159MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");160MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");161MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");162MODULE_FIRMWARE("amdgpu/polaris12_me.bin");163MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");164MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");165MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");166MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");167MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");168MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");169 170MODULE_FIRMWARE("amdgpu/vegam_ce.bin");171MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");172MODULE_FIRMWARE("amdgpu/vegam_me.bin");173MODULE_FIRMWARE("amdgpu/vegam_mec.bin");174MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");175MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");176 177static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =178{179	{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},180	{mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},181	{mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},182	{mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},183	{mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},184	{mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},185	{mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},186	{mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},187	{mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},188	{mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},189	{mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},190	{mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},191	{mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},192	{mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},193	{mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},194	{mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}195};196 197static const u32 golden_settings_tonga_a11[] =198{199	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,200	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,201	mmDB_DEBUG2, 0xf00fffff, 0x00000400,202	mmGB_GPU_ID, 0x0000000f, 0x00000000,203	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,204	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,205	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,206	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,207	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,208	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,209	mmTCC_CTRL, 0x00100000, 0xf31fff7f,210	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,211	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,212	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,213	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,214	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,215};216 217static const u32 tonga_golden_common_all[] =218{219	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,220	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,221	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,222	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,223	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,224	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,225	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,226	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF227};228 229static const u32 tonga_mgcg_cgcg_init[] =230{231	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,232	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,233	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,234	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,235	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,236	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,237	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,238	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,239	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,240	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,241	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,242	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,243	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,244	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,245	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,246	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,247	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,248	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,249	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,250	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,251	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,252	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,253	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,254	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,255	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,256	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,257	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,258	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,259	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,260	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,261	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,262	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,263	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,264	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,265	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,266	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,267	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,268	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,269	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,270	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,271	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,272	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,273	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,274	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,275	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,276	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,277	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,278	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,279	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,280	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,281	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,282	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,283	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,284	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,285	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,286	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,287	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,288	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,289	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,290	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,291	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,292	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,293	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,294	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,295	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,296	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,297	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,298	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,299	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,300	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,301	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,302	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,303	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,304	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,305	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,306};307 308static const u32 golden_settings_vegam_a11[] =309{310	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,311	mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,312	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,313	mmDB_DEBUG2, 0xf00fffff, 0x00000400,314	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,315	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,316	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,317	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,318	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,319	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,320	mmSQ_CONFIG, 0x07f80000, 0x01180000,321	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,322	mmTCC_CTRL, 0x00100000, 0xf31fff7f,323	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,324	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,325	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,326	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,327};328 329static const u32 vegam_golden_common_all[] =330{331	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,332	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,333	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,334	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,335	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,336	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,337};338 339static const u32 golden_settings_polaris11_a11[] =340{341	mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,342	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,343	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,344	mmDB_DEBUG2, 0xf00fffff, 0x00000400,345	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,346	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,347	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,348	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,349	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,350	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,351	mmSQ_CONFIG, 0x07f80000, 0x01180000,352	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,353	mmTCC_CTRL, 0x00100000, 0xf31fff7f,354	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,355	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,356	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,357	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,358};359 360static const u32 polaris11_golden_common_all[] =361{362	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,363	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,364	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,365	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,366	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,367	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,368};369 370static const u32 golden_settings_polaris10_a11[] =371{372	mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,373	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,374	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,375	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,376	mmDB_DEBUG2, 0xf00fffff, 0x00000400,377	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,378	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,379	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,380	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,381	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,382	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,383	mmSQ_CONFIG, 0x07f80000, 0x07180000,384	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,385	mmTCC_CTRL, 0x00100000, 0xf31fff7f,386	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,387	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,388	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,389};390 391static const u32 polaris10_golden_common_all[] =392{393	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,394	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,395	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,396	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,397	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,398	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,399	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,400	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,401};402 403static const u32 fiji_golden_common_all[] =404{405	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,406	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,407	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,408	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,409	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,410	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,411	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,412	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,413	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,414	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,415};416 417static const u32 golden_settings_fiji_a10[] =418{419	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,420	mmDB_DEBUG2, 0xf00fffff, 0x00000400,421	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,422	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,423	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,424	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,425	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,426	mmTCC_CTRL, 0x00100000, 0xf31fff7f,427	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,428	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,429	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,430};431 432static const u32 fiji_mgcg_cgcg_init[] =433{434	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,435	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,436	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,437	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,438	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,439	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,440	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,441	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,442	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,443	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,444	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,445	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,446	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,447	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,448	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,449	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,450	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,451	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,452	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,453	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,454	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,455	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,456	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,457	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,458	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,459	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,460	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,461	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,462	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,463	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,464	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,465	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,466	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,467	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,468	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,469};470 471static const u32 golden_settings_iceland_a11[] =472{473	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,474	mmDB_DEBUG2, 0xf00fffff, 0x00000400,475	mmDB_DEBUG3, 0xc0000000, 0xc0000000,476	mmGB_GPU_ID, 0x0000000f, 0x00000000,477	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,478	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,479	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,480	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,481	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,482	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,483	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,484	mmTCC_CTRL, 0x00100000, 0xf31fff7f,485	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,486	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,487	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,488	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,489};490 491static const u32 iceland_golden_common_all[] =492{493	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,494	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,495	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,496	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,497	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,498	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,499	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,500	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF501};502 503static const u32 iceland_mgcg_cgcg_init[] =504{505	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,506	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,507	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,508	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,509	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,510	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,511	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,512	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,513	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,514	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,515	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,516	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,517	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,518	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,519	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,520	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,521	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,522	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,523	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,524	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,525	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,526	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,527	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,528	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,529	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,530	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,531	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,532	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,533	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,534	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,535	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,536	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,537	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,538	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,539	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,540	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,541	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,542	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,543	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,544	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,545	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,546	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,547	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,548	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,549	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,550	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,551	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,552	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,553	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,554	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,555	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,556	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,557	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,558	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,559	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,560	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,561	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,562	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,563	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,564	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,565	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,566	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,567	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,568	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,569};570 571static const u32 cz_golden_settings_a11[] =572{573	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,574	mmDB_DEBUG2, 0xf00fffff, 0x00000400,575	mmGB_GPU_ID, 0x0000000f, 0x00000000,576	mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,577	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,578	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,579	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,580	mmTA_CNTL_AUX, 0x000f000f, 0x00010000,581	mmTCC_CTRL, 0x00100000, 0xf31fff7f,582	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,583	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,584	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302585};586 587static const u32 cz_golden_common_all[] =588{589	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,590	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,591	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,592	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,593	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,594	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,595	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,596	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF597};598 599static const u32 cz_mgcg_cgcg_init[] =600{601	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,602	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,603	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,604	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,605	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,606	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,607	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,608	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,609	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,610	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,611	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,612	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,613	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,614	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,615	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,616	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,617	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,618	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,619	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,620	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,621	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,622	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,623	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,624	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,625	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,626	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,627	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,628	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,629	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,630	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,631	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,632	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,633	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,634	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,635	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,636	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,637	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,638	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,639	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,640	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,641	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,642	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,643	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,644	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,645	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,646	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,647	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,648	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,649	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,650	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,651	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,652	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,653	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,654	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,655	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,656	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,657	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,658	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,659	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,660	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,661	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,662	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,663	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,664	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,665	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,666	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,667	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,668	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,669	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,670	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,671	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,672	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,673	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,674	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,675	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,676};677 678static const u32 stoney_golden_settings_a11[] =679{680	mmDB_DEBUG2, 0xf00fffff, 0x00000400,681	mmGB_GPU_ID, 0x0000000f, 0x00000000,682	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,683	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,684	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,685	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,686	mmTCC_CTRL, 0x00100000, 0xf31fff7f,687	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,688	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,689	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,690};691 692static const u32 stoney_golden_common_all[] =693{694	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,695	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,696	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,697	mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,698	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,699	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,700	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,701	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,702};703 704static const u32 stoney_mgcg_cgcg_init[] =705{706	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,707	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,708	mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,709	mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,710	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,711};712 713 714static const char * const sq_edc_source_names[] = {715	"SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",716	"SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",717	"SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",718	"SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",719	"SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",720	"SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",721	"SQ_EDC_INFO_SOURCE_TA: EDC source is TA",722};723 724static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);725static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);726static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);727static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);728static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);729static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);730static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);731static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);732 733#define CG_ACLK_CNTL__ACLK_DIVIDER_MASK                    0x0000007fL734#define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT                  0x00000000L735 736static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)737{738	uint32_t data;739 740	switch (adev->asic_type) {741	case CHIP_TOPAZ:742		amdgpu_device_program_register_sequence(adev,743							iceland_mgcg_cgcg_init,744							ARRAY_SIZE(iceland_mgcg_cgcg_init));745		amdgpu_device_program_register_sequence(adev,746							golden_settings_iceland_a11,747							ARRAY_SIZE(golden_settings_iceland_a11));748		amdgpu_device_program_register_sequence(adev,749							iceland_golden_common_all,750							ARRAY_SIZE(iceland_golden_common_all));751		break;752	case CHIP_FIJI:753		amdgpu_device_program_register_sequence(adev,754							fiji_mgcg_cgcg_init,755							ARRAY_SIZE(fiji_mgcg_cgcg_init));756		amdgpu_device_program_register_sequence(adev,757							golden_settings_fiji_a10,758							ARRAY_SIZE(golden_settings_fiji_a10));759		amdgpu_device_program_register_sequence(adev,760							fiji_golden_common_all,761							ARRAY_SIZE(fiji_golden_common_all));762		break;763 764	case CHIP_TONGA:765		amdgpu_device_program_register_sequence(adev,766							tonga_mgcg_cgcg_init,767							ARRAY_SIZE(tonga_mgcg_cgcg_init));768		amdgpu_device_program_register_sequence(adev,769							golden_settings_tonga_a11,770							ARRAY_SIZE(golden_settings_tonga_a11));771		amdgpu_device_program_register_sequence(adev,772							tonga_golden_common_all,773							ARRAY_SIZE(tonga_golden_common_all));774		break;775	case CHIP_VEGAM:776		amdgpu_device_program_register_sequence(adev,777							golden_settings_vegam_a11,778							ARRAY_SIZE(golden_settings_vegam_a11));779		amdgpu_device_program_register_sequence(adev,780							vegam_golden_common_all,781							ARRAY_SIZE(vegam_golden_common_all));782		break;783	case CHIP_POLARIS11:784	case CHIP_POLARIS12:785		amdgpu_device_program_register_sequence(adev,786							golden_settings_polaris11_a11,787							ARRAY_SIZE(golden_settings_polaris11_a11));788		amdgpu_device_program_register_sequence(adev,789							polaris11_golden_common_all,790							ARRAY_SIZE(polaris11_golden_common_all));791		break;792	case CHIP_POLARIS10:793		amdgpu_device_program_register_sequence(adev,794							golden_settings_polaris10_a11,795							ARRAY_SIZE(golden_settings_polaris10_a11));796		amdgpu_device_program_register_sequence(adev,797							polaris10_golden_common_all,798							ARRAY_SIZE(polaris10_golden_common_all));799		data = RREG32_SMC(ixCG_ACLK_CNTL);800		data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK;801		data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT;802		WREG32_SMC(ixCG_ACLK_CNTL, data);803		if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) &&804		    ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||805		     (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||806		     (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) {807			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);808			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);809		}810		break;811	case CHIP_CARRIZO:812		amdgpu_device_program_register_sequence(adev,813							cz_mgcg_cgcg_init,814							ARRAY_SIZE(cz_mgcg_cgcg_init));815		amdgpu_device_program_register_sequence(adev,816							cz_golden_settings_a11,817							ARRAY_SIZE(cz_golden_settings_a11));818		amdgpu_device_program_register_sequence(adev,819							cz_golden_common_all,820							ARRAY_SIZE(cz_golden_common_all));821		break;822	case CHIP_STONEY:823		amdgpu_device_program_register_sequence(adev,824							stoney_mgcg_cgcg_init,825							ARRAY_SIZE(stoney_mgcg_cgcg_init));826		amdgpu_device_program_register_sequence(adev,827							stoney_golden_settings_a11,828							ARRAY_SIZE(stoney_golden_settings_a11));829		amdgpu_device_program_register_sequence(adev,830							stoney_golden_common_all,831							ARRAY_SIZE(stoney_golden_common_all));832		break;833	default:834		break;835	}836}837 838static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)839{840	struct amdgpu_device *adev = ring->adev;841	uint32_t tmp = 0;842	unsigned i;843	int r;844 845	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);846	r = amdgpu_ring_alloc(ring, 3);847	if (r)848		return r;849 850	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));851	amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START);852	amdgpu_ring_write(ring, 0xDEADBEEF);853	amdgpu_ring_commit(ring);854 855	for (i = 0; i < adev->usec_timeout; i++) {856		tmp = RREG32(mmSCRATCH_REG0);857		if (tmp == 0xDEADBEEF)858			break;859		udelay(1);860	}861 862	if (i >= adev->usec_timeout)863		r = -ETIMEDOUT;864 865	return r;866}867 868static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)869{870	struct amdgpu_device *adev = ring->adev;871	struct amdgpu_ib ib;872	struct dma_fence *f = NULL;873 874	unsigned int index;875	uint64_t gpu_addr;876	uint32_t tmp;877	long r;878 879	r = amdgpu_device_wb_get(adev, &index);880	if (r)881		return r;882 883	gpu_addr = adev->wb.gpu_addr + (index * 4);884	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);885	memset(&ib, 0, sizeof(ib));886 887	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);888	if (r)889		goto err1;890 891	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);892	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;893	ib.ptr[2] = lower_32_bits(gpu_addr);894	ib.ptr[3] = upper_32_bits(gpu_addr);895	ib.ptr[4] = 0xDEADBEEF;896	ib.length_dw = 5;897 898	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);899	if (r)900		goto err2;901 902	r = dma_fence_wait_timeout(f, false, timeout);903	if (r == 0) {904		r = -ETIMEDOUT;905		goto err2;906	} else if (r < 0) {907		goto err2;908	}909 910	tmp = adev->wb.wb[index];911	if (tmp == 0xDEADBEEF)912		r = 0;913	else914		r = -EINVAL;915 916err2:917	amdgpu_ib_free(adev, &ib, NULL);918	dma_fence_put(f);919err1:920	amdgpu_device_wb_free(adev, index);921	return r;922}923 924 925static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)926{927	amdgpu_ucode_release(&adev->gfx.pfp_fw);928	amdgpu_ucode_release(&adev->gfx.me_fw);929	amdgpu_ucode_release(&adev->gfx.ce_fw);930	amdgpu_ucode_release(&adev->gfx.rlc_fw);931	amdgpu_ucode_release(&adev->gfx.mec_fw);932	if ((adev->asic_type != CHIP_STONEY) &&933	    (adev->asic_type != CHIP_TOPAZ))934		amdgpu_ucode_release(&adev->gfx.mec2_fw);935 936	kfree(adev->gfx.rlc.register_list_format);937}938 939static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)940{941	const char *chip_name;942	int err;943	struct amdgpu_firmware_info *info = NULL;944	const struct common_firmware_header *header = NULL;945	const struct gfx_firmware_header_v1_0 *cp_hdr;946	const struct rlc_firmware_header_v2_0 *rlc_hdr;947	unsigned int *tmp = NULL, i;948 949	DRM_DEBUG("\n");950 951	switch (adev->asic_type) {952	case CHIP_TOPAZ:953		chip_name = "topaz";954		break;955	case CHIP_TONGA:956		chip_name = "tonga";957		break;958	case CHIP_CARRIZO:959		chip_name = "carrizo";960		break;961	case CHIP_FIJI:962		chip_name = "fiji";963		break;964	case CHIP_STONEY:965		chip_name = "stoney";966		break;967	case CHIP_POLARIS10:968		chip_name = "polaris10";969		break;970	case CHIP_POLARIS11:971		chip_name = "polaris11";972		break;973	case CHIP_POLARIS12:974		chip_name = "polaris12";975		break;976	case CHIP_VEGAM:977		chip_name = "vegam";978		break;979	default:980		BUG();981	}982 983	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {984		err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,985					   "amdgpu/%s_pfp_2.bin", chip_name);986		if (err == -ENODEV) {987			err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,988						   "amdgpu/%s_pfp.bin", chip_name);989		}990	} else {991		err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,992					   "amdgpu/%s_pfp.bin", chip_name);993	}994	if (err)995		goto out;996	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;997	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);998	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);999 1000	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {1001		err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,1002					   "amdgpu/%s_me_2.bin", chip_name);1003		if (err == -ENODEV) {1004			err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,1005						   "amdgpu/%s_me.bin", chip_name);1006		}1007	} else {1008		err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,1009					   "amdgpu/%s_me.bin", chip_name);1010	}1011	if (err)1012		goto out;1013	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;1014	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);1015 1016	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);1017 1018	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {1019		err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,1020					   "amdgpu/%s_ce_2.bin", chip_name);1021		if (err == -ENODEV) {1022			err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,1023						   "amdgpu/%s_ce.bin", chip_name);1024		}1025	} else {1026		err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,1027					   "amdgpu/%s_ce.bin", chip_name);1028	}1029	if (err)1030		goto out;1031	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;1032	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);1033	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);1034 1035	/*1036	 * Support for MCBP/Virtualization in combination with chained IBs is1037	 * formal released on feature version #461038	 */1039	if (adev->gfx.ce_feature_version >= 46 &&1040	    adev->gfx.pfp_feature_version >= 46) {1041		adev->virt.chained_ib_support = true;1042		DRM_INFO("Chained IB support enabled!\n");1043	} else1044		adev->virt.chained_ib_support = false;1045 1046	err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,1047				   "amdgpu/%s_rlc.bin", chip_name);1048	if (err)1049		goto out;1050	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;1051	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);1052	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);1053 1054	adev->gfx.rlc.save_and_restore_offset =1055			le32_to_cpu(rlc_hdr->save_and_restore_offset);1056	adev->gfx.rlc.clear_state_descriptor_offset =1057			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);1058	adev->gfx.rlc.avail_scratch_ram_locations =1059			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);1060	adev->gfx.rlc.reg_restore_list_size =1061			le32_to_cpu(rlc_hdr->reg_restore_list_size);1062	adev->gfx.rlc.reg_list_format_start =1063			le32_to_cpu(rlc_hdr->reg_list_format_start);1064	adev->gfx.rlc.reg_list_format_separate_start =1065			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);1066	adev->gfx.rlc.starting_offsets_start =1067			le32_to_cpu(rlc_hdr->starting_offsets_start);1068	adev->gfx.rlc.reg_list_format_size_bytes =1069			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);1070	adev->gfx.rlc.reg_list_size_bytes =1071			le32_to_cpu(rlc_hdr->reg_list_size_bytes);1072 1073	adev->gfx.rlc.register_list_format =1074			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +1075					adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);1076 1077	if (!adev->gfx.rlc.register_list_format) {1078		err = -ENOMEM;1079		goto out;1080	}1081 1082	tmp = (unsigned int *)((uintptr_t)rlc_hdr +1083			le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));1084	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)1085		adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);1086 1087	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;1088 1089	tmp = (unsigned int *)((uintptr_t)rlc_hdr +1090			le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));1091	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)1092		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);1093 1094	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {1095		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,1096					   "amdgpu/%s_mec_2.bin", chip_name);1097		if (err == -ENODEV) {1098			err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,1099						   "amdgpu/%s_mec.bin", chip_name);1100		}1101	} else {1102		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,1103					   "amdgpu/%s_mec.bin", chip_name);1104	}1105	if (err)1106		goto out;1107	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;1108	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);1109	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);1110 1111	if ((adev->asic_type != CHIP_STONEY) &&1112	    (adev->asic_type != CHIP_TOPAZ)) {1113		if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {1114			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,1115						   "amdgpu/%s_mec2_2.bin", chip_name);1116			if (err == -ENODEV) {1117				err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,1118							   "amdgpu/%s_mec2.bin", chip_name);1119			}1120		} else {1121			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,1122						   "amdgpu/%s_mec2.bin", chip_name);1123		}1124		if (!err) {1125			cp_hdr = (const struct gfx_firmware_header_v1_0 *)1126				adev->gfx.mec2_fw->data;1127			adev->gfx.mec2_fw_version =1128				le32_to_cpu(cp_hdr->header.ucode_version);1129			adev->gfx.mec2_feature_version =1130				le32_to_cpu(cp_hdr->ucode_feature_version);1131		} else {1132			err = 0;1133			adev->gfx.mec2_fw = NULL;1134		}1135	}1136 1137	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];1138	info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;1139	info->fw = adev->gfx.pfp_fw;1140	header = (const struct common_firmware_header *)info->fw->data;1141	adev->firmware.fw_size +=1142		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1143 1144	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];1145	info->ucode_id = AMDGPU_UCODE_ID_CP_ME;1146	info->fw = adev->gfx.me_fw;1147	header = (const struct common_firmware_header *)info->fw->data;1148	adev->firmware.fw_size +=1149		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1150 1151	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];1152	info->ucode_id = AMDGPU_UCODE_ID_CP_CE;1153	info->fw = adev->gfx.ce_fw;1154	header = (const struct common_firmware_header *)info->fw->data;1155	adev->firmware.fw_size +=1156		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1157 1158	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];1159	info->ucode_id = AMDGPU_UCODE_ID_RLC_G;1160	info->fw = adev->gfx.rlc_fw;1161	header = (const struct common_firmware_header *)info->fw->data;1162	adev->firmware.fw_size +=1163		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1164 1165	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];1166	info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;1167	info->fw = adev->gfx.mec_fw;1168	header = (const struct common_firmware_header *)info->fw->data;1169	adev->firmware.fw_size +=1170		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1171 1172	/* we need account JT in */1173	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;1174	adev->firmware.fw_size +=1175		ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);1176 1177	if (amdgpu_sriov_vf(adev)) {1178		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];1179		info->ucode_id = AMDGPU_UCODE_ID_STORAGE;1180		info->fw = adev->gfx.mec_fw;1181		adev->firmware.fw_size +=1182			ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);1183	}1184 1185	if (adev->gfx.mec2_fw) {1186		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];1187		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;1188		info->fw = adev->gfx.mec2_fw;1189		header = (const struct common_firmware_header *)info->fw->data;1190		adev->firmware.fw_size +=1191			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);1192	}1193 1194out:1195	if (err) {1196		dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name);1197		amdgpu_ucode_release(&adev->gfx.pfp_fw);1198		amdgpu_ucode_release(&adev->gfx.me_fw);1199		amdgpu_ucode_release(&adev->gfx.ce_fw);1200		amdgpu_ucode_release(&adev->gfx.rlc_fw);1201		amdgpu_ucode_release(&adev->gfx.mec_fw);1202		amdgpu_ucode_release(&adev->gfx.mec2_fw);1203	}1204	return err;1205}1206 1207static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,1208				    volatile u32 *buffer)1209{1210	u32 count = 0, i;1211	const struct cs_section_def *sect = NULL;1212	const struct cs_extent_def *ext = NULL;1213 1214	if (adev->gfx.rlc.cs_data == NULL)1215		return;1216	if (buffer == NULL)1217		return;1218 1219	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));1220	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);1221 1222	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));1223	buffer[count++] = cpu_to_le32(0x80000000);1224	buffer[count++] = cpu_to_le32(0x80000000);1225 1226	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {1227		for (ext = sect->section; ext->extent != NULL; ++ext) {1228			if (sect->id == SECT_CONTEXT) {1229				buffer[count++] =1230					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));1231				buffer[count++] = cpu_to_le32(ext->reg_index -1232						PACKET3_SET_CONTEXT_REG_START);1233				for (i = 0; i < ext->reg_count; i++)1234					buffer[count++] = cpu_to_le32(ext->extent[i]);1235			} else {1236				return;1237			}1238		}1239	}1240 1241	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));1242	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -1243			PACKET3_SET_CONTEXT_REG_START);1244	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);1245	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);1246 1247	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));1248	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);1249 1250	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));1251	buffer[count++] = cpu_to_le32(0);1252}1253 1254static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)1255{1256	if (adev->asic_type == CHIP_CARRIZO)1257		return 5;1258	else1259		return 4;1260}1261 1262static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)1263{1264	const struct cs_section_def *cs_data;1265	int r;1266 1267	adev->gfx.rlc.cs_data = vi_cs_data;1268 1269	cs_data = adev->gfx.rlc.cs_data;1270 1271	if (cs_data) {1272		/* init clear state block */1273		r = amdgpu_gfx_rlc_init_csb(adev);1274		if (r)1275			return r;1276	}1277 1278	if ((adev->asic_type == CHIP_CARRIZO) ||1279	    (adev->asic_type == CHIP_STONEY)) {1280		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */1281		r = amdgpu_gfx_rlc_init_cpt(adev);1282		if (r)1283			return r;1284	}1285 1286	/* init spm vmid with 0xf */1287	if (adev->gfx.rlc.funcs->update_spm_vmid)1288		adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);1289 1290	return 0;1291}1292 1293static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)1294{1295	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);1296}1297 1298static int gfx_v8_0_mec_init(struct amdgpu_device *adev)1299{1300	int r;1301	u32 *hpd;1302	size_t mec_hpd_size;1303 1304	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);1305 1306	/* take ownership of the relevant compute queues */1307	amdgpu_gfx_compute_queue_acquire(adev);1308 1309	mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;1310	if (mec_hpd_size) {1311		r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,1312					      AMDGPU_GEM_DOMAIN_VRAM |1313					      AMDGPU_GEM_DOMAIN_GTT,1314					      &adev->gfx.mec.hpd_eop_obj,1315					      &adev->gfx.mec.hpd_eop_gpu_addr,1316					      (void **)&hpd);1317		if (r) {1318			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);1319			return r;1320		}1321 1322		memset(hpd, 0, mec_hpd_size);1323 1324		amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);1325		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);1326	}1327 1328	return 0;1329}1330 1331static const u32 vgpr_init_compute_shader[] =1332{1333	0x7e000209, 0x7e020208,1334	0x7e040207, 0x7e060206,1335	0x7e080205, 0x7e0a0204,1336	0x7e0c0203, 0x7e0e0202,1337	0x7e100201, 0x7e120200,1338	0x7e140209, 0x7e160208,1339	0x7e180207, 0x7e1a0206,1340	0x7e1c0205, 0x7e1e0204,1341	0x7e200203, 0x7e220202,1342	0x7e240201, 0x7e260200,1343	0x7e280209, 0x7e2a0208,1344	0x7e2c0207, 0x7e2e0206,1345	0x7e300205, 0x7e320204,1346	0x7e340203, 0x7e360202,1347	0x7e380201, 0x7e3a0200,1348	0x7e3c0209, 0x7e3e0208,1349	0x7e400207, 0x7e420206,1350	0x7e440205, 0x7e460204,1351	0x7e480203, 0x7e4a0202,1352	0x7e4c0201, 0x7e4e0200,1353	0x7e500209, 0x7e520208,1354	0x7e540207, 0x7e560206,1355	0x7e580205, 0x7e5a0204,1356	0x7e5c0203, 0x7e5e0202,1357	0x7e600201, 0x7e620200,1358	0x7e640209, 0x7e660208,1359	0x7e680207, 0x7e6a0206,1360	0x7e6c0205, 0x7e6e0204,1361	0x7e700203, 0x7e720202,1362	0x7e740201, 0x7e760200,1363	0x7e780209, 0x7e7a0208,1364	0x7e7c0207, 0x7e7e0206,1365	0xbf8a0000, 0xbf810000,1366};1367 1368static const u32 sgpr_init_compute_shader[] =1369{1370	0xbe8a0100, 0xbe8c0102,1371	0xbe8e0104, 0xbe900106,1372	0xbe920108, 0xbe940100,1373	0xbe960102, 0xbe980104,1374	0xbe9a0106, 0xbe9c0108,1375	0xbe9e0100, 0xbea00102,1376	0xbea20104, 0xbea40106,1377	0xbea60108, 0xbea80100,1378	0xbeaa0102, 0xbeac0104,1379	0xbeae0106, 0xbeb00108,1380	0xbeb20100, 0xbeb40102,1381	0xbeb60104, 0xbeb80106,1382	0xbeba0108, 0xbebc0100,1383	0xbebe0102, 0xbec00104,1384	0xbec20106, 0xbec40108,1385	0xbec60100, 0xbec80102,1386	0xbee60004, 0xbee70005,1387	0xbeea0006, 0xbeeb0007,1388	0xbee80008, 0xbee90009,1389	0xbefc0000, 0xbf8a0000,1390	0xbf810000, 0x00000000,1391};1392 1393static const u32 vgpr_init_regs[] =1394{1395	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,1396	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */1397	mmCOMPUTE_NUM_THREAD_X, 256*4,1398	mmCOMPUTE_NUM_THREAD_Y, 1,1399	mmCOMPUTE_NUM_THREAD_Z, 1,1400	mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */1401	mmCOMPUTE_PGM_RSRC2, 20,1402	mmCOMPUTE_USER_DATA_0, 0xedcedc00,1403	mmCOMPUTE_USER_DATA_1, 0xedcedc01,1404	mmCOMPUTE_USER_DATA_2, 0xedcedc02,1405	mmCOMPUTE_USER_DATA_3, 0xedcedc03,1406	mmCOMPUTE_USER_DATA_4, 0xedcedc04,1407	mmCOMPUTE_USER_DATA_5, 0xedcedc05,1408	mmCOMPUTE_USER_DATA_6, 0xedcedc06,1409	mmCOMPUTE_USER_DATA_7, 0xedcedc07,1410	mmCOMPUTE_USER_DATA_8, 0xedcedc08,1411	mmCOMPUTE_USER_DATA_9, 0xedcedc09,1412};1413 1414static const u32 sgpr1_init_regs[] =1415{1416	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,1417	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */1418	mmCOMPUTE_NUM_THREAD_X, 256*5,1419	mmCOMPUTE_NUM_THREAD_Y, 1,1420	mmCOMPUTE_NUM_THREAD_Z, 1,1421	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */1422	mmCOMPUTE_PGM_RSRC2, 20,1423	mmCOMPUTE_USER_DATA_0, 0xedcedc00,1424	mmCOMPUTE_USER_DATA_1, 0xedcedc01,1425	mmCOMPUTE_USER_DATA_2, 0xedcedc02,1426	mmCOMPUTE_USER_DATA_3, 0xedcedc03,1427	mmCOMPUTE_USER_DATA_4, 0xedcedc04,1428	mmCOMPUTE_USER_DATA_5, 0xedcedc05,1429	mmCOMPUTE_USER_DATA_6, 0xedcedc06,1430	mmCOMPUTE_USER_DATA_7, 0xedcedc07,1431	mmCOMPUTE_USER_DATA_8, 0xedcedc08,1432	mmCOMPUTE_USER_DATA_9, 0xedcedc09,1433};1434 1435static const u32 sgpr2_init_regs[] =1436{1437	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,1438	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,1439	mmCOMPUTE_NUM_THREAD_X, 256*5,1440	mmCOMPUTE_NUM_THREAD_Y, 1,1441	mmCOMPUTE_NUM_THREAD_Z, 1,1442	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */1443	mmCOMPUTE_PGM_RSRC2, 20,1444	mmCOMPUTE_USER_DATA_0, 0xedcedc00,1445	mmCOMPUTE_USER_DATA_1, 0xedcedc01,1446	mmCOMPUTE_USER_DATA_2, 0xedcedc02,1447	mmCOMPUTE_USER_DATA_3, 0xedcedc03,1448	mmCOMPUTE_USER_DATA_4, 0xedcedc04,1449	mmCOMPUTE_USER_DATA_5, 0xedcedc05,1450	mmCOMPUTE_USER_DATA_6, 0xedcedc06,1451	mmCOMPUTE_USER_DATA_7, 0xedcedc07,1452	mmCOMPUTE_USER_DATA_8, 0xedcedc08,1453	mmCOMPUTE_USER_DATA_9, 0xedcedc09,1454};1455 1456static const u32 sec_ded_counter_registers[] =1457{1458	mmCPC_EDC_ATC_CNT,1459	mmCPC_EDC_SCRATCH_CNT,1460	mmCPC_EDC_UCODE_CNT,1461	mmCPF_EDC_ATC_CNT,1462	mmCPF_EDC_ROQ_CNT,1463	mmCPF_EDC_TAG_CNT,1464	mmCPG_EDC_ATC_CNT,1465	mmCPG_EDC_DMA_CNT,1466	mmCPG_EDC_TAG_CNT,1467	mmDC_EDC_CSINVOC_CNT,1468	mmDC_EDC_RESTORE_CNT,1469	mmDC_EDC_STATE_CNT,1470	mmGDS_EDC_CNT,1471	mmGDS_EDC_GRBM_CNT,1472	mmGDS_EDC_OA_DED,1473	mmSPI_EDC_CNT,1474	mmSQC_ATC_EDC_GATCL1_CNT,1475	mmSQC_EDC_CNT,1476	mmSQ_EDC_DED_CNT,1477	mmSQ_EDC_INFO,1478	mmSQ_EDC_SEC_CNT,1479	mmTCC_EDC_CNT,1480	mmTCP_ATC_EDC_GATCL1_CNT,1481	mmTCP_EDC_CNT,1482	mmTD_EDC_CNT1483};1484 1485static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)1486{1487	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];1488	struct amdgpu_ib ib;1489	struct dma_fence *f = NULL;1490	int r, i;1491	u32 tmp;1492	unsigned total_size, vgpr_offset, sgpr_offset;1493	u64 gpu_addr;1494 1495	/* only supported on CZ */1496	if (adev->asic_type != CHIP_CARRIZO)1497		return 0;1498 1499	/* bail if the compute ring is not ready */1500	if (!ring->sched.ready)1501		return 0;1502 1503	tmp = RREG32(mmGB_EDC_MODE);1504	WREG32(mmGB_EDC_MODE, 0);1505 1506	total_size =1507		(((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;1508	total_size +=1509		(((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;1510	total_size +=1511		(((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;1512	total_size = ALIGN(total_size, 256);1513	vgpr_offset = total_size;1514	total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);1515	sgpr_offset = total_size;1516	total_size += sizeof(sgpr_init_compute_shader);1517 1518	/* allocate an indirect buffer to put the commands in */1519	memset(&ib, 0, sizeof(ib));1520	r = amdgpu_ib_get(adev, NULL, total_size,1521					AMDGPU_IB_POOL_DIRECT, &ib);1522	if (r) {1523		DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);1524		return r;1525	}1526 1527	/* load the compute shaders */1528	for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)1529		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];1530 1531	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)1532		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];1533 1534	/* init the ib length to 0 */1535	ib.length_dw = 0;1536 1537	/* VGPR */1538	/* write the register state for the compute dispatch */1539	for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {1540		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);1541		ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;1542		ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];1543	}1544	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */1545	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;1546	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);1547	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;1548	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);1549	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);1550 1551	/* write dispatch packet */1552	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);1553	ib.ptr[ib.length_dw++] = 8; /* x */1554	ib.ptr[ib.length_dw++] = 1; /* y */1555	ib.ptr[ib.length_dw++] = 1; /* z */1556	ib.ptr[ib.length_dw++] =1557		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);1558 1559	/* write CS partial flush packet */1560	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);1561	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);1562 1563	/* SGPR1 */1564	/* write the register state for the compute dispatch */1565	for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {1566		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);1567		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;1568		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];1569	}1570	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */1571	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;1572	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);1573	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;1574	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);1575	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);1576 1577	/* write dispatch packet */1578	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);1579	ib.ptr[ib.length_dw++] = 8; /* x */1580	ib.ptr[ib.length_dw++] = 1; /* y */1581	ib.ptr[ib.length_dw++] = 1; /* z */1582	ib.ptr[ib.length_dw++] =1583		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);1584 1585	/* write CS partial flush packet */1586	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);1587	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);1588 1589	/* SGPR2 */1590	/* write the register state for the compute dispatch */1591	for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {1592		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);1593		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;1594		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];1595	}1596	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */1597	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;1598	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);1599	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;1600	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);1601	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);1602 1603	/* write dispatch packet */1604	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);1605	ib.ptr[ib.length_dw++] = 8; /* x */1606	ib.ptr[ib.length_dw++] = 1; /* y */1607	ib.ptr[ib.length_dw++] = 1; /* z */1608	ib.ptr[ib.length_dw++] =1609		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);1610 1611	/* write CS partial flush packet */1612	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);1613	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);1614 1615	/* shedule the ib on the ring */1616	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);1617	if (r) {1618		DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);1619		goto fail;1620	}1621 1622	/* wait for the GPU to finish processing the IB */1623	r = dma_fence_wait(f, false);1624	if (r) {1625		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);1626		goto fail;1627	}1628 1629	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);1630	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);1631	WREG32(mmGB_EDC_MODE, tmp);1632 1633	tmp = RREG32(mmCC_GC_EDC_CONFIG);1634	tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;1635	WREG32(mmCC_GC_EDC_CONFIG, tmp);1636 1637 1638	/* read back registers to clear the counters */1639	for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)1640		RREG32(sec_ded_counter_registers[i]);1641 1642fail:1643	amdgpu_ib_free(adev, &ib, NULL);1644	dma_fence_put(f);1645 1646	return r;1647}1648 1649static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)1650{1651	u32 gb_addr_config;1652	u32 mc_arb_ramcfg;1653	u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;1654	u32 tmp;1655	int ret;1656 1657	switch (adev->asic_type) {1658	case CHIP_TOPAZ:1659		adev->gfx.config.max_shader_engines = 1;1660		adev->gfx.config.max_tile_pipes = 2;1661		adev->gfx.config.max_cu_per_sh = 6;1662		adev->gfx.config.max_sh_per_se = 1;1663		adev->gfx.config.max_backends_per_se = 2;1664		adev->gfx.config.max_texture_channel_caches = 2;1665		adev->gfx.config.max_gprs = 256;1666		adev->gfx.config.max_gs_threads = 32;1667		adev->gfx.config.max_hw_contexts = 8;1668 1669		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1670		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1671		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1672		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1673		gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;1674		break;1675	case CHIP_FIJI:1676		adev->gfx.config.max_shader_engines = 4;1677		adev->gfx.config.max_tile_pipes = 16;1678		adev->gfx.config.max_cu_per_sh = 16;1679		adev->gfx.config.max_sh_per_se = 1;1680		adev->gfx.config.max_backends_per_se = 4;1681		adev->gfx.config.max_texture_channel_caches = 16;1682		adev->gfx.config.max_gprs = 256;1683		adev->gfx.config.max_gs_threads = 32;1684		adev->gfx.config.max_hw_contexts = 8;1685 1686		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1687		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1688		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1689		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1690		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;1691		break;1692	case CHIP_POLARIS11:1693	case CHIP_POLARIS12:1694		ret = amdgpu_atombios_get_gfx_info(adev);1695		if (ret)1696			return ret;1697		adev->gfx.config.max_gprs = 256;1698		adev->gfx.config.max_gs_threads = 32;1699		adev->gfx.config.max_hw_contexts = 8;1700 1701		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1702		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1703		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1704		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1705		gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;1706		break;1707	case CHIP_POLARIS10:1708	case CHIP_VEGAM:1709		ret = amdgpu_atombios_get_gfx_info(adev);1710		if (ret)1711			return ret;1712		adev->gfx.config.max_gprs = 256;1713		adev->gfx.config.max_gs_threads = 32;1714		adev->gfx.config.max_hw_contexts = 8;1715 1716		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1717		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1718		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1719		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1720		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;1721		break;1722	case CHIP_TONGA:1723		adev->gfx.config.max_shader_engines = 4;1724		adev->gfx.config.max_tile_pipes = 8;1725		adev->gfx.config.max_cu_per_sh = 8;1726		adev->gfx.config.max_sh_per_se = 1;1727		adev->gfx.config.max_backends_per_se = 2;1728		adev->gfx.config.max_texture_channel_caches = 8;1729		adev->gfx.config.max_gprs = 256;1730		adev->gfx.config.max_gs_threads = 32;1731		adev->gfx.config.max_hw_contexts = 8;1732 1733		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1734		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1735		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1736		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1737		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;1738		break;1739	case CHIP_CARRIZO:1740		adev->gfx.config.max_shader_engines = 1;1741		adev->gfx.config.max_tile_pipes = 2;1742		adev->gfx.config.max_sh_per_se = 1;1743		adev->gfx.config.max_backends_per_se = 2;1744		adev->gfx.config.max_cu_per_sh = 8;1745		adev->gfx.config.max_texture_channel_caches = 2;1746		adev->gfx.config.max_gprs = 256;1747		adev->gfx.config.max_gs_threads = 32;1748		adev->gfx.config.max_hw_contexts = 8;1749 1750		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1751		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1752		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1753		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1754		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;1755		break;1756	case CHIP_STONEY:1757		adev->gfx.config.max_shader_engines = 1;1758		adev->gfx.config.max_tile_pipes = 2;1759		adev->gfx.config.max_sh_per_se = 1;1760		adev->gfx.config.max_backends_per_se = 1;1761		adev->gfx.config.max_cu_per_sh = 3;1762		adev->gfx.config.max_texture_channel_caches = 2;1763		adev->gfx.config.max_gprs = 256;1764		adev->gfx.config.max_gs_threads = 16;1765		adev->gfx.config.max_hw_contexts = 8;1766 1767		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1768		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1769		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1770		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1771		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;1772		break;1773	default:1774		adev->gfx.config.max_shader_engines = 2;1775		adev->gfx.config.max_tile_pipes = 4;1776		adev->gfx.config.max_cu_per_sh = 2;1777		adev->gfx.config.max_sh_per_se = 1;1778		adev->gfx.config.max_backends_per_se = 2;1779		adev->gfx.config.max_texture_channel_caches = 4;1780		adev->gfx.config.max_gprs = 256;1781		adev->gfx.config.max_gs_threads = 32;1782		adev->gfx.config.max_hw_contexts = 8;1783 1784		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;1785		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;1786		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;1787		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;1788		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;1789		break;1790	}1791 1792	adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);1793	mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;1794 1795	adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg,1796				MC_ARB_RAMCFG, NOOFBANK);1797	adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg,1798				MC_ARB_RAMCFG, NOOFRANKS);1799 1800	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;1801	adev->gfx.config.mem_max_burst_length_bytes = 256;1802	if (adev->flags & AMD_IS_APU) {1803		/* Get memory bank mapping mode. */1804		tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);1805		dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);1806		dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);1807 1808		tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);1809		dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);1810		dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);1811 1812		/* Validate settings in case only one DIMM installed. */1813		if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))1814			dimm00_addr_map = 0;1815		if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))1816			dimm01_addr_map = 0;1817		if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))1818			dimm10_addr_map = 0;1819		if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))1820			dimm11_addr_map = 0;1821 1822		/* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */1823		/* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */1824		if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))1825			adev->gfx.config.mem_row_size_in_kb = 2;1826		else1827			adev->gfx.config.mem_row_size_in_kb = 1;1828	} else {1829		tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);1830		adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;1831		if (adev->gfx.config.mem_row_size_in_kb > 4)1832			adev->gfx.config.mem_row_size_in_kb = 4;1833	}1834 1835	adev->gfx.config.shader_engine_tile_size = 32;1836	adev->gfx.config.num_gpus = 1;1837	adev->gfx.config.multi_gpu_tile_size = 64;1838 1839	/* fix up row size */1840	switch (adev->gfx.config.mem_row_size_in_kb) {1841	case 1:1842	default:1843		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);1844		break;1845	case 2:1846		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);1847		break;1848	case 4:1849		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);1850		break;1851	}1852	adev->gfx.config.gb_addr_config = gb_addr_config;1853 1854	return 0;1855}1856 1857static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,1858					int mec, int pipe, int queue)1859{1860	int r;1861	unsigned irq_type;1862	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];1863	unsigned int hw_prio;1864 1865	ring = &adev->gfx.compute_ring[ring_id];1866 1867	/* mec0 is me1 */1868	ring->me = mec + 1;1869	ring->pipe = pipe;1870	ring->queue = queue;1871 1872	ring->ring_obj = NULL;1873	ring->use_doorbell = true;1874	ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;1875	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr1876				+ (ring_id * GFX8_MEC_HPD_SIZE);1877	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);1878 1879	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP1880		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)1881		+ ring->pipe;1882 1883	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?1884			AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;1885	/* type-2 packets are deprecated on MEC, use type-3 instead */1886	r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,1887			     hw_prio, NULL);1888	if (r)1889		return r;1890 1891 1892	return 0;1893}1894 1895static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);1896 1897static int gfx_v8_0_sw_init(void *handle)1898{1899	int i, j, k, r, ring_id;1900	int xcc_id = 0;1901	struct amdgpu_ring *ring;1902	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1903 1904	switch (adev->asic_type) {1905	case CHIP_TONGA:1906	case CHIP_CARRIZO:1907	case CHIP_FIJI:1908	case CHIP_POLARIS10:1909	case CHIP_POLARIS11:1910	case CHIP_POLARIS12:1911	case CHIP_VEGAM:1912		adev->gfx.mec.num_mec = 2;1913		break;1914	case CHIP_TOPAZ:1915	case CHIP_STONEY:1916	default:1917		adev->gfx.mec.num_mec = 1;1918		break;1919	}1920 1921	adev->gfx.mec.num_pipe_per_mec = 4;1922	adev->gfx.mec.num_queue_per_pipe = 8;1923 1924	/* EOP Event */1925	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);1926	if (r)1927		return r;1928 1929	/* Privileged reg */1930	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,1931			      &adev->gfx.priv_reg_irq);1932	if (r)1933		return r;1934 1935	/* Privileged inst */1936	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,1937			      &adev->gfx.priv_inst_irq);1938	if (r)1939		return r;1940 1941	/* Add CP EDC/ECC irq  */1942	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,1943			      &adev->gfx.cp_ecc_error_irq);1944	if (r)1945		return r;1946 1947	/* SQ interrupts. */1948	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,1949			      &adev->gfx.sq_irq);1950	if (r) {1951		DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);1952		return r;1953	}1954 1955	INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);1956 1957	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;1958 1959	r = gfx_v8_0_init_microcode(adev);1960	if (r) {1961		DRM_ERROR("Failed to load gfx firmware!\n");1962		return r;1963	}1964 1965	r = adev->gfx.rlc.funcs->init(adev);1966	if (r) {1967		DRM_ERROR("Failed to init rlc BOs!\n");1968		return r;1969	}1970 1971	r = gfx_v8_0_mec_init(adev);1972	if (r) {1973		DRM_ERROR("Failed to init MEC BOs!\n");1974		return r;1975	}1976 1977	/* set up the gfx ring */1978	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {1979		ring = &adev->gfx.gfx_ring[i];1980		ring->ring_obj = NULL;1981		sprintf(ring->name, "gfx");1982		/* no gfx doorbells on iceland */1983		if (adev->asic_type != CHIP_TOPAZ) {1984			ring->use_doorbell = true;1985			ring->doorbell_index = adev->doorbell_index.gfx_ring0;1986		}1987 1988		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,1989				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,1990				     AMDGPU_RING_PRIO_DEFAULT, NULL);1991		if (r)1992			return r;1993	}1994 1995 1996	/* set up the compute queues - allocate horizontally across pipes */1997	ring_id = 0;1998	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {1999		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {2000			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {2001				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,2002								     k, j))2003					continue;2004 2005				r = gfx_v8_0_compute_ring_init(adev,2006								ring_id,2007								i, k, j);2008				if (r)2009					return r;2010 2011				ring_id++;2012			}2013		}2014	}2015 2016	r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE, 0);2017	if (r) {2018		DRM_ERROR("Failed to init KIQ BOs!\n");2019		return r;2020	}2021 2022	r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);2023	if (r)2024		return r;2025 2026	/* create MQD for all compute queues as well as KIQ for SRIOV case */2027	r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation), 0);2028	if (r)2029		return r;2030 2031	adev->gfx.ce_ram_size = 0x8000;2032 2033	r = gfx_v8_0_gpu_early_init(adev);2034	if (r)2035		return r;2036 2037	return 0;2038}2039 2040static int gfx_v8_0_sw_fini(void *handle)2041{2042	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2043	int i;2044 2045	for (i = 0; i < adev->gfx.num_gfx_rings; i++)2046		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);2047	for (i = 0; i < adev->gfx.num_compute_rings; i++)2048		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);2049 2050	amdgpu_gfx_mqd_sw_fini(adev, 0);2051	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);2052	amdgpu_gfx_kiq_fini(adev, 0);2053 2054	gfx_v8_0_mec_fini(adev);2055	amdgpu_gfx_rlc_fini(adev);2056	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,2057				&adev->gfx.rlc.clear_state_gpu_addr,2058				(void **)&adev->gfx.rlc.cs_ptr);2059	if ((adev->asic_type == CHIP_CARRIZO) ||2060	    (adev->asic_type == CHIP_STONEY)) {2061		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,2062				&adev->gfx.rlc.cp_table_gpu_addr,2063				(void **)&adev->gfx.rlc.cp_table_ptr);2064	}2065	gfx_v8_0_free_microcode(adev);2066 2067	return 0;2068}2069 2070static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)2071{2072	uint32_t *modearray, *mod2array;2073	const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);2074	const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);2075	u32 reg_offset;2076 2077	modearray = adev->gfx.config.tile_mode_array;2078	mod2array = adev->gfx.config.macrotile_mode_array;2079 2080	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)2081		modearray[reg_offset] = 0;2082 2083	for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)2084		mod2array[reg_offset] = 0;2085 2086	switch (adev->asic_type) {2087	case CHIP_TOPAZ:2088		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2089				PIPE_CONFIG(ADDR_SURF_P2) |2090				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |2091				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2092		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2093				PIPE_CONFIG(ADDR_SURF_P2) |2094				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |2095				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2096		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2097				PIPE_CONFIG(ADDR_SURF_P2) |2098				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |2099				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2100		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2101				PIPE_CONFIG(ADDR_SURF_P2) |2102				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |2103				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2104		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2105				PIPE_CONFIG(ADDR_SURF_P2) |2106				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2107				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2108		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2109				PIPE_CONFIG(ADDR_SURF_P2) |2110				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2111				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2112		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2113				PIPE_CONFIG(ADDR_SURF_P2) |2114				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2115				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2116		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |2117				PIPE_CONFIG(ADDR_SURF_P2));2118		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2119				PIPE_CONFIG(ADDR_SURF_P2) |2120				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2121				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2122		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2123				 PIPE_CONFIG(ADDR_SURF_P2) |2124				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2125				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2126		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2127				 PIPE_CONFIG(ADDR_SURF_P2) |2128				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2129				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2130		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2131				 PIPE_CONFIG(ADDR_SURF_P2) |2132				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2133				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2134		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2135				 PIPE_CONFIG(ADDR_SURF_P2) |2136				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2137				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2138		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |2139				 PIPE_CONFIG(ADDR_SURF_P2) |2140				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2141				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2142		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2143				 PIPE_CONFIG(ADDR_SURF_P2) |2144				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2145				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2146		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2147				 PIPE_CONFIG(ADDR_SURF_P2) |2148				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2149				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2150		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2151				 PIPE_CONFIG(ADDR_SURF_P2) |2152				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2153				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2154		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2155				 PIPE_CONFIG(ADDR_SURF_P2) |2156				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2157				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2158		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |2159				 PIPE_CONFIG(ADDR_SURF_P2) |2160				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2161				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2162		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2163				 PIPE_CONFIG(ADDR_SURF_P2) |2164				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2165				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2166		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2167				 PIPE_CONFIG(ADDR_SURF_P2) |2168				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2169				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2170		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |2171				 PIPE_CONFIG(ADDR_SURF_P2) |2172				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2173				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2174		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |2175				 PIPE_CONFIG(ADDR_SURF_P2) |2176				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2177				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2178		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2179				 PIPE_CONFIG(ADDR_SURF_P2) |2180				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2181				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2182		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2183				 PIPE_CONFIG(ADDR_SURF_P2) |2184				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2185				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2186		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2187				 PIPE_CONFIG(ADDR_SURF_P2) |2188				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2189				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2190 2191		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |2192				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2193				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2194				NUM_BANKS(ADDR_SURF_8_BANK));2195		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |2196				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2197				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2198				NUM_BANKS(ADDR_SURF_8_BANK));2199		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |2200				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2201				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2202				NUM_BANKS(ADDR_SURF_8_BANK));2203		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2204				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2205				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2206				NUM_BANKS(ADDR_SURF_8_BANK));2207		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2208				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2209				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2210				NUM_BANKS(ADDR_SURF_8_BANK));2211		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2212				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2213				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2214				NUM_BANKS(ADDR_SURF_8_BANK));2215		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2216				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2217				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2218				NUM_BANKS(ADDR_SURF_8_BANK));2219		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |2220				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |2221				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2222				NUM_BANKS(ADDR_SURF_16_BANK));2223		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |2224				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2225				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2226				NUM_BANKS(ADDR_SURF_16_BANK));2227		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |2228				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2229				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2230				 NUM_BANKS(ADDR_SURF_16_BANK));2231		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |2232				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2233				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2234				 NUM_BANKS(ADDR_SURF_16_BANK));2235		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2236				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2237				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2238				 NUM_BANKS(ADDR_SURF_16_BANK));2239		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2240				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2241				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2242				 NUM_BANKS(ADDR_SURF_16_BANK));2243		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2244				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2245				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2246				 NUM_BANKS(ADDR_SURF_8_BANK));2247 2248		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)2249			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&2250			    reg_offset != 23)2251				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);2252 2253		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)2254			if (reg_offset != 7)2255				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);2256 2257		break;2258	case CHIP_FIJI:2259	case CHIP_VEGAM:2260		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2261				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2262				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |2263				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2264		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2265				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2266				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |2267				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2268		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2269				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2270				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |2271				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2272		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2273				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2274				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |2275				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2276		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2277				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2278				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2279				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2280		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2281				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2282				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2283				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2284		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2285				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2286				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2287				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2288		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2289				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2290				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2291				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2292		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |2293				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));2294		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2295				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2296				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2297				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2298		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2299				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2300				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2301				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2302		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2303				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2304				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2305				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2306		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2307				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2308				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2309				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2310		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2311				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2312				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2313				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2314		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2315				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2316				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2317				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2318		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |2319				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2320				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2321				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2322		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2323				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2324				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2325				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2326		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2327				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2328				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2329				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2330		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2331				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2332				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2333				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2334		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2335				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2336				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2337				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2338		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2339				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2340				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2341				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2342		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |2343				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2344				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2345				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2346		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2347				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2348				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2349				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2350		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2351				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2352				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2353				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2354		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2355				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2356				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2357				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2358		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |2359				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2360				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2361				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2362		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |2363				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2364				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2365				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2366		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2367				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2368				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2369				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2370		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2371				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2372				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2373				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2374		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2375				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |2376				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2377				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2378		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2379				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2380				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2381				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2382 2383		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2384				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2385				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2386				NUM_BANKS(ADDR_SURF_8_BANK));2387		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2388				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2389				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2390				NUM_BANKS(ADDR_SURF_8_BANK));2391		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2392				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2393				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2394				NUM_BANKS(ADDR_SURF_8_BANK));2395		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2396				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2397				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2398				NUM_BANKS(ADDR_SURF_8_BANK));2399		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2400				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2401				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2402				NUM_BANKS(ADDR_SURF_8_BANK));2403		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2404				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2405				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2406				NUM_BANKS(ADDR_SURF_8_BANK));2407		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2408				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2409				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2410				NUM_BANKS(ADDR_SURF_8_BANK));2411		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2412				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |2413				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2414				NUM_BANKS(ADDR_SURF_8_BANK));2415		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2416				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2417				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2418				NUM_BANKS(ADDR_SURF_8_BANK));2419		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2420				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2421				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2422				 NUM_BANKS(ADDR_SURF_8_BANK));2423		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2424				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2425				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2426				 NUM_BANKS(ADDR_SURF_8_BANK));2427		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2428				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2429				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2430				 NUM_BANKS(ADDR_SURF_8_BANK));2431		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2432				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2433				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2434				 NUM_BANKS(ADDR_SURF_8_BANK));2435		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2436				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2437				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2438				 NUM_BANKS(ADDR_SURF_4_BANK));2439 2440		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)2441			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);2442 2443		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)2444			if (reg_offset != 7)2445				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);2446 2447		break;2448	case CHIP_TONGA:2449		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2450				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2451				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |2452				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2453		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2454				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2455				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |2456				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2457		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2458				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2459				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |2460				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2461		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2462				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2463				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |2464				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2465		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2466				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2467				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2468				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2469		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2470				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2471				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2472				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2473		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2474				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2475				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2476				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2477		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2478				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2479				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2480				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2481		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |2482				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));2483		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2484				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2485				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2486				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2487		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2488				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2489				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2490				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2491		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2492				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2493				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2494				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2495		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2496				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2497				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2498				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2499		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2500				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2501				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2502				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2503		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2504				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2505				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2506				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2507		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |2508				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2509				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2510				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2511		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2512				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2513				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2514				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2515		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2516				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2517				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2518				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2519		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2520				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2521				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2522				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2523		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2524				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2525				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2526				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2527		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2528				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2529				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2530				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2531		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |2532				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2533				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2534				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2535		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2536				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2537				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2538				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2539		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2540				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2541				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2542				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2543		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2544				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2545				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2546				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2547		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |2548				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2549				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2550				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2551		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |2552				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2553				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2554				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2555		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2556				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2557				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2558				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2559		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2560				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2561				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2562				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2563		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2564				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2565				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2566				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2567		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2568				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |2569				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2570				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2571 2572		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2573				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2574				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2575				NUM_BANKS(ADDR_SURF_16_BANK));2576		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2577				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2578				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2579				NUM_BANKS(ADDR_SURF_16_BANK));2580		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2581				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2582				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2583				NUM_BANKS(ADDR_SURF_16_BANK));2584		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2585				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2586				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2587				NUM_BANKS(ADDR_SURF_16_BANK));2588		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2589				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2590				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2591				NUM_BANKS(ADDR_SURF_16_BANK));2592		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2593				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2594				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2595				NUM_BANKS(ADDR_SURF_16_BANK));2596		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2597				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2598				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2599				NUM_BANKS(ADDR_SURF_16_BANK));2600		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2601				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |2602				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2603				NUM_BANKS(ADDR_SURF_16_BANK));2604		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2605				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2606				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2607				NUM_BANKS(ADDR_SURF_16_BANK));2608		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2609				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2610				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2611				 NUM_BANKS(ADDR_SURF_16_BANK));2612		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2613				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2614				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2615				 NUM_BANKS(ADDR_SURF_16_BANK));2616		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2617				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2618				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2619				 NUM_BANKS(ADDR_SURF_8_BANK));2620		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2621				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2622				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2623				 NUM_BANKS(ADDR_SURF_4_BANK));2624		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2625				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2626				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2627				 NUM_BANKS(ADDR_SURF_4_BANK));2628 2629		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)2630			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);2631 2632		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)2633			if (reg_offset != 7)2634				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);2635 2636		break;2637	case CHIP_POLARIS11:2638	case CHIP_POLARIS12:2639		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2640				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2641				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |2642				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2643		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2644				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2645				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |2646				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2647		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2648				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2649				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |2650				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2651		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2652				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2653				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |2654				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2655		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2656				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2657				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2658				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2659		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2660				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2661				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2662				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2663		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2664				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2665				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2666				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2667		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2668				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2669				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2670				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2671		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |2672				PIPE_CONFIG(ADDR_SURF_P4_16x16));2673		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2674				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2675				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2676				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2677		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2678				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2679				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2680				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2681		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2682				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2683				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2684				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2685		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2686				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2687				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2688				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2689		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2690				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2691				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2692				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2693		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2694				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2695				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2696				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2697		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |2698				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2699				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2700				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2701		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2702				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2703				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2704				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2705		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2706				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2707				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2708				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2709		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2710				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2711				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2712				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2713		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2714				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2715				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2716				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2717		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2718				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2719				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2720				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2721		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |2722				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2723				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2724				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2725		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2726				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2727				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2728				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2729		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2730				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2731				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2732				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2733		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2734				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2735				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2736				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2737		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |2738				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2739				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2740				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2741		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |2742				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2743				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2744				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2745		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2746				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2747				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2748				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2749		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2750				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2751				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2752				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2753		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2754				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2755				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2756				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2757		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2758				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2759				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2760				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2761 2762		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2763				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2764				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2765				NUM_BANKS(ADDR_SURF_16_BANK));2766 2767		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2768				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2769				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2770				NUM_BANKS(ADDR_SURF_16_BANK));2771 2772		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2773				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2774				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2775				NUM_BANKS(ADDR_SURF_16_BANK));2776 2777		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2778				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2779				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2780				NUM_BANKS(ADDR_SURF_16_BANK));2781 2782		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2783				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2784				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2785				NUM_BANKS(ADDR_SURF_16_BANK));2786 2787		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2788				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2789				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2790				NUM_BANKS(ADDR_SURF_16_BANK));2791 2792		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2793				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2794				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2795				NUM_BANKS(ADDR_SURF_16_BANK));2796 2797		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |2798				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |2799				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2800				NUM_BANKS(ADDR_SURF_16_BANK));2801 2802		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |2803				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2804				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2805				NUM_BANKS(ADDR_SURF_16_BANK));2806 2807		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2808				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2809				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2810				NUM_BANKS(ADDR_SURF_16_BANK));2811 2812		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2813				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2814				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2815				NUM_BANKS(ADDR_SURF_16_BANK));2816 2817		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2818				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2819				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2820				NUM_BANKS(ADDR_SURF_16_BANK));2821 2822		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2823				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2824				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2825				NUM_BANKS(ADDR_SURF_8_BANK));2826 2827		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2828				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2829				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2830				NUM_BANKS(ADDR_SURF_4_BANK));2831 2832		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)2833			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);2834 2835		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)2836			if (reg_offset != 7)2837				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);2838 2839		break;2840	case CHIP_POLARIS10:2841		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2842				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2843				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |2844				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2845		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2846				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2847				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |2848				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2849		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2850				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2851				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |2852				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2853		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2854				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2855				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |2856				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2857		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2858				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2859				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2860				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2861		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2862				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2863				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2864				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2865		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2866				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2867				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2868				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2869		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2870				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2871				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |2872				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));2873		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |2874				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));2875		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2876				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2877				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2878				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2879		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2880				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2881				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2882				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2883		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2884				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2885				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2886				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2887		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2888				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2889				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |2890				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2891		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2892				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2893				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2894				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2895		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2896				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2897				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2898				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2899		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |2900				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2901				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2902				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2903		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2904				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2905				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2906				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2907		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2908				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2909				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2910				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2911		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2912				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2913				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2914				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2915		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |2916				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2917				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2918				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2919		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2920				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2921				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2922				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2923		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |2924				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2925				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2926				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2927		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2928				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2929				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2930				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2931		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |2932				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2933				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2934				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2935		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |2936				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2937				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |2938				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2939		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |2940				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2941				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2942				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2943		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |2944				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2945				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |2946				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));2947		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |2948				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2949				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2950				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2951		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |2952				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2953				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2954				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));2955		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2956				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |2957				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2958				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2959		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |2960				PIPE_CONFIG(ADDR_SURF_P4_16x16) |2961				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |2962				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));2963 2964		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2965				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2966				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2967				NUM_BANKS(ADDR_SURF_16_BANK));2968 2969		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2970				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2971				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2972				NUM_BANKS(ADDR_SURF_16_BANK));2973 2974		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2975				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2976				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2977				NUM_BANKS(ADDR_SURF_16_BANK));2978 2979		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2980				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |2981				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |2982				NUM_BANKS(ADDR_SURF_16_BANK));2983 2984		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2985				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |2986				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |2987				NUM_BANKS(ADDR_SURF_16_BANK));2988 2989		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2990				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2991				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2992				NUM_BANKS(ADDR_SURF_16_BANK));2993 2994		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |2995				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |2996				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |2997				NUM_BANKS(ADDR_SURF_16_BANK));2998 2999		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3000				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |3001				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3002				NUM_BANKS(ADDR_SURF_16_BANK));3003 3004		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3005				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3006				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3007				NUM_BANKS(ADDR_SURF_16_BANK));3008 3009		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3010				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3011				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3012				NUM_BANKS(ADDR_SURF_16_BANK));3013 3014		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3015				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3016				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3017				NUM_BANKS(ADDR_SURF_16_BANK));3018 3019		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3020				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3021				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |3022				NUM_BANKS(ADDR_SURF_8_BANK));3023 3024		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3025				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3026				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |3027				NUM_BANKS(ADDR_SURF_4_BANK));3028 3029		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3030				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3031				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |3032				NUM_BANKS(ADDR_SURF_4_BANK));3033 3034		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)3035			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);3036 3037		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)3038			if (reg_offset != 7)3039				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);3040 3041		break;3042	case CHIP_STONEY:3043		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3044				PIPE_CONFIG(ADDR_SURF_P2) |3045				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |3046				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3047		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3048				PIPE_CONFIG(ADDR_SURF_P2) |3049				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |3050				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3051		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3052				PIPE_CONFIG(ADDR_SURF_P2) |3053				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |3054				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3055		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3056				PIPE_CONFIG(ADDR_SURF_P2) |3057				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |3058				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3059		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3060				PIPE_CONFIG(ADDR_SURF_P2) |3061				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3062				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3063		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3064				PIPE_CONFIG(ADDR_SURF_P2) |3065				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3066				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3067		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3068				PIPE_CONFIG(ADDR_SURF_P2) |3069				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3070				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3071		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |3072				PIPE_CONFIG(ADDR_SURF_P2));3073		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3074				PIPE_CONFIG(ADDR_SURF_P2) |3075				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3076				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3077		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3078				 PIPE_CONFIG(ADDR_SURF_P2) |3079				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3080				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3081		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3082				 PIPE_CONFIG(ADDR_SURF_P2) |3083				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3084				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3085		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3086				 PIPE_CONFIG(ADDR_SURF_P2) |3087				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3088				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3089		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3090				 PIPE_CONFIG(ADDR_SURF_P2) |3091				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3092				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3093		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |3094				 PIPE_CONFIG(ADDR_SURF_P2) |3095				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3096				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3097		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3098				 PIPE_CONFIG(ADDR_SURF_P2) |3099				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3100				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3101		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |3102				 PIPE_CONFIG(ADDR_SURF_P2) |3103				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3104				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3105		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |3106				 PIPE_CONFIG(ADDR_SURF_P2) |3107				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3108				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3109		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |3110				 PIPE_CONFIG(ADDR_SURF_P2) |3111				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3112				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3113		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |3114				 PIPE_CONFIG(ADDR_SURF_P2) |3115				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3116				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3117		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |3118				 PIPE_CONFIG(ADDR_SURF_P2) |3119				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3120				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3121		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |3122				 PIPE_CONFIG(ADDR_SURF_P2) |3123				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3124				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3125		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |3126				 PIPE_CONFIG(ADDR_SURF_P2) |3127				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3128				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3129		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |3130				 PIPE_CONFIG(ADDR_SURF_P2) |3131				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3132				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3133		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3134				 PIPE_CONFIG(ADDR_SURF_P2) |3135				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3136				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3137		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3138				 PIPE_CONFIG(ADDR_SURF_P2) |3139				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3140				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3141		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3142				 PIPE_CONFIG(ADDR_SURF_P2) |3143				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3144				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3145 3146		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3147				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3148				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3149				NUM_BANKS(ADDR_SURF_8_BANK));3150		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3151				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3152				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3153				NUM_BANKS(ADDR_SURF_8_BANK));3154		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3155				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3156				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3157				NUM_BANKS(ADDR_SURF_8_BANK));3158		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3159				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3160				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3161				NUM_BANKS(ADDR_SURF_8_BANK));3162		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3163				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3164				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3165				NUM_BANKS(ADDR_SURF_8_BANK));3166		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3167				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3168				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3169				NUM_BANKS(ADDR_SURF_8_BANK));3170		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3171				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3172				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3173				NUM_BANKS(ADDR_SURF_8_BANK));3174		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |3175				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |3176				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3177				NUM_BANKS(ADDR_SURF_16_BANK));3178		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |3179				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3180				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3181				NUM_BANKS(ADDR_SURF_16_BANK));3182		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |3183				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3184				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3185				 NUM_BANKS(ADDR_SURF_16_BANK));3186		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |3187				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3188				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3189				 NUM_BANKS(ADDR_SURF_16_BANK));3190		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3191				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3192				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3193				 NUM_BANKS(ADDR_SURF_16_BANK));3194		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3195				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3196				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3197				 NUM_BANKS(ADDR_SURF_16_BANK));3198		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3199				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3200				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3201				 NUM_BANKS(ADDR_SURF_8_BANK));3202 3203		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)3204			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&3205			    reg_offset != 23)3206				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);3207 3208		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)3209			if (reg_offset != 7)3210				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);3211 3212		break;3213	default:3214		dev_warn(adev->dev,3215			 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",3216			 adev->asic_type);3217		fallthrough;3218 3219	case CHIP_CARRIZO:3220		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3221				PIPE_CONFIG(ADDR_SURF_P2) |3222				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |3223				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3224		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3225				PIPE_CONFIG(ADDR_SURF_P2) |3226				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |3227				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3228		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3229				PIPE_CONFIG(ADDR_SURF_P2) |3230				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |3231				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3232		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3233				PIPE_CONFIG(ADDR_SURF_P2) |3234				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |3235				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3236		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3237				PIPE_CONFIG(ADDR_SURF_P2) |3238				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3239				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3240		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3241				PIPE_CONFIG(ADDR_SURF_P2) |3242				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3243				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3244		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3245				PIPE_CONFIG(ADDR_SURF_P2) |3246				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |3247				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));3248		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |3249				PIPE_CONFIG(ADDR_SURF_P2));3250		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3251				PIPE_CONFIG(ADDR_SURF_P2) |3252				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3253				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3254		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3255				 PIPE_CONFIG(ADDR_SURF_P2) |3256				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3257				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3258		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3259				 PIPE_CONFIG(ADDR_SURF_P2) |3260				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |3261				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3262		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3263				 PIPE_CONFIG(ADDR_SURF_P2) |3264				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3265				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3266		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3267				 PIPE_CONFIG(ADDR_SURF_P2) |3268				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3269				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3270		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |3271				 PIPE_CONFIG(ADDR_SURF_P2) |3272				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3273				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3274		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3275				 PIPE_CONFIG(ADDR_SURF_P2) |3276				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3277				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3278		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |3279				 PIPE_CONFIG(ADDR_SURF_P2) |3280				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3281				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3282		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |3283				 PIPE_CONFIG(ADDR_SURF_P2) |3284				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3285				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3286		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |3287				 PIPE_CONFIG(ADDR_SURF_P2) |3288				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3289				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3290		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |3291				 PIPE_CONFIG(ADDR_SURF_P2) |3292				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3293				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3294		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |3295				 PIPE_CONFIG(ADDR_SURF_P2) |3296				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3297				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3298		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |3299				 PIPE_CONFIG(ADDR_SURF_P2) |3300				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |3301				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3302		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |3303				 PIPE_CONFIG(ADDR_SURF_P2) |3304				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3305				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3306		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |3307				 PIPE_CONFIG(ADDR_SURF_P2) |3308				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |3309				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));3310		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |3311				 PIPE_CONFIG(ADDR_SURF_P2) |3312				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3313				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3314		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |3315				 PIPE_CONFIG(ADDR_SURF_P2) |3316				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3317				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));3318		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |3319				 PIPE_CONFIG(ADDR_SURF_P2) |3320				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |3321				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));3322 3323		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3324				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3325				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3326				NUM_BANKS(ADDR_SURF_8_BANK));3327		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3328				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3329				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3330				NUM_BANKS(ADDR_SURF_8_BANK));3331		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3332				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3333				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3334				NUM_BANKS(ADDR_SURF_8_BANK));3335		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3336				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3337				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3338				NUM_BANKS(ADDR_SURF_8_BANK));3339		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3340				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3341				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3342				NUM_BANKS(ADDR_SURF_8_BANK));3343		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3344				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3345				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3346				NUM_BANKS(ADDR_SURF_8_BANK));3347		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3348				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3349				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3350				NUM_BANKS(ADDR_SURF_8_BANK));3351		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |3352				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |3353				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3354				NUM_BANKS(ADDR_SURF_16_BANK));3355		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |3356				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3357				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3358				NUM_BANKS(ADDR_SURF_16_BANK));3359		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |3360				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |3361				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3362				 NUM_BANKS(ADDR_SURF_16_BANK));3363		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |3364				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3365				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3366				 NUM_BANKS(ADDR_SURF_16_BANK));3367		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3368				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |3369				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3370				 NUM_BANKS(ADDR_SURF_16_BANK));3371		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3372				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3373				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |3374				 NUM_BANKS(ADDR_SURF_16_BANK));3375		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |3376				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |3377				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |3378				 NUM_BANKS(ADDR_SURF_8_BANK));3379 3380		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)3381			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&3382			    reg_offset != 23)3383				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);3384 3385		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)3386			if (reg_offset != 7)3387				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);3388 3389		break;3390	}3391}3392 3393static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,3394				  u32 se_num, u32 sh_num, u32 instance,3395				  int xcc_id)3396{3397	u32 data;3398 3399	if (instance == 0xffffffff)3400		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);3401	else3402		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);3403 3404	if (se_num == 0xffffffff)3405		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);3406	else3407		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);3408 3409	if (sh_num == 0xffffffff)3410		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);3411	else3412		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);3413 3414	WREG32(mmGRBM_GFX_INDEX, data);3415}3416 3417static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,3418				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)3419{3420	vi_srbm_select(adev, me, pipe, q, vm);3421}3422 3423static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)3424{3425	u32 data, mask;3426 3427	data =  RREG32(mmCC_RB_BACKEND_DISABLE) |3428		RREG32(mmGC_USER_RB_BACKEND_DISABLE);3429 3430	data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);3431 3432	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /3433					 adev->gfx.config.max_sh_per_se);3434 3435	return (~data) & mask;3436}3437 3438static void3439gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)3440{3441	switch (adev->asic_type) {3442	case CHIP_FIJI:3443	case CHIP_VEGAM:3444		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |3445			  RB_XSEL2(1) | PKR_MAP(2) |3446			  PKR_XSEL(1) | PKR_YSEL(1) |3447			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);3448		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |3449			   SE_PAIR_YSEL(2);3450		break;3451	case CHIP_TONGA:3452	case CHIP_POLARIS10:3453		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |3454			  SE_XSEL(1) | SE_YSEL(1);3455		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |3456			   SE_PAIR_YSEL(2);3457		break;3458	case CHIP_TOPAZ:3459	case CHIP_CARRIZO:3460		*rconf |= RB_MAP_PKR0(2);3461		*rconf1 |= 0x0;3462		break;3463	case CHIP_POLARIS11:3464	case CHIP_POLARIS12:3465		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |3466			  SE_XSEL(1) | SE_YSEL(1);3467		*rconf1 |= 0x0;3468		break;3469	case CHIP_STONEY:3470		*rconf |= 0x0;3471		*rconf1 |= 0x0;3472		break;3473	default:3474		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);3475		break;3476	}3477}3478 3479static void3480gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,3481					u32 raster_config, u32 raster_config_1,3482					unsigned rb_mask, unsigned num_rb)3483{3484	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);3485	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);3486	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);3487	unsigned rb_per_se = num_rb / num_se;3488	unsigned se_mask[4];3489	unsigned se;3490 3491	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;3492	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;3493	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;3494	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;3495 3496	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));3497	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));3498	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));3499 3500	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||3501			     (!se_mask[2] && !se_mask[3]))) {3502		raster_config_1 &= ~SE_PAIR_MAP_MASK;3503 3504		if (!se_mask[0] && !se_mask[1]) {3505			raster_config_1 |=3506				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);3507		} else {3508			raster_config_1 |=3509				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);3510		}3511	}3512 3513	for (se = 0; se < num_se; se++) {3514		unsigned raster_config_se = raster_config;3515		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);3516		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;3517		int idx = (se / 2) * 2;3518 3519		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {3520			raster_config_se &= ~SE_MAP_MASK;3521 3522			if (!se_mask[idx]) {3523				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);3524			} else {3525				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);3526			}3527		}3528 3529		pkr0_mask &= rb_mask;3530		pkr1_mask &= rb_mask;3531		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {3532			raster_config_se &= ~PKR_MAP_MASK;3533 3534			if (!pkr0_mask) {3535				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);3536			} else {3537				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);3538			}3539		}3540 3541		if (rb_per_se >= 2) {3542			unsigned rb0_mask = 1 << (se * rb_per_se);3543			unsigned rb1_mask = rb0_mask << 1;3544 3545			rb0_mask &= rb_mask;3546			rb1_mask &= rb_mask;3547			if (!rb0_mask || !rb1_mask) {3548				raster_config_se &= ~RB_MAP_PKR0_MASK;3549 3550				if (!rb0_mask) {3551					raster_config_se |=3552						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);3553				} else {3554					raster_config_se |=3555						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);3556				}3557			}3558 3559			if (rb_per_se > 2) {3560				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);3561				rb1_mask = rb0_mask << 1;3562				rb0_mask &= rb_mask;3563				rb1_mask &= rb_mask;3564				if (!rb0_mask || !rb1_mask) {3565					raster_config_se &= ~RB_MAP_PKR1_MASK;3566 3567					if (!rb0_mask) {3568						raster_config_se |=3569							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);3570					} else {3571						raster_config_se |=3572							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);3573					}3574				}3575			}3576		}3577 3578		/* GRBM_GFX_INDEX has a different offset on VI */3579		gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0);3580		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);3581		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);3582	}3583 3584	/* GRBM_GFX_INDEX has a different offset on VI */3585	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);3586}3587 3588static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)3589{3590	int i, j;3591	u32 data;3592	u32 raster_config = 0, raster_config_1 = 0;3593	u32 active_rbs = 0;3594	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /3595					adev->gfx.config.max_sh_per_se;3596	unsigned num_rb_pipes;3597 3598	mutex_lock(&adev->grbm_idx_mutex);3599	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {3600		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {3601			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);3602			data = gfx_v8_0_get_rb_active_bitmap(adev);3603			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *3604					       rb_bitmap_width_per_sh);3605		}3606	}3607	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);3608 3609	adev->gfx.config.backend_enable_mask = active_rbs;3610	adev->gfx.config.num_rbs = hweight32(active_rbs);3611 3612	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *3613			     adev->gfx.config.max_shader_engines, 16);3614 3615	gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);3616 3617	if (!adev->gfx.config.backend_enable_mask ||3618			adev->gfx.config.num_rbs >= num_rb_pipes) {3619		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);3620		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);3621	} else {3622		gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,3623							adev->gfx.config.backend_enable_mask,3624							num_rb_pipes);3625	}3626 3627	/* cache the values for userspace */3628	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {3629		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {3630			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);3631			adev->gfx.config.rb_config[i][j].rb_backend_disable =3632				RREG32(mmCC_RB_BACKEND_DISABLE);3633			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =3634				RREG32(mmGC_USER_RB_BACKEND_DISABLE);3635			adev->gfx.config.rb_config[i][j].raster_config =3636				RREG32(mmPA_SC_RASTER_CONFIG);3637			adev->gfx.config.rb_config[i][j].raster_config_1 =3638				RREG32(mmPA_SC_RASTER_CONFIG_1);3639		}3640	}3641	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);3642	mutex_unlock(&adev->grbm_idx_mutex);3643}3644 3645#define DEFAULT_SH_MEM_BASES	(0x6000)3646/**3647 * gfx_v8_0_init_compute_vmid - gart enable3648 *3649 * @adev: amdgpu_device pointer3650 *3651 * Initialize compute vmid sh_mem registers3652 *3653 */3654static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)3655{3656	int i;3657	uint32_t sh_mem_config;3658	uint32_t sh_mem_bases;3659 3660	/*3661	 * Configure apertures:3662	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)3663	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)3664	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)3665	 */3666	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);3667 3668	sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<3669			SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |3670			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<3671			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |3672			MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |3673			SH_MEM_CONFIG__PRIVATE_ATC_MASK;3674 3675	mutex_lock(&adev->srbm_mutex);3676	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {3677		vi_srbm_select(adev, 0, 0, 0, i);3678		/* CP and shaders */3679		WREG32(mmSH_MEM_CONFIG, sh_mem_config);3680		WREG32(mmSH_MEM_APE1_BASE, 1);3681		WREG32(mmSH_MEM_APE1_LIMIT, 0);3682		WREG32(mmSH_MEM_BASES, sh_mem_bases);3683	}3684	vi_srbm_select(adev, 0, 0, 0, 0);3685	mutex_unlock(&adev->srbm_mutex);3686 3687	/* Initialize all compute VMIDs to have no GDS, GWS, or OA3688	   access. These should be enabled by FW for target VMIDs. */3689	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {3690		WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);3691		WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);3692		WREG32(amdgpu_gds_reg_offset[i].gws, 0);3693		WREG32(amdgpu_gds_reg_offset[i].oa, 0);3694	}3695}3696 3697static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev)3698{3699	int vmid;3700 3701	/*3702	 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA3703	 * access. Compute VMIDs should be enabled by FW for target VMIDs,3704	 * the driver can enable them for graphics. VMID0 should maintain3705	 * access so that HWS firmware can save/restore entries.3706	 */3707	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {3708		WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0);3709		WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0);3710		WREG32(amdgpu_gds_reg_offset[vmid].gws, 0);3711		WREG32(amdgpu_gds_reg_offset[vmid].oa, 0);3712	}3713}3714 3715static void gfx_v8_0_config_init(struct amdgpu_device *adev)3716{3717	switch (adev->asic_type) {3718	default:3719		adev->gfx.config.double_offchip_lds_buf = 1;3720		break;3721	case CHIP_CARRIZO:3722	case CHIP_STONEY:3723		adev->gfx.config.double_offchip_lds_buf = 0;3724		break;3725	}3726}3727 3728static void gfx_v8_0_constants_init(struct amdgpu_device *adev)3729{3730	u32 tmp, sh_static_mem_cfg;3731	int i;3732 3733	WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);3734	WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);3735	WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);3736	WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);3737 3738	gfx_v8_0_tiling_mode_table_init(adev);3739	gfx_v8_0_setup_rb(adev);3740	gfx_v8_0_get_cu_info(adev);3741	gfx_v8_0_config_init(adev);3742 3743	/* XXX SH_MEM regs */3744	/* where to put LDS, scratch, GPUVM in FSA64 space */3745	sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,3746				   SWIZZLE_ENABLE, 1);3747	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,3748				   ELEMENT_SIZE, 1);3749	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,3750				   INDEX_STRIDE, 3);3751	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);3752 3753	mutex_lock(&adev->srbm_mutex);3754	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {3755		vi_srbm_select(adev, 0, 0, 0, i);3756		/* CP and shaders */3757		if (i == 0) {3758			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);3759			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);3760			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,3761					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);3762			WREG32(mmSH_MEM_CONFIG, tmp);3763			WREG32(mmSH_MEM_BASES, 0);3764		} else {3765			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);3766			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);3767			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,3768					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);3769			WREG32(mmSH_MEM_CONFIG, tmp);3770			tmp = adev->gmc.shared_aperture_start >> 48;3771			WREG32(mmSH_MEM_BASES, tmp);3772		}3773 3774		WREG32(mmSH_MEM_APE1_BASE, 1);3775		WREG32(mmSH_MEM_APE1_LIMIT, 0);3776	}3777	vi_srbm_select(adev, 0, 0, 0, 0);3778	mutex_unlock(&adev->srbm_mutex);3779 3780	gfx_v8_0_init_compute_vmid(adev);3781	gfx_v8_0_init_gds_vmid(adev);3782 3783	mutex_lock(&adev->grbm_idx_mutex);3784	/*3785	 * making sure that the following register writes will be broadcasted3786	 * to all the shaders3787	 */3788	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);3789 3790	WREG32(mmPA_SC_FIFO_SIZE,3791		   (adev->gfx.config.sc_prim_fifo_size_frontend <<3792			PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |3793		   (adev->gfx.config.sc_prim_fifo_size_backend <<3794			PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |3795		   (adev->gfx.config.sc_hiz_tile_fifo_size <<3796			PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |3797		   (adev->gfx.config.sc_earlyz_tile_fifo_size <<3798			PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));3799 3800	tmp = RREG32(mmSPI_ARB_PRIORITY);3801	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);3802	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);3803	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);3804	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);3805	WREG32(mmSPI_ARB_PRIORITY, tmp);3806 3807	mutex_unlock(&adev->grbm_idx_mutex);3808 3809}3810 3811static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)3812{3813	u32 i, j, k;3814	u32 mask;3815 3816	mutex_lock(&adev->grbm_idx_mutex);3817	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {3818		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {3819			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);3820			for (k = 0; k < adev->usec_timeout; k++) {3821				if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)3822					break;3823				udelay(1);3824			}3825			if (k == adev->usec_timeout) {3826				gfx_v8_0_select_se_sh(adev, 0xffffffff,3827						      0xffffffff, 0xffffffff, 0);3828				mutex_unlock(&adev->grbm_idx_mutex);3829				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",3830					 i, j);3831				return;3832			}3833		}3834	}3835	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);3836	mutex_unlock(&adev->grbm_idx_mutex);3837 3838	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |3839		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |3840		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |3841		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;3842	for (k = 0; k < adev->usec_timeout; k++) {3843		if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)3844			break;3845		udelay(1);3846	}3847}3848 3849static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,3850					       bool enable)3851{3852	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);3853 3854	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);3855	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);3856	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);3857	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);3858 3859	WREG32(mmCP_INT_CNTL_RING0, tmp);3860}3861 3862static void gfx_v8_0_init_csb(struct amdgpu_device *adev)3863{3864	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);3865	/* csib */3866	WREG32(mmRLC_CSIB_ADDR_HI,3867			adev->gfx.rlc.clear_state_gpu_addr >> 32);3868	WREG32(mmRLC_CSIB_ADDR_LO,3869			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);3870	WREG32(mmRLC_CSIB_LENGTH,3871			adev->gfx.rlc.clear_state_size);3872}3873 3874static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,3875				int ind_offset,3876				int list_size,3877				int *unique_indices,3878				int *indices_count,3879				int max_indices,3880				int *ind_start_offsets,3881				int *offset_count,3882				int max_offset)3883{3884	int indices;3885	bool new_entry = true;3886 3887	for (; ind_offset < list_size; ind_offset++) {3888 3889		if (new_entry) {3890			new_entry = false;3891			ind_start_offsets[*offset_count] = ind_offset;3892			*offset_count = *offset_count + 1;3893			BUG_ON(*offset_count >= max_offset);3894		}3895 3896		if (register_list_format[ind_offset] == 0xFFFFFFFF) {3897			new_entry = true;3898			continue;3899		}3900 3901		ind_offset += 2;3902 3903		/* look for the matching indice */3904		for (indices = 0;3905			indices < *indices_count;3906			indices++) {3907			if (unique_indices[indices] ==3908				register_list_format[ind_offset])3909				break;3910		}3911 3912		if (indices >= *indices_count) {3913			unique_indices[*indices_count] =3914				register_list_format[ind_offset];3915			indices = *indices_count;3916			*indices_count = *indices_count + 1;3917			BUG_ON(*indices_count >= max_indices);3918		}3919 3920		register_list_format[ind_offset] = indices;3921	}3922}3923 3924static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)3925{3926	int i, temp, data;3927	int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};3928	int indices_count = 0;3929	int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};3930	int offset_count = 0;3931 3932	int list_size;3933	unsigned int *register_list_format =3934		kmemdup(adev->gfx.rlc.register_list_format,3935			adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);3936	if (!register_list_format)3937		return -ENOMEM;3938 3939	gfx_v8_0_parse_ind_reg_list(register_list_format,3940				RLC_FormatDirectRegListLength,3941				adev->gfx.rlc.reg_list_format_size_bytes >> 2,3942				unique_indices,3943				&indices_count,3944				ARRAY_SIZE(unique_indices),3945				indirect_start_offsets,3946				&offset_count,3947				ARRAY_SIZE(indirect_start_offsets));3948 3949	/* save and restore list */3950	WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);3951 3952	WREG32(mmRLC_SRM_ARAM_ADDR, 0);3953	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)3954		WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);3955 3956	/* indirect list */3957	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);3958	for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)3959		WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);3960 3961	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;3962	list_size = list_size >> 1;3963	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);3964	WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);3965 3966	/* starting offsets starts */3967	WREG32(mmRLC_GPM_SCRATCH_ADDR,3968		adev->gfx.rlc.starting_offsets_start);3969	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)3970		WREG32(mmRLC_GPM_SCRATCH_DATA,3971				indirect_start_offsets[i]);3972 3973	/* unique indices */3974	temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;3975	data = mmRLC_SRM_INDEX_CNTL_DATA_0;3976	for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {3977		if (unique_indices[i] != 0) {3978			WREG32(temp + i, unique_indices[i] & 0x3FFFF);3979			WREG32(data + i, unique_indices[i] >> 20);3980		}3981	}3982	kfree(register_list_format);3983 3984	return 0;3985}3986 3987static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)3988{3989	WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);3990}3991 3992static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)3993{3994	uint32_t data;3995 3996	WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);3997 3998	data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);3999	data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);4000	data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);4001	data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);4002	WREG32(mmRLC_PG_DELAY, data);4003 4004	WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);4005	WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);4006 4007}4008 4009static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,4010						bool enable)4011{4012	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);4013}4014 4015static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,4016						  bool enable)4017{4018	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);4019}4020 4021static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)4022{4023	WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);4024}4025 4026static void gfx_v8_0_init_pg(struct amdgpu_device *adev)4027{4028	if ((adev->asic_type == CHIP_CARRIZO) ||4029	    (adev->asic_type == CHIP_STONEY)) {4030		gfx_v8_0_init_csb(adev);4031		gfx_v8_0_init_save_restore_list(adev);4032		gfx_v8_0_enable_save_restore_machine(adev);4033		WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);4034		gfx_v8_0_init_power_gating(adev);4035		WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);4036	} else if ((adev->asic_type == CHIP_POLARIS11) ||4037		   (adev->asic_type == CHIP_POLARIS12) ||4038		   (adev->asic_type == CHIP_VEGAM)) {4039		gfx_v8_0_init_csb(adev);4040		gfx_v8_0_init_save_restore_list(adev);4041		gfx_v8_0_enable_save_restore_machine(adev);4042		gfx_v8_0_init_power_gating(adev);4043	}4044 4045}4046 4047static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)4048{4049	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);4050 4051	gfx_v8_0_enable_gui_idle_interrupt(adev, false);4052	gfx_v8_0_wait_for_rlc_serdes(adev);4053}4054 4055static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)4056{4057	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);4058	udelay(50);4059 4060	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);4061	udelay(50);4062}4063 4064static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)4065{4066	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);4067 4068	/* carrizo do enable cp interrupt after cp inited */4069	if (!(adev->flags & AMD_IS_APU))4070		gfx_v8_0_enable_gui_idle_interrupt(adev, true);4071 4072	udelay(50);4073}4074 4075static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)4076{4077	if (amdgpu_sriov_vf(adev)) {4078		gfx_v8_0_init_csb(adev);4079		return 0;4080	}4081 4082	adev->gfx.rlc.funcs->stop(adev);4083	adev->gfx.rlc.funcs->reset(adev);4084	gfx_v8_0_init_pg(adev);4085	adev->gfx.rlc.funcs->start(adev);4086 4087	return 0;4088}4089 4090static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)4091{4092	u32 tmp = RREG32(mmCP_ME_CNTL);4093 4094	if (enable) {4095		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);4096		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);4097		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);4098	} else {4099		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);4100		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);4101		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);4102	}4103	WREG32(mmCP_ME_CNTL, tmp);4104	udelay(50);4105}4106 4107static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)4108{4109	u32 count = 0;4110	const struct cs_section_def *sect = NULL;4111	const struct cs_extent_def *ext = NULL;4112 4113	/* begin clear state */4114	count += 2;4115	/* context control state */4116	count += 3;4117 4118	for (sect = vi_cs_data; sect->section != NULL; ++sect) {4119		for (ext = sect->section; ext->extent != NULL; ++ext) {4120			if (sect->id == SECT_CONTEXT)4121				count += 2 + ext->reg_count;4122			else4123				return 0;4124		}4125	}4126	/* pa_sc_raster_config/pa_sc_raster_config1 */4127	count += 4;4128	/* end clear state */4129	count += 2;4130	/* clear state */4131	count += 2;4132 4133	return count;4134}4135 4136static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)4137{4138	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];4139	const struct cs_section_def *sect = NULL;4140	const struct cs_extent_def *ext = NULL;4141	int r, i;4142 4143	/* init the CP */4144	WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);4145	WREG32(mmCP_ENDIAN_SWAP, 0);4146	WREG32(mmCP_DEVICE_ID, 1);4147 4148	gfx_v8_0_cp_gfx_enable(adev, true);4149 4150	r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);4151	if (r) {4152		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);4153		return r;4154	}4155 4156	/* clear state buffer */4157	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));4158	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);4159 4160	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));4161	amdgpu_ring_write(ring, 0x80000000);4162	amdgpu_ring_write(ring, 0x80000000);4163 4164	for (sect = vi_cs_data; sect->section != NULL; ++sect) {4165		for (ext = sect->section; ext->extent != NULL; ++ext) {4166			if (sect->id == SECT_CONTEXT) {4167				amdgpu_ring_write(ring,4168				       PACKET3(PACKET3_SET_CONTEXT_REG,4169					       ext->reg_count));4170				amdgpu_ring_write(ring,4171				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);4172				for (i = 0; i < ext->reg_count; i++)4173					amdgpu_ring_write(ring, ext->extent[i]);4174			}4175		}4176	}4177 4178	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));4179	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);4180	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);4181	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);4182 4183	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));4184	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);4185 4186	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));4187	amdgpu_ring_write(ring, 0);4188 4189	/* init the CE partitions */4190	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));4191	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));4192	amdgpu_ring_write(ring, 0x8000);4193	amdgpu_ring_write(ring, 0x8000);4194 4195	amdgpu_ring_commit(ring);4196 4197	return 0;4198}4199static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)4200{4201	u32 tmp;4202	/* no gfx doorbells on iceland */4203	if (adev->asic_type == CHIP_TOPAZ)4204		return;4205 4206	tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);4207 4208	if (ring->use_doorbell) {4209		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,4210				DOORBELL_OFFSET, ring->doorbell_index);4211		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,4212						DOORBELL_HIT, 0);4213		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,4214					    DOORBELL_EN, 1);4215	} else {4216		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);4217	}4218 4219	WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);4220 4221	if (adev->flags & AMD_IS_APU)4222		return;4223 4224	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,4225					DOORBELL_RANGE_LOWER,4226					adev->doorbell_index.gfx_ring0);4227	WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);4228 4229	WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,4230		CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);4231}4232 4233static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)4234{4235	struct amdgpu_ring *ring;4236	u32 tmp;4237	u32 rb_bufsz;4238	u64 rb_addr, rptr_addr, wptr_gpu_addr;4239 4240	/* Set the write pointer delay */4241	WREG32(mmCP_RB_WPTR_DELAY, 0);4242 4243	/* set the RB to use vmid 0 */4244	WREG32(mmCP_RB_VMID, 0);4245 4246	/* Set ring buffer size */4247	ring = &adev->gfx.gfx_ring[0];4248	rb_bufsz = order_base_2(ring->ring_size / 8);4249	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);4250	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);4251	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);4252	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);4253#ifdef __BIG_ENDIAN4254	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);4255#endif4256	WREG32(mmCP_RB0_CNTL, tmp);4257 4258	/* Initialize the ring buffer's read and write pointers */4259	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);4260	ring->wptr = 0;4261	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));4262 4263	/* set the wb address wether it's enabled or not */4264	rptr_addr = ring->rptr_gpu_addr;4265	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));4266	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);4267 4268	wptr_gpu_addr = ring->wptr_gpu_addr;4269	WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));4270	WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));4271	mdelay(1);4272	WREG32(mmCP_RB0_CNTL, tmp);4273 4274	rb_addr = ring->gpu_addr >> 8;4275	WREG32(mmCP_RB0_BASE, rb_addr);4276	WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));4277 4278	gfx_v8_0_set_cpg_door_bell(adev, ring);4279	/* start the ring */4280	amdgpu_ring_clear_ring(ring);4281	gfx_v8_0_cp_gfx_start(adev);4282 4283	return 0;4284}4285 4286static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)4287{4288	if (enable) {4289		WREG32(mmCP_MEC_CNTL, 0);4290	} else {4291		WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));4292		adev->gfx.kiq[0].ring.sched.ready = false;4293	}4294	udelay(50);4295}4296 4297/* KIQ functions */4298static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)4299{4300	uint32_t tmp;4301	struct amdgpu_device *adev = ring->adev;4302 4303	/* tell RLC which is KIQ queue */4304	tmp = RREG32(mmRLC_CP_SCHEDULERS);4305	tmp &= 0xffffff00;4306	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);4307	WREG32(mmRLC_CP_SCHEDULERS, tmp);4308	tmp |= 0x80;4309	WREG32(mmRLC_CP_SCHEDULERS, tmp);4310}4311 4312static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)4313{4314	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;4315	uint64_t queue_mask = 0;4316	int r, i;4317 4318	for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {4319		if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap))4320			continue;4321 4322		/* This situation may be hit in the future if a new HW4323		 * generation exposes more than 64 queues. If so, the4324		 * definition of queue_mask needs updating */4325		if (WARN_ON(i >= (sizeof(queue_mask)*8))) {4326			DRM_ERROR("Invalid KCQ enabled: %d\n", i);4327			break;4328		}4329 4330		queue_mask |= (1ull << i);4331	}4332 4333	r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);4334	if (r) {4335		DRM_ERROR("Failed to lock KIQ (%d).\n", r);4336		return r;4337	}4338	/* set resources */4339	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));4340	amdgpu_ring_write(kiq_ring, 0);	/* vmid_mask:0 queue_type:0 (KIQ) */4341	amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));	/* queue mask lo */4342	amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));	/* queue mask hi */4343	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */4344	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */4345	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */4346	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */4347	for (i = 0; i < adev->gfx.num_compute_rings; i++) {4348		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];4349		uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);4350		uint64_t wptr_addr = ring->wptr_gpu_addr;4351 4352		/* map queues */4353		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));4354		/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/4355		amdgpu_ring_write(kiq_ring,4356				  PACKET3_MAP_QUEUES_NUM_QUEUES(1));4357		amdgpu_ring_write(kiq_ring,4358				  PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |4359				  PACKET3_MAP_QUEUES_QUEUE(ring->queue) |4360				  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |4361				  PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */4362		amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));4363		amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));4364		amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));4365		amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));4366	}4367 4368	amdgpu_ring_commit(kiq_ring);4369 4370	return 0;4371}4372 4373static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)4374{4375	int i, r = 0;4376 4377	if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {4378		WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);4379		for (i = 0; i < adev->usec_timeout; i++) {4380			if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))4381				break;4382			udelay(1);4383		}4384		if (i == adev->usec_timeout)4385			r = -ETIMEDOUT;4386	}4387	WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);4388	WREG32(mmCP_HQD_PQ_RPTR, 0);4389	WREG32(mmCP_HQD_PQ_WPTR, 0);4390 4391	return r;4392}4393 4394static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd)4395{4396	struct amdgpu_device *adev = ring->adev;4397 4398	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {4399		if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {4400			mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;4401			mqd->cp_hqd_queue_priority =4402				AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;4403		}4404	}4405}4406 4407static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)4408{4409	struct amdgpu_device *adev = ring->adev;4410	struct vi_mqd *mqd = ring->mqd_ptr;4411	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;4412	uint32_t tmp;4413 4414	mqd->header = 0xC0310800;4415	mqd->compute_pipelinestat_enable = 0x00000001;4416	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;4417	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;4418	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;4419	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;4420	mqd->compute_misc_reserved = 0x00000003;4421	mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr4422						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));4423	mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr4424						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));4425	eop_base_addr = ring->eop_gpu_addr >> 8;4426	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;4427	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);4428 4429	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */4430	tmp = RREG32(mmCP_HQD_EOP_CONTROL);4431	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,4432			(order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));4433 4434	mqd->cp_hqd_eop_control = tmp;4435 4436	/* enable doorbell? */4437	tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),4438			    CP_HQD_PQ_DOORBELL_CONTROL,4439			    DOORBELL_EN,4440			    ring->use_doorbell ? 1 : 0);4441 4442	mqd->cp_hqd_pq_doorbell_control = tmp;4443 4444	/* set the pointer to the MQD */4445	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;4446	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);4447 4448	/* set MQD vmid to 0 */4449	tmp = RREG32(mmCP_MQD_CONTROL);4450	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);4451	mqd->cp_mqd_control = tmp;4452 4453	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */4454	hqd_gpu_addr = ring->gpu_addr >> 8;4455	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;4456	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);4457 4458	/* set up the HQD, this is similar to CP_RB0_CNTL */4459	tmp = RREG32(mmCP_HQD_PQ_CONTROL);4460	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,4461			    (order_base_2(ring->ring_size / 4) - 1));4462	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,4463			(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));4464#ifdef __BIG_ENDIAN4465	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);4466#endif4467	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);4468	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);4469	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);4470	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);4471	mqd->cp_hqd_pq_control = tmp;4472 4473	/* set the wb address whether it's enabled or not */4474	wb_gpu_addr = ring->rptr_gpu_addr;4475	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;4476	mqd->cp_hqd_pq_rptr_report_addr_hi =4477		upper_32_bits(wb_gpu_addr) & 0xffff;4478 4479	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */4480	wb_gpu_addr = ring->wptr_gpu_addr;4481	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;4482	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;4483 4484	tmp = 0;4485	/* enable the doorbell if requested */4486	if (ring->use_doorbell) {4487		tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);4488		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,4489				DOORBELL_OFFSET, ring->doorbell_index);4490 4491		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,4492					 DOORBELL_EN, 1);4493		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,4494					 DOORBELL_SOURCE, 0);4495		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,4496					 DOORBELL_HIT, 0);4497	}4498 4499	mqd->cp_hqd_pq_doorbell_control = tmp;4500 4501	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */4502	ring->wptr = 0;4503	mqd->cp_hqd_pq_wptr = ring->wptr;4504	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);4505 4506	/* set the vmid for the queue */4507	mqd->cp_hqd_vmid = 0;4508 4509	tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);4510	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);4511	mqd->cp_hqd_persistent_state = tmp;4512 4513	/* set MTYPE */4514	tmp = RREG32(mmCP_HQD_IB_CONTROL);4515	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);4516	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);4517	mqd->cp_hqd_ib_control = tmp;4518 4519	tmp = RREG32(mmCP_HQD_IQ_TIMER);4520	tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);4521	mqd->cp_hqd_iq_timer = tmp;4522 4523	tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);4524	tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);4525	mqd->cp_hqd_ctx_save_control = tmp;4526 4527	/* defaults */4528	mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);4529	mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);4530	mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);4531	mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);4532	mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);4533	mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);4534	mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);4535	mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);4536	mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);4537	mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);4538	mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);4539	mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);4540 4541	/* set static priority for a queue/ring */4542	gfx_v8_0_mqd_set_priority(ring, mqd);4543	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);4544 4545	/* map_queues packet doesn't need activate the queue,4546	 * so only kiq need set this field.4547	 */4548	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)4549		mqd->cp_hqd_active = 1;4550 4551	return 0;4552}4553 4554static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,4555			struct vi_mqd *mqd)4556{4557	uint32_t mqd_reg;4558	uint32_t *mqd_data;4559 4560	/* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */4561	mqd_data = &mqd->cp_mqd_base_addr_lo;4562 4563	/* disable wptr polling */4564	WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);4565 4566	/* program all HQD registers */4567	for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)4568		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);4569 4570	/* Tonga errata: EOP RPTR/WPTR should be left unmodified.4571	 * This is safe since EOP RPTR==WPTR for any inactive HQD4572	 * on ASICs that do not support context-save.4573	 * EOP writes/reads can start anywhere in the ring.4574	 */4575	if (adev->asic_type != CHIP_TONGA) {4576		WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);4577		WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);4578		WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);4579	}4580 4581	for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)4582		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);4583 4584	/* activate the HQD */4585	for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)4586		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);4587 4588	return 0;4589}4590 4591static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)4592{4593	struct amdgpu_device *adev = ring->adev;4594	struct vi_mqd *mqd = ring->mqd_ptr;4595 4596	gfx_v8_0_kiq_setting(ring);4597 4598	if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */4599		/* reset MQD to a clean status */4600		if (adev->gfx.kiq[0].mqd_backup)4601			memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation));4602 4603		/* reset ring buffer */4604		ring->wptr = 0;4605		amdgpu_ring_clear_ring(ring);4606		mutex_lock(&adev->srbm_mutex);4607		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);4608		gfx_v8_0_mqd_commit(adev, mqd);4609		vi_srbm_select(adev, 0, 0, 0, 0);4610		mutex_unlock(&adev->srbm_mutex);4611	} else {4612		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));4613		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;4614		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;4615		if (amdgpu_sriov_vf(adev) && adev->in_suspend)4616			amdgpu_ring_clear_ring(ring);4617		mutex_lock(&adev->srbm_mutex);4618		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);4619		gfx_v8_0_mqd_init(ring);4620		gfx_v8_0_mqd_commit(adev, mqd);4621		vi_srbm_select(adev, 0, 0, 0, 0);4622		mutex_unlock(&adev->srbm_mutex);4623 4624		if (adev->gfx.kiq[0].mqd_backup)4625			memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation));4626	}4627 4628	return 0;4629}4630 4631static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)4632{4633	struct amdgpu_device *adev = ring->adev;4634	struct vi_mqd *mqd = ring->mqd_ptr;4635	int mqd_idx = ring - &adev->gfx.compute_ring[0];4636 4637	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {4638		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));4639		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;4640		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;4641		mutex_lock(&adev->srbm_mutex);4642		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);4643		gfx_v8_0_mqd_init(ring);4644		vi_srbm_select(adev, 0, 0, 0, 0);4645		mutex_unlock(&adev->srbm_mutex);4646 4647		if (adev->gfx.mec.mqd_backup[mqd_idx])4648			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));4649	} else {4650		/* restore MQD to a clean status */4651		if (adev->gfx.mec.mqd_backup[mqd_idx])4652			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));4653		/* reset ring buffer */4654		ring->wptr = 0;4655		amdgpu_ring_clear_ring(ring);4656	}4657	return 0;4658}4659 4660static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)4661{4662	if (adev->asic_type > CHIP_TONGA) {4663		WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2);4664		WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2);4665	}4666	/* enable doorbells */4667	WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);4668}4669 4670static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)4671{4672	struct amdgpu_ring *ring;4673	int r;4674 4675	ring = &adev->gfx.kiq[0].ring;4676 4677	r = amdgpu_bo_reserve(ring->mqd_obj, false);4678	if (unlikely(r != 0))4679		return r;4680 4681	r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);4682	if (unlikely(r != 0)) {4683		amdgpu_bo_unreserve(ring->mqd_obj);4684		return r;4685	}4686 4687	gfx_v8_0_kiq_init_queue(ring);4688	amdgpu_bo_kunmap(ring->mqd_obj);4689	ring->mqd_ptr = NULL;4690	amdgpu_bo_unreserve(ring->mqd_obj);4691	return 0;4692}4693 4694static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)4695{4696	struct amdgpu_ring *ring = NULL;4697	int r = 0, i;4698 4699	gfx_v8_0_cp_compute_enable(adev, true);4700 4701	for (i = 0; i < adev->gfx.num_compute_rings; i++) {4702		ring = &adev->gfx.compute_ring[i];4703 4704		r = amdgpu_bo_reserve(ring->mqd_obj, false);4705		if (unlikely(r != 0))4706			goto done;4707		r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);4708		if (!r) {4709			r = gfx_v8_0_kcq_init_queue(ring);4710			amdgpu_bo_kunmap(ring->mqd_obj);4711			ring->mqd_ptr = NULL;4712		}4713		amdgpu_bo_unreserve(ring->mqd_obj);4714		if (r)4715			goto done;4716	}4717 4718	gfx_v8_0_set_mec_doorbell_range(adev);4719 4720	r = gfx_v8_0_kiq_kcq_enable(adev);4721	if (r)4722		goto done;4723 4724done:4725	return r;4726}4727 4728static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)4729{4730	int r, i;4731	struct amdgpu_ring *ring;4732 4733	/* collect all the ring_tests here, gfx, kiq, compute */4734	ring = &adev->gfx.gfx_ring[0];4735	r = amdgpu_ring_test_helper(ring);4736	if (r)4737		return r;4738 4739	ring = &adev->gfx.kiq[0].ring;4740	r = amdgpu_ring_test_helper(ring);4741	if (r)4742		return r;4743 4744	for (i = 0; i < adev->gfx.num_compute_rings; i++) {4745		ring = &adev->gfx.compute_ring[i];4746		amdgpu_ring_test_helper(ring);4747	}4748 4749	return 0;4750}4751 4752static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)4753{4754	int r;4755 4756	if (!(adev->flags & AMD_IS_APU))4757		gfx_v8_0_enable_gui_idle_interrupt(adev, false);4758 4759	r = gfx_v8_0_kiq_resume(adev);4760	if (r)4761		return r;4762 4763	r = gfx_v8_0_cp_gfx_resume(adev);4764	if (r)4765		return r;4766 4767	r = gfx_v8_0_kcq_resume(adev);4768	if (r)4769		return r;4770 4771	r = gfx_v8_0_cp_test_all_rings(adev);4772	if (r)4773		return r;4774 4775	gfx_v8_0_enable_gui_idle_interrupt(adev, true);4776 4777	return 0;4778}4779 4780static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)4781{4782	gfx_v8_0_cp_gfx_enable(adev, enable);4783	gfx_v8_0_cp_compute_enable(adev, enable);4784}4785 4786static int gfx_v8_0_hw_init(void *handle)4787{4788	int r;4789	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4790 4791	gfx_v8_0_init_golden_registers(adev);4792	gfx_v8_0_constants_init(adev);4793 4794	r = adev->gfx.rlc.funcs->resume(adev);4795	if (r)4796		return r;4797 4798	r = gfx_v8_0_cp_resume(adev);4799 4800	return r;4801}4802 4803static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)4804{4805	int r, i;4806	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;4807 4808	r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);4809	if (r)4810		DRM_ERROR("Failed to lock KIQ (%d).\n", r);4811 4812	for (i = 0; i < adev->gfx.num_compute_rings; i++) {4813		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];4814 4815		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));4816		amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */4817						PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */4818						PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |4819						PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |4820						PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));4821		amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));4822		amdgpu_ring_write(kiq_ring, 0);4823		amdgpu_ring_write(kiq_ring, 0);4824		amdgpu_ring_write(kiq_ring, 0);4825	}4826	r = amdgpu_ring_test_helper(kiq_ring);4827	if (r)4828		DRM_ERROR("KCQ disable failed\n");4829 4830	return r;4831}4832 4833static bool gfx_v8_0_is_idle(void *handle)4834{4835	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4836 4837	if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)4838		|| RREG32(mmGRBM_STATUS2) != 0x8)4839		return false;4840	else4841		return true;4842}4843 4844static bool gfx_v8_0_rlc_is_idle(void *handle)4845{4846	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4847 4848	if (RREG32(mmGRBM_STATUS2) != 0x8)4849		return false;4850	else4851		return true;4852}4853 4854static int gfx_v8_0_wait_for_rlc_idle(void *handle)4855{4856	unsigned int i;4857	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4858 4859	for (i = 0; i < adev->usec_timeout; i++) {4860		if (gfx_v8_0_rlc_is_idle(handle))4861			return 0;4862 4863		udelay(1);4864	}4865	return -ETIMEDOUT;4866}4867 4868static int gfx_v8_0_wait_for_idle(void *handle)4869{4870	unsigned int i;4871	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4872 4873	for (i = 0; i < adev->usec_timeout; i++) {4874		if (gfx_v8_0_is_idle(handle))4875			return 0;4876 4877		udelay(1);4878	}4879	return -ETIMEDOUT;4880}4881 4882static int gfx_v8_0_hw_fini(void *handle)4883{4884	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4885 4886	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);4887	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);4888 4889	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);4890 4891	amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);4892 4893	/* disable KCQ to avoid CPC touch memory not valid anymore */4894	gfx_v8_0_kcq_disable(adev);4895 4896	if (amdgpu_sriov_vf(adev)) {4897		pr_debug("For SRIOV client, shouldn't do anything.\n");4898		return 0;4899	}4900	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);4901	if (!gfx_v8_0_wait_for_idle(adev))4902		gfx_v8_0_cp_enable(adev, false);4903	else4904		pr_err("cp is busy, skip halt cp\n");4905	if (!gfx_v8_0_wait_for_rlc_idle(adev))4906		adev->gfx.rlc.funcs->stop(adev);4907	else4908		pr_err("rlc is busy, skip halt rlc\n");4909	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);4910 4911	return 0;4912}4913 4914static int gfx_v8_0_suspend(void *handle)4915{4916	return gfx_v8_0_hw_fini(handle);4917}4918 4919static int gfx_v8_0_resume(void *handle)4920{4921	return gfx_v8_0_hw_init(handle);4922}4923 4924static bool gfx_v8_0_check_soft_reset(void *handle)4925{4926	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4927	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;4928	u32 tmp;4929 4930	/* GRBM_STATUS */4931	tmp = RREG32(mmGRBM_STATUS);4932	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |4933		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |4934		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |4935		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |4936		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |4937		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |4938		   GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {4939		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4940						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);4941		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4942						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);4943		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,4944						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);4945	}4946 4947	/* GRBM_STATUS2 */4948	tmp = RREG32(mmGRBM_STATUS2);4949	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))4950		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4951						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);4952 4953	if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||4954	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||4955	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {4956		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,4957						SOFT_RESET_CPF, 1);4958		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,4959						SOFT_RESET_CPC, 1);4960		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,4961						SOFT_RESET_CPG, 1);4962		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,4963						SOFT_RESET_GRBM, 1);4964	}4965 4966	/* SRBM_STATUS */4967	tmp = RREG32(mmSRBM_STATUS);4968	if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))4969		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,4970						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);4971	if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))4972		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,4973						SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);4974 4975	if (grbm_soft_reset || srbm_soft_reset) {4976		adev->gfx.grbm_soft_reset = grbm_soft_reset;4977		adev->gfx.srbm_soft_reset = srbm_soft_reset;4978		return true;4979	} else {4980		adev->gfx.grbm_soft_reset = 0;4981		adev->gfx.srbm_soft_reset = 0;4982		return false;4983	}4984}4985 4986static int gfx_v8_0_pre_soft_reset(void *handle)4987{4988	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4989	u32 grbm_soft_reset = 0;4990 4991	if ((!adev->gfx.grbm_soft_reset) &&4992	    (!adev->gfx.srbm_soft_reset))4993		return 0;4994 4995	grbm_soft_reset = adev->gfx.grbm_soft_reset;4996 4997	/* stop the rlc */4998	adev->gfx.rlc.funcs->stop(adev);4999 5000	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||5001	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))5002		/* Disable GFX parsing/prefetching */5003		gfx_v8_0_cp_gfx_enable(adev, false);5004 5005	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||5006	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||5007	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||5008	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {5009		int i;5010 5011		for (i = 0; i < adev->gfx.num_compute_rings; i++) {5012			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];5013 5014			mutex_lock(&adev->srbm_mutex);5015			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);5016			gfx_v8_0_deactivate_hqd(adev, 2);5017			vi_srbm_select(adev, 0, 0, 0, 0);5018			mutex_unlock(&adev->srbm_mutex);5019		}5020		/* Disable MEC parsing/prefetching */5021		gfx_v8_0_cp_compute_enable(adev, false);5022	}5023 5024	return 0;5025}5026 5027static int gfx_v8_0_soft_reset(void *handle)5028{5029	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5030	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;5031	u32 tmp;5032 5033	if ((!adev->gfx.grbm_soft_reset) &&5034	    (!adev->gfx.srbm_soft_reset))5035		return 0;5036 5037	grbm_soft_reset = adev->gfx.grbm_soft_reset;5038	srbm_soft_reset = adev->gfx.srbm_soft_reset;5039 5040	if (grbm_soft_reset || srbm_soft_reset) {5041		tmp = RREG32(mmGMCON_DEBUG);5042		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);5043		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);5044		WREG32(mmGMCON_DEBUG, tmp);5045		udelay(50);5046	}5047 5048	if (grbm_soft_reset) {5049		tmp = RREG32(mmGRBM_SOFT_RESET);5050		tmp |= grbm_soft_reset;5051		dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);5052		WREG32(mmGRBM_SOFT_RESET, tmp);5053		tmp = RREG32(mmGRBM_SOFT_RESET);5054 5055		udelay(50);5056 5057		tmp &= ~grbm_soft_reset;5058		WREG32(mmGRBM_SOFT_RESET, tmp);5059		tmp = RREG32(mmGRBM_SOFT_RESET);5060	}5061 5062	if (srbm_soft_reset) {5063		tmp = RREG32(mmSRBM_SOFT_RESET);5064		tmp |= srbm_soft_reset;5065		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);5066		WREG32(mmSRBM_SOFT_RESET, tmp);5067		tmp = RREG32(mmSRBM_SOFT_RESET);5068 5069		udelay(50);5070 5071		tmp &= ~srbm_soft_reset;5072		WREG32(mmSRBM_SOFT_RESET, tmp);5073		tmp = RREG32(mmSRBM_SOFT_RESET);5074	}5075 5076	if (grbm_soft_reset || srbm_soft_reset) {5077		tmp = RREG32(mmGMCON_DEBUG);5078		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);5079		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);5080		WREG32(mmGMCON_DEBUG, tmp);5081	}5082 5083	/* Wait a little for things to settle down */5084	udelay(50);5085 5086	return 0;5087}5088 5089static int gfx_v8_0_post_soft_reset(void *handle)5090{5091	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5092	u32 grbm_soft_reset = 0;5093 5094	if ((!adev->gfx.grbm_soft_reset) &&5095	    (!adev->gfx.srbm_soft_reset))5096		return 0;5097 5098	grbm_soft_reset = adev->gfx.grbm_soft_reset;5099 5100	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||5101	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||5102	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||5103	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {5104		int i;5105 5106		for (i = 0; i < adev->gfx.num_compute_rings; i++) {5107			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];5108 5109			mutex_lock(&adev->srbm_mutex);5110			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);5111			gfx_v8_0_deactivate_hqd(adev, 2);5112			vi_srbm_select(adev, 0, 0, 0, 0);5113			mutex_unlock(&adev->srbm_mutex);5114		}5115		gfx_v8_0_kiq_resume(adev);5116		gfx_v8_0_kcq_resume(adev);5117	}5118 5119	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||5120	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))5121		gfx_v8_0_cp_gfx_resume(adev);5122 5123	gfx_v8_0_cp_test_all_rings(adev);5124 5125	adev->gfx.rlc.funcs->start(adev);5126 5127	return 0;5128}5129 5130/**5131 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot5132 *5133 * @adev: amdgpu_device pointer5134 *5135 * Fetches a GPU clock counter snapshot.5136 * Returns the 64 bit clock counter snapshot.5137 */5138static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)5139{5140	uint64_t clock;5141 5142	mutex_lock(&adev->gfx.gpu_clock_mutex);5143	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);5144	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |5145		((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);5146	mutex_unlock(&adev->gfx.gpu_clock_mutex);5147	return clock;5148}5149 5150static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,5151					  uint32_t vmid,5152					  uint32_t gds_base, uint32_t gds_size,5153					  uint32_t gws_base, uint32_t gws_size,5154					  uint32_t oa_base, uint32_t oa_size)5155{5156	/* GDS Base */5157	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5158	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5159				WRITE_DATA_DST_SEL(0)));5160	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);5161	amdgpu_ring_write(ring, 0);5162	amdgpu_ring_write(ring, gds_base);5163 5164	/* GDS Size */5165	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5166	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5167				WRITE_DATA_DST_SEL(0)));5168	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);5169	amdgpu_ring_write(ring, 0);5170	amdgpu_ring_write(ring, gds_size);5171 5172	/* GWS */5173	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5174	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5175				WRITE_DATA_DST_SEL(0)));5176	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);5177	amdgpu_ring_write(ring, 0);5178	amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);5179 5180	/* OA */5181	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5182	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5183				WRITE_DATA_DST_SEL(0)));5184	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);5185	amdgpu_ring_write(ring, 0);5186	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));5187}5188 5189static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)5190{5191	WREG32(mmSQ_IND_INDEX,5192		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |5193		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |5194		(address << SQ_IND_INDEX__INDEX__SHIFT) |5195		(SQ_IND_INDEX__FORCE_READ_MASK));5196	return RREG32(mmSQ_IND_DATA);5197}5198 5199static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,5200			   uint32_t wave, uint32_t thread,5201			   uint32_t regno, uint32_t num, uint32_t *out)5202{5203	WREG32(mmSQ_IND_INDEX,5204		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |5205		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |5206		(regno << SQ_IND_INDEX__INDEX__SHIFT) |5207		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |5208		(SQ_IND_INDEX__FORCE_READ_MASK) |5209		(SQ_IND_INDEX__AUTO_INCR_MASK));5210	while (num--)5211		*(out++) = RREG32(mmSQ_IND_DATA);5212}5213 5214static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)5215{5216	/* type 0 wave data */5217	dst[(*no_fields)++] = 0;5218	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);5219	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);5220	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);5221	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);5222	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);5223	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);5224	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);5225	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);5226	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);5227	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);5228	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);5229	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);5230	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);5231	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);5232	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);5233	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);5234	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);5235	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);5236	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);5237}5238 5239static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,5240				     uint32_t wave, uint32_t start,5241				     uint32_t size, uint32_t *dst)5242{5243	wave_read_regs(5244		adev, simd, wave, 0,5245		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);5246}5247 5248 5249static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {5250	.get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,5251	.select_se_sh = &gfx_v8_0_select_se_sh,5252	.read_wave_data = &gfx_v8_0_read_wave_data,5253	.read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,5254	.select_me_pipe_q = &gfx_v8_0_select_me_pipe_q5255};5256 5257static int gfx_v8_0_early_init(void *handle)5258{5259	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5260 5261	adev->gfx.xcc_mask = 1;5262	adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;5263	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),5264					  AMDGPU_MAX_COMPUTE_RINGS);5265	adev->gfx.funcs = &gfx_v8_0_gfx_funcs;5266	gfx_v8_0_set_ring_funcs(adev);5267	gfx_v8_0_set_irq_funcs(adev);5268	gfx_v8_0_set_gds_init(adev);5269	gfx_v8_0_set_rlc_funcs(adev);5270 5271	return 0;5272}5273 5274static int gfx_v8_0_late_init(void *handle)5275{5276	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5277	int r;5278 5279	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);5280	if (r)5281		return r;5282 5283	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);5284	if (r)5285		return r;5286 5287	/* requires IBs so do in late init after IB pool is initialized */5288	r = gfx_v8_0_do_edc_gpr_workarounds(adev);5289	if (r)5290		return r;5291 5292	r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);5293	if (r) {5294		DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);5295		return r;5296	}5297 5298	r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);5299	if (r) {5300		DRM_ERROR(5301			"amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",5302			r);5303		return r;5304	}5305 5306	return 0;5307}5308 5309static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,5310						       bool enable)5311{5312	if ((adev->asic_type == CHIP_POLARIS11) ||5313	    (adev->asic_type == CHIP_POLARIS12) ||5314	    (adev->asic_type == CHIP_VEGAM))5315		/* Send msg to SMU via Powerplay */5316		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);5317 5318	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);5319}5320 5321static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,5322							bool enable)5323{5324	WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);5325}5326 5327static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,5328		bool enable)5329{5330	WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);5331}5332 5333static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,5334					  bool enable)5335{5336	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);5337}5338 5339static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,5340						bool enable)5341{5342	WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);5343 5344	/* Read any GFX register to wake up GFX. */5345	if (!enable)5346		RREG32(mmDB_RENDER_CONTROL);5347}5348 5349static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,5350					  bool enable)5351{5352	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {5353		cz_enable_gfx_cg_power_gating(adev, true);5354		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)5355			cz_enable_gfx_pipeline_power_gating(adev, true);5356	} else {5357		cz_enable_gfx_cg_power_gating(adev, false);5358		cz_enable_gfx_pipeline_power_gating(adev, false);5359	}5360}5361 5362static int gfx_v8_0_set_powergating_state(void *handle,5363					  enum amd_powergating_state state)5364{5365	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5366	bool enable = (state == AMD_PG_STATE_GATE);5367 5368	if (amdgpu_sriov_vf(adev))5369		return 0;5370 5371	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |5372				AMD_PG_SUPPORT_RLC_SMU_HS |5373				AMD_PG_SUPPORT_CP |5374				AMD_PG_SUPPORT_GFX_DMG))5375		amdgpu_gfx_rlc_enter_safe_mode(adev, 0);5376	switch (adev->asic_type) {5377	case CHIP_CARRIZO:5378	case CHIP_STONEY:5379 5380		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {5381			cz_enable_sck_slow_down_on_power_up(adev, true);5382			cz_enable_sck_slow_down_on_power_down(adev, true);5383		} else {5384			cz_enable_sck_slow_down_on_power_up(adev, false);5385			cz_enable_sck_slow_down_on_power_down(adev, false);5386		}5387		if (adev->pg_flags & AMD_PG_SUPPORT_CP)5388			cz_enable_cp_power_gating(adev, true);5389		else5390			cz_enable_cp_power_gating(adev, false);5391 5392		cz_update_gfx_cg_power_gating(adev, enable);5393 5394		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)5395			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);5396		else5397			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);5398 5399		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)5400			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);5401		else5402			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);5403		break;5404	case CHIP_POLARIS11:5405	case CHIP_POLARIS12:5406	case CHIP_VEGAM:5407		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)5408			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);5409		else5410			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);5411 5412		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)5413			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);5414		else5415			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);5416 5417		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)5418			polaris11_enable_gfx_quick_mg_power_gating(adev, true);5419		else5420			polaris11_enable_gfx_quick_mg_power_gating(adev, false);5421		break;5422	default:5423		break;5424	}5425	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |5426				AMD_PG_SUPPORT_RLC_SMU_HS |5427				AMD_PG_SUPPORT_CP |5428				AMD_PG_SUPPORT_GFX_DMG))5429		amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5430	return 0;5431}5432 5433static void gfx_v8_0_get_clockgating_state(void *handle, u64 *flags)5434{5435	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5436	int data;5437 5438	if (amdgpu_sriov_vf(adev))5439		*flags = 0;5440 5441	/* AMD_CG_SUPPORT_GFX_MGCG */5442	data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5443	if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))5444		*flags |= AMD_CG_SUPPORT_GFX_MGCG;5445 5446	/* AMD_CG_SUPPORT_GFX_CGLG */5447	data = RREG32(mmRLC_CGCG_CGLS_CTRL);5448	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)5449		*flags |= AMD_CG_SUPPORT_GFX_CGCG;5450 5451	/* AMD_CG_SUPPORT_GFX_CGLS */5452	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)5453		*flags |= AMD_CG_SUPPORT_GFX_CGLS;5454 5455	/* AMD_CG_SUPPORT_GFX_CGTS */5456	data = RREG32(mmCGTS_SM_CTRL_REG);5457	if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))5458		*flags |= AMD_CG_SUPPORT_GFX_CGTS;5459 5460	/* AMD_CG_SUPPORT_GFX_CGTS_LS */5461	if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))5462		*flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;5463 5464	/* AMD_CG_SUPPORT_GFX_RLC_LS */5465	data = RREG32(mmRLC_MEM_SLP_CNTL);5466	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)5467		*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;5468 5469	/* AMD_CG_SUPPORT_GFX_CP_LS */5470	data = RREG32(mmCP_MEM_SLP_CNTL);5471	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)5472		*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;5473}5474 5475static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,5476				     uint32_t reg_addr, uint32_t cmd)5477{5478	uint32_t data;5479 5480	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);5481 5482	WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);5483	WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);5484 5485	data = RREG32(mmRLC_SERDES_WR_CTRL);5486	if (adev->asic_type == CHIP_STONEY)5487		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |5488			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |5489			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |5490			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |5491			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |5492			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |5493			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |5494			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |5495			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);5496	else5497		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |5498			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |5499			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |5500			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |5501			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |5502			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |5503			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |5504			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |5505			  RLC_SERDES_WR_CTRL__BPM_DATA_MASK |5506			  RLC_SERDES_WR_CTRL__REG_ADDR_MASK |5507			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);5508	data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |5509		 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |5510		 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |5511		 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));5512 5513	WREG32(mmRLC_SERDES_WR_CTRL, data);5514}5515 5516#define MSG_ENTER_RLC_SAFE_MODE     15517#define MSG_EXIT_RLC_SAFE_MODE      05518#define RLC_GPR_REG2__REQ_MASK 0x000000015519#define RLC_GPR_REG2__REQ__SHIFT 05520#define RLC_GPR_REG2__MESSAGE__SHIFT 0x000000015521#define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e5522 5523static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)5524{5525	uint32_t rlc_setting;5526 5527	rlc_setting = RREG32(mmRLC_CNTL);5528	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))5529		return false;5530 5531	return true;5532}5533 5534static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)5535{5536	uint32_t data;5537	unsigned i;5538	data = RREG32(mmRLC_CNTL);5539	data |= RLC_SAFE_MODE__CMD_MASK;5540	data &= ~RLC_SAFE_MODE__MESSAGE_MASK;5541	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);5542	WREG32(mmRLC_SAFE_MODE, data);5543 5544	/* wait for RLC_SAFE_MODE */5545	for (i = 0; i < adev->usec_timeout; i++) {5546		if ((RREG32(mmRLC_GPM_STAT) &5547		     (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |5548		      RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==5549		    (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |5550		     RLC_GPM_STAT__GFX_POWER_STATUS_MASK))5551			break;5552		udelay(1);5553	}5554	for (i = 0; i < adev->usec_timeout; i++) {5555		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))5556			break;5557		udelay(1);5558	}5559}5560 5561static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)5562{5563	uint32_t data;5564	unsigned i;5565 5566	data = RREG32(mmRLC_CNTL);5567	data |= RLC_SAFE_MODE__CMD_MASK;5568	data &= ~RLC_SAFE_MODE__MESSAGE_MASK;5569	WREG32(mmRLC_SAFE_MODE, data);5570 5571	for (i = 0; i < adev->usec_timeout; i++) {5572		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))5573			break;5574		udelay(1);5575	}5576}5577 5578static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)5579{5580	u32 data;5581 5582	amdgpu_gfx_off_ctrl(adev, false);5583 5584	if (amdgpu_sriov_is_pp_one_vf(adev))5585		data = RREG32_NO_KIQ(mmRLC_SPM_VMID);5586	else5587		data = RREG32(mmRLC_SPM_VMID);5588 5589	data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK;5590	data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT;5591 5592	if (amdgpu_sriov_is_pp_one_vf(adev))5593		WREG32_NO_KIQ(mmRLC_SPM_VMID, data);5594	else5595		WREG32(mmRLC_SPM_VMID, data);5596 5597	amdgpu_gfx_off_ctrl(adev, true);5598}5599 5600static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {5601	.is_rlc_enabled = gfx_v8_0_is_rlc_enabled,5602	.set_safe_mode = gfx_v8_0_set_safe_mode,5603	.unset_safe_mode = gfx_v8_0_unset_safe_mode,5604	.init = gfx_v8_0_rlc_init,5605	.get_csb_size = gfx_v8_0_get_csb_size,5606	.get_csb_buffer = gfx_v8_0_get_csb_buffer,5607	.get_cp_table_num = gfx_v8_0_cp_jump_table_num,5608	.resume = gfx_v8_0_rlc_resume,5609	.stop = gfx_v8_0_rlc_stop,5610	.reset = gfx_v8_0_rlc_reset,5611	.start = gfx_v8_0_rlc_start,5612	.update_spm_vmid = gfx_v8_0_update_spm_vmid5613};5614 5615static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,5616						      bool enable)5617{5618	uint32_t temp, data;5619 5620	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);5621 5622	/* It is disabled by HW by default */5623	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {5624		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {5625			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)5626				/* 1 - RLC memory Light sleep */5627				WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);5628 5629			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)5630				WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);5631		}5632 5633		/* 3 - RLC_CGTT_MGCG_OVERRIDE */5634		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5635		if (adev->flags & AMD_IS_APU)5636			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |5637				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |5638				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);5639		else5640			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |5641				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |5642				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |5643				  RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);5644 5645		if (temp != data)5646			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);5647 5648		/* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5649		gfx_v8_0_wait_for_rlc_serdes(adev);5650 5651		/* 5 - clear mgcg override */5652		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);5653 5654		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {5655			/* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */5656			temp = data = RREG32(mmCGTS_SM_CTRL_REG);5657			data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);5658			data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);5659			data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;5660			data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;5661			if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&5662			    (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))5663				data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;5664			data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;5665			data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);5666			if (temp != data)5667				WREG32(mmCGTS_SM_CTRL_REG, data);5668		}5669		udelay(50);5670 5671		/* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5672		gfx_v8_0_wait_for_rlc_serdes(adev);5673	} else {5674		/* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */5675		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5676		data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |5677				RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |5678				RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |5679				RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);5680		if (temp != data)5681			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);5682 5683		/* 2 - disable MGLS in RLC */5684		data = RREG32(mmRLC_MEM_SLP_CNTL);5685		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {5686			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;5687			WREG32(mmRLC_MEM_SLP_CNTL, data);5688		}5689 5690		/* 3 - disable MGLS in CP */5691		data = RREG32(mmCP_MEM_SLP_CNTL);5692		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {5693			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;5694			WREG32(mmCP_MEM_SLP_CNTL, data);5695		}5696 5697		/* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */5698		temp = data = RREG32(mmCGTS_SM_CTRL_REG);5699		data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |5700				CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);5701		if (temp != data)5702			WREG32(mmCGTS_SM_CTRL_REG, data);5703 5704		/* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5705		gfx_v8_0_wait_for_rlc_serdes(adev);5706 5707		/* 6 - set mgcg override */5708		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);5709 5710		udelay(50);5711 5712		/* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5713		gfx_v8_0_wait_for_rlc_serdes(adev);5714	}5715 5716	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5717}5718 5719static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,5720						      bool enable)5721{5722	uint32_t temp, temp1, data, data1;5723 5724	temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);5725 5726	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);5727 5728	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {5729		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5730		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;5731		if (temp1 != data1)5732			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);5733 5734		/* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5735		gfx_v8_0_wait_for_rlc_serdes(adev);5736 5737		/* 2 - clear cgcg override */5738		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);5739 5740		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5741		gfx_v8_0_wait_for_rlc_serdes(adev);5742 5743		/* 3 - write cmd to set CGLS */5744		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);5745 5746		/* 4 - enable cgcg */5747		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;5748 5749		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {5750			/* enable cgls*/5751			data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;5752 5753			temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5754			data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;5755 5756			if (temp1 != data1)5757				WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);5758		} else {5759			data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;5760		}5761 5762		if (temp != data)5763			WREG32(mmRLC_CGCG_CGLS_CTRL, data);5764 5765		/* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/5766		 * Cmp_busy/GFX_Idle interrupts5767		 */5768		gfx_v8_0_enable_gui_idle_interrupt(adev, true);5769	} else {5770		/* disable cntx_empty_int_enable & GFX Idle interrupt */5771		gfx_v8_0_enable_gui_idle_interrupt(adev, false);5772 5773		/* TEST CGCG */5774		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);5775		data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |5776				RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);5777		if (temp1 != data1)5778			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);5779 5780		/* read gfx register to wake up cgcg */5781		RREG32(mmCB_CGTT_SCLK_CTRL);5782		RREG32(mmCB_CGTT_SCLK_CTRL);5783		RREG32(mmCB_CGTT_SCLK_CTRL);5784		RREG32(mmCB_CGTT_SCLK_CTRL);5785 5786		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5787		gfx_v8_0_wait_for_rlc_serdes(adev);5788 5789		/* write cmd to Set CGCG Override */5790		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);5791 5792		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */5793		gfx_v8_0_wait_for_rlc_serdes(adev);5794 5795		/* write cmd to Clear CGLS */5796		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);5797 5798		/* disable cgcg, cgls should be disabled too. */5799		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |5800			  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);5801		if (temp != data)5802			WREG32(mmRLC_CGCG_CGLS_CTRL, data);5803		/* enable interrupts again for PG */5804		gfx_v8_0_enable_gui_idle_interrupt(adev, true);5805	}5806 5807	gfx_v8_0_wait_for_rlc_serdes(adev);5808 5809	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5810}5811static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,5812					    bool enable)5813{5814	if (enable) {5815		/* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)5816		 * ===  MGCG + MGLS + TS(CG/LS) ===5817		 */5818		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);5819		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);5820	} else {5821		/* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)5822		 * ===  CGCG + CGLS ===5823		 */5824		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);5825		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);5826	}5827	return 0;5828}5829 5830static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,5831					  enum amd_clockgating_state state)5832{5833	uint32_t msg_id, pp_state = 0;5834	uint32_t pp_support_state = 0;5835 5836	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {5837		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {5838			pp_support_state = PP_STATE_SUPPORT_LS;5839			pp_state = PP_STATE_LS;5840		}5841		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {5842			pp_support_state |= PP_STATE_SUPPORT_CG;5843			pp_state |= PP_STATE_CG;5844		}5845		if (state == AMD_CG_STATE_UNGATE)5846			pp_state = 0;5847 5848		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5849				PP_BLOCK_GFX_CG,5850				pp_support_state,5851				pp_state);5852		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5853	}5854 5855	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {5856		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {5857			pp_support_state = PP_STATE_SUPPORT_LS;5858			pp_state = PP_STATE_LS;5859		}5860 5861		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {5862			pp_support_state |= PP_STATE_SUPPORT_CG;5863			pp_state |= PP_STATE_CG;5864		}5865 5866		if (state == AMD_CG_STATE_UNGATE)5867			pp_state = 0;5868 5869		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5870				PP_BLOCK_GFX_MG,5871				pp_support_state,5872				pp_state);5873		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5874	}5875 5876	return 0;5877}5878 5879static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,5880					  enum amd_clockgating_state state)5881{5882 5883	uint32_t msg_id, pp_state = 0;5884	uint32_t pp_support_state = 0;5885 5886	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {5887		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {5888			pp_support_state = PP_STATE_SUPPORT_LS;5889			pp_state = PP_STATE_LS;5890		}5891		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {5892			pp_support_state |= PP_STATE_SUPPORT_CG;5893			pp_state |= PP_STATE_CG;5894		}5895		if (state == AMD_CG_STATE_UNGATE)5896			pp_state = 0;5897 5898		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5899				PP_BLOCK_GFX_CG,5900				pp_support_state,5901				pp_state);5902		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5903	}5904 5905	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {5906		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {5907			pp_support_state = PP_STATE_SUPPORT_LS;5908			pp_state = PP_STATE_LS;5909		}5910		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {5911			pp_support_state |= PP_STATE_SUPPORT_CG;5912			pp_state |= PP_STATE_CG;5913		}5914		if (state == AMD_CG_STATE_UNGATE)5915			pp_state = 0;5916 5917		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5918				PP_BLOCK_GFX_3D,5919				pp_support_state,5920				pp_state);5921		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5922	}5923 5924	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {5925		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {5926			pp_support_state = PP_STATE_SUPPORT_LS;5927			pp_state = PP_STATE_LS;5928		}5929 5930		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {5931			pp_support_state |= PP_STATE_SUPPORT_CG;5932			pp_state |= PP_STATE_CG;5933		}5934 5935		if (state == AMD_CG_STATE_UNGATE)5936			pp_state = 0;5937 5938		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5939				PP_BLOCK_GFX_MG,5940				pp_support_state,5941				pp_state);5942		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5943	}5944 5945	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {5946		pp_support_state = PP_STATE_SUPPORT_LS;5947 5948		if (state == AMD_CG_STATE_UNGATE)5949			pp_state = 0;5950		else5951			pp_state = PP_STATE_LS;5952 5953		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5954				PP_BLOCK_GFX_RLC,5955				pp_support_state,5956				pp_state);5957		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5958	}5959 5960	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {5961		pp_support_state = PP_STATE_SUPPORT_LS;5962 5963		if (state == AMD_CG_STATE_UNGATE)5964			pp_state = 0;5965		else5966			pp_state = PP_STATE_LS;5967		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,5968			PP_BLOCK_GFX_CP,5969			pp_support_state,5970			pp_state);5971		amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);5972	}5973 5974	return 0;5975}5976 5977static int gfx_v8_0_set_clockgating_state(void *handle,5978					  enum amd_clockgating_state state)5979{5980	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5981 5982	if (amdgpu_sriov_vf(adev))5983		return 0;5984 5985	switch (adev->asic_type) {5986	case CHIP_FIJI:5987	case CHIP_CARRIZO:5988	case CHIP_STONEY:5989		gfx_v8_0_update_gfx_clock_gating(adev,5990						 state == AMD_CG_STATE_GATE);5991		break;5992	case CHIP_TONGA:5993		gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);5994		break;5995	case CHIP_POLARIS10:5996	case CHIP_POLARIS11:5997	case CHIP_POLARIS12:5998	case CHIP_VEGAM:5999		gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);6000		break;6001	default:6002		break;6003	}6004	return 0;6005}6006 6007static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)6008{6009	return *ring->rptr_cpu_addr;6010}6011 6012static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)6013{6014	struct amdgpu_device *adev = ring->adev;6015 6016	if (ring->use_doorbell)6017		/* XXX check if swapping is necessary on BE */6018		return *ring->wptr_cpu_addr;6019	else6020		return RREG32(mmCP_RB0_WPTR);6021}6022 6023static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)6024{6025	struct amdgpu_device *adev = ring->adev;6026 6027	if (ring->use_doorbell) {6028		/* XXX check if swapping is necessary on BE */6029		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);6030		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));6031	} else {6032		WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));6033		(void)RREG32(mmCP_RB0_WPTR);6034	}6035}6036 6037static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)6038{6039	u32 ref_and_mask, reg_mem_engine;6040 6041	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||6042	    (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {6043		switch (ring->me) {6044		case 1:6045			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;6046			break;6047		case 2:6048			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;6049			break;6050		default:6051			return;6052		}6053		reg_mem_engine = 0;6054	} else {6055		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;6056		reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */6057	}6058 6059	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));6060	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */6061				 WAIT_REG_MEM_FUNCTION(3) |  /* == */6062				 reg_mem_engine));6063	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);6064	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);6065	amdgpu_ring_write(ring, ref_and_mask);6066	amdgpu_ring_write(ring, ref_and_mask);6067	amdgpu_ring_write(ring, 0x20); /* poll interval */6068}6069 6070static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)6071{6072	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));6073	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |6074		EVENT_INDEX(4));6075 6076	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));6077	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |6078		EVENT_INDEX(0));6079}6080 6081static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,6082					struct amdgpu_job *job,6083					struct amdgpu_ib *ib,6084					uint32_t flags)6085{6086	unsigned vmid = AMDGPU_JOB_GET_VMID(job);6087	u32 header, control = 0;6088 6089	if (ib->flags & AMDGPU_IB_FLAG_CE)6090		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);6091	else6092		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);6093 6094	control |= ib->length_dw | (vmid << 24);6095 6096	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {6097		control |= INDIRECT_BUFFER_PRE_ENB(1);6098 6099		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)6100			gfx_v8_0_ring_emit_de_meta(ring);6101	}6102 6103	amdgpu_ring_write(ring, header);6104	amdgpu_ring_write(ring,6105#ifdef __BIG_ENDIAN6106			  (2 << 0) |6107#endif6108			  (ib->gpu_addr & 0xFFFFFFFC));6109	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);6110	amdgpu_ring_write(ring, control);6111}6112 6113static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,6114					  struct amdgpu_job *job,6115					  struct amdgpu_ib *ib,6116					  uint32_t flags)6117{6118	unsigned vmid = AMDGPU_JOB_GET_VMID(job);6119	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);6120 6121	/* Currently, there is a high possibility to get wave ID mismatch6122	 * between ME and GDS, leading to a hw deadlock, because ME generates6123	 * different wave IDs than the GDS expects. This situation happens6124	 * randomly when at least 5 compute pipes use GDS ordered append.6125	 * The wave IDs generated by ME are also wrong after suspend/resume.6126	 * Those are probably bugs somewhere else in the kernel driver.6127	 *6128	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and6129	 * GDS to 0 for this ring (me/pipe).6130	 */6131	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {6132		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));6133		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);6134		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);6135	}6136 6137	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));6138	amdgpu_ring_write(ring,6139#ifdef __BIG_ENDIAN6140				(2 << 0) |6141#endif6142				(ib->gpu_addr & 0xFFFFFFFC));6143	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);6144	amdgpu_ring_write(ring, control);6145}6146 6147static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,6148					 u64 seq, unsigned flags)6149{6150	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;6151	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;6152	bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;6153 6154	/* Workaround for cache flush problems. First send a dummy EOP6155	 * event down the pipe with seq one below.6156	 */6157	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));6158	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |6159				 EOP_TC_ACTION_EN |6160				 EOP_TC_WB_ACTION_EN |6161				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |6162				 EVENT_INDEX(5)));6163	amdgpu_ring_write(ring, addr & 0xfffffffc);6164	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |6165				DATA_SEL(1) | INT_SEL(0));6166	amdgpu_ring_write(ring, lower_32_bits(seq - 1));6167	amdgpu_ring_write(ring, upper_32_bits(seq - 1));6168 6169	/* Then send the real EOP event down the pipe:6170	 * EVENT_WRITE_EOP - flush caches, send int */6171	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));6172	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |6173				 EOP_TC_ACTION_EN |6174				 EOP_TC_WB_ACTION_EN |6175				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |6176				 EVENT_INDEX(5) |6177				 (exec ? EOP_EXEC : 0)));6178	amdgpu_ring_write(ring, addr & 0xfffffffc);6179	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |6180			  DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));6181	amdgpu_ring_write(ring, lower_32_bits(seq));6182	amdgpu_ring_write(ring, upper_32_bits(seq));6183 6184}6185 6186static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)6187{6188	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);6189	uint32_t seq = ring->fence_drv.sync_seq;6190	uint64_t addr = ring->fence_drv.gpu_addr;6191 6192	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));6193	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */6194				 WAIT_REG_MEM_FUNCTION(3) | /* equal */6195				 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */6196	amdgpu_ring_write(ring, addr & 0xfffffffc);6197	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);6198	amdgpu_ring_write(ring, seq);6199	amdgpu_ring_write(ring, 0xffffffff);6200	amdgpu_ring_write(ring, 4); /* poll interval */6201}6202 6203static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,6204					unsigned vmid, uint64_t pd_addr)6205{6206	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);6207 6208	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);6209 6210	/* wait for the invalidate to complete */6211	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));6212	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */6213				 WAIT_REG_MEM_FUNCTION(0) |  /* always */6214				 WAIT_REG_MEM_ENGINE(0))); /* me */6215	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);6216	amdgpu_ring_write(ring, 0);6217	amdgpu_ring_write(ring, 0); /* ref */6218	amdgpu_ring_write(ring, 0); /* mask */6219	amdgpu_ring_write(ring, 0x20); /* poll interval */6220 6221	/* compute doesn't have PFP */6222	if (usepfp) {6223		/* sync PFP to ME, otherwise we might get invalid PFP reads */6224		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));6225		amdgpu_ring_write(ring, 0x0);6226	}6227}6228 6229static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)6230{6231	return *ring->wptr_cpu_addr;6232}6233 6234static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)6235{6236	struct amdgpu_device *adev = ring->adev;6237 6238	/* XXX check if swapping is necessary on BE */6239	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);6240	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));6241}6242 6243static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,6244					     u64 addr, u64 seq,6245					     unsigned flags)6246{6247	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;6248	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;6249 6250	/* RELEASE_MEM - flush caches, send int */6251	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));6252	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |6253				 EOP_TC_ACTION_EN |6254				 EOP_TC_WB_ACTION_EN |6255				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |6256				 EVENT_INDEX(5)));6257	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));6258	amdgpu_ring_write(ring, addr & 0xfffffffc);6259	amdgpu_ring_write(ring, upper_32_bits(addr));6260	amdgpu_ring_write(ring, lower_32_bits(seq));6261	amdgpu_ring_write(ring, upper_32_bits(seq));6262}6263 6264static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,6265					 u64 seq, unsigned int flags)6266{6267	/* we only allocate 32bit for each seq wb address */6268	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);6269 6270	/* write fence seq to the "addr" */6271	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));6272	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |6273				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));6274	amdgpu_ring_write(ring, lower_32_bits(addr));6275	amdgpu_ring_write(ring, upper_32_bits(addr));6276	amdgpu_ring_write(ring, lower_32_bits(seq));6277 6278	if (flags & AMDGPU_FENCE_FLAG_INT) {6279		/* set register to trigger INT */6280		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));6281		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |6282					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));6283		amdgpu_ring_write(ring, mmCPC_INT_STATUS);6284		amdgpu_ring_write(ring, 0);6285		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */6286	}6287}6288 6289static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)6290{6291	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));6292	amdgpu_ring_write(ring, 0);6293}6294 6295static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)6296{6297	uint32_t dw2 = 0;6298 6299	if (amdgpu_sriov_vf(ring->adev))6300		gfx_v8_0_ring_emit_ce_meta(ring);6301 6302	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */6303	if (flags & AMDGPU_HAVE_CTX_SWITCH) {6304		gfx_v8_0_ring_emit_vgt_flush(ring);6305		/* set load_global_config & load_global_uconfig */6306		dw2 |= 0x8001;6307		/* set load_cs_sh_regs */6308		dw2 |= 0x01000000;6309		/* set load_per_context_state & load_gfx_sh_regs for GFX */6310		dw2 |= 0x10002;6311 6312		/* set load_ce_ram if preamble presented */6313		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)6314			dw2 |= 0x10000000;6315	} else {6316		/* still load_ce_ram if this is the first time preamble presented6317		 * although there is no context switch happens.6318		 */6319		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)6320			dw2 |= 0x10000000;6321	}6322 6323	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));6324	amdgpu_ring_write(ring, dw2);6325	amdgpu_ring_write(ring, 0);6326}6327 6328static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,6329						  uint64_t addr)6330{6331	unsigned ret;6332 6333	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));6334	amdgpu_ring_write(ring, lower_32_bits(addr));6335	amdgpu_ring_write(ring, upper_32_bits(addr));6336	/* discard following DWs if *cond_exec_gpu_addr==0 */6337	amdgpu_ring_write(ring, 0);6338	ret = ring->wptr & ring->buf_mask;6339	/* patch dummy value later */6340	amdgpu_ring_write(ring, 0);6341	return ret;6342}6343 6344static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,6345				    uint32_t reg_val_offs)6346{6347	struct amdgpu_device *adev = ring->adev;6348 6349	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));6350	amdgpu_ring_write(ring, 0 |	/* src: register*/6351				(5 << 8) |	/* dst: memory */6352				(1 << 20));	/* write confirm */6353	amdgpu_ring_write(ring, reg);6354	amdgpu_ring_write(ring, 0);6355	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +6356				reg_val_offs * 4));6357	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +6358				reg_val_offs * 4));6359}6360 6361static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,6362				  uint32_t val)6363{6364	uint32_t cmd;6365 6366	switch (ring->funcs->type) {6367	case AMDGPU_RING_TYPE_GFX:6368		cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;6369		break;6370	case AMDGPU_RING_TYPE_KIQ:6371		cmd = 1 << 16; /* no inc addr */6372		break;6373	default:6374		cmd = WR_CONFIRM;6375		break;6376	}6377 6378	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));6379	amdgpu_ring_write(ring, cmd);6380	amdgpu_ring_write(ring, reg);6381	amdgpu_ring_write(ring, 0);6382	amdgpu_ring_write(ring, val);6383}6384 6385static void gfx_v8_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,6386				  int mem_space, int opt, uint32_t addr0,6387				  uint32_t addr1, uint32_t ref, uint32_t mask,6388				  uint32_t inv)6389{6390	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));6391	amdgpu_ring_write(ring,6392			  /* memory (1) or register (0) */6393			  (WAIT_REG_MEM_MEM_SPACE(mem_space) |6394			   WAIT_REG_MEM_OPERATION(opt) | /* wait */6395			   WAIT_REG_MEM_FUNCTION(3) |  /* equal */6396			   WAIT_REG_MEM_ENGINE(eng_sel)));6397 6398	if (mem_space)6399		BUG_ON(addr0 & 0x3); /* Dword align */6400	amdgpu_ring_write(ring, addr0);6401	amdgpu_ring_write(ring, addr1);6402	amdgpu_ring_write(ring, ref);6403	amdgpu_ring_write(ring, mask);6404	amdgpu_ring_write(ring, inv); /* poll interval */6405}6406 6407static void gfx_v8_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,6408					uint32_t val, uint32_t mask)6409{6410	gfx_v8_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);6411}6412 6413static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)6414{6415	struct amdgpu_device *adev = ring->adev;6416	uint32_t value = 0;6417 6418	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);6419	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);6420	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);6421	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);6422	WREG32(mmSQ_CMD, value);6423}6424 6425static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,6426						 enum amdgpu_interrupt_state state)6427{6428	WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,6429		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);6430}6431 6432static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,6433						     int me, int pipe,6434						     enum amdgpu_interrupt_state state)6435{6436	u32 mec_int_cntl, mec_int_cntl_reg;6437 6438	/*6439	 * amdgpu controls only the first MEC. That's why this function only6440	 * handles the setting of interrupts for this specific MEC. All other6441	 * pipes' interrupts are set by amdkfd.6442	 */6443 6444	if (me == 1) {6445		switch (pipe) {6446		case 0:6447			mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;6448			break;6449		case 1:6450			mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;6451			break;6452		case 2:6453			mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;6454			break;6455		case 3:6456			mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;6457			break;6458		default:6459			DRM_DEBUG("invalid pipe %d\n", pipe);6460			return;6461		}6462	} else {6463		DRM_DEBUG("invalid me %d\n", me);6464		return;6465	}6466 6467	switch (state) {6468	case AMDGPU_IRQ_STATE_DISABLE:6469		mec_int_cntl = RREG32(mec_int_cntl_reg);6470		mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;6471		WREG32(mec_int_cntl_reg, mec_int_cntl);6472		break;6473	case AMDGPU_IRQ_STATE_ENABLE:6474		mec_int_cntl = RREG32(mec_int_cntl_reg);6475		mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;6476		WREG32(mec_int_cntl_reg, mec_int_cntl);6477		break;6478	default:6479		break;6480	}6481}6482 6483static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,6484					     struct amdgpu_irq_src *source,6485					     unsigned type,6486					     enum amdgpu_interrupt_state state)6487{6488	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,6489		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);6490 6491	return 0;6492}6493 6494static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,6495					      struct amdgpu_irq_src *source,6496					      unsigned type,6497					      enum amdgpu_interrupt_state state)6498{6499	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,6500		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);6501 6502	return 0;6503}6504 6505static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,6506					    struct amdgpu_irq_src *src,6507					    unsigned type,6508					    enum amdgpu_interrupt_state state)6509{6510	switch (type) {6511	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:6512		gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);6513		break;6514	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:6515		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);6516		break;6517	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:6518		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);6519		break;6520	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:6521		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);6522		break;6523	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:6524		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);6525		break;6526	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:6527		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);6528		break;6529	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:6530		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);6531		break;6532	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:6533		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);6534		break;6535	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:6536		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);6537		break;6538	default:6539		break;6540	}6541	return 0;6542}6543 6544static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,6545					 struct amdgpu_irq_src *source,6546					 unsigned int type,6547					 enum amdgpu_interrupt_state state)6548{6549	int enable_flag;6550 6551	switch (state) {6552	case AMDGPU_IRQ_STATE_DISABLE:6553		enable_flag = 0;6554		break;6555 6556	case AMDGPU_IRQ_STATE_ENABLE:6557		enable_flag = 1;6558		break;6559 6560	default:6561		return -EINVAL;6562	}6563 6564	WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);6565	WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);6566	WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);6567	WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);6568	WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);6569	WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6570		     enable_flag);6571	WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6572		     enable_flag);6573	WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6574		     enable_flag);6575	WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6576		     enable_flag);6577	WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6578		     enable_flag);6579	WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6580		     enable_flag);6581	WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6582		     enable_flag);6583	WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,6584		     enable_flag);6585 6586	return 0;6587}6588 6589static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,6590				     struct amdgpu_irq_src *source,6591				     unsigned int type,6592				     enum amdgpu_interrupt_state state)6593{6594	int enable_flag;6595 6596	switch (state) {6597	case AMDGPU_IRQ_STATE_DISABLE:6598		enable_flag = 1;6599		break;6600 6601	case AMDGPU_IRQ_STATE_ENABLE:6602		enable_flag = 0;6603		break;6604 6605	default:6606		return -EINVAL;6607	}6608 6609	WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,6610		     enable_flag);6611 6612	return 0;6613}6614 6615static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,6616			    struct amdgpu_irq_src *source,6617			    struct amdgpu_iv_entry *entry)6618{6619	int i;6620	u8 me_id, pipe_id, queue_id;6621	struct amdgpu_ring *ring;6622 6623	DRM_DEBUG("IH: CP EOP\n");6624	me_id = (entry->ring_id & 0x0c) >> 2;6625	pipe_id = (entry->ring_id & 0x03) >> 0;6626	queue_id = (entry->ring_id & 0x70) >> 4;6627 6628	switch (me_id) {6629	case 0:6630		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);6631		break;6632	case 1:6633	case 2:6634		for (i = 0; i < adev->gfx.num_compute_rings; i++) {6635			ring = &adev->gfx.compute_ring[i];6636			/* Per-queue interrupt is supported for MEC starting from VI.6637			  * The interrupt can only be enabled/disabled per pipe instead of per queue.6638			  */6639			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))6640				amdgpu_fence_process(ring);6641		}6642		break;6643	}6644	return 0;6645}6646 6647static void gfx_v8_0_fault(struct amdgpu_device *adev,6648			   struct amdgpu_iv_entry *entry)6649{6650	u8 me_id, pipe_id, queue_id;6651	struct amdgpu_ring *ring;6652	int i;6653 6654	me_id = (entry->ring_id & 0x0c) >> 2;6655	pipe_id = (entry->ring_id & 0x03) >> 0;6656	queue_id = (entry->ring_id & 0x70) >> 4;6657 6658	switch (me_id) {6659	case 0:6660		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);6661		break;6662	case 1:6663	case 2:6664		for (i = 0; i < adev->gfx.num_compute_rings; i++) {6665			ring = &adev->gfx.compute_ring[i];6666			if (ring->me == me_id && ring->pipe == pipe_id &&6667			    ring->queue == queue_id)6668				drm_sched_fault(&ring->sched);6669		}6670		break;6671	}6672}6673 6674static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,6675				 struct amdgpu_irq_src *source,6676				 struct amdgpu_iv_entry *entry)6677{6678	DRM_ERROR("Illegal register access in command stream\n");6679	gfx_v8_0_fault(adev, entry);6680	return 0;6681}6682 6683static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,6684				  struct amdgpu_irq_src *source,6685				  struct amdgpu_iv_entry *entry)6686{6687	DRM_ERROR("Illegal instruction in command stream\n");6688	gfx_v8_0_fault(adev, entry);6689	return 0;6690}6691 6692static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,6693				     struct amdgpu_irq_src *source,6694				     struct amdgpu_iv_entry *entry)6695{6696	DRM_ERROR("CP EDC/ECC error detected.");6697	return 0;6698}6699 6700static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data,6701				  bool from_wq)6702{6703	u32 enc, se_id, sh_id, cu_id;6704	char type[20];6705	int sq_edc_source = -1;6706 6707	enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);6708	se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);6709 6710	switch (enc) {6711		case 0:6712			DRM_INFO("SQ general purpose intr detected:"6713					"se_id %d, immed_overflow %d, host_reg_overflow %d,"6714					"host_cmd_overflow %d, cmd_timestamp %d,"6715					"reg_timestamp %d, thread_trace_buff_full %d,"6716					"wlt %d, thread_trace %d.\n",6717					se_id,6718					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),6719					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),6720					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),6721					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),6722					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),6723					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),6724					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),6725					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)6726					);6727			break;6728		case 1:6729		case 2:6730 6731			cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);6732			sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);6733 6734			/*6735			 * This function can be called either directly from ISR6736			 * or from BH in which case we can access SQ_EDC_INFO6737			 * instance6738			 */6739			if (from_wq) {6740				mutex_lock(&adev->grbm_idx_mutex);6741				gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id, 0);6742 6743				sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);6744 6745				gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);6746				mutex_unlock(&adev->grbm_idx_mutex);6747			}6748 6749			if (enc == 1)6750				sprintf(type, "instruction intr");6751			else6752				sprintf(type, "EDC/ECC error");6753 6754			DRM_INFO(6755				"SQ %s detected: "6756					"se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "6757					"trap %s, sq_ed_info.source %s.\n",6758					type, se_id, sh_id, cu_id,6759					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),6760					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),6761					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),6762					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",6763					(sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"6764				);6765			break;6766		default:6767			DRM_ERROR("SQ invalid encoding type\n.");6768	}6769}6770 6771static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)6772{6773 6774	struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);6775	struct sq_work *sq_work = container_of(work, struct sq_work, work);6776 6777	gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true);6778}6779 6780static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,6781			   struct amdgpu_irq_src *source,6782			   struct amdgpu_iv_entry *entry)6783{6784	unsigned ih_data = entry->src_data[0];6785 6786	/*6787	 * Try to submit work so SQ_EDC_INFO can be accessed from6788	 * BH. If previous work submission hasn't finished yet6789	 * just print whatever info is possible directly from the ISR.6790	 */6791	if (work_pending(&adev->gfx.sq_work.work)) {6792		gfx_v8_0_parse_sq_irq(adev, ih_data, false);6793	} else {6794		adev->gfx.sq_work.ih_data = ih_data;6795		schedule_work(&adev->gfx.sq_work.work);6796	}6797 6798	return 0;6799}6800 6801static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring)6802{6803	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));6804	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |6805			  PACKET3_TC_ACTION_ENA |6806			  PACKET3_SH_KCACHE_ACTION_ENA |6807			  PACKET3_SH_ICACHE_ACTION_ENA |6808			  PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */6809	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */6810	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE */6811	amdgpu_ring_write(ring, 0x0000000A); /* poll interval */6812}6813 6814static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring)6815{6816	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));6817	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |6818			  PACKET3_TC_ACTION_ENA |6819			  PACKET3_SH_KCACHE_ACTION_ENA |6820			  PACKET3_SH_ICACHE_ACTION_ENA |6821			  PACKET3_TC_WB_ACTION_ENA);  /* CP_COHER_CNTL */6822	amdgpu_ring_write(ring, 0xffffffff);	/* CP_COHER_SIZE */6823	amdgpu_ring_write(ring, 0xff);		/* CP_COHER_SIZE_HI */6824	amdgpu_ring_write(ring, 0);		/* CP_COHER_BASE */6825	amdgpu_ring_write(ring, 0);		/* CP_COHER_BASE_HI */6826	amdgpu_ring_write(ring, 0x0000000A);	/* poll interval */6827}6828 6829 6830/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */6831#define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT	0x0000007f6832static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring,6833					uint32_t pipe, bool enable)6834{6835	uint32_t val;6836	uint32_t wcl_cs_reg;6837 6838	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT;6839 6840	switch (pipe) {6841	case 0:6842		wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0;6843		break;6844	case 1:6845		wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1;6846		break;6847	case 2:6848		wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2;6849		break;6850	case 3:6851		wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3;6852		break;6853	default:6854		DRM_DEBUG("invalid pipe %d\n", pipe);6855		return;6856	}6857 6858	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);6859 6860}6861 6862#define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT	0x07ffffff6863static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)6864{6865	struct amdgpu_device *adev = ring->adev;6866	uint32_t val;6867	int i;6868 6869	/* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit6870	 * number of gfx waves. Setting 5 bit will make sure gfx only gets6871	 * around 25% of gpu resources.6872	 */6873	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;6874	amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val);6875 6876	/* Restrict waves for normal/low priority compute queues as well6877	 * to get best QoS for high priority compute jobs.6878	 *6879	 * amdgpu controls only 1st ME(0-3 CS pipes).6880	 */6881	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {6882		if (i != ring->pipe)6883			gfx_v8_0_emit_wave_limit_cs(ring, i, enable);6884 6885	}6886 6887}6888 6889static int gfx_v8_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)6890{6891	struct amdgpu_device *adev = ring->adev;6892	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];6893	struct amdgpu_ring *kiq_ring = &kiq->ring;6894	unsigned long flags;6895	u32 tmp;6896	int r;6897 6898	if (amdgpu_sriov_vf(adev))6899		return -EINVAL;6900 6901	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)6902		return -EINVAL;6903 6904	spin_lock_irqsave(&kiq->ring_lock, flags);6905 6906	if (amdgpu_ring_alloc(kiq_ring, 5)) {6907		spin_unlock_irqrestore(&kiq->ring_lock, flags);6908		return -ENOMEM;6909	}6910 6911	tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);6912	gfx_v8_0_ring_emit_wreg(kiq_ring, mmCP_VMID_RESET, tmp);6913	amdgpu_ring_commit(kiq_ring);6914 6915	spin_unlock_irqrestore(&kiq->ring_lock, flags);6916 6917	r = amdgpu_ring_test_ring(kiq_ring);6918	if (r)6919		return r;6920 6921	if (amdgpu_ring_alloc(ring, 7 + 12 + 5))6922		return -ENOMEM;6923	gfx_v8_0_ring_emit_fence_gfx(ring, ring->fence_drv.gpu_addr,6924				     ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);6925	gfx_v8_0_ring_emit_reg_wait(ring, mmCP_VMID_RESET, 0, 0xffff);6926	gfx_v8_0_ring_emit_wreg(ring, mmCP_VMID_RESET, 0);6927 6928	return amdgpu_ring_test_ring(ring);6929}6930 6931static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {6932	.name = "gfx_v8_0",6933	.early_init = gfx_v8_0_early_init,6934	.late_init = gfx_v8_0_late_init,6935	.sw_init = gfx_v8_0_sw_init,6936	.sw_fini = gfx_v8_0_sw_fini,6937	.hw_init = gfx_v8_0_hw_init,6938	.hw_fini = gfx_v8_0_hw_fini,6939	.suspend = gfx_v8_0_suspend,6940	.resume = gfx_v8_0_resume,6941	.is_idle = gfx_v8_0_is_idle,6942	.wait_for_idle = gfx_v8_0_wait_for_idle,6943	.check_soft_reset = gfx_v8_0_check_soft_reset,6944	.pre_soft_reset = gfx_v8_0_pre_soft_reset,6945	.soft_reset = gfx_v8_0_soft_reset,6946	.post_soft_reset = gfx_v8_0_post_soft_reset,6947	.set_clockgating_state = gfx_v8_0_set_clockgating_state,6948	.set_powergating_state = gfx_v8_0_set_powergating_state,6949	.get_clockgating_state = gfx_v8_0_get_clockgating_state,6950	.dump_ip_state = NULL,6951	.print_ip_state = NULL,6952};6953 6954static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {6955	.type = AMDGPU_RING_TYPE_GFX,6956	.align_mask = 0xff,6957	.nop = PACKET3(PACKET3_NOP, 0x3FFF),6958	.support_64bit_ptrs = false,6959	.get_rptr = gfx_v8_0_ring_get_rptr,6960	.get_wptr = gfx_v8_0_ring_get_wptr_gfx,6961	.set_wptr = gfx_v8_0_ring_set_wptr_gfx,6962	.emit_frame_size = /* maximum 215dw if count 16 IBs in */6963		5 +  /* COND_EXEC */6964		7 +  /* PIPELINE_SYNC */6965		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */6966		12 +  /* FENCE for VM_FLUSH */6967		20 + /* GDS switch */6968		4 + /* double SWITCH_BUFFER,6969		       the first COND_EXEC jump to the place just6970			   prior to this double SWITCH_BUFFER  */6971		5 + /* COND_EXEC */6972		7 +	 /*	HDP_flush */6973		4 +	 /*	VGT_flush */6974		14 + /*	CE_META */6975		31 + /*	DE_META */6976		3 + /* CNTX_CTRL */6977		5 + /* HDP_INVL */6978		12 + 12 + /* FENCE x2 */6979		2 + /* SWITCH_BUFFER */6980		5, /* SURFACE_SYNC */6981	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_gfx */6982	.emit_ib = gfx_v8_0_ring_emit_ib_gfx,6983	.emit_fence = gfx_v8_0_ring_emit_fence_gfx,6984	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,6985	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,6986	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,6987	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,6988	.test_ring = gfx_v8_0_ring_test_ring,6989	.test_ib = gfx_v8_0_ring_test_ib,6990	.insert_nop = amdgpu_ring_insert_nop,6991	.pad_ib = amdgpu_ring_generic_pad_ib,6992	.emit_switch_buffer = gfx_v8_ring_emit_sb,6993	.emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,6994	.init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,6995	.emit_wreg = gfx_v8_0_ring_emit_wreg,6996	.soft_recovery = gfx_v8_0_ring_soft_recovery,6997	.emit_mem_sync = gfx_v8_0_emit_mem_sync,6998	.reset = gfx_v8_0_reset_kgq,6999};7000 7001static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {7002	.type = AMDGPU_RING_TYPE_COMPUTE,7003	.align_mask = 0xff,7004	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7005	.support_64bit_ptrs = false,7006	.get_rptr = gfx_v8_0_ring_get_rptr,7007	.get_wptr = gfx_v8_0_ring_get_wptr_compute,7008	.set_wptr = gfx_v8_0_ring_set_wptr_compute,7009	.emit_frame_size =7010		20 + /* gfx_v8_0_ring_emit_gds_switch */7011		7 + /* gfx_v8_0_ring_emit_hdp_flush */7012		5 + /* hdp_invalidate */7013		7 + /* gfx_v8_0_ring_emit_pipeline_sync */7014		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */7015		7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */7016		7 + /* gfx_v8_0_emit_mem_sync_compute */7017		5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */7018		15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */7019	.emit_ib_size =	7, /* gfx_v8_0_ring_emit_ib_compute */7020	.emit_ib = gfx_v8_0_ring_emit_ib_compute,7021	.emit_fence = gfx_v8_0_ring_emit_fence_compute,7022	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,7023	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,7024	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,7025	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,7026	.test_ring = gfx_v8_0_ring_test_ring,7027	.test_ib = gfx_v8_0_ring_test_ib,7028	.insert_nop = amdgpu_ring_insert_nop,7029	.pad_ib = amdgpu_ring_generic_pad_ib,7030	.emit_wreg = gfx_v8_0_ring_emit_wreg,7031	.soft_recovery = gfx_v8_0_ring_soft_recovery,7032	.emit_mem_sync = gfx_v8_0_emit_mem_sync_compute,7033	.emit_wave_limit = gfx_v8_0_emit_wave_limit,7034};7035 7036static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {7037	.type = AMDGPU_RING_TYPE_KIQ,7038	.align_mask = 0xff,7039	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7040	.support_64bit_ptrs = false,7041	.get_rptr = gfx_v8_0_ring_get_rptr,7042	.get_wptr = gfx_v8_0_ring_get_wptr_compute,7043	.set_wptr = gfx_v8_0_ring_set_wptr_compute,7044	.emit_frame_size =7045		20 + /* gfx_v8_0_ring_emit_gds_switch */7046		7 + /* gfx_v8_0_ring_emit_hdp_flush */7047		5 + /* hdp_invalidate */7048		7 + /* gfx_v8_0_ring_emit_pipeline_sync */7049		17 + /* gfx_v8_0_ring_emit_vm_flush */7050		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */7051	.emit_ib_size =	7, /* gfx_v8_0_ring_emit_ib_compute */7052	.emit_fence = gfx_v8_0_ring_emit_fence_kiq,7053	.test_ring = gfx_v8_0_ring_test_ring,7054	.insert_nop = amdgpu_ring_insert_nop,7055	.pad_ib = amdgpu_ring_generic_pad_ib,7056	.emit_rreg = gfx_v8_0_ring_emit_rreg,7057	.emit_wreg = gfx_v8_0_ring_emit_wreg,7058};7059 7060static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)7061{7062	int i;7063 7064	adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq;7065 7066	for (i = 0; i < adev->gfx.num_gfx_rings; i++)7067		adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;7068 7069	for (i = 0; i < adev->gfx.num_compute_rings; i++)7070		adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;7071}7072 7073static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {7074	.set = gfx_v8_0_set_eop_interrupt_state,7075	.process = gfx_v8_0_eop_irq,7076};7077 7078static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {7079	.set = gfx_v8_0_set_priv_reg_fault_state,7080	.process = gfx_v8_0_priv_reg_irq,7081};7082 7083static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {7084	.set = gfx_v8_0_set_priv_inst_fault_state,7085	.process = gfx_v8_0_priv_inst_irq,7086};7087 7088static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {7089	.set = gfx_v8_0_set_cp_ecc_int_state,7090	.process = gfx_v8_0_cp_ecc_error_irq,7091};7092 7093static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {7094	.set = gfx_v8_0_set_sq_int_state,7095	.process = gfx_v8_0_sq_irq,7096};7097 7098static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)7099{7100	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;7101	adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;7102 7103	adev->gfx.priv_reg_irq.num_types = 1;7104	adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;7105 7106	adev->gfx.priv_inst_irq.num_types = 1;7107	adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;7108 7109	adev->gfx.cp_ecc_error_irq.num_types = 1;7110	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;7111 7112	adev->gfx.sq_irq.num_types = 1;7113	adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;7114}7115 7116static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)7117{7118	adev->gfx.rlc.funcs = &iceland_rlc_funcs;7119}7120 7121static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)7122{7123	/* init asci gds info */7124	adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);7125	adev->gds.gws_size = 64;7126	adev->gds.oa_size = 16;7127	adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);7128}7129 7130static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,7131						 u32 bitmap)7132{7133	u32 data;7134 7135	if (!bitmap)7136		return;7137 7138	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;7139	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;7140 7141	WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);7142}7143 7144static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)7145{7146	u32 data, mask;7147 7148	data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |7149		RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);7150 7151	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);7152 7153	return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;7154}7155 7156static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)7157{7158	int i, j, k, counter, active_cu_number = 0;7159	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;7160	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;7161	unsigned disable_masks[4 * 2];7162	u32 ao_cu_num;7163 7164	memset(cu_info, 0, sizeof(*cu_info));7165 7166	if (adev->flags & AMD_IS_APU)7167		ao_cu_num = 2;7168	else7169		ao_cu_num = adev->gfx.config.max_cu_per_sh;7170 7171	amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);7172 7173	mutex_lock(&adev->grbm_idx_mutex);7174	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {7175		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {7176			mask = 1;7177			ao_bitmap = 0;7178			counter = 0;7179			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);7180			if (i < 4 && j < 2)7181				gfx_v8_0_set_user_cu_inactive_bitmap(7182					adev, disable_masks[i * 2 + j]);7183			bitmap = gfx_v8_0_get_cu_active_bitmap(adev);7184			cu_info->bitmap[0][i][j] = bitmap;7185 7186			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {7187				if (bitmap & mask) {7188					if (counter < ao_cu_num)7189						ao_bitmap |= mask;7190					counter ++;7191				}7192				mask <<= 1;7193			}7194			active_cu_number += counter;7195			if (i < 2 && j < 2)7196				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));7197			cu_info->ao_cu_bitmap[i][j] = ao_bitmap;7198		}7199	}7200	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);7201	mutex_unlock(&adev->grbm_idx_mutex);7202 7203	cu_info->number = active_cu_number;7204	cu_info->ao_cu_mask = ao_cu_mask;7205	cu_info->simd_per_cu = NUM_SIMD_PER_CU;7206	cu_info->max_waves_per_simd = 10;7207	cu_info->max_scratch_slots_per_cu = 32;7208	cu_info->wave_front_size = 64;7209	cu_info->lds_size = 64;7210}7211 7212const struct amdgpu_ip_block_version gfx_v8_0_ip_block =7213{7214	.type = AMD_IP_BLOCK_TYPE_GFX,7215	.major = 8,7216	.minor = 0,7217	.rev = 0,7218	.funcs = &gfx_v8_0_ip_funcs,7219};7220 7221const struct amdgpu_ip_block_version gfx_v8_1_ip_block =7222{7223	.type = AMD_IP_BLOCK_TYPE_GFX,7224	.major = 8,7225	.minor = 1,7226	.rev = 0,7227	.funcs = &gfx_v8_0_ip_funcs,7228};7229 7230static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)7231{7232	uint64_t ce_payload_addr;7233	int cnt_ce;7234	union {7235		struct vi_ce_ib_state regular;7236		struct vi_ce_ib_state_chained_ib chained;7237	} ce_payload = {};7238 7239	if (ring->adev->virt.chained_ib_support) {7240		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +7241			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);7242		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;7243	} else {7244		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +7245			offsetof(struct vi_gfx_meta_data, ce_payload);7246		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;7247	}7248 7249	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));7250	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |7251				WRITE_DATA_DST_SEL(8) |7252				WR_CONFIRM) |7253				WRITE_DATA_CACHE_POLICY(0));7254	amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));7255	amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));7256	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);7257}7258 7259static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)7260{7261	uint64_t de_payload_addr, gds_addr, csa_addr;7262	int cnt_de;7263	union {7264		struct vi_de_ib_state regular;7265		struct vi_de_ib_state_chained_ib chained;7266	} de_payload = {};7267 7268	csa_addr = amdgpu_csa_vaddr(ring->adev);7269	gds_addr = csa_addr + 4096;7270	if (ring->adev->virt.chained_ib_support) {7271		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);7272		de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);7273		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);7274		cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;7275	} else {7276		de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);7277		de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);7278		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);7279		cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;7280	}7281 7282	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));7283	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |7284				WRITE_DATA_DST_SEL(8) |7285				WR_CONFIRM) |7286				WRITE_DATA_CACHE_POLICY(0));7287	amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));7288	amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));7289	amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);7290}7291