brintos / linux-shallow public Read only

1/*2 * Copyright 2015 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include <linux/slab.h>26#include <linux/module.h>27#include <linux/pci.h>28 29#include <drm/amdgpu_drm.h>30 31#include "amdgpu.h"32#include "amdgpu_atombios.h"33#include "amdgpu_ih.h"34#include "amdgpu_uvd.h"35#include "amdgpu_vce.h"36#include "atom.h"37#include "amd_pcie.h"38#include "si_dpm.h"39#include "sid.h"40#include "si_ih.h"41#include "gfx_v6_0.h"42#include "gmc_v6_0.h"43#include "si_dma.h"44#include "dce_v6_0.h"45#include "si.h"46#include "uvd_v3_1.h"47#include "amdgpu_vkms.h"48#include "gca/gfx_6_0_d.h"49#include "oss/oss_1_0_d.h"50#include "oss/oss_1_0_sh_mask.h"51#include "gmc/gmc_6_0_d.h"52#include "dce/dce_6_0_d.h"53#include "uvd/uvd_4_0_d.h"54#include "bif/bif_3_0_d.h"55#include "bif/bif_3_0_sh_mask.h"56 57#include "amdgpu_dm.h"58 59static const u32 tahiti_golden_registers[] =60{61	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,62	mmCB_HW_CONTROL, 0x00010000, 0x00018208,63	mmDB_DEBUG, 0xffffffff, 0x00000000,64	mmDB_DEBUG2, 0xf00fffff, 0x00000400,65	mmDB_DEBUG3, 0x0002021c, 0x00020200,66	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,67	0x340c, 0x000000c0, 0x00800040,68	0x360c, 0x000000c0, 0x00800040,69	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,70	mmFBC_MISC, 0x00200000, 0x50100000,71	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,72	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,73	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,74	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,75	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,76	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,77	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,78	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,79	0x000c, 0xffffffff, 0x0040,80	0x000d, 0x00000040, 0x00004040,81	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,82	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,83	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,84	mmSX_DEBUG_1, 0x0000007f, 0x00000020,85	mmTA_CNTL_AUX, 0x00010000, 0x00010000,86	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,87	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,88	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,89	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,90	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,91	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,92	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,93	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,94	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,95	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,96	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,97};98 99static const u32 tahiti_golden_registers2[] =100{101	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,102};103 104static const u32 tahiti_golden_rlc_registers[] =105{106	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,107	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,108	0x311f, 0xffffffff, 0x10104040,109	0x3122, 0xffffffff, 0x0100000a,110	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,111	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,112	mmUVD_CGC_GATE, 0x00000008, 0x00000000,113};114 115static const u32 pitcairn_golden_registers[] =116{117	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,118	mmCB_HW_CONTROL, 0x00010000, 0x00018208,119	mmDB_DEBUG, 0xffffffff, 0x00000000,120	mmDB_DEBUG2, 0xf00fffff, 0x00000400,121	mmDB_DEBUG3, 0x0002021c, 0x00020200,122	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,123	0x340c, 0x000300c0, 0x00800040,124	0x360c, 0x000300c0, 0x00800040,125	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,126	mmFBC_MISC, 0x00200000, 0x50100000,127	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,128	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,129	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,130	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,131	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,132	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,133	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,134	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,135	0x000c, 0xffffffff, 0x0040,136	0x000d, 0x00000040, 0x00004040,137	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,138	mmSX_DEBUG_1, 0x0000007f, 0x00000020,139	mmTA_CNTL_AUX, 0x00010000, 0x00010000,140	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,141	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,142	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,143	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,144	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,145	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,146	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,147	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,148	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,149};150 151static const u32 pitcairn_golden_rlc_registers[] =152{153	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,154	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,155	0x311f, 0xffffffff, 0x10102020,156	0x3122, 0xffffffff, 0x01000020,157	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,158	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,159};160 161static const u32 verde_pg_init[] =162{163	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,164	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,165	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,166	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,167	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,168	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,169	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,170	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,171	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,172	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,173	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,174	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,175	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,176	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,177	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,178	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,179	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,180	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,181	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,182	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,183	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,184	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,185	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,186	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,187	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,188	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,189	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,190	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,191	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,192	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,193	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,194	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,195	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,196	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,197	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,198	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,199	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,200	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,201	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,202	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,203	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,204	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,205	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,206	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,207	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,208	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,209	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,210	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,211	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,212	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,213	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,214	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,215	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,216	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,217	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,218	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,219	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,220	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,221	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,222	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,223	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,224	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,225	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,226	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,227	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,228	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,229	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,230	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,231	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,232	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,233	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,234	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,235	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,236	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,237	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,238	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,239	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,240	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,241	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,242	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,243	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,244	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,245	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,246	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,247	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,248	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,249	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,250	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,251	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,252	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,253	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,254	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,255	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,256	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,257	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,258	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,259	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,260	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,261	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,262	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,263	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,264	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,265	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,266	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,267	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,268	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,269	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,270	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,271	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,272	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,273	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,274	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,275	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,276	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,277	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,278	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,279	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,280	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,281	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,282	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,283	mmGMCON_MISC2, 0xfc00, 0x2000,284	mmGMCON_MISC3, 0xffffffff, 0xfc0,285	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,286};287 288static const u32 verde_golden_rlc_registers[] =289{290	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,291	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,292	0x311f, 0xffffffff, 0x10808020,293	0x3122, 0xffffffff, 0x00800008,294	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,295	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,296};297 298static const u32 verde_golden_registers[] =299{300	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,301	mmCB_HW_CONTROL, 0x00010000, 0x00018208,302	mmDB_DEBUG, 0xffffffff, 0x00000000,303	mmDB_DEBUG2, 0xf00fffff, 0x00000400,304	mmDB_DEBUG3, 0x0002021c, 0x00020200,305	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,306	0x340c, 0x000300c0, 0x00800040,307	0x360c, 0x000300c0, 0x00800040,308	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,309	mmFBC_MISC, 0x00200000, 0x50100000,310	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,311	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,312	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,313	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,314	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,315	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,316	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,317	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,318	0x000c, 0xffffffff, 0x0040,319	0x000d, 0x00000040, 0x00004040,320	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,321	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,322	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,323	mmSX_DEBUG_1, 0x0000007f, 0x00000020,324	mmTA_CNTL_AUX, 0x00010000, 0x00010000,325	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,326	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,327	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,328	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,329	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,330	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,331	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,332	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,333	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,334};335 336static const u32 oland_golden_registers[] =337{338	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,339	mmCB_HW_CONTROL, 0x00010000, 0x00018208,340	mmDB_DEBUG, 0xffffffff, 0x00000000,341	mmDB_DEBUG2, 0xf00fffff, 0x00000400,342	mmDB_DEBUG3, 0x0002021c, 0x00020200,343	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,344	0x340c, 0x000300c0, 0x00800040,345	0x360c, 0x000300c0, 0x00800040,346	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,347	mmFBC_MISC, 0x00200000, 0x50100000,348	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,349	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,350	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,351	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,352	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,353	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,354	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,355	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,356	0x000c, 0xffffffff, 0x0040,357	0x000d, 0x00000040, 0x00004040,358	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,359	mmSX_DEBUG_1, 0x0000007f, 0x00000020,360	mmTA_CNTL_AUX, 0x00010000, 0x00010000,361	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,362	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,363	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,364	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,365	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,366	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,367	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,368	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,369	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,370 371};372 373static const u32 oland_golden_rlc_registers[] =374{375	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,376	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,377	0x311f, 0xffffffff, 0x10104040,378	0x3122, 0xffffffff, 0x0100000a,379	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,380	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,381};382 383static const u32 hainan_golden_registers[] =384{385	0x17bc, 0x00000030, 0x00000011,386	mmCB_HW_CONTROL, 0x00010000, 0x00018208,387	mmDB_DEBUG, 0xffffffff, 0x00000000,388	mmDB_DEBUG2, 0xf00fffff, 0x00000400,389	mmDB_DEBUG3, 0x0002021c, 0x00020200,390	0x031e, 0x00000080, 0x00000000,391	0x3430, 0xff000fff, 0x00000100,392	0x340c, 0x000300c0, 0x00800040,393	0x3630, 0xff000fff, 0x00000100,394	0x360c, 0x000300c0, 0x00800040,395	0x16ec, 0x000000f0, 0x00000070,396	0x16f0, 0x00200000, 0x50100000,397	0x1c0c, 0x31000311, 0x00000011,398	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,399	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,400	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,401	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,402	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,403	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,404	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,405	0x000c, 0xffffffff, 0x0040,406	0x000d, 0x00000040, 0x00004040,407	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,408	mmSX_DEBUG_1, 0x0000007f, 0x00000020,409	mmTA_CNTL_AUX, 0x00010000, 0x00010000,410	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,411	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,412	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,413	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,414	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,415	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,416	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,417	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,418	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,419};420 421static const u32 hainan_golden_registers2[] =422{423	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,424};425 426static const u32 tahiti_mgcg_cgcg_init[] =427{428	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,429	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,430	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,431	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,432	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,433	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,434	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,435	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,436	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,437	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,438	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,439	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,440	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,441	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,442	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,443	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,444	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,445	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,446	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,447	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,448	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,449	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,450	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,451	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,452	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,453	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,454	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,455	0x2458, 0xffffffff, 0x00010000,456	0x2459, 0xffffffff, 0x00030002,457	0x245a, 0xffffffff, 0x00040007,458	0x245b, 0xffffffff, 0x00060005,459	0x245c, 0xffffffff, 0x00090008,460	0x245d, 0xffffffff, 0x00020001,461	0x245e, 0xffffffff, 0x00040003,462	0x245f, 0xffffffff, 0x00000007,463	0x2460, 0xffffffff, 0x00060005,464	0x2461, 0xffffffff, 0x00090008,465	0x2462, 0xffffffff, 0x00030002,466	0x2463, 0xffffffff, 0x00050004,467	0x2464, 0xffffffff, 0x00000008,468	0x2465, 0xffffffff, 0x00070006,469	0x2466, 0xffffffff, 0x000a0009,470	0x2467, 0xffffffff, 0x00040003,471	0x2468, 0xffffffff, 0x00060005,472	0x2469, 0xffffffff, 0x00000009,473	0x246a, 0xffffffff, 0x00080007,474	0x246b, 0xffffffff, 0x000b000a,475	0x246c, 0xffffffff, 0x00050004,476	0x246d, 0xffffffff, 0x00070006,477	0x246e, 0xffffffff, 0x0008000b,478	0x246f, 0xffffffff, 0x000a0009,479	0x2470, 0xffffffff, 0x000d000c,480	0x2471, 0xffffffff, 0x00060005,481	0x2472, 0xffffffff, 0x00080007,482	0x2473, 0xffffffff, 0x0000000b,483	0x2474, 0xffffffff, 0x000a0009,484	0x2475, 0xffffffff, 0x000d000c,485	0x2476, 0xffffffff, 0x00070006,486	0x2477, 0xffffffff, 0x00090008,487	0x2478, 0xffffffff, 0x0000000c,488	0x2479, 0xffffffff, 0x000b000a,489	0x247a, 0xffffffff, 0x000e000d,490	0x247b, 0xffffffff, 0x00080007,491	0x247c, 0xffffffff, 0x000a0009,492	0x247d, 0xffffffff, 0x0000000d,493	0x247e, 0xffffffff, 0x000c000b,494	0x247f, 0xffffffff, 0x000f000e,495	0x2480, 0xffffffff, 0x00090008,496	0x2481, 0xffffffff, 0x000b000a,497	0x2482, 0xffffffff, 0x000c000f,498	0x2483, 0xffffffff, 0x000e000d,499	0x2484, 0xffffffff, 0x00110010,500	0x2485, 0xffffffff, 0x000a0009,501	0x2486, 0xffffffff, 0x000c000b,502	0x2487, 0xffffffff, 0x0000000f,503	0x2488, 0xffffffff, 0x000e000d,504	0x2489, 0xffffffff, 0x00110010,505	0x248a, 0xffffffff, 0x000b000a,506	0x248b, 0xffffffff, 0x000d000c,507	0x248c, 0xffffffff, 0x00000010,508	0x248d, 0xffffffff, 0x000f000e,509	0x248e, 0xffffffff, 0x00120011,510	0x248f, 0xffffffff, 0x000c000b,511	0x2490, 0xffffffff, 0x000e000d,512	0x2491, 0xffffffff, 0x00000011,513	0x2492, 0xffffffff, 0x0010000f,514	0x2493, 0xffffffff, 0x00130012,515	0x2494, 0xffffffff, 0x000d000c,516	0x2495, 0xffffffff, 0x000f000e,517	0x2496, 0xffffffff, 0x00100013,518	0x2497, 0xffffffff, 0x00120011,519	0x2498, 0xffffffff, 0x00150014,520	0x2499, 0xffffffff, 0x000e000d,521	0x249a, 0xffffffff, 0x0010000f,522	0x249b, 0xffffffff, 0x00000013,523	0x249c, 0xffffffff, 0x00120011,524	0x249d, 0xffffffff, 0x00150014,525	0x249e, 0xffffffff, 0x000f000e,526	0x249f, 0xffffffff, 0x00110010,527	0x24a0, 0xffffffff, 0x00000014,528	0x24a1, 0xffffffff, 0x00130012,529	0x24a2, 0xffffffff, 0x00160015,530	0x24a3, 0xffffffff, 0x0010000f,531	0x24a4, 0xffffffff, 0x00120011,532	0x24a5, 0xffffffff, 0x00000015,533	0x24a6, 0xffffffff, 0x00140013,534	0x24a7, 0xffffffff, 0x00170016,535	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,536	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,537	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,538	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,539	0x000c, 0xffffffff, 0x0000001c,540	0x000d, 0x000f0000, 0x000f0000,541	0x0583, 0xffffffff, 0x00000100,542	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,543	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,544	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,545	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,546	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,547	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,548	0x157a, 0x00000001, 0x00000001,549	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,550	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,551	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,552	0x3430, 0xfffffff0, 0x00000100,553	0x3630, 0xfffffff0, 0x00000100,554};555static const u32 pitcairn_mgcg_cgcg_init[] =556{557	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,558	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,559	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,560	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,561	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,562	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,563	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,564	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,565	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,566	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,567	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,568	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,569	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,570	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,571	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,572	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,573	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,574	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,575	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,576	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,577	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,578	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,579	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,580	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,581	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,582	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,583	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,584	0x2458, 0xffffffff, 0x00010000,585	0x2459, 0xffffffff, 0x00030002,586	0x245a, 0xffffffff, 0x00040007,587	0x245b, 0xffffffff, 0x00060005,588	0x245c, 0xffffffff, 0x00090008,589	0x245d, 0xffffffff, 0x00020001,590	0x245e, 0xffffffff, 0x00040003,591	0x245f, 0xffffffff, 0x00000007,592	0x2460, 0xffffffff, 0x00060005,593	0x2461, 0xffffffff, 0x00090008,594	0x2462, 0xffffffff, 0x00030002,595	0x2463, 0xffffffff, 0x00050004,596	0x2464, 0xffffffff, 0x00000008,597	0x2465, 0xffffffff, 0x00070006,598	0x2466, 0xffffffff, 0x000a0009,599	0x2467, 0xffffffff, 0x00040003,600	0x2468, 0xffffffff, 0x00060005,601	0x2469, 0xffffffff, 0x00000009,602	0x246a, 0xffffffff, 0x00080007,603	0x246b, 0xffffffff, 0x000b000a,604	0x246c, 0xffffffff, 0x00050004,605	0x246d, 0xffffffff, 0x00070006,606	0x246e, 0xffffffff, 0x0008000b,607	0x246f, 0xffffffff, 0x000a0009,608	0x2470, 0xffffffff, 0x000d000c,609	0x2480, 0xffffffff, 0x00090008,610	0x2481, 0xffffffff, 0x000b000a,611	0x2482, 0xffffffff, 0x000c000f,612	0x2483, 0xffffffff, 0x000e000d,613	0x2484, 0xffffffff, 0x00110010,614	0x2485, 0xffffffff, 0x000a0009,615	0x2486, 0xffffffff, 0x000c000b,616	0x2487, 0xffffffff, 0x0000000f,617	0x2488, 0xffffffff, 0x000e000d,618	0x2489, 0xffffffff, 0x00110010,619	0x248a, 0xffffffff, 0x000b000a,620	0x248b, 0xffffffff, 0x000d000c,621	0x248c, 0xffffffff, 0x00000010,622	0x248d, 0xffffffff, 0x000f000e,623	0x248e, 0xffffffff, 0x00120011,624	0x248f, 0xffffffff, 0x000c000b,625	0x2490, 0xffffffff, 0x000e000d,626	0x2491, 0xffffffff, 0x00000011,627	0x2492, 0xffffffff, 0x0010000f,628	0x2493, 0xffffffff, 0x00130012,629	0x2494, 0xffffffff, 0x000d000c,630	0x2495, 0xffffffff, 0x000f000e,631	0x2496, 0xffffffff, 0x00100013,632	0x2497, 0xffffffff, 0x00120011,633	0x2498, 0xffffffff, 0x00150014,634	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,635	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,636	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,637	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,638	0x000c, 0xffffffff, 0x0000001c,639	0x000d, 0x000f0000, 0x000f0000,640	0x0583, 0xffffffff, 0x00000100,641	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,642	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,643	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,644	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,645	0x157a, 0x00000001, 0x00000001,646	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,647	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,648	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,649	0x3430, 0xfffffff0, 0x00000100,650	0x3630, 0xfffffff0, 0x00000100,651};652 653static const u32 verde_mgcg_cgcg_init[] =654{655	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,656	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,657	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,658	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,659	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,660	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,661	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,662	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,663	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,664	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,665	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,666	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,667	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,668	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,669	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,670	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,671	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,672	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,673	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,674	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,675	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,676	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,677	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,678	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,679	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,680	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,681	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,682	0x2458, 0xffffffff, 0x00010000,683	0x2459, 0xffffffff, 0x00030002,684	0x245a, 0xffffffff, 0x00040007,685	0x245b, 0xffffffff, 0x00060005,686	0x245c, 0xffffffff, 0x00090008,687	0x245d, 0xffffffff, 0x00020001,688	0x245e, 0xffffffff, 0x00040003,689	0x245f, 0xffffffff, 0x00000007,690	0x2460, 0xffffffff, 0x00060005,691	0x2461, 0xffffffff, 0x00090008,692	0x2462, 0xffffffff, 0x00030002,693	0x2463, 0xffffffff, 0x00050004,694	0x2464, 0xffffffff, 0x00000008,695	0x2465, 0xffffffff, 0x00070006,696	0x2466, 0xffffffff, 0x000a0009,697	0x2467, 0xffffffff, 0x00040003,698	0x2468, 0xffffffff, 0x00060005,699	0x2469, 0xffffffff, 0x00000009,700	0x246a, 0xffffffff, 0x00080007,701	0x246b, 0xffffffff, 0x000b000a,702	0x246c, 0xffffffff, 0x00050004,703	0x246d, 0xffffffff, 0x00070006,704	0x246e, 0xffffffff, 0x0008000b,705	0x246f, 0xffffffff, 0x000a0009,706	0x2470, 0xffffffff, 0x000d000c,707	0x2480, 0xffffffff, 0x00090008,708	0x2481, 0xffffffff, 0x000b000a,709	0x2482, 0xffffffff, 0x000c000f,710	0x2483, 0xffffffff, 0x000e000d,711	0x2484, 0xffffffff, 0x00110010,712	0x2485, 0xffffffff, 0x000a0009,713	0x2486, 0xffffffff, 0x000c000b,714	0x2487, 0xffffffff, 0x0000000f,715	0x2488, 0xffffffff, 0x000e000d,716	0x2489, 0xffffffff, 0x00110010,717	0x248a, 0xffffffff, 0x000b000a,718	0x248b, 0xffffffff, 0x000d000c,719	0x248c, 0xffffffff, 0x00000010,720	0x248d, 0xffffffff, 0x000f000e,721	0x248e, 0xffffffff, 0x00120011,722	0x248f, 0xffffffff, 0x000c000b,723	0x2490, 0xffffffff, 0x000e000d,724	0x2491, 0xffffffff, 0x00000011,725	0x2492, 0xffffffff, 0x0010000f,726	0x2493, 0xffffffff, 0x00130012,727	0x2494, 0xffffffff, 0x000d000c,728	0x2495, 0xffffffff, 0x000f000e,729	0x2496, 0xffffffff, 0x00100013,730	0x2497, 0xffffffff, 0x00120011,731	0x2498, 0xffffffff, 0x00150014,732	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,733	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,734	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,735	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,736	0x000c, 0xffffffff, 0x0000001c,737	0x000d, 0x000f0000, 0x000f0000,738	0x0583, 0xffffffff, 0x00000100,739	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,740	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,741	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,742	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,743	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,744	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,745	0x157a, 0x00000001, 0x00000001,746	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,747	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,748	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,749	0x3430, 0xfffffff0, 0x00000100,750	0x3630, 0xfffffff0, 0x00000100,751};752 753static const u32 oland_mgcg_cgcg_init[] =754{755	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,756	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,757	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,758	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,759	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,760	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,761	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,762	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,763	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,764	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,765	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,766	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,767	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,768	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,769	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,770	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,771	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,772	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,773	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,774	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,775	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,776	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,777	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,778	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,779	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,780	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,781	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,782	0x2458, 0xffffffff, 0x00010000,783	0x2459, 0xffffffff, 0x00030002,784	0x245a, 0xffffffff, 0x00040007,785	0x245b, 0xffffffff, 0x00060005,786	0x245c, 0xffffffff, 0x00090008,787	0x245d, 0xffffffff, 0x00020001,788	0x245e, 0xffffffff, 0x00040003,789	0x245f, 0xffffffff, 0x00000007,790	0x2460, 0xffffffff, 0x00060005,791	0x2461, 0xffffffff, 0x00090008,792	0x2462, 0xffffffff, 0x00030002,793	0x2463, 0xffffffff, 0x00050004,794	0x2464, 0xffffffff, 0x00000008,795	0x2465, 0xffffffff, 0x00070006,796	0x2466, 0xffffffff, 0x000a0009,797	0x2467, 0xffffffff, 0x00040003,798	0x2468, 0xffffffff, 0x00060005,799	0x2469, 0xffffffff, 0x00000009,800	0x246a, 0xffffffff, 0x00080007,801	0x246b, 0xffffffff, 0x000b000a,802	0x246c, 0xffffffff, 0x00050004,803	0x246d, 0xffffffff, 0x00070006,804	0x246e, 0xffffffff, 0x0008000b,805	0x246f, 0xffffffff, 0x000a0009,806	0x2470, 0xffffffff, 0x000d000c,807	0x2471, 0xffffffff, 0x00060005,808	0x2472, 0xffffffff, 0x00080007,809	0x2473, 0xffffffff, 0x0000000b,810	0x2474, 0xffffffff, 0x000a0009,811	0x2475, 0xffffffff, 0x000d000c,812	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,813	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,814	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,815	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,816	0x000c, 0xffffffff, 0x0000001c,817	0x000d, 0x000f0000, 0x000f0000,818	0x0583, 0xffffffff, 0x00000100,819	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,820	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,821	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,822	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,823	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,824	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,825	0x157a, 0x00000001, 0x00000001,826	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,827	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,828	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,829	0x3430, 0xfffffff0, 0x00000100,830	0x3630, 0xfffffff0, 0x00000100,831};832 833static const u32 hainan_mgcg_cgcg_init[] =834{835	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,836	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,837	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,838	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,839	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,840	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,841	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,842	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,843	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,844	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,845	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,846	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,847	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,848	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,849	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,850	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,851	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,852	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,853	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,854	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,855	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,856	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,857	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,858	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,859	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,860	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,861	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,862	0x2458, 0xffffffff, 0x00010000,863	0x2459, 0xffffffff, 0x00030002,864	0x245a, 0xffffffff, 0x00040007,865	0x245b, 0xffffffff, 0x00060005,866	0x245c, 0xffffffff, 0x00090008,867	0x245d, 0xffffffff, 0x00020001,868	0x245e, 0xffffffff, 0x00040003,869	0x245f, 0xffffffff, 0x00000007,870	0x2460, 0xffffffff, 0x00060005,871	0x2461, 0xffffffff, 0x00090008,872	0x2462, 0xffffffff, 0x00030002,873	0x2463, 0xffffffff, 0x00050004,874	0x2464, 0xffffffff, 0x00000008,875	0x2465, 0xffffffff, 0x00070006,876	0x2466, 0xffffffff, 0x000a0009,877	0x2467, 0xffffffff, 0x00040003,878	0x2468, 0xffffffff, 0x00060005,879	0x2469, 0xffffffff, 0x00000009,880	0x246a, 0xffffffff, 0x00080007,881	0x246b, 0xffffffff, 0x000b000a,882	0x246c, 0xffffffff, 0x00050004,883	0x246d, 0xffffffff, 0x00070006,884	0x246e, 0xffffffff, 0x0008000b,885	0x246f, 0xffffffff, 0x000a0009,886	0x2470, 0xffffffff, 0x000d000c,887	0x2471, 0xffffffff, 0x00060005,888	0x2472, 0xffffffff, 0x00080007,889	0x2473, 0xffffffff, 0x0000000b,890	0x2474, 0xffffffff, 0x000a0009,891	0x2475, 0xffffffff, 0x000d000c,892	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,893	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,894	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,895	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,896	0x000c, 0xffffffff, 0x0000001c,897	0x000d, 0x000f0000, 0x000f0000,898	0x0583, 0xffffffff, 0x00000100,899	0x0409, 0xffffffff, 0x00000100,900	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,901	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,902	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,903	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,904	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,905	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,906	0x3430, 0xfffffff0, 0x00000100,907	0x3630, 0xfffffff0, 0x00000100,908};909 910/* XXX: update when we support VCE */911#if 0912/* tahiti, pitcarin, verde */913static const struct amdgpu_video_codec_info tahiti_video_codecs_encode_array[] =914{915	{916		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,917		.max_width = 2048,918		.max_height = 1152,919		.max_pixels_per_frame = 2048 * 1152,920		.max_level = 0,921	},922};923 924static const struct amdgpu_video_codecs tahiti_video_codecs_encode =925{926	.codec_count = ARRAY_SIZE(tahiti_video_codecs_encode_array),927	.codec_array = tahiti_video_codecs_encode_array,928};929#else930static const struct amdgpu_video_codecs tahiti_video_codecs_encode =931{932	.codec_count = 0,933	.codec_array = NULL,934};935#endif936/* oland and hainan don't support encode */937static const struct amdgpu_video_codecs hainan_video_codecs_encode =938{939	.codec_count = 0,940	.codec_array = NULL,941};942 943/* tahiti, pitcarin, verde, oland */944static const struct amdgpu_video_codec_info tahiti_video_codecs_decode_array[] =945{946	{947		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,948		.max_width = 2048,949		.max_height = 1152,950		.max_pixels_per_frame = 2048 * 1152,951		.max_level = 3,952	},953	{954		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,955		.max_width = 2048,956		.max_height = 1152,957		.max_pixels_per_frame = 2048 * 1152,958		.max_level = 5,959	},960	{961		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,962		.max_width = 2048,963		.max_height = 1152,964		.max_pixels_per_frame = 2048 * 1152,965		.max_level = 41,966	},967	{968		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,969		.max_width = 2048,970		.max_height = 1152,971		.max_pixels_per_frame = 2048 * 1152,972		.max_level = 4,973	},974};975 976static const struct amdgpu_video_codecs tahiti_video_codecs_decode =977{978	.codec_count = ARRAY_SIZE(tahiti_video_codecs_decode_array),979	.codec_array = tahiti_video_codecs_decode_array,980};981 982/* hainan doesn't support decode */983static const struct amdgpu_video_codecs hainan_video_codecs_decode =984{985	.codec_count = 0,986	.codec_array = NULL,987};988 989static int si_query_video_codecs(struct amdgpu_device *adev, bool encode,990				 const struct amdgpu_video_codecs **codecs)991{992	switch (adev->asic_type) {993	case CHIP_VERDE:994	case CHIP_TAHITI:995	case CHIP_PITCAIRN:996		if (encode)997			*codecs = &tahiti_video_codecs_encode;998		else999			*codecs = &tahiti_video_codecs_decode;1000		return 0;1001	case CHIP_OLAND:1002		if (encode)1003			*codecs = &hainan_video_codecs_encode;1004		else1005			*codecs = &tahiti_video_codecs_decode;1006		return 0;1007	case CHIP_HAINAN:1008		if (encode)1009			*codecs = &hainan_video_codecs_encode;1010		else1011			*codecs = &hainan_video_codecs_decode;1012		return 0;1013	default:1014		return -EINVAL;1015	}1016}1017 1018static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)1019{1020	unsigned long flags;1021	u32 r;1022 1023	spin_lock_irqsave(&adev->pcie_idx_lock, flags);1024	WREG32(AMDGPU_PCIE_INDEX, reg);1025	(void)RREG32(AMDGPU_PCIE_INDEX);1026	r = RREG32(AMDGPU_PCIE_DATA);1027	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);1028	return r;1029}1030 1031static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)1032{1033	unsigned long flags;1034 1035	spin_lock_irqsave(&adev->pcie_idx_lock, flags);1036	WREG32(AMDGPU_PCIE_INDEX, reg);1037	(void)RREG32(AMDGPU_PCIE_INDEX);1038	WREG32(AMDGPU_PCIE_DATA, v);1039	(void)RREG32(AMDGPU_PCIE_DATA);1040	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);1041}1042 1043static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)1044{1045	unsigned long flags;1046	u32 r;1047 1048	spin_lock_irqsave(&adev->pcie_idx_lock, flags);1049	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));1050	(void)RREG32(PCIE_PORT_INDEX);1051	r = RREG32(PCIE_PORT_DATA);1052	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);1053	return r;1054}1055 1056static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)1057{1058	unsigned long flags;1059 1060	spin_lock_irqsave(&adev->pcie_idx_lock, flags);1061	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));1062	(void)RREG32(PCIE_PORT_INDEX);1063	WREG32(PCIE_PORT_DATA, (v));1064	(void)RREG32(PCIE_PORT_DATA);1065	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);1066}1067 1068static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)1069{1070	unsigned long flags;1071	u32 r;1072 1073	spin_lock_irqsave(&adev->smc_idx_lock, flags);1074	WREG32(SMC_IND_INDEX_0, (reg));1075	r = RREG32(SMC_IND_DATA_0);1076	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);1077	return r;1078}1079 1080static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)1081{1082	unsigned long flags;1083 1084	spin_lock_irqsave(&adev->smc_idx_lock, flags);1085	WREG32(SMC_IND_INDEX_0, (reg));1086	WREG32(SMC_IND_DATA_0, (v));1087	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);1088}1089 1090static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)1091{1092	unsigned long flags;1093	u32 r;1094 1095	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);1096	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));1097	r = RREG32(mmUVD_CTX_DATA);1098	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);1099	return r;1100}1101 1102static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)1103{1104	unsigned long flags;1105 1106	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);1107	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));1108	WREG32(mmUVD_CTX_DATA, (v));1109	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);1110}1111 1112static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {1113	{GRBM_STATUS},1114	{mmGRBM_STATUS2},1115	{mmGRBM_STATUS_SE0},1116	{mmGRBM_STATUS_SE1},1117	{mmSRBM_STATUS},1118	{mmSRBM_STATUS2},1119	{DMA_STATUS_REG + DMA0_REGISTER_OFFSET},1120	{DMA_STATUS_REG + DMA1_REGISTER_OFFSET},1121	{mmCP_STAT},1122	{mmCP_STALLED_STAT1},1123	{mmCP_STALLED_STAT2},1124	{mmCP_STALLED_STAT3},1125	{GB_ADDR_CONFIG},1126	{MC_ARB_RAMCFG},1127	{GB_TILE_MODE0},1128	{GB_TILE_MODE1},1129	{GB_TILE_MODE2},1130	{GB_TILE_MODE3},1131	{GB_TILE_MODE4},1132	{GB_TILE_MODE5},1133	{GB_TILE_MODE6},1134	{GB_TILE_MODE7},1135	{GB_TILE_MODE8},1136	{GB_TILE_MODE9},1137	{GB_TILE_MODE10},1138	{GB_TILE_MODE11},1139	{GB_TILE_MODE12},1140	{GB_TILE_MODE13},1141	{GB_TILE_MODE14},1142	{GB_TILE_MODE15},1143	{GB_TILE_MODE16},1144	{GB_TILE_MODE17},1145	{GB_TILE_MODE18},1146	{GB_TILE_MODE19},1147	{GB_TILE_MODE20},1148	{GB_TILE_MODE21},1149	{GB_TILE_MODE22},1150	{GB_TILE_MODE23},1151	{GB_TILE_MODE24},1152	{GB_TILE_MODE25},1153	{GB_TILE_MODE26},1154	{GB_TILE_MODE27},1155	{GB_TILE_MODE28},1156	{GB_TILE_MODE29},1157	{GB_TILE_MODE30},1158	{GB_TILE_MODE31},1159	{CC_RB_BACKEND_DISABLE, true},1160	{GC_USER_RB_BACKEND_DISABLE, true},1161	{PA_SC_RASTER_CONFIG, true},1162};1163 1164static uint32_t si_get_register_value(struct amdgpu_device *adev,1165				      bool indexed, u32 se_num,1166				      u32 sh_num, u32 reg_offset)1167{1168	if (indexed) {1169		uint32_t val;1170		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;1171		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;1172 1173		switch (reg_offset) {1174		case mmCC_RB_BACKEND_DISABLE:1175			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;1176		case mmGC_USER_RB_BACKEND_DISABLE:1177			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;1178		case mmPA_SC_RASTER_CONFIG:1179			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;1180		}1181 1182		mutex_lock(&adev->grbm_idx_mutex);1183		if (se_num != 0xffffffff || sh_num != 0xffffffff)1184			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);1185 1186		val = RREG32(reg_offset);1187 1188		if (se_num != 0xffffffff || sh_num != 0xffffffff)1189			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);1190		mutex_unlock(&adev->grbm_idx_mutex);1191		return val;1192	} else {1193		unsigned idx;1194 1195		switch (reg_offset) {1196		case mmGB_ADDR_CONFIG:1197			return adev->gfx.config.gb_addr_config;1198		case mmMC_ARB_RAMCFG:1199			return adev->gfx.config.mc_arb_ramcfg;1200		case mmGB_TILE_MODE0:1201		case mmGB_TILE_MODE1:1202		case mmGB_TILE_MODE2:1203		case mmGB_TILE_MODE3:1204		case mmGB_TILE_MODE4:1205		case mmGB_TILE_MODE5:1206		case mmGB_TILE_MODE6:1207		case mmGB_TILE_MODE7:1208		case mmGB_TILE_MODE8:1209		case mmGB_TILE_MODE9:1210		case mmGB_TILE_MODE10:1211		case mmGB_TILE_MODE11:1212		case mmGB_TILE_MODE12:1213		case mmGB_TILE_MODE13:1214		case mmGB_TILE_MODE14:1215		case mmGB_TILE_MODE15:1216		case mmGB_TILE_MODE16:1217		case mmGB_TILE_MODE17:1218		case mmGB_TILE_MODE18:1219		case mmGB_TILE_MODE19:1220		case mmGB_TILE_MODE20:1221		case mmGB_TILE_MODE21:1222		case mmGB_TILE_MODE22:1223		case mmGB_TILE_MODE23:1224		case mmGB_TILE_MODE24:1225		case mmGB_TILE_MODE25:1226		case mmGB_TILE_MODE26:1227		case mmGB_TILE_MODE27:1228		case mmGB_TILE_MODE28:1229		case mmGB_TILE_MODE29:1230		case mmGB_TILE_MODE30:1231		case mmGB_TILE_MODE31:1232			idx = (reg_offset - mmGB_TILE_MODE0);1233			return adev->gfx.config.tile_mode_array[idx];1234		default:1235			return RREG32(reg_offset);1236		}1237	}1238}1239static int si_read_register(struct amdgpu_device *adev, u32 se_num,1240			     u32 sh_num, u32 reg_offset, u32 *value)1241{1242	uint32_t i;1243 1244	*value = 0;1245	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {1246		bool indexed = si_allowed_read_registers[i].grbm_indexed;1247 1248		if (reg_offset != si_allowed_read_registers[i].reg_offset)1249			continue;1250 1251		*value = si_get_register_value(adev, indexed, se_num, sh_num,1252					       reg_offset);1253		return 0;1254	}1255	return -EINVAL;1256}1257 1258static bool si_read_disabled_bios(struct amdgpu_device *adev)1259{1260	u32 bus_cntl;1261	u32 d1vga_control = 0;1262	u32 d2vga_control = 0;1263	u32 vga_render_control = 0;1264	u32 rom_cntl;1265	bool r;1266 1267	bus_cntl = RREG32(R600_BUS_CNTL);1268	if (adev->mode_info.num_crtc) {1269		d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);1270		d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);1271		vga_render_control = RREG32(VGA_RENDER_CONTROL);1272	}1273	rom_cntl = RREG32(R600_ROM_CNTL);1274 1275	/* enable the rom */1276	WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));1277	if (adev->mode_info.num_crtc) {1278		/* Disable VGA mode */1279		WREG32(AVIVO_D1VGA_CONTROL,1280		       (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |1281					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));1282		WREG32(AVIVO_D2VGA_CONTROL,1283		       (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |1284					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));1285		WREG32(VGA_RENDER_CONTROL,1286		       (vga_render_control & C_000300_VGA_VSTATUS_CNTL));1287	}1288	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);1289 1290	r = amdgpu_read_bios(adev);1291 1292	/* restore regs */1293	WREG32(R600_BUS_CNTL, bus_cntl);1294	if (adev->mode_info.num_crtc) {1295		WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);1296		WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);1297		WREG32(VGA_RENDER_CONTROL, vga_render_control);1298	}1299	WREG32(R600_ROM_CNTL, rom_cntl);1300	return r;1301}1302 1303#define mmROM_INDEX 0x2A1304#define mmROM_DATA  0x2B1305 1306static bool si_read_bios_from_rom(struct amdgpu_device *adev,1307				  u8 *bios, u32 length_bytes)1308{1309	u32 *dw_ptr;1310	u32 i, length_dw;1311 1312	if (bios == NULL)1313		return false;1314	if (length_bytes == 0)1315		return false;1316	/* APU vbios image is part of sbios image */1317	if (adev->flags & AMD_IS_APU)1318		return false;1319 1320	dw_ptr = (u32 *)bios;1321	length_dw = ALIGN(length_bytes, 4) / 4;1322	/* set rom index to 0 */1323	WREG32(mmROM_INDEX, 0);1324	for (i = 0; i < length_dw; i++)1325		dw_ptr[i] = RREG32(mmROM_DATA);1326 1327	return true;1328}1329 1330static void si_set_clk_bypass_mode(struct amdgpu_device *adev)1331{1332	u32 tmp, i;1333 1334	tmp = RREG32(CG_SPLL_FUNC_CNTL);1335	tmp |= SPLL_BYPASS_EN;1336	WREG32(CG_SPLL_FUNC_CNTL, tmp);1337 1338	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);1339	tmp |= SPLL_CTLREQ_CHG;1340	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);1341 1342	for (i = 0; i < adev->usec_timeout; i++) {1343		if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)1344			break;1345		udelay(1);1346	}1347 1348	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);1349	tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);1350	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);1351 1352	tmp = RREG32(MPLL_CNTL_MODE);1353	tmp &= ~MPLL_MCLK_SEL;1354	WREG32(MPLL_CNTL_MODE, tmp);1355}1356 1357static void si_spll_powerdown(struct amdgpu_device *adev)1358{1359	u32 tmp;1360 1361	tmp = RREG32(SPLL_CNTL_MODE);1362	tmp |= SPLL_SW_DIR_CONTROL;1363	WREG32(SPLL_CNTL_MODE, tmp);1364 1365	tmp = RREG32(CG_SPLL_FUNC_CNTL);1366	tmp |= SPLL_RESET;1367	WREG32(CG_SPLL_FUNC_CNTL, tmp);1368 1369	tmp = RREG32(CG_SPLL_FUNC_CNTL);1370	tmp |= SPLL_SLEEP;1371	WREG32(CG_SPLL_FUNC_CNTL, tmp);1372 1373	tmp = RREG32(SPLL_CNTL_MODE);1374	tmp &= ~SPLL_SW_DIR_CONTROL;1375	WREG32(SPLL_CNTL_MODE, tmp);1376}1377 1378static int si_gpu_pci_config_reset(struct amdgpu_device *adev)1379{1380	u32 i;1381	int r = -EINVAL;1382 1383	amdgpu_atombios_scratch_regs_engine_hung(adev, true);1384 1385	/* set mclk/sclk to bypass */1386	si_set_clk_bypass_mode(adev);1387	/* powerdown spll */1388	si_spll_powerdown(adev);1389	/* disable BM */1390	pci_clear_master(adev->pdev);1391	/* reset */1392	amdgpu_device_pci_config_reset(adev);1393 1394	udelay(100);1395 1396	/* wait for asic to come out of reset */1397	for (i = 0; i < adev->usec_timeout; i++) {1398		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {1399			/* enable BM */1400			pci_set_master(adev->pdev);1401			adev->has_hw_reset = true;1402			r = 0;1403			break;1404		}1405		udelay(1);1406	}1407	amdgpu_atombios_scratch_regs_engine_hung(adev, false);1408 1409	return r;1410}1411 1412static int si_asic_supports_baco(struct amdgpu_device *adev)1413{1414	return 0;1415}1416 1417static enum amd_reset_method1418si_asic_reset_method(struct amdgpu_device *adev)1419{1420	if (amdgpu_reset_method == AMD_RESET_METHOD_PCI)1421		return amdgpu_reset_method;1422	else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&1423		 amdgpu_reset_method != -1)1424		dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",1425			 amdgpu_reset_method);1426 1427	return AMD_RESET_METHOD_LEGACY;1428}1429 1430static int si_asic_reset(struct amdgpu_device *adev)1431{1432	int r;1433 1434	switch (si_asic_reset_method(adev)) {1435	case AMD_RESET_METHOD_PCI:1436		dev_info(adev->dev, "PCI reset\n");1437		r = amdgpu_device_pci_reset(adev);1438		break;1439	default:1440		dev_info(adev->dev, "PCI CONFIG reset\n");1441		r = si_gpu_pci_config_reset(adev);1442		break;1443	}1444 1445	return r;1446}1447 1448static u32 si_get_config_memsize(struct amdgpu_device *adev)1449{1450	return RREG32(mmCONFIG_MEMSIZE);1451}1452 1453static void si_vga_set_state(struct amdgpu_device *adev, bool state)1454{1455	uint32_t temp;1456 1457	temp = RREG32(CONFIG_CNTL);1458	if (!state) {1459		temp &= ~(1<<0);1460		temp |= (1<<1);1461	} else {1462		temp &= ~(1<<1);1463	}1464	WREG32(CONFIG_CNTL, temp);1465}1466 1467static u32 si_get_xclk(struct amdgpu_device *adev)1468{1469	u32 reference_clock = adev->clock.spll.reference_freq;1470	u32 tmp;1471 1472	tmp = RREG32(CG_CLKPIN_CNTL_2);1473	if (tmp & MUX_TCLK_TO_XCLK)1474		return TCLK;1475 1476	tmp = RREG32(CG_CLKPIN_CNTL);1477	if (tmp & XTALIN_DIVIDE)1478		return reference_clock / 4;1479 1480	return reference_clock;1481}1482 1483static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)1484{1485	if (!ring || !ring->funcs->emit_wreg) {1486		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);1487		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);1488	} else {1489		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);1490	}1491}1492 1493static void si_invalidate_hdp(struct amdgpu_device *adev,1494			      struct amdgpu_ring *ring)1495{1496	if (!ring || !ring->funcs->emit_wreg) {1497		WREG32(mmHDP_DEBUG0, 1);1498		RREG32(mmHDP_DEBUG0);1499	} else {1500		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);1501	}1502}1503 1504static bool si_need_full_reset(struct amdgpu_device *adev)1505{1506	/* change this when we support soft reset */1507	return true;1508}1509 1510static bool si_need_reset_on_init(struct amdgpu_device *adev)1511{1512	return false;1513}1514 1515static int si_get_pcie_lanes(struct amdgpu_device *adev)1516{1517	u32 link_width_cntl;1518 1519	if (adev->flags & AMD_IS_APU)1520		return 0;1521 1522	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);1523 1524	switch ((link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT) {1525	case LC_LINK_WIDTH_X1:1526		return 1;1527	case LC_LINK_WIDTH_X2:1528		return 2;1529	case LC_LINK_WIDTH_X4:1530		return 4;1531	case LC_LINK_WIDTH_X8:1532		return 8;1533	case LC_LINK_WIDTH_X0:1534	case LC_LINK_WIDTH_X16:1535	default:1536		return 16;1537	}1538}1539 1540static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)1541{1542	u32 link_width_cntl, mask;1543 1544	if (adev->flags & AMD_IS_APU)1545		return;1546 1547	switch (lanes) {1548	case 0:1549		mask = LC_LINK_WIDTH_X0;1550		break;1551	case 1:1552		mask = LC_LINK_WIDTH_X1;1553		break;1554	case 2:1555		mask = LC_LINK_WIDTH_X2;1556		break;1557	case 4:1558		mask = LC_LINK_WIDTH_X4;1559		break;1560	case 8:1561		mask = LC_LINK_WIDTH_X8;1562		break;1563	case 16:1564		mask = LC_LINK_WIDTH_X16;1565		break;1566	default:1567		DRM_ERROR("invalid pcie lane request: %d\n", lanes);1568		return;1569	}1570 1571	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);1572	link_width_cntl &= ~LC_LINK_WIDTH_MASK;1573	link_width_cntl |= mask << LC_LINK_WIDTH_SHIFT;1574	link_width_cntl |= (LC_RECONFIG_NOW |1575			    LC_RECONFIG_ARC_MISSING_ESCAPE);1576 1577	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);1578}1579 1580static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,1581			      uint64_t *count1)1582{1583	uint32_t perfctr = 0;1584	uint64_t cnt0_of, cnt1_of;1585	int tmp;1586 1587	/* This reports 0 on APUs, so return to avoid writing/reading registers1588	 * that may or may not be different from their GPU counterparts1589	 */1590	if (adev->flags & AMD_IS_APU)1591		return;1592 1593	/* Set the 2 events that we wish to watch, defined above */1594	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */1595	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);1596	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);1597 1598	/* Write to enable desired perf counters */1599	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);1600	/* Zero out and enable the perf counters1601	 * Write 0x5:1602	 * Bit 0 = Start all counters(1)1603	 * Bit 2 = Global counter reset enable(1)1604	 */1605	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);1606 1607	msleep(1000);1608 1609	/* Load the shadow and disable the perf counters1610	 * Write 0x2:1611	 * Bit 0 = Stop counters(0)1612	 * Bit 1 = Load the shadow counters(1)1613	 */1614	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);1615 1616	/* Read register values to get any >32bit overflow */1617	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);1618	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);1619	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);1620 1621	/* Get the values and add the overflow */1622	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);1623	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);1624}1625 1626static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)1627{1628	uint64_t nak_r, nak_g;1629 1630	/* Get the number of NAKs received and generated */1631	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);1632	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);1633 1634	/* Add the total number of NAKs, i.e the number of replays */1635	return (nak_r + nak_g);1636}1637 1638static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,1639				   unsigned cg_upll_func_cntl)1640{1641	unsigned i;1642 1643	/* Make sure UPLL_CTLREQ is deasserted */1644	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);1645 1646	mdelay(10);1647 1648	/* Assert UPLL_CTLREQ */1649	WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);1650 1651	/* Wait for CTLACK and CTLACK2 to get asserted */1652	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {1653		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;1654 1655		if ((RREG32(cg_upll_func_cntl) & mask) == mask)1656			break;1657		mdelay(10);1658	}1659 1660	/* Deassert UPLL_CTLREQ */1661	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);1662 1663	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {1664		DRM_ERROR("Timeout setting UVD clocks!\n");1665		return -ETIMEDOUT;1666	}1667 1668	return 0;1669}1670 1671static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,1672					  unsigned target_freq,1673					  unsigned pd_min,1674					  unsigned pd_even)1675{1676	unsigned post_div = vco_freq / target_freq;1677 1678	/* Adjust to post divider minimum value */1679	if (post_div < pd_min)1680		post_div = pd_min;1681 1682	/* We alway need a frequency less than or equal the target */1683	if ((vco_freq / post_div) > target_freq)1684		post_div += 1;1685 1686	/* Post dividers above a certain value must be even */1687	if (post_div > pd_even && post_div % 2)1688		post_div += 1;1689 1690	return post_div;1691}1692 1693/**1694 * si_calc_upll_dividers - calc UPLL clock dividers1695 *1696 * @adev: amdgpu_device pointer1697 * @vclk: wanted VCLK1698 * @dclk: wanted DCLK1699 * @vco_min: minimum VCO frequency1700 * @vco_max: maximum VCO frequency1701 * @fb_factor: factor to multiply vco freq with1702 * @fb_mask: limit and bitmask for feedback divider1703 * @pd_min: post divider minimum1704 * @pd_max: post divider maximum1705 * @pd_even: post divider must be even above this value1706 * @optimal_fb_div: resulting feedback divider1707 * @optimal_vclk_div: resulting vclk post divider1708 * @optimal_dclk_div: resulting dclk post divider1709 *1710 * Calculate dividers for UVDs UPLL (except APUs).1711 * Returns zero on success; -EINVAL on error.1712 */1713static int si_calc_upll_dividers(struct amdgpu_device *adev,1714				 unsigned vclk, unsigned dclk,1715				 unsigned vco_min, unsigned vco_max,1716				 unsigned fb_factor, unsigned fb_mask,1717				 unsigned pd_min, unsigned pd_max,1718				 unsigned pd_even,1719				 unsigned *optimal_fb_div,1720				 unsigned *optimal_vclk_div,1721				 unsigned *optimal_dclk_div)1722{1723	unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;1724 1725	/* Start off with something large */1726	unsigned optimal_score = ~0;1727 1728	/* Loop through vco from low to high */1729	vco_min = max(max(vco_min, vclk), dclk);1730	for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {1731		uint64_t fb_div = (uint64_t)vco_freq * fb_factor;1732		unsigned vclk_div, dclk_div, score;1733 1734		do_div(fb_div, ref_freq);1735 1736		/* fb div out of range ? */1737		if (fb_div > fb_mask)1738			break; /* It can oly get worse */1739 1740		fb_div &= fb_mask;1741 1742		/* Calc vclk divider with current vco freq */1743		vclk_div = si_uvd_calc_upll_post_div(vco_freq, vclk,1744						     pd_min, pd_even);1745		if (vclk_div > pd_max)1746			break; /* vco is too big, it has to stop */1747 1748		/* Calc dclk divider with current vco freq */1749		dclk_div = si_uvd_calc_upll_post_div(vco_freq, dclk,1750						     pd_min, pd_even);1751		if (dclk_div > pd_max)1752			break; /* vco is too big, it has to stop */1753 1754		/* Calc score with current vco freq */1755		score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);1756 1757		/* Determine if this vco setting is better than current optimal settings */1758		if (score < optimal_score) {1759			*optimal_fb_div = fb_div;1760			*optimal_vclk_div = vclk_div;1761			*optimal_dclk_div = dclk_div;1762			optimal_score = score;1763			if (optimal_score == 0)1764				break; /* It can't get better than this */1765		}1766	}1767 1768	/* Did we found a valid setup ? */1769	if (optimal_score == ~0)1770		return -EINVAL;1771 1772	return 0;1773}1774 1775static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)1776{1777	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;1778	int r;1779 1780	/* Bypass vclk and dclk with bclk */1781	WREG32_P(CG_UPLL_FUNC_CNTL_2,1782		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),1783		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));1784 1785	/* Put PLL in bypass mode */1786	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);1787 1788	if (!vclk || !dclk) {1789		/* Keep the Bypass mode */1790		return 0;1791	}1792 1793	r = si_calc_upll_dividers(adev, vclk, dclk, 125000, 250000,1794				  16384, 0x03FFFFFF, 0, 128, 5,1795				  &fb_div, &vclk_div, &dclk_div);1796	if (r)1797		return r;1798 1799	/* Set RESET_ANTI_MUX to 0 */1800	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);1801 1802	/* Set VCO_MODE to 1 */1803	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);1804 1805	/* Disable sleep mode */1806	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);1807 1808	/* Deassert UPLL_RESET */1809	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);1810 1811	mdelay(1);1812 1813	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);1814	if (r)1815		return r;1816 1817	/* Assert UPLL_RESET again */1818	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);1819 1820	/* Disable spread spectrum. */1821	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);1822 1823	/* Set feedback divider */1824	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);1825 1826	/* Set ref divider to 0 */1827	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);1828 1829	if (fb_div < 307200)1830		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);1831	else1832		WREG32_P(CG_UPLL_FUNC_CNTL_4,1833			 UPLL_SPARE_ISPARE9,1834			 ~UPLL_SPARE_ISPARE9);1835 1836	/* Set PDIV_A and PDIV_B */1837	WREG32_P(CG_UPLL_FUNC_CNTL_2,1838		 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),1839		 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));1840 1841	/* Give the PLL some time to settle */1842	mdelay(15);1843 1844	/* Deassert PLL_RESET */1845	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);1846 1847	mdelay(15);1848 1849	/* Switch from bypass mode to normal mode */1850	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);1851 1852	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);1853	if (r)1854		return r;1855 1856	/* Switch VCLK and DCLK selection */1857	WREG32_P(CG_UPLL_FUNC_CNTL_2,1858		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),1859		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));1860 1861	mdelay(100);1862 1863	return 0;1864}1865 1866static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)1867{1868	unsigned i;1869 1870	/* Make sure VCEPLL_CTLREQ is deasserted */1871	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);1872 1873	mdelay(10);1874 1875	/* Assert UPLL_CTLREQ */1876	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);1877 1878	/* Wait for CTLACK and CTLACK2 to get asserted */1879	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {1880		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;1881 1882		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)1883			break;1884		mdelay(10);1885	}1886 1887	/* Deassert UPLL_CTLREQ */1888	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);1889 1890	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {1891		DRM_ERROR("Timeout setting UVD clocks!\n");1892		return -ETIMEDOUT;1893	}1894 1895	return 0;1896}1897 1898static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)1899{1900	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;1901	int r;1902 1903	/* Bypass evclk and ecclk with bclk */1904	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,1905		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),1906		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));1907 1908	/* Put PLL in bypass mode */1909	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,1910		     ~VCEPLL_BYPASS_EN_MASK);1911 1912	if (!evclk || !ecclk) {1913		/* Keep the Bypass mode, put PLL to sleep */1914		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,1915			     ~VCEPLL_SLEEP_MASK);1916		return 0;1917	}1918 1919	r = si_calc_upll_dividers(adev, evclk, ecclk, 125000, 250000,1920				  16384, 0x03FFFFFF, 0, 128, 5,1921				  &fb_div, &evclk_div, &ecclk_div);1922	if (r)1923		return r;1924 1925	/* Set RESET_ANTI_MUX to 0 */1926	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);1927 1928	/* Set VCO_MODE to 1 */1929	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,1930		     ~VCEPLL_VCO_MODE_MASK);1931 1932	/* Toggle VCEPLL_SLEEP to 1 then back to 0 */1933	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,1934		     ~VCEPLL_SLEEP_MASK);1935	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);1936 1937	/* Deassert VCEPLL_RESET */1938	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);1939 1940	mdelay(1);1941 1942	r = si_vce_send_vcepll_ctlreq(adev);1943	if (r)1944		return r;1945 1946	/* Assert VCEPLL_RESET again */1947	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);1948 1949	/* Disable spread spectrum. */1950	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);1951 1952	/* Set feedback divider */1953	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,1954		     VCEPLL_FB_DIV(fb_div),1955		     ~VCEPLL_FB_DIV_MASK);1956 1957	/* Set ref divider to 0 */1958	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);1959 1960	/* Set PDIV_A and PDIV_B */1961	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,1962		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),1963		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));1964 1965	/* Give the PLL some time to settle */1966	mdelay(15);1967 1968	/* Deassert PLL_RESET */1969	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);1970 1971	mdelay(15);1972 1973	/* Switch from bypass mode to normal mode */1974	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);1975 1976	r = si_vce_send_vcepll_ctlreq(adev);1977	if (r)1978		return r;1979 1980	/* Switch VCLK and DCLK selection */1981	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,1982		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),1983		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));1984 1985	mdelay(100);1986 1987	return 0;1988}1989 1990static void si_pre_asic_init(struct amdgpu_device *adev)1991{1992}1993 1994static const struct amdgpu_asic_funcs si_asic_funcs =1995{1996	.read_disabled_bios = &si_read_disabled_bios,1997	.read_bios_from_rom = &si_read_bios_from_rom,1998	.read_register = &si_read_register,1999	.reset = &si_asic_reset,2000	.reset_method = &si_asic_reset_method,2001	.set_vga_state = &si_vga_set_state,2002	.get_xclk = &si_get_xclk,2003	.set_uvd_clocks = &si_set_uvd_clocks,2004	.set_vce_clocks = &si_set_vce_clocks,2005	.get_pcie_lanes = &si_get_pcie_lanes,2006	.set_pcie_lanes = &si_set_pcie_lanes,2007	.get_config_memsize = &si_get_config_memsize,2008	.flush_hdp = &si_flush_hdp,2009	.invalidate_hdp = &si_invalidate_hdp,2010	.need_full_reset = &si_need_full_reset,2011	.get_pcie_usage = &si_get_pcie_usage,2012	.need_reset_on_init = &si_need_reset_on_init,2013	.get_pcie_replay_count = &si_get_pcie_replay_count,2014	.supports_baco = &si_asic_supports_baco,2015	.pre_asic_init = &si_pre_asic_init,2016	.query_video_codecs = &si_query_video_codecs,2017};2018 2019static uint32_t si_get_rev_id(struct amdgpu_device *adev)2020{2021	return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)2022		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;2023}2024 2025static int si_common_early_init(void *handle)2026{2027	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2028 2029	adev->smc_rreg = &si_smc_rreg;2030	adev->smc_wreg = &si_smc_wreg;2031	adev->pcie_rreg = &si_pcie_rreg;2032	adev->pcie_wreg = &si_pcie_wreg;2033	adev->pciep_rreg = &si_pciep_rreg;2034	adev->pciep_wreg = &si_pciep_wreg;2035	adev->uvd_ctx_rreg = si_uvd_ctx_rreg;2036	adev->uvd_ctx_wreg = si_uvd_ctx_wreg;2037	adev->didt_rreg = NULL;2038	adev->didt_wreg = NULL;2039 2040	adev->asic_funcs = &si_asic_funcs;2041 2042	adev->rev_id = si_get_rev_id(adev);2043	adev->external_rev_id = 0xFF;2044	switch (adev->asic_type) {2045	case CHIP_TAHITI:2046		adev->cg_flags =2047			AMD_CG_SUPPORT_GFX_MGCG |2048			AMD_CG_SUPPORT_GFX_MGLS |2049			/*AMD_CG_SUPPORT_GFX_CGCG |*/2050			AMD_CG_SUPPORT_GFX_CGLS |2051			AMD_CG_SUPPORT_GFX_CGTS |2052			AMD_CG_SUPPORT_GFX_CP_LS |2053			AMD_CG_SUPPORT_MC_MGCG |2054			AMD_CG_SUPPORT_SDMA_MGCG |2055			AMD_CG_SUPPORT_BIF_LS |2056			AMD_CG_SUPPORT_VCE_MGCG |2057			AMD_CG_SUPPORT_UVD_MGCG |2058			AMD_CG_SUPPORT_HDP_LS |2059			AMD_CG_SUPPORT_HDP_MGCG;2060		adev->pg_flags = 0;2061		adev->external_rev_id = (adev->rev_id == 0) ? 1 :2062					(adev->rev_id == 1) ? 5 : 6;2063		break;2064	case CHIP_PITCAIRN:2065		adev->cg_flags =2066			AMD_CG_SUPPORT_GFX_MGCG |2067			AMD_CG_SUPPORT_GFX_MGLS |2068			/*AMD_CG_SUPPORT_GFX_CGCG |*/2069			AMD_CG_SUPPORT_GFX_CGLS |2070			AMD_CG_SUPPORT_GFX_CGTS |2071			AMD_CG_SUPPORT_GFX_CP_LS |2072			AMD_CG_SUPPORT_GFX_RLC_LS |2073			AMD_CG_SUPPORT_MC_LS |2074			AMD_CG_SUPPORT_MC_MGCG |2075			AMD_CG_SUPPORT_SDMA_MGCG |2076			AMD_CG_SUPPORT_BIF_LS |2077			AMD_CG_SUPPORT_VCE_MGCG |2078			AMD_CG_SUPPORT_UVD_MGCG |2079			AMD_CG_SUPPORT_HDP_LS |2080			AMD_CG_SUPPORT_HDP_MGCG;2081		adev->pg_flags = 0;2082		adev->external_rev_id = adev->rev_id + 20;2083		break;2084 2085	case CHIP_VERDE:2086		adev->cg_flags =2087			AMD_CG_SUPPORT_GFX_MGCG |2088			AMD_CG_SUPPORT_GFX_MGLS |2089			AMD_CG_SUPPORT_GFX_CGLS |2090			AMD_CG_SUPPORT_GFX_CGTS |2091			AMD_CG_SUPPORT_GFX_CGTS_LS |2092			AMD_CG_SUPPORT_GFX_CP_LS |2093			AMD_CG_SUPPORT_MC_LS |2094			AMD_CG_SUPPORT_MC_MGCG |2095			AMD_CG_SUPPORT_SDMA_MGCG |2096			AMD_CG_SUPPORT_SDMA_LS |2097			AMD_CG_SUPPORT_BIF_LS |2098			AMD_CG_SUPPORT_VCE_MGCG |2099			AMD_CG_SUPPORT_UVD_MGCG |2100			AMD_CG_SUPPORT_HDP_LS |2101			AMD_CG_SUPPORT_HDP_MGCG;2102		adev->pg_flags = 0;2103		//???2104		adev->external_rev_id = adev->rev_id + 40;2105		break;2106	case CHIP_OLAND:2107		adev->cg_flags =2108			AMD_CG_SUPPORT_GFX_MGCG |2109			AMD_CG_SUPPORT_GFX_MGLS |2110			/*AMD_CG_SUPPORT_GFX_CGCG |*/2111			AMD_CG_SUPPORT_GFX_CGLS |2112			AMD_CG_SUPPORT_GFX_CGTS |2113			AMD_CG_SUPPORT_GFX_CP_LS |2114			AMD_CG_SUPPORT_GFX_RLC_LS |2115			AMD_CG_SUPPORT_MC_LS |2116			AMD_CG_SUPPORT_MC_MGCG |2117			AMD_CG_SUPPORT_SDMA_MGCG |2118			AMD_CG_SUPPORT_BIF_LS |2119			AMD_CG_SUPPORT_UVD_MGCG |2120			AMD_CG_SUPPORT_HDP_LS |2121			AMD_CG_SUPPORT_HDP_MGCG;2122		adev->pg_flags = 0;2123		adev->external_rev_id = 60;2124		break;2125	case CHIP_HAINAN:2126		adev->cg_flags =2127			AMD_CG_SUPPORT_GFX_MGCG |2128			AMD_CG_SUPPORT_GFX_MGLS |2129			/*AMD_CG_SUPPORT_GFX_CGCG |*/2130			AMD_CG_SUPPORT_GFX_CGLS |2131			AMD_CG_SUPPORT_GFX_CGTS |2132			AMD_CG_SUPPORT_GFX_CP_LS |2133			AMD_CG_SUPPORT_GFX_RLC_LS |2134			AMD_CG_SUPPORT_MC_LS |2135			AMD_CG_SUPPORT_MC_MGCG |2136			AMD_CG_SUPPORT_SDMA_MGCG |2137			AMD_CG_SUPPORT_BIF_LS |2138			AMD_CG_SUPPORT_HDP_LS |2139			AMD_CG_SUPPORT_HDP_MGCG;2140		adev->pg_flags = 0;2141		adev->external_rev_id = 70;2142		break;2143 2144	default:2145		return -EINVAL;2146	}2147 2148	return 0;2149}2150 2151static int si_common_sw_init(void *handle)2152{2153	return 0;2154}2155 2156static int si_common_sw_fini(void *handle)2157{2158	return 0;2159}2160 2161 2162static void si_init_golden_registers(struct amdgpu_device *adev)2163{2164	switch (adev->asic_type) {2165	case CHIP_TAHITI:2166		amdgpu_device_program_register_sequence(adev,2167							tahiti_golden_registers,2168							ARRAY_SIZE(tahiti_golden_registers));2169		amdgpu_device_program_register_sequence(adev,2170							tahiti_golden_rlc_registers,2171							ARRAY_SIZE(tahiti_golden_rlc_registers));2172		amdgpu_device_program_register_sequence(adev,2173							tahiti_mgcg_cgcg_init,2174							ARRAY_SIZE(tahiti_mgcg_cgcg_init));2175		amdgpu_device_program_register_sequence(adev,2176							tahiti_golden_registers2,2177							ARRAY_SIZE(tahiti_golden_registers2));2178		break;2179	case CHIP_PITCAIRN:2180		amdgpu_device_program_register_sequence(adev,2181							pitcairn_golden_registers,2182							ARRAY_SIZE(pitcairn_golden_registers));2183		amdgpu_device_program_register_sequence(adev,2184							pitcairn_golden_rlc_registers,2185							ARRAY_SIZE(pitcairn_golden_rlc_registers));2186		amdgpu_device_program_register_sequence(adev,2187							pitcairn_mgcg_cgcg_init,2188							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));2189		break;2190	case CHIP_VERDE:2191		amdgpu_device_program_register_sequence(adev,2192							verde_golden_registers,2193							ARRAY_SIZE(verde_golden_registers));2194		amdgpu_device_program_register_sequence(adev,2195							verde_golden_rlc_registers,2196							ARRAY_SIZE(verde_golden_rlc_registers));2197		amdgpu_device_program_register_sequence(adev,2198							verde_mgcg_cgcg_init,2199							ARRAY_SIZE(verde_mgcg_cgcg_init));2200		amdgpu_device_program_register_sequence(adev,2201							verde_pg_init,2202							ARRAY_SIZE(verde_pg_init));2203		break;2204	case CHIP_OLAND:2205		amdgpu_device_program_register_sequence(adev,2206							oland_golden_registers,2207							ARRAY_SIZE(oland_golden_registers));2208		amdgpu_device_program_register_sequence(adev,2209							oland_golden_rlc_registers,2210							ARRAY_SIZE(oland_golden_rlc_registers));2211		amdgpu_device_program_register_sequence(adev,2212							oland_mgcg_cgcg_init,2213							ARRAY_SIZE(oland_mgcg_cgcg_init));2214		break;2215	case CHIP_HAINAN:2216		amdgpu_device_program_register_sequence(adev,2217							hainan_golden_registers,2218							ARRAY_SIZE(hainan_golden_registers));2219		amdgpu_device_program_register_sequence(adev,2220							hainan_golden_registers2,2221							ARRAY_SIZE(hainan_golden_registers2));2222		amdgpu_device_program_register_sequence(adev,2223							hainan_mgcg_cgcg_init,2224							ARRAY_SIZE(hainan_mgcg_cgcg_init));2225		break;2226 2227 2228	default:2229		BUG();2230	}2231}2232 2233static void si_pcie_gen3_enable(struct amdgpu_device *adev)2234{2235	struct pci_dev *root = adev->pdev->bus->self;2236	u32 speed_cntl, current_data_rate;2237	int i;2238	u16 tmp16;2239 2240	if (pci_is_root_bus(adev->pdev->bus))2241		return;2242 2243	if (amdgpu_pcie_gen2 == 0)2244		return;2245 2246	if (adev->flags & AMD_IS_APU)2247		return;2248 2249	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |2250					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))2251		return;2252 2253	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);2254	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>2255		LC_CURRENT_DATA_RATE_SHIFT;2256	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {2257		if (current_data_rate == 2) {2258			DRM_INFO("PCIE gen 3 link speeds already enabled\n");2259			return;2260		}2261		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");2262	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {2263		if (current_data_rate == 1) {2264			DRM_INFO("PCIE gen 2 link speeds already enabled\n");2265			return;2266		}2267		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");2268	}2269 2270	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))2271		return;2272 2273	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {2274		if (current_data_rate != 2) {2275			u16 bridge_cfg, gpu_cfg;2276			u16 bridge_cfg2, gpu_cfg2;2277			u32 max_lw, current_lw, tmp;2278 2279			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);2280			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);2281 2282			tmp = RREG32_PCIE(PCIE_LC_STATUS1);2283			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;2284			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;2285 2286			if (current_lw < max_lw) {2287				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);2288				if (tmp & LC_RENEGOTIATION_SUPPORT) {2289					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);2290					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);2291					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;2292					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);2293				}2294			}2295 2296			for (i = 0; i < 10; i++) {2297				pcie_capability_read_word(adev->pdev,2298							  PCI_EXP_DEVSTA,2299							  &tmp16);2300				if (tmp16 & PCI_EXP_DEVSTA_TRPND)2301					break;2302 2303				pcie_capability_read_word(root, PCI_EXP_LNKCTL,2304							  &bridge_cfg);2305				pcie_capability_read_word(adev->pdev,2306							  PCI_EXP_LNKCTL,2307							  &gpu_cfg);2308 2309				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,2310							  &bridge_cfg2);2311				pcie_capability_read_word(adev->pdev,2312							  PCI_EXP_LNKCTL2,2313							  &gpu_cfg2);2314 2315				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);2316				tmp |= LC_SET_QUIESCE;2317				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);2318 2319				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);2320				tmp |= LC_REDO_EQ;2321				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);2322 2323				mdelay(100);2324 2325				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,2326								   PCI_EXP_LNKCTL_HAWD,2327								   bridge_cfg &2328								   PCI_EXP_LNKCTL_HAWD);2329				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,2330								   PCI_EXP_LNKCTL_HAWD,2331								   gpu_cfg &2332								   PCI_EXP_LNKCTL_HAWD);2333 2334				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,2335								   PCI_EXP_LNKCTL2_ENTER_COMP |2336								   PCI_EXP_LNKCTL2_TX_MARGIN,2337								   bridge_cfg2 &2338								   (PCI_EXP_LNKCTL2_ENTER_COMP |2339								    PCI_EXP_LNKCTL2_TX_MARGIN));2340				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,2341								   PCI_EXP_LNKCTL2_ENTER_COMP |2342								   PCI_EXP_LNKCTL2_TX_MARGIN,2343								   gpu_cfg2 &2344								   (PCI_EXP_LNKCTL2_ENTER_COMP |2345								    PCI_EXP_LNKCTL2_TX_MARGIN));2346 2347				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);2348				tmp &= ~LC_SET_QUIESCE;2349				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);2350			}2351		}2352	}2353 2354	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;2355	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;2356	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);2357 2358	tmp16 = 0;2359	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)2360		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */2361	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)2362		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */2363	else2364		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */2365	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,2366					   PCI_EXP_LNKCTL2_TLS, tmp16);2367 2368	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);2369	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;2370	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);2371 2372	for (i = 0; i < adev->usec_timeout; i++) {2373		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);2374		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)2375			break;2376		udelay(1);2377	}2378}2379 2380static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)2381{2382	unsigned long flags;2383	u32 r;2384 2385	spin_lock_irqsave(&adev->pcie_idx_lock, flags);2386	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));2387	r = RREG32(EVERGREEN_PIF_PHY0_DATA);2388	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);2389	return r;2390}2391 2392static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)2393{2394	unsigned long flags;2395 2396	spin_lock_irqsave(&adev->pcie_idx_lock, flags);2397	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));2398	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));2399	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);2400}2401 2402static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)2403{2404	unsigned long flags;2405	u32 r;2406 2407	spin_lock_irqsave(&adev->pcie_idx_lock, flags);2408	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));2409	r = RREG32(EVERGREEN_PIF_PHY1_DATA);2410	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);2411	return r;2412}2413 2414static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)2415{2416	unsigned long flags;2417 2418	spin_lock_irqsave(&adev->pcie_idx_lock, flags);2419	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));2420	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));2421	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);2422}2423static void si_program_aspm(struct amdgpu_device *adev)2424{2425	u32 data, orig;2426	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;2427	bool disable_clkreq = false;2428 2429	if (!amdgpu_device_should_use_aspm(adev))2430		return;2431 2432	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);2433	data &= ~LC_XMIT_N_FTS_MASK;2434	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;2435	if (orig != data)2436		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);2437 2438	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);2439	data |= LC_GO_TO_RECOVERY;2440	if (orig != data)2441		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);2442 2443	orig = data = RREG32_PCIE(PCIE_P_CNTL);2444	data |= P_IGNORE_EDB_ERR;2445	if (orig != data)2446		WREG32_PCIE(PCIE_P_CNTL, data);2447 2448	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);2449	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);2450	data |= LC_PMI_TO_L1_DIS;2451	if (!disable_l0s)2452		data |= LC_L0S_INACTIVITY(7);2453 2454	if (!disable_l1) {2455		data |= LC_L1_INACTIVITY(7);2456		data &= ~LC_PMI_TO_L1_DIS;2457		if (orig != data)2458			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);2459 2460		if (!disable_plloff_in_l1) {2461			bool clk_req_support;2462 2463			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);2464			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);2465			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);2466			if (orig != data)2467				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);2468 2469			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);2470			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);2471			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);2472			if (orig != data)2473				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);2474 2475			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);2476			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);2477			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);2478			if (orig != data)2479				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);2480 2481			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);2482			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);2483			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);2484			if (orig != data)2485				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);2486 2487			if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {2488				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);2489				data &= ~PLL_RAMP_UP_TIME_0_MASK;2490				if (orig != data)2491					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);2492 2493				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);2494				data &= ~PLL_RAMP_UP_TIME_1_MASK;2495				if (orig != data)2496					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);2497 2498				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);2499				data &= ~PLL_RAMP_UP_TIME_2_MASK;2500				if (orig != data)2501					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);2502 2503				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);2504				data &= ~PLL_RAMP_UP_TIME_3_MASK;2505				if (orig != data)2506					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);2507 2508				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);2509				data &= ~PLL_RAMP_UP_TIME_0_MASK;2510				if (orig != data)2511					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);2512 2513				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);2514				data &= ~PLL_RAMP_UP_TIME_1_MASK;2515				if (orig != data)2516					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);2517 2518				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);2519				data &= ~PLL_RAMP_UP_TIME_2_MASK;2520				if (orig != data)2521					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);2522 2523				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);2524				data &= ~PLL_RAMP_UP_TIME_3_MASK;2525				if (orig != data)2526					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);2527			}2528			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);2529			data &= ~LC_DYN_LANES_PWR_STATE_MASK;2530			data |= LC_DYN_LANES_PWR_STATE(3);2531			if (orig != data)2532				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);2533 2534			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);2535			data &= ~LS2_EXIT_TIME_MASK;2536			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))2537				data |= LS2_EXIT_TIME(5);2538			if (orig != data)2539				si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);2540 2541			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);2542			data &= ~LS2_EXIT_TIME_MASK;2543			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))2544				data |= LS2_EXIT_TIME(5);2545			if (orig != data)2546				si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);2547 2548			if (!disable_clkreq &&2549			    !pci_is_root_bus(adev->pdev->bus)) {2550				struct pci_dev *root = adev->pdev->bus->self;2551				u32 lnkcap;2552 2553				clk_req_support = false;2554				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);2555				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)2556					clk_req_support = true;2557			} else {2558				clk_req_support = false;2559			}2560 2561			if (clk_req_support) {2562				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);2563				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;2564				if (orig != data)2565					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);2566 2567				orig = data = RREG32(THM_CLK_CNTL);2568				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);2569				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);2570				if (orig != data)2571					WREG32(THM_CLK_CNTL, data);2572 2573				orig = data = RREG32(MISC_CLK_CNTL);2574				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);2575				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);2576				if (orig != data)2577					WREG32(MISC_CLK_CNTL, data);2578 2579				orig = data = RREG32(CG_CLKPIN_CNTL);2580				data &= ~BCLK_AS_XCLK;2581				if (orig != data)2582					WREG32(CG_CLKPIN_CNTL, data);2583 2584				orig = data = RREG32(CG_CLKPIN_CNTL_2);2585				data &= ~FORCE_BIF_REFCLK_EN;2586				if (orig != data)2587					WREG32(CG_CLKPIN_CNTL_2, data);2588 2589				orig = data = RREG32(MPLL_BYPASSCLK_SEL);2590				data &= ~MPLL_CLKOUT_SEL_MASK;2591				data |= MPLL_CLKOUT_SEL(4);2592				if (orig != data)2593					WREG32(MPLL_BYPASSCLK_SEL, data);2594 2595				orig = data = RREG32(SPLL_CNTL_MODE);2596				data &= ~SPLL_REFCLK_SEL_MASK;2597				if (orig != data)2598					WREG32(SPLL_CNTL_MODE, data);2599			}2600		}2601	} else {2602		if (orig != data)2603			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);2604	}2605 2606	orig = data = RREG32_PCIE(PCIE_CNTL2);2607	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;2608	if (orig != data)2609		WREG32_PCIE(PCIE_CNTL2, data);2610 2611	if (!disable_l0s) {2612		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);2613		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {2614			data = RREG32_PCIE(PCIE_LC_STATUS1);2615			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {2616				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);2617				data &= ~LC_L0S_INACTIVITY_MASK;2618				if (orig != data)2619					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);2620			}2621		}2622	}2623}2624 2625static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)2626{2627	int readrq;2628	u16 v;2629 2630	readrq = pcie_get_readrq(adev->pdev);2631	v = ffs(readrq) - 8;2632	if ((v == 0) || (v == 6) || (v == 7))2633		pcie_set_readrq(adev->pdev, 512);2634}2635 2636static int si_common_hw_init(void *handle)2637{2638	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2639 2640	si_fix_pci_max_read_req_size(adev);2641	si_init_golden_registers(adev);2642	si_pcie_gen3_enable(adev);2643	si_program_aspm(adev);2644 2645	return 0;2646}2647 2648static int si_common_hw_fini(void *handle)2649{2650	return 0;2651}2652 2653static int si_common_suspend(void *handle)2654{2655	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2656 2657	return si_common_hw_fini(adev);2658}2659 2660static int si_common_resume(void *handle)2661{2662	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2663 2664	return si_common_hw_init(adev);2665}2666 2667static bool si_common_is_idle(void *handle)2668{2669	return true;2670}2671 2672static int si_common_wait_for_idle(void *handle)2673{2674	return 0;2675}2676 2677static int si_common_soft_reset(void *handle)2678{2679	return 0;2680}2681 2682static int si_common_set_clockgating_state(void *handle,2683					    enum amd_clockgating_state state)2684{2685	return 0;2686}2687 2688static int si_common_set_powergating_state(void *handle,2689					    enum amd_powergating_state state)2690{2691	return 0;2692}2693 2694static const struct amd_ip_funcs si_common_ip_funcs = {2695	.name = "si_common",2696	.early_init = si_common_early_init,2697	.late_init = NULL,2698	.sw_init = si_common_sw_init,2699	.sw_fini = si_common_sw_fini,2700	.hw_init = si_common_hw_init,2701	.hw_fini = si_common_hw_fini,2702	.suspend = si_common_suspend,2703	.resume = si_common_resume,2704	.is_idle = si_common_is_idle,2705	.wait_for_idle = si_common_wait_for_idle,2706	.soft_reset = si_common_soft_reset,2707	.set_clockgating_state = si_common_set_clockgating_state,2708	.set_powergating_state = si_common_set_powergating_state,2709	.dump_ip_state = NULL,2710	.print_ip_state = NULL,2711};2712 2713static const struct amdgpu_ip_block_version si_common_ip_block =2714{2715	.type = AMD_IP_BLOCK_TYPE_COMMON,2716	.major = 1,2717	.minor = 0,2718	.rev = 0,2719	.funcs = &si_common_ip_funcs,2720};2721 2722int si_set_ip_blocks(struct amdgpu_device *adev)2723{2724	switch (adev->asic_type) {2725	case CHIP_VERDE:2726	case CHIP_TAHITI:2727	case CHIP_PITCAIRN:2728		amdgpu_device_ip_block_add(adev, &si_common_ip_block);2729		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);2730		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);2731		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);2732		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);2733		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);2734		if (adev->enable_virtual_display)2735			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);2736#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)2737		else if (amdgpu_device_has_dc_support(adev))2738			amdgpu_device_ip_block_add(adev, &dm_ip_block);2739#endif2740		else2741			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);2742		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);2743		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */2744		break;2745	case CHIP_OLAND:2746		amdgpu_device_ip_block_add(adev, &si_common_ip_block);2747		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);2748		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);2749		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);2750		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);2751		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);2752		if (adev->enable_virtual_display)2753			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);2754#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)2755		else if (amdgpu_device_has_dc_support(adev))2756			amdgpu_device_ip_block_add(adev, &dm_ip_block);2757#endif2758		else2759			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);2760		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);2761		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */2762		break;2763	case CHIP_HAINAN:2764		amdgpu_device_ip_block_add(adev, &si_common_ip_block);2765		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);2766		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);2767		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);2768		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);2769		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);2770		if (adev->enable_virtual_display)2771			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);2772		break;2773	default:2774		BUG();2775	}2776	return 0;2777}2778 2779