brintos / linux-shallow public Read only

1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/delay.h>25#include <linux/kernel.h>26#include <linux/firmware.h>27#include <linux/module.h>28#include <linux/pci.h>29 30#include "amdgpu.h"31#include "amdgpu_gfx.h"32#include "soc15.h"33#include "soc15d.h"34#include "amdgpu_atomfirmware.h"35#include "amdgpu_pm.h"36 37#include "gc/gc_9_0_offset.h"38#include "gc/gc_9_0_sh_mask.h"39 40#include "vega10_enum.h"41 42#include "soc15_common.h"43#include "clearstate_gfx9.h"44#include "v9_structs.h"45 46#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"47 48#include "amdgpu_ras.h"49 50#include "amdgpu_ring_mux.h"51#include "gfx_v9_4.h"52#include "gfx_v9_0.h"53#include "gfx_v9_0_cleaner_shader.h"54#include "gfx_v9_4_2.h"55 56#include "asic_reg/pwr/pwr_10_0_offset.h"57#include "asic_reg/pwr/pwr_10_0_sh_mask.h"58#include "asic_reg/gc/gc_9_0_default.h"59 60#define GFX9_NUM_GFX_RINGS     161#define GFX9_NUM_SW_GFX_RINGS  262#define GFX9_MEC_HPD_SIZE 409663#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L64#define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L65 66#define mmGCEA_PROBE_MAP                        0x070c67#define mmGCEA_PROBE_MAP_BASE_IDX               068 69MODULE_FIRMWARE("amdgpu/vega10_ce.bin");70MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");71MODULE_FIRMWARE("amdgpu/vega10_me.bin");72MODULE_FIRMWARE("amdgpu/vega10_mec.bin");73MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");74MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");75 76MODULE_FIRMWARE("amdgpu/vega12_ce.bin");77MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");78MODULE_FIRMWARE("amdgpu/vega12_me.bin");79MODULE_FIRMWARE("amdgpu/vega12_mec.bin");80MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");81MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");82 83MODULE_FIRMWARE("amdgpu/vega20_ce.bin");84MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");85MODULE_FIRMWARE("amdgpu/vega20_me.bin");86MODULE_FIRMWARE("amdgpu/vega20_mec.bin");87MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");88MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");89 90MODULE_FIRMWARE("amdgpu/raven_ce.bin");91MODULE_FIRMWARE("amdgpu/raven_pfp.bin");92MODULE_FIRMWARE("amdgpu/raven_me.bin");93MODULE_FIRMWARE("amdgpu/raven_mec.bin");94MODULE_FIRMWARE("amdgpu/raven_mec2.bin");95MODULE_FIRMWARE("amdgpu/raven_rlc.bin");96 97MODULE_FIRMWARE("amdgpu/picasso_ce.bin");98MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");99MODULE_FIRMWARE("amdgpu/picasso_me.bin");100MODULE_FIRMWARE("amdgpu/picasso_mec.bin");101MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");102MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");103MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");104 105MODULE_FIRMWARE("amdgpu/raven2_ce.bin");106MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");107MODULE_FIRMWARE("amdgpu/raven2_me.bin");108MODULE_FIRMWARE("amdgpu/raven2_mec.bin");109MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");110MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");111MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");112 113MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");114MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");115 116MODULE_FIRMWARE("amdgpu/renoir_ce.bin");117MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");118MODULE_FIRMWARE("amdgpu/renoir_me.bin");119MODULE_FIRMWARE("amdgpu/renoir_mec.bin");120MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");121 122MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");123MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");124MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");125MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");126MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");127MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");128 129MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");130MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");131MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");132MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");133MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");134 135#define mmTCP_CHAN_STEER_0_ARCT								0x0b03136#define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX							0137#define mmTCP_CHAN_STEER_1_ARCT								0x0b04138#define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX							0139#define mmTCP_CHAN_STEER_2_ARCT								0x0b09140#define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX							0141#define mmTCP_CHAN_STEER_3_ARCT								0x0b0a142#define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX							0143#define mmTCP_CHAN_STEER_4_ARCT								0x0b0b144#define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX							0145#define mmTCP_CHAN_STEER_5_ARCT								0x0b0c146#define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX							0147 148#define mmGOLDEN_TSC_COUNT_UPPER_Renoir                0x0025149#define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX       1150#define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026151#define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1152 153static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {154	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),155	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),156	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),157	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),158	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),159	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),160	SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),161	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),162	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),163	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),164	SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),165	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),166	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),167	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),168	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),169	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),170	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),171	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),172	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),173	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),174	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),175	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),176	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),177	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),178	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),179	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),180	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),181	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),182	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),183	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),184	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),185	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),186	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),187	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),188	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),189	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),190	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),191	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),192	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),193	SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),194	SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),195	SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),196	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),197	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),198	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),199	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),200	SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),201	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),202	SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),203	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),204	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),205	SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),206	SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),207	SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),208	SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),209	SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),210	SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),211	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),212	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),213	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),214	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),215	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),216	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),217	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),218	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),219	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),220	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),221	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),222	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),223	SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),224	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),225	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),226	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),227	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),228	/* cp header registers */229	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),230	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),231	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME2_HEADER_DUMP),232	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),233	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),234	/* SE status registers */235	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),236	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),237	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),238	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3)239};240 241static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {242	/* compute queue registers */243	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),244	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),245	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),246	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),247	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),248	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),249	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),250	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),251	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),252	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),253	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),254	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),255	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),256	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),257	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),258	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),259	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),260	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),261	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),262	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),263	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),264	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),265	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),266	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),267	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),268	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),269	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),270	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),271	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),272	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),273	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),274	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),275	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),276	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),277	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),278	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),279	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),280};281 282enum ta_ras_gfx_subblock {283	/*CPC*/284	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,285	TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,286	TA_RAS_BLOCK__GFX_CPC_UCODE,287	TA_RAS_BLOCK__GFX_DC_STATE_ME1,288	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,289	TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,290	TA_RAS_BLOCK__GFX_DC_STATE_ME2,291	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,292	TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,293	TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,294	/* CPF*/295	TA_RAS_BLOCK__GFX_CPF_INDEX_START,296	TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,297	TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,298	TA_RAS_BLOCK__GFX_CPF_TAG,299	TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,300	/* CPG*/301	TA_RAS_BLOCK__GFX_CPG_INDEX_START,302	TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,303	TA_RAS_BLOCK__GFX_CPG_DMA_TAG,304	TA_RAS_BLOCK__GFX_CPG_TAG,305	TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,306	/* GDS*/307	TA_RAS_BLOCK__GFX_GDS_INDEX_START,308	TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,309	TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,310	TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,311	TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,312	TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,313	TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,314	/* SPI*/315	TA_RAS_BLOCK__GFX_SPI_SR_MEM,316	/* SQ*/317	TA_RAS_BLOCK__GFX_SQ_INDEX_START,318	TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,319	TA_RAS_BLOCK__GFX_SQ_LDS_D,320	TA_RAS_BLOCK__GFX_SQ_LDS_I,321	TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/322	TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,323	/* SQC (3 ranges)*/324	TA_RAS_BLOCK__GFX_SQC_INDEX_START,325	/* SQC range 0*/326	TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,327	TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =328		TA_RAS_BLOCK__GFX_SQC_INDEX0_START,329	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,330	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,331	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,332	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,333	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,334	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,335	TA_RAS_BLOCK__GFX_SQC_INDEX0_END =336		TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,337	/* SQC range 1*/338	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,339	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =340		TA_RAS_BLOCK__GFX_SQC_INDEX1_START,341	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,342	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,343	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,344	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,345	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,346	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,347	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,348	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,349	TA_RAS_BLOCK__GFX_SQC_INDEX1_END =350		TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,351	/* SQC range 2*/352	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,353	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =354		TA_RAS_BLOCK__GFX_SQC_INDEX2_START,355	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,356	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,357	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,358	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,359	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,360	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,361	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,362	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,363	TA_RAS_BLOCK__GFX_SQC_INDEX2_END =364		TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,365	TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,366	/* TA*/367	TA_RAS_BLOCK__GFX_TA_INDEX_START,368	TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,369	TA_RAS_BLOCK__GFX_TA_FS_AFIFO,370	TA_RAS_BLOCK__GFX_TA_FL_LFIFO,371	TA_RAS_BLOCK__GFX_TA_FX_LFIFO,372	TA_RAS_BLOCK__GFX_TA_FS_CFIFO,373	TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,374	/* TCA*/375	TA_RAS_BLOCK__GFX_TCA_INDEX_START,376	TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,377	TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,378	TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,379	/* TCC (5 sub-ranges)*/380	TA_RAS_BLOCK__GFX_TCC_INDEX_START,381	/* TCC range 0*/382	TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,383	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,384	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,385	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,386	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,387	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,388	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,389	TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,390	TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,391	TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,392	/* TCC range 1*/393	TA_RAS_BLOCK__GFX_TCC_INDEX1_START,394	TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,395	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,396	TA_RAS_BLOCK__GFX_TCC_INDEX1_END =397		TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,398	/* TCC range 2*/399	TA_RAS_BLOCK__GFX_TCC_INDEX2_START,400	TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,401	TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,402	TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,403	TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,404	TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,405	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,406	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,407	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,408	TA_RAS_BLOCK__GFX_TCC_INDEX2_END =409		TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,410	/* TCC range 3*/411	TA_RAS_BLOCK__GFX_TCC_INDEX3_START,412	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,413	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,414	TA_RAS_BLOCK__GFX_TCC_INDEX3_END =415		TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,416	/* TCC range 4*/417	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,418	TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =419		TA_RAS_BLOCK__GFX_TCC_INDEX4_START,420	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,421	TA_RAS_BLOCK__GFX_TCC_INDEX4_END =422		TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,423	TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,424	/* TCI*/425	TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,426	/* TCP*/427	TA_RAS_BLOCK__GFX_TCP_INDEX_START,428	TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,429	TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,430	TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,431	TA_RAS_BLOCK__GFX_TCP_VM_FIFO,432	TA_RAS_BLOCK__GFX_TCP_DB_RAM,433	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,434	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,435	TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,436	/* TD*/437	TA_RAS_BLOCK__GFX_TD_INDEX_START,438	TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,439	TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,440	TA_RAS_BLOCK__GFX_TD_CS_FIFO,441	TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,442	/* EA (3 sub-ranges)*/443	TA_RAS_BLOCK__GFX_EA_INDEX_START,444	/* EA range 0*/445	TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,446	TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,447	TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,448	TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,449	TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,450	TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,451	TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,452	TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,453	TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,454	TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,455	/* EA range 1*/456	TA_RAS_BLOCK__GFX_EA_INDEX1_START,457	TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,458	TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,459	TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,460	TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,461	TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,462	TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,463	TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,464	TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,465	/* EA range 2*/466	TA_RAS_BLOCK__GFX_EA_INDEX2_START,467	TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,468	TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,469	TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,470	TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,471	TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,472	TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,473	/* UTC VM L2 bank*/474	TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,475	/* UTC VM walker*/476	TA_RAS_BLOCK__UTC_VML2_WALKER,477	/* UTC ATC L2 2MB cache*/478	TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,479	/* UTC ATC L2 4KB cache*/480	TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,481	TA_RAS_BLOCK__GFX_MAX482};483 484struct ras_gfx_subblock {485	unsigned char *name;486	int ta_subblock;487	int hw_supported_error_type;488	int sw_supported_error_type;489};490 491#define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \492	[AMDGPU_RAS_BLOCK__##subblock] = {                                     \493		#subblock,                                                     \494		TA_RAS_BLOCK__##subblock,                                      \495		((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \496		(((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \497	}498 499static const struct ras_gfx_subblock ras_gfx_subblocks[] = {500	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),501	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),502	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),503	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),504	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),505	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),506	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),507	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),508	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),509	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),510	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),511	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),512	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),513	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),514	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),515	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),516	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,517			     0),518	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,519			     0),520	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),521	AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),522	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),523	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),524	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),525	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),526	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),527	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,528			     0, 0),529	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,530			     0),531	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,532			     0, 0),533	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,534			     0),535	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,536			     0, 0),537	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,538			     0),539	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,540			     1),541	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,542			     0, 0, 0),543	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,544			     0),545	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,546			     0),547	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,548			     0),549	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,550			     0),551	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,552			     0),553	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,554			     0, 0),555	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,556			     0),557	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,558			     0),559	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,560			     0, 0, 0),561	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,562			     0),563	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,564			     0),565	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,566			     0),567	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,568			     0),569	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,570			     0),571	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,572			     0, 0),573	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,574			     0),575	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),576	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),577	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),578	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),579	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),580	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),581	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),582	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),583	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,584			     1),585	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,586			     1),587	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,588			     1),589	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,590			     0),591	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,592			     0),593	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),594	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),595	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),596	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),597	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),598	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),599	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),600	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),601	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),602	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),603	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),604	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,605			     0),606	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),607	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,608			     0),609	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,610			     0, 0),611	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,612			     0),613	AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),614	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),615	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),616	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),617	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),618	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),619	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),620	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),621	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),622	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),623	AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),624	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),625	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),626	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),627	AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),628	AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),629	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),630	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),631	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),632	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),633	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),634	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),635	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),636	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),637	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),638	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),639	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),640	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),641	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),642	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),643	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),644	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),645	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),646	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),647};648 649static const struct soc15_reg_golden golden_settings_gc_9_0[] =650{651	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),652	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),653	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),654	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),655	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),656	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),657	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),658	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),659	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),660	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),661	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),662	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),663	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),664	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),665	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),666	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),667	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),668	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),669	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),670	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)671};672 673static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =674{675	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),676	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),677	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),678	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),679	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),680	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),681	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),682	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),683	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),684	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),685	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),686	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),687	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),688	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),689	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),690	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),691	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),692	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)693};694 695static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =696{697	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),698	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),699	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),700	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),701	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),702	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),703	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),704	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),705	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),706	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),707	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)708};709 710static const struct soc15_reg_golden golden_settings_gc_9_1[] =711{712	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),713	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),714	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),715	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),716	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),717	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),718	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),719	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),720	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),721	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),722	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),723	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),724	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),725	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),726	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),727	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),728	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),729	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),730	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),731	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),732	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),733	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),734	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),735	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)736};737 738static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =739{740	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),741	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),742	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),743	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),744	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),745	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),746	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)747};748 749static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =750{751	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),752	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),753	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),754	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),755	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),756	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),757	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),758	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),759	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),760	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),761	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),762	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),763	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),764	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),765	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),766	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),767	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),768	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),769	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),770};771 772static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =773{774	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),775	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),776	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),777	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),778	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),779	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),780	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),781	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),782	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),783	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),784	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),785	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),786};787 788static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =789{790	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),791	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),792	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)793};794 795static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =796{797	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),798	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),799	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),800	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),801	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),802	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),803	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),804	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),805	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),806	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),807	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),808	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),809	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),810	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),811	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),812	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)813};814 815static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =816{817	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),818	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),819	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),820	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),821	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),822	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),823	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),824	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),825	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),826	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),827	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),828	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),829	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)830};831 832static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =833{834	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),835	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),836	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),837	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),838	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),839	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),840	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),841	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),842	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),843	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),844	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)845};846 847static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {848	{SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},849	{SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},850};851 852static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =853{854	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,855	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,856	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,857	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,858	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,859	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,860	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,861	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,862};863 864static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =865{866	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,867	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,868	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,869	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,870	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,871	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,872	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,873	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,874};875 876#define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042877#define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041878#define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042879#define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041880 881static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);882static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);883static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);884static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);885static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,886				struct amdgpu_cu_info *cu_info);887static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);888static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);889static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);890static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,891					  void *ras_error_status);892static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,893				     void *inject_if, uint32_t instance_mask);894static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);895static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,896					      unsigned int vmid);897static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);898static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);899 900static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,901				uint64_t queue_mask)902{903	struct amdgpu_device *adev = kiq_ring->adev;904	u64 shader_mc_addr;905 906	/* Cleaner shader MC address */907	shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;908 909	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));910	amdgpu_ring_write(kiq_ring,911		PACKET3_SET_RESOURCES_VMID_MASK(0) |912		/* vmid_mask:0* queue_type:0 (KIQ) */913		PACKET3_SET_RESOURCES_QUEUE_TYPE(0));914	amdgpu_ring_write(kiq_ring,915			lower_32_bits(queue_mask));	/* queue mask lo */916	amdgpu_ring_write(kiq_ring,917			upper_32_bits(queue_mask));	/* queue mask hi */918	amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */919	amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */920	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */921	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */922}923 924static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,925				 struct amdgpu_ring *ring)926{927	uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);928	uint64_t wptr_addr = ring->wptr_gpu_addr;929	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;930 931	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));932	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/933	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */934			 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */935			 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */936			 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |937			 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |938			 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |939			 /*queue_type: normal compute queue */940			 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |941			 /* alloc format: all_on_one_pipe */942			 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |943			 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |944			 /* num_queues: must be 1 */945			 PACKET3_MAP_QUEUES_NUM_QUEUES(1));946	amdgpu_ring_write(kiq_ring,947			PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));948	amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));949	amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));950	amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));951	amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));952}953 954static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,955				   struct amdgpu_ring *ring,956				   enum amdgpu_unmap_queues_action action,957				   u64 gpu_addr, u64 seq)958{959	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;960 961	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));962	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */963			  PACKET3_UNMAP_QUEUES_ACTION(action) |964			  PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |965			  PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |966			  PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));967	amdgpu_ring_write(kiq_ring,968			PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));969 970	if (action == PREEMPT_QUEUES_NO_UNMAP) {971		amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));972		amdgpu_ring_write(kiq_ring, 0);973		amdgpu_ring_write(kiq_ring, 0);974 975	} else {976		amdgpu_ring_write(kiq_ring, 0);977		amdgpu_ring_write(kiq_ring, 0);978		amdgpu_ring_write(kiq_ring, 0);979	}980}981 982static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,983				   struct amdgpu_ring *ring,984				   u64 addr,985				   u64 seq)986{987	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;988 989	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));990	amdgpu_ring_write(kiq_ring,991			  PACKET3_QUERY_STATUS_CONTEXT_ID(0) |992			  PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |993			  PACKET3_QUERY_STATUS_COMMAND(2));994	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */995	amdgpu_ring_write(kiq_ring,996			PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |997			PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));998	amdgpu_ring_write(kiq_ring, lower_32_bits(addr));999	amdgpu_ring_write(kiq_ring, upper_32_bits(addr));1000	amdgpu_ring_write(kiq_ring, lower_32_bits(seq));1001	amdgpu_ring_write(kiq_ring, upper_32_bits(seq));1002}1003 1004static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,1005				uint16_t pasid, uint32_t flush_type,1006				bool all_hub)1007{1008	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));1009	amdgpu_ring_write(kiq_ring,1010			PACKET3_INVALIDATE_TLBS_DST_SEL(1) |1011			PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |1012			PACKET3_INVALIDATE_TLBS_PASID(pasid) |1013			PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));1014}1015 1016 1017static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,1018					uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,1019					uint32_t xcc_id, uint32_t vmid)1020{1021	struct amdgpu_device *adev = kiq_ring->adev;1022	unsigned i;1023 1024	/* enter save mode */1025	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);1026	mutex_lock(&adev->srbm_mutex);1027	soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);1028 1029	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {1030		WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);1031		WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);1032		/* wait till dequeue take effects */1033		for (i = 0; i < adev->usec_timeout; i++) {1034			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))1035				break;1036			udelay(1);1037		}1038		if (i >= adev->usec_timeout)1039			dev_err(adev->dev, "fail to wait on hqd deactive\n");1040	} else {1041		dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);1042	}1043 1044	soc15_grbm_select(adev, 0, 0, 0, 0, 0);1045	mutex_unlock(&adev->srbm_mutex);1046	/* exit safe mode */1047	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);1048}1049 1050static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {1051	.kiq_set_resources = gfx_v9_0_kiq_set_resources,1052	.kiq_map_queues = gfx_v9_0_kiq_map_queues,1053	.kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,1054	.kiq_query_status = gfx_v9_0_kiq_query_status,1055	.kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,1056	.kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,1057	.set_resources_size = 8,1058	.map_queues_size = 7,1059	.unmap_queues_size = 6,1060	.query_status_size = 7,1061	.invalidate_tlbs_size = 2,1062};1063 1064static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)1065{1066	adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;1067}1068 1069static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)1070{1071	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {1072	case IP_VERSION(9, 0, 1):1073		soc15_program_register_sequence(adev,1074						golden_settings_gc_9_0,1075						ARRAY_SIZE(golden_settings_gc_9_0));1076		soc15_program_register_sequence(adev,1077						golden_settings_gc_9_0_vg10,1078						ARRAY_SIZE(golden_settings_gc_9_0_vg10));1079		break;1080	case IP_VERSION(9, 2, 1):1081		soc15_program_register_sequence(adev,1082						golden_settings_gc_9_2_1,1083						ARRAY_SIZE(golden_settings_gc_9_2_1));1084		soc15_program_register_sequence(adev,1085						golden_settings_gc_9_2_1_vg12,1086						ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));1087		break;1088	case IP_VERSION(9, 4, 0):1089		soc15_program_register_sequence(adev,1090						golden_settings_gc_9_0,1091						ARRAY_SIZE(golden_settings_gc_9_0));1092		soc15_program_register_sequence(adev,1093						golden_settings_gc_9_0_vg20,1094						ARRAY_SIZE(golden_settings_gc_9_0_vg20));1095		break;1096	case IP_VERSION(9, 4, 1):1097		soc15_program_register_sequence(adev,1098						golden_settings_gc_9_4_1_arct,1099						ARRAY_SIZE(golden_settings_gc_9_4_1_arct));1100		break;1101	case IP_VERSION(9, 2, 2):1102	case IP_VERSION(9, 1, 0):1103		soc15_program_register_sequence(adev, golden_settings_gc_9_1,1104						ARRAY_SIZE(golden_settings_gc_9_1));1105		if (adev->apu_flags & AMD_APU_IS_RAVEN2)1106			soc15_program_register_sequence(adev,1107							golden_settings_gc_9_1_rv2,1108							ARRAY_SIZE(golden_settings_gc_9_1_rv2));1109		else1110			soc15_program_register_sequence(adev,1111							golden_settings_gc_9_1_rv1,1112							ARRAY_SIZE(golden_settings_gc_9_1_rv1));1113		break;1114	 case IP_VERSION(9, 3, 0):1115		soc15_program_register_sequence(adev,1116						golden_settings_gc_9_1_rn,1117						ARRAY_SIZE(golden_settings_gc_9_1_rn));1118		return; /* for renoir, don't need common goldensetting */1119	case IP_VERSION(9, 4, 2):1120		gfx_v9_4_2_init_golden_registers(adev,1121						 adev->smuio.funcs->get_die_id(adev));1122		break;1123	default:1124		break;1125	}1126 1127	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&1128	    (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))1129		soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,1130						(const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));1131}1132 1133static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,1134				       bool wc, uint32_t reg, uint32_t val)1135{1136	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));1137	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |1138				WRITE_DATA_DST_SEL(0) |1139				(wc ? WR_CONFIRM : 0));1140	amdgpu_ring_write(ring, reg);1141	amdgpu_ring_write(ring, 0);1142	amdgpu_ring_write(ring, val);1143}1144 1145static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,1146				  int mem_space, int opt, uint32_t addr0,1147				  uint32_t addr1, uint32_t ref, uint32_t mask,1148				  uint32_t inv)1149{1150	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));1151	amdgpu_ring_write(ring,1152				 /* memory (1) or register (0) */1153				 (WAIT_REG_MEM_MEM_SPACE(mem_space) |1154				 WAIT_REG_MEM_OPERATION(opt) | /* wait */1155				 WAIT_REG_MEM_FUNCTION(3) |  /* equal */1156				 WAIT_REG_MEM_ENGINE(eng_sel)));1157 1158	if (mem_space)1159		BUG_ON(addr0 & 0x3); /* Dword align */1160	amdgpu_ring_write(ring, addr0);1161	amdgpu_ring_write(ring, addr1);1162	amdgpu_ring_write(ring, ref);1163	amdgpu_ring_write(ring, mask);1164	amdgpu_ring_write(ring, inv); /* poll interval */1165}1166 1167static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)1168{1169	struct amdgpu_device *adev = ring->adev;1170	uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);1171	uint32_t tmp = 0;1172	unsigned i;1173	int r;1174 1175	WREG32(scratch, 0xCAFEDEAD);1176	r = amdgpu_ring_alloc(ring, 3);1177	if (r)1178		return r;1179 1180	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));1181	amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);1182	amdgpu_ring_write(ring, 0xDEADBEEF);1183	amdgpu_ring_commit(ring);1184 1185	for (i = 0; i < adev->usec_timeout; i++) {1186		tmp = RREG32(scratch);1187		if (tmp == 0xDEADBEEF)1188			break;1189		udelay(1);1190	}1191 1192	if (i >= adev->usec_timeout)1193		r = -ETIMEDOUT;1194	return r;1195}1196 1197static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)1198{1199	struct amdgpu_device *adev = ring->adev;1200	struct amdgpu_ib ib;1201	struct dma_fence *f = NULL;1202 1203	unsigned index;1204	uint64_t gpu_addr;1205	uint32_t tmp;1206	long r;1207 1208	r = amdgpu_device_wb_get(adev, &index);1209	if (r)1210		return r;1211 1212	gpu_addr = adev->wb.gpu_addr + (index * 4);1213	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);1214	memset(&ib, 0, sizeof(ib));1215 1216	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);1217	if (r)1218		goto err1;1219 1220	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);1221	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;1222	ib.ptr[2] = lower_32_bits(gpu_addr);1223	ib.ptr[3] = upper_32_bits(gpu_addr);1224	ib.ptr[4] = 0xDEADBEEF;1225	ib.length_dw = 5;1226 1227	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);1228	if (r)1229		goto err2;1230 1231	r = dma_fence_wait_timeout(f, false, timeout);1232	if (r == 0) {1233		r = -ETIMEDOUT;1234		goto err2;1235	} else if (r < 0) {1236		goto err2;1237	}1238 1239	tmp = adev->wb.wb[index];1240	if (tmp == 0xDEADBEEF)1241		r = 0;1242	else1243		r = -EINVAL;1244 1245err2:1246	amdgpu_ib_free(adev, &ib, NULL);1247	dma_fence_put(f);1248err1:1249	amdgpu_device_wb_free(adev, index);1250	return r;1251}1252 1253 1254static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)1255{1256	amdgpu_ucode_release(&adev->gfx.pfp_fw);1257	amdgpu_ucode_release(&adev->gfx.me_fw);1258	amdgpu_ucode_release(&adev->gfx.ce_fw);1259	amdgpu_ucode_release(&adev->gfx.rlc_fw);1260	amdgpu_ucode_release(&adev->gfx.mec_fw);1261	amdgpu_ucode_release(&adev->gfx.mec2_fw);1262 1263	kfree(adev->gfx.rlc.register_list_format);1264}1265 1266static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)1267{1268	adev->gfx.me_fw_write_wait = false;1269	adev->gfx.mec_fw_write_wait = false;1270 1271	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&1272	    ((adev->gfx.mec_fw_version < 0x000001a5) ||1273	     (adev->gfx.mec_feature_version < 46) ||1274	     (adev->gfx.pfp_fw_version < 0x000000b7) ||1275	     (adev->gfx.pfp_feature_version < 46)))1276		DRM_WARN_ONCE("CP firmware version too old, please update!");1277 1278	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {1279	case IP_VERSION(9, 0, 1):1280		if ((adev->gfx.me_fw_version >= 0x0000009c) &&1281		    (adev->gfx.me_feature_version >= 42) &&1282		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&1283		    (adev->gfx.pfp_feature_version >= 42))1284			adev->gfx.me_fw_write_wait = true;1285 1286		if ((adev->gfx.mec_fw_version >=  0x00000193) &&1287		    (adev->gfx.mec_feature_version >= 42))1288			adev->gfx.mec_fw_write_wait = true;1289		break;1290	case IP_VERSION(9, 2, 1):1291		if ((adev->gfx.me_fw_version >= 0x0000009c) &&1292		    (adev->gfx.me_feature_version >= 44) &&1293		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&1294		    (adev->gfx.pfp_feature_version >= 44))1295			adev->gfx.me_fw_write_wait = true;1296 1297		if ((adev->gfx.mec_fw_version >=  0x00000196) &&1298		    (adev->gfx.mec_feature_version >= 44))1299			adev->gfx.mec_fw_write_wait = true;1300		break;1301	case IP_VERSION(9, 4, 0):1302		if ((adev->gfx.me_fw_version >= 0x0000009c) &&1303		    (adev->gfx.me_feature_version >= 44) &&1304		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&1305		    (adev->gfx.pfp_feature_version >= 44))1306			adev->gfx.me_fw_write_wait = true;1307 1308		if ((adev->gfx.mec_fw_version >=  0x00000197) &&1309		    (adev->gfx.mec_feature_version >= 44))1310			adev->gfx.mec_fw_write_wait = true;1311		break;1312	case IP_VERSION(9, 1, 0):1313	case IP_VERSION(9, 2, 2):1314		if ((adev->gfx.me_fw_version >= 0x0000009c) &&1315		    (adev->gfx.me_feature_version >= 42) &&1316		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&1317		    (adev->gfx.pfp_feature_version >= 42))1318			adev->gfx.me_fw_write_wait = true;1319 1320		if ((adev->gfx.mec_fw_version >=  0x00000192) &&1321		    (adev->gfx.mec_feature_version >= 42))1322			adev->gfx.mec_fw_write_wait = true;1323		break;1324	default:1325		adev->gfx.me_fw_write_wait = true;1326		adev->gfx.mec_fw_write_wait = true;1327		break;1328	}1329}1330 1331struct amdgpu_gfxoff_quirk {1332	u16 chip_vendor;1333	u16 chip_device;1334	u16 subsys_vendor;1335	u16 subsys_device;1336	u8 revision;1337};1338 1339static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {1340	/* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */1341	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },1342	/* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */1343	{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },1344	/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */1345	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },1346	/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */1347	{ 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },1348	/* https://bbs.openkylin.top/t/topic/171497 */1349	{ 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },1350	/* HP 705G4 DM with R5 2400G */1351	{ 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },1352	{ 0, 0, 0, 0, 0 },1353};1354 1355static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)1356{1357	const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;1358 1359	while (p && p->chip_device != 0) {1360		if (pdev->vendor == p->chip_vendor &&1361		    pdev->device == p->chip_device &&1362		    pdev->subsystem_vendor == p->subsys_vendor &&1363		    pdev->subsystem_device == p->subsys_device &&1364		    pdev->revision == p->revision) {1365			return true;1366		}1367		++p;1368	}1369	return false;1370}1371 1372static bool is_raven_kicker(struct amdgpu_device *adev)1373{1374	if (adev->pm.fw_version >= 0x41e2b)1375		return true;1376	else1377		return false;1378}1379 1380static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)1381{1382	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&1383	    (adev->gfx.me_fw_version >= 0x000000a5) &&1384	    (adev->gfx.me_feature_version >= 52))1385		return true;1386	else1387		return false;1388}1389 1390static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)1391{1392	if (gfx_v9_0_should_disable_gfxoff(adev->pdev))1393		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;1394 1395	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {1396	case IP_VERSION(9, 0, 1):1397	case IP_VERSION(9, 2, 1):1398	case IP_VERSION(9, 4, 0):1399		break;1400	case IP_VERSION(9, 2, 2):1401	case IP_VERSION(9, 1, 0):1402		if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||1403		      (adev->apu_flags & AMD_APU_IS_PICASSO)) &&1404		    ((!is_raven_kicker(adev) &&1405		      adev->gfx.rlc_fw_version < 531) ||1406		     (adev->gfx.rlc_feature_version < 1) ||1407		     !adev->gfx.rlc.is_rlc_v2_1))1408			adev->pm.pp_feature &= ~PP_GFXOFF_MASK;1409 1410		if (adev->pm.pp_feature & PP_GFXOFF_MASK)1411			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |1412				AMD_PG_SUPPORT_CP |1413				AMD_PG_SUPPORT_RLC_SMU_HS;1414		break;1415	case IP_VERSION(9, 3, 0):1416		if (adev->pm.pp_feature & PP_GFXOFF_MASK)1417			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |1418				AMD_PG_SUPPORT_CP |1419				AMD_PG_SUPPORT_RLC_SMU_HS;1420		break;1421	default:1422		break;1423	}1424}1425 1426static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,1427					  char *chip_name)1428{1429	int err;1430 1431	err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,1432				   "amdgpu/%s_pfp.bin", chip_name);1433	if (err)1434		goto out;1435	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);1436 1437	err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,1438				   "amdgpu/%s_me.bin", chip_name);1439	if (err)1440		goto out;1441	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);1442 1443	err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,1444				   "amdgpu/%s_ce.bin", chip_name);1445	if (err)1446		goto out;1447	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);1448 1449out:1450	if (err) {1451		amdgpu_ucode_release(&adev->gfx.pfp_fw);1452		amdgpu_ucode_release(&adev->gfx.me_fw);1453		amdgpu_ucode_release(&adev->gfx.ce_fw);1454	}1455	return err;1456}1457 1458static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,1459				       char *chip_name)1460{1461	int err;1462	const struct rlc_firmware_header_v2_0 *rlc_hdr;1463	uint16_t version_major;1464	uint16_t version_minor;1465	uint32_t smu_version;1466 1467	/*1468	 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin1469	 * instead of picasso_rlc.bin.1470	 * Judgment method:1471	 * PCO AM4: revision >= 0xC8 && revision <= 0xCF1472	 *          or revision >= 0xD8 && revision <= 0xDF1473	 * otherwise is PCO FP51474	 */1475	if (!strcmp(chip_name, "picasso") &&1476		(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||1477		((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))1478		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,1479					   "amdgpu/%s_rlc_am4.bin", chip_name);1480	else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&1481		(smu_version >= 0x41e2b))1482		/**1483		*SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.1484		*/1485		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,1486					   "amdgpu/%s_kicker_rlc.bin", chip_name);1487	else1488		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,1489					   "amdgpu/%s_rlc.bin", chip_name);1490	if (err)1491		goto out;1492 1493	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;1494	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);1495	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);1496	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);1497out:1498	if (err)1499		amdgpu_ucode_release(&adev->gfx.rlc_fw);1500 1501	return err;1502}1503 1504static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)1505{1506	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||1507	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||1508	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))1509		return false;1510 1511	return true;1512}1513 1514static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,1515					      char *chip_name)1516{1517	int err;1518 1519	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))1520		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,1521					   "amdgpu/%s_sjt_mec.bin", chip_name);1522	else1523		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,1524					   "amdgpu/%s_mec.bin", chip_name);1525	if (err)1526		goto out;1527 1528	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);1529	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);1530 1531	if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {1532		if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))1533			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,1534						   "amdgpu/%s_sjt_mec2.bin", chip_name);1535		else1536			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,1537						   "amdgpu/%s_mec2.bin", chip_name);1538		if (!err) {1539			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);1540			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);1541		} else {1542			err = 0;1543			amdgpu_ucode_release(&adev->gfx.mec2_fw);1544		}1545	} else {1546		adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;1547		adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;1548	}1549 1550	gfx_v9_0_check_if_need_gfxoff(adev);1551	gfx_v9_0_check_fw_write_wait(adev);1552 1553out:1554	if (err)1555		amdgpu_ucode_release(&adev->gfx.mec_fw);1556	return err;1557}1558 1559static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)1560{1561	char ucode_prefix[30];1562	int r;1563 1564	DRM_DEBUG("\n");1565	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));1566 1567	/* No CPG in Arcturus */1568	if (adev->gfx.num_gfx_rings) {1569		r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);1570		if (r)1571			return r;1572	}1573 1574	r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);1575	if (r)1576		return r;1577 1578	r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);1579	if (r)1580		return r;1581 1582	return r;1583}1584 1585static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)1586{1587	u32 count = 0;1588	const struct cs_section_def *sect = NULL;1589	const struct cs_extent_def *ext = NULL;1590 1591	/* begin clear state */1592	count += 2;1593	/* context control state */1594	count += 3;1595 1596	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {1597		for (ext = sect->section; ext->extent != NULL; ++ext) {1598			if (sect->id == SECT_CONTEXT)1599				count += 2 + ext->reg_count;1600			else1601				return 0;1602		}1603	}1604 1605	/* end clear state */1606	count += 2;1607	/* clear state */1608	count += 2;1609 1610	return count;1611}1612 1613static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,1614				    volatile u32 *buffer)1615{1616	u32 count = 0, i;1617	const struct cs_section_def *sect = NULL;1618	const struct cs_extent_def *ext = NULL;1619 1620	if (adev->gfx.rlc.cs_data == NULL)1621		return;1622	if (buffer == NULL)1623		return;1624 1625	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));1626	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);1627 1628	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));1629	buffer[count++] = cpu_to_le32(0x80000000);1630	buffer[count++] = cpu_to_le32(0x80000000);1631 1632	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {1633		for (ext = sect->section; ext->extent != NULL; ++ext) {1634			if (sect->id == SECT_CONTEXT) {1635				buffer[count++] =1636					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));1637				buffer[count++] = cpu_to_le32(ext->reg_index -1638						PACKET3_SET_CONTEXT_REG_START);1639				for (i = 0; i < ext->reg_count; i++)1640					buffer[count++] = cpu_to_le32(ext->extent[i]);1641			} else {1642				return;1643			}1644		}1645	}1646 1647	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));1648	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);1649 1650	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));1651	buffer[count++] = cpu_to_le32(0);1652}1653 1654static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)1655{1656	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;1657	uint32_t pg_always_on_cu_num = 2;1658	uint32_t always_on_cu_num;1659	uint32_t i, j, k;1660	uint32_t mask, cu_bitmap, counter;1661 1662	if (adev->flags & AMD_IS_APU)1663		always_on_cu_num = 4;1664	else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))1665		always_on_cu_num = 8;1666	else1667		always_on_cu_num = 12;1668 1669	mutex_lock(&adev->grbm_idx_mutex);1670	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {1671		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {1672			mask = 1;1673			cu_bitmap = 0;1674			counter = 0;1675			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);1676 1677			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {1678				if (cu_info->bitmap[0][i][j] & mask) {1679					if (counter == pg_always_on_cu_num)1680						WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);1681					if (counter < always_on_cu_num)1682						cu_bitmap |= mask;1683					else1684						break;1685					counter++;1686				}1687				mask <<= 1;1688			}1689 1690			WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);1691			cu_info->ao_cu_bitmap[i][j] = cu_bitmap;1692		}1693	}1694	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);1695	mutex_unlock(&adev->grbm_idx_mutex);1696}1697 1698static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)1699{1700	uint32_t data;1701 1702	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */1703	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);1704	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);1705	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);1706	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));1707 1708	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */1709	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);1710 1711	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */1712	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);1713 1714	mutex_lock(&adev->grbm_idx_mutex);1715	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/1716	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);1717	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);1718 1719	/* set mmRLC_LB_PARAMS = 0x003F_1006 */1720	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);1721	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);1722	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);1723	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);1724 1725	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */1726	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);1727	data &= 0x0000FFFF;1728	data |= 0x00C00000;1729	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);1730 1731	/*1732	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),1733	 * programmed in gfx_v9_0_init_always_on_cu_mask()1734	 */1735 1736	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,1737	 * but used for RLC_LB_CNTL configuration */1738	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;1739	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);1740	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);1741	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);1742	mutex_unlock(&adev->grbm_idx_mutex);1743 1744	gfx_v9_0_init_always_on_cu_mask(adev);1745}1746 1747static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)1748{1749	uint32_t data;1750 1751	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */1752	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);1753	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);1754	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);1755	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));1756 1757	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */1758	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);1759 1760	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */1761	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);1762 1763	mutex_lock(&adev->grbm_idx_mutex);1764	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/1765	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);1766	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);1767 1768	/* set mmRLC_LB_PARAMS = 0x003F_1006 */1769	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);1770	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);1771	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);1772	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);1773 1774	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */1775	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);1776	data &= 0x0000FFFF;1777	data |= 0x00C00000;1778	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);1779 1780	/*1781	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),1782	 * programmed in gfx_v9_0_init_always_on_cu_mask()1783	 */1784 1785	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,1786	 * but used for RLC_LB_CNTL configuration */1787	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;1788	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);1789	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);1790	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);1791	mutex_unlock(&adev->grbm_idx_mutex);1792 1793	gfx_v9_0_init_always_on_cu_mask(adev);1794}1795 1796static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)1797{1798	WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);1799}1800 1801static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)1802{1803	if (gfx_v9_0_load_mec2_fw_bin_support(adev))1804		return 5;1805	else1806		return 4;1807}1808 1809static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)1810{1811	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;1812 1813	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];1814	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);1815	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);1816	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);1817	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);1818	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);1819	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);1820	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);1821	adev->gfx.rlc.rlcg_reg_access_supported = true;1822}1823 1824static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)1825{1826	const struct cs_section_def *cs_data;1827	int r;1828 1829	adev->gfx.rlc.cs_data = gfx9_cs_data;1830 1831	cs_data = adev->gfx.rlc.cs_data;1832 1833	if (cs_data) {1834		/* init clear state block */1835		r = amdgpu_gfx_rlc_init_csb(adev);1836		if (r)1837			return r;1838	}1839 1840	if (adev->flags & AMD_IS_APU) {1841		/* TODO: double check the cp_table_size for RV */1842		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */1843		r = amdgpu_gfx_rlc_init_cpt(adev);1844		if (r)1845			return r;1846	}1847 1848	return 0;1849}1850 1851static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)1852{1853	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);1854	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);1855}1856 1857static int gfx_v9_0_mec_init(struct amdgpu_device *adev)1858{1859	int r;1860	u32 *hpd;1861	const __le32 *fw_data;1862	unsigned fw_size;1863	u32 *fw;1864	size_t mec_hpd_size;1865 1866	const struct gfx_firmware_header_v1_0 *mec_hdr;1867 1868	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);1869 1870	/* take ownership of the relevant compute queues */1871	amdgpu_gfx_compute_queue_acquire(adev);1872	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;1873	if (mec_hpd_size) {1874		r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,1875					      AMDGPU_GEM_DOMAIN_VRAM |1876					      AMDGPU_GEM_DOMAIN_GTT,1877					      &adev->gfx.mec.hpd_eop_obj,1878					      &adev->gfx.mec.hpd_eop_gpu_addr,1879					      (void **)&hpd);1880		if (r) {1881			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);1882			gfx_v9_0_mec_fini(adev);1883			return r;1884		}1885 1886		memset(hpd, 0, mec_hpd_size);1887 1888		amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);1889		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);1890	}1891 1892	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;1893 1894	fw_data = (const __le32 *)1895		(adev->gfx.mec_fw->data +1896		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));1897	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);1898 1899	r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,1900				      PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,1901				      &adev->gfx.mec.mec_fw_obj,1902				      &adev->gfx.mec.mec_fw_gpu_addr,1903				      (void **)&fw);1904	if (r) {1905		dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);1906		gfx_v9_0_mec_fini(adev);1907		return r;1908	}1909 1910	memcpy(fw, fw_data, fw_size);1911 1912	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);1913	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);1914 1915	return 0;1916}1917 1918static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)1919{1920	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,1921		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |1922		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |1923		(address << SQ_IND_INDEX__INDEX__SHIFT) |1924		(SQ_IND_INDEX__FORCE_READ_MASK));1925	return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);1926}1927 1928static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,1929			   uint32_t wave, uint32_t thread,1930			   uint32_t regno, uint32_t num, uint32_t *out)1931{1932	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,1933		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |1934		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |1935		(regno << SQ_IND_INDEX__INDEX__SHIFT) |1936		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |1937		(SQ_IND_INDEX__FORCE_READ_MASK) |1938		(SQ_IND_INDEX__AUTO_INCR_MASK));1939	while (num--)1940		*(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);1941}1942 1943static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)1944{1945	/* type 1 wave data */1946	dst[(*no_fields)++] = 1;1947	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);1948	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);1949	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);1950	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);1951	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);1952	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);1953	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);1954	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);1955	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);1956	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);1957	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);1958	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);1959	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);1960	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);1961	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);1962}1963 1964static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,1965				     uint32_t wave, uint32_t start,1966				     uint32_t size, uint32_t *dst)1967{1968	wave_read_regs(1969		adev, simd, wave, 0,1970		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);1971}1972 1973static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,1974				     uint32_t wave, uint32_t thread,1975				     uint32_t start, uint32_t size,1976				     uint32_t *dst)1977{1978	wave_read_regs(1979		adev, simd, wave, thread,1980		start + SQIND_WAVE_VGPRS_OFFSET, size, dst);1981}1982 1983static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,1984				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)1985{1986	soc15_grbm_select(adev, me, pipe, q, vm, 0);1987}1988 1989static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {1990        .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,1991        .select_se_sh = &gfx_v9_0_select_se_sh,1992        .read_wave_data = &gfx_v9_0_read_wave_data,1993        .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,1994        .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,1995        .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,1996};1997 1998const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {1999		.ras_error_inject = &gfx_v9_0_ras_error_inject,2000		.query_ras_error_count = &gfx_v9_0_query_ras_error_count,2001		.reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,2002};2003 2004static struct amdgpu_gfx_ras gfx_v9_0_ras = {2005	.ras_block = {2006		.hw_ops = &gfx_v9_0_ras_ops,2007	},2008};2009 2010static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)2011{2012	u32 gb_addr_config;2013	int err;2014 2015	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {2016	case IP_VERSION(9, 0, 1):2017		adev->gfx.config.max_hw_contexts = 8;2018		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2019		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2020		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2021		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2022		gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;2023		break;2024	case IP_VERSION(9, 2, 1):2025		adev->gfx.config.max_hw_contexts = 8;2026		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2027		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2028		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2029		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2030		gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;2031		DRM_INFO("fix gfx.config for vega12\n");2032		break;2033	case IP_VERSION(9, 4, 0):2034		adev->gfx.ras = &gfx_v9_0_ras;2035		adev->gfx.config.max_hw_contexts = 8;2036		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2037		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2038		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2039		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2040		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);2041		gb_addr_config &= ~0xf3e777ff;2042		gb_addr_config |= 0x22014042;2043		/* check vbios table if gpu info is not available */2044		err = amdgpu_atomfirmware_get_gfx_info(adev);2045		if (err)2046			return err;2047		break;2048	case IP_VERSION(9, 2, 2):2049	case IP_VERSION(9, 1, 0):2050		adev->gfx.config.max_hw_contexts = 8;2051		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2052		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2053		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2054		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2055		if (adev->apu_flags & AMD_APU_IS_RAVEN2)2056			gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;2057		else2058			gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;2059		break;2060	case IP_VERSION(9, 4, 1):2061		adev->gfx.ras = &gfx_v9_4_ras;2062		adev->gfx.config.max_hw_contexts = 8;2063		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2064		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2065		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2066		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2067		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);2068		gb_addr_config &= ~0xf3e777ff;2069		gb_addr_config |= 0x22014042;2070		break;2071	case IP_VERSION(9, 3, 0):2072		adev->gfx.config.max_hw_contexts = 8;2073		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2074		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2075		adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;2076		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2077		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);2078		gb_addr_config &= ~0xf3e777ff;2079		gb_addr_config |= 0x22010042;2080		break;2081	case IP_VERSION(9, 4, 2):2082		adev->gfx.ras = &gfx_v9_4_2_ras;2083		adev->gfx.config.max_hw_contexts = 8;2084		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;2085		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;2086		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;2087		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;2088		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);2089		gb_addr_config &= ~0xf3e777ff;2090		gb_addr_config |= 0x22014042;2091		/* check vbios table if gpu info is not available */2092		err = amdgpu_atomfirmware_get_gfx_info(adev);2093		if (err)2094			return err;2095		break;2096	default:2097		BUG();2098		break;2099	}2100 2101	adev->gfx.config.gb_addr_config = gb_addr_config;2102 2103	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<2104			REG_GET_FIELD(2105					adev->gfx.config.gb_addr_config,2106					GB_ADDR_CONFIG,2107					NUM_PIPES);2108 2109	adev->gfx.config.max_tile_pipes =2110		adev->gfx.config.gb_addr_config_fields.num_pipes;2111 2112	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<2113			REG_GET_FIELD(2114					adev->gfx.config.gb_addr_config,2115					GB_ADDR_CONFIG,2116					NUM_BANKS);2117	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<2118			REG_GET_FIELD(2119					adev->gfx.config.gb_addr_config,2120					GB_ADDR_CONFIG,2121					MAX_COMPRESSED_FRAGS);2122	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<2123			REG_GET_FIELD(2124					adev->gfx.config.gb_addr_config,2125					GB_ADDR_CONFIG,2126					NUM_RB_PER_SE);2127	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<2128			REG_GET_FIELD(2129					adev->gfx.config.gb_addr_config,2130					GB_ADDR_CONFIG,2131					NUM_SHADER_ENGINES);2132	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +2133			REG_GET_FIELD(2134					adev->gfx.config.gb_addr_config,2135					GB_ADDR_CONFIG,2136					PIPE_INTERLEAVE_SIZE));2137 2138	return 0;2139}2140 2141static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,2142				      int mec, int pipe, int queue)2143{2144	unsigned irq_type;2145	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];2146	unsigned int hw_prio;2147 2148	ring = &adev->gfx.compute_ring[ring_id];2149 2150	/* mec0 is me1 */2151	ring->me = mec + 1;2152	ring->pipe = pipe;2153	ring->queue = queue;2154 2155	ring->ring_obj = NULL;2156	ring->use_doorbell = true;2157	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;2158	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr2159				+ (ring_id * GFX9_MEC_HPD_SIZE);2160	ring->vm_hub = AMDGPU_GFXHUB(0);2161	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);2162 2163	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP2164		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)2165		+ ring->pipe;2166	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?2167			AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;2168	/* type-2 packets are deprecated on MEC, use type-3 instead */2169	return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,2170				hw_prio, NULL);2171}2172 2173static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)2174{2175	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);2176	uint32_t *ptr;2177	uint32_t inst;2178 2179	ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);2180	if (!ptr) {2181		DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");2182		adev->gfx.ip_dump_core = NULL;2183	} else {2184		adev->gfx.ip_dump_core = ptr;2185	}2186 2187	/* Allocate memory for compute queue registers for all the instances */2188	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);2189	inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *2190		adev->gfx.mec.num_queue_per_pipe;2191 2192	ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);2193	if (!ptr) {2194		DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");2195		adev->gfx.ip_dump_compute_queues = NULL;2196	} else {2197		adev->gfx.ip_dump_compute_queues = ptr;2198	}2199}2200 2201static int gfx_v9_0_sw_init(void *handle)2202{2203	int i, j, k, r, ring_id;2204	int xcc_id = 0;2205	struct amdgpu_ring *ring;2206	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2207	unsigned int hw_prio;2208 2209	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {2210	case IP_VERSION(9, 0, 1):2211	case IP_VERSION(9, 2, 1):2212	case IP_VERSION(9, 4, 0):2213	case IP_VERSION(9, 2, 2):2214	case IP_VERSION(9, 1, 0):2215	case IP_VERSION(9, 4, 1):2216	case IP_VERSION(9, 3, 0):2217	case IP_VERSION(9, 4, 2):2218		adev->gfx.mec.num_mec = 2;2219		break;2220	default:2221		adev->gfx.mec.num_mec = 1;2222		break;2223	}2224 2225	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {2226	default:2227		adev->gfx.enable_cleaner_shader = false;2228		break;2229	}2230 2231	adev->gfx.mec.num_pipe_per_mec = 4;2232	adev->gfx.mec.num_queue_per_pipe = 8;2233 2234	/* EOP Event */2235	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);2236	if (r)2237		return r;2238 2239	/* Bad opcode Event */2240	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,2241			      GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,2242			      &adev->gfx.bad_op_irq);2243	if (r)2244		return r;2245 2246	/* Privileged reg */2247	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,2248			      &adev->gfx.priv_reg_irq);2249	if (r)2250		return r;2251 2252	/* Privileged inst */2253	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,2254			      &adev->gfx.priv_inst_irq);2255	if (r)2256		return r;2257 2258	/* ECC error */2259	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,2260			      &adev->gfx.cp_ecc_error_irq);2261	if (r)2262		return r;2263 2264	/* FUE error */2265	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,2266			      &adev->gfx.cp_ecc_error_irq);2267	if (r)2268		return r;2269 2270	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;2271 2272	if (adev->gfx.rlc.funcs) {2273		if (adev->gfx.rlc.funcs->init) {2274			r = adev->gfx.rlc.funcs->init(adev);2275			if (r) {2276				dev_err(adev->dev, "Failed to init rlc BOs!\n");2277				return r;2278			}2279		}2280	}2281 2282	r = gfx_v9_0_mec_init(adev);2283	if (r) {2284		DRM_ERROR("Failed to init MEC BOs!\n");2285		return r;2286	}2287 2288	/* set up the gfx ring */2289	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {2290		ring = &adev->gfx.gfx_ring[i];2291		ring->ring_obj = NULL;2292		if (!i)2293			sprintf(ring->name, "gfx");2294		else2295			sprintf(ring->name, "gfx_%d", i);2296		ring->use_doorbell = true;2297		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;2298 2299		/* disable scheduler on the real ring */2300		ring->no_scheduler = adev->gfx.mcbp;2301		ring->vm_hub = AMDGPU_GFXHUB(0);2302		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,2303				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,2304				     AMDGPU_RING_PRIO_DEFAULT, NULL);2305		if (r)2306			return r;2307	}2308 2309	/* set up the software rings */2310	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {2311		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {2312			ring = &adev->gfx.sw_gfx_ring[i];2313			ring->ring_obj = NULL;2314			sprintf(ring->name, amdgpu_sw_ring_name(i));2315			ring->use_doorbell = true;2316			ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;2317			ring->is_sw_ring = true;2318			hw_prio = amdgpu_sw_ring_priority(i);2319			ring->vm_hub = AMDGPU_GFXHUB(0);2320			r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,2321					     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,2322					     NULL);2323			if (r)2324				return r;2325			ring->wptr = 0;2326		}2327 2328		/* init the muxer and add software rings */2329		r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],2330					 GFX9_NUM_SW_GFX_RINGS);2331		if (r) {2332			DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);2333			return r;2334		}2335		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {2336			r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,2337							&adev->gfx.sw_gfx_ring[i]);2338			if (r) {2339				DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);2340				return r;2341			}2342		}2343	}2344 2345	/* set up the compute queues - allocate horizontally across pipes */2346	ring_id = 0;2347	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {2348		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {2349			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {2350				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,2351								     k, j))2352					continue;2353 2354				r = gfx_v9_0_compute_ring_init(adev,2355							       ring_id,2356							       i, k, j);2357				if (r)2358					return r;2359 2360				ring_id++;2361			}2362		}2363	}2364 2365	r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);2366	if (r) {2367		DRM_ERROR("Failed to init KIQ BOs!\n");2368		return r;2369	}2370 2371	r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);2372	if (r)2373		return r;2374 2375	/* create MQD for all compute queues as wel as KIQ for SRIOV case */2376	r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);2377	if (r)2378		return r;2379 2380	adev->gfx.ce_ram_size = 0x8000;2381 2382	r = gfx_v9_0_gpu_early_init(adev);2383	if (r)2384		return r;2385 2386	if (amdgpu_gfx_ras_sw_init(adev)) {2387		dev_err(adev->dev, "Failed to initialize gfx ras block!\n");2388		return -EINVAL;2389	}2390 2391	gfx_v9_0_alloc_ip_dump(adev);2392 2393	r = amdgpu_gfx_sysfs_isolation_shader_init(adev);2394	if (r)2395		return r;2396 2397	return 0;2398}2399 2400 2401static int gfx_v9_0_sw_fini(void *handle)2402{2403	int i;2404	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2405 2406	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {2407		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)2408			amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);2409		amdgpu_ring_mux_fini(&adev->gfx.muxer);2410	}2411 2412	for (i = 0; i < adev->gfx.num_gfx_rings; i++)2413		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);2414	for (i = 0; i < adev->gfx.num_compute_rings; i++)2415		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);2416 2417	amdgpu_gfx_mqd_sw_fini(adev, 0);2418	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);2419	amdgpu_gfx_kiq_fini(adev, 0);2420 2421	gfx_v9_0_mec_fini(adev);2422	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,2423				&adev->gfx.rlc.clear_state_gpu_addr,2424				(void **)&adev->gfx.rlc.cs_ptr);2425	if (adev->flags & AMD_IS_APU) {2426		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,2427				&adev->gfx.rlc.cp_table_gpu_addr,2428				(void **)&adev->gfx.rlc.cp_table_ptr);2429	}2430	gfx_v9_0_free_microcode(adev);2431 2432	amdgpu_gfx_sysfs_isolation_shader_fini(adev);2433 2434	kfree(adev->gfx.ip_dump_core);2435	kfree(adev->gfx.ip_dump_compute_queues);2436 2437	return 0;2438}2439 2440 2441static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)2442{2443	/* TODO */2444}2445 2446void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,2447			   u32 instance, int xcc_id)2448{2449	u32 data;2450 2451	if (instance == 0xffffffff)2452		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);2453	else2454		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);2455 2456	if (se_num == 0xffffffff)2457		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);2458	else2459		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);2460 2461	if (sh_num == 0xffffffff)2462		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);2463	else2464		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);2465 2466	WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);2467}2468 2469static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)2470{2471	u32 data, mask;2472 2473	data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);2474	data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);2475 2476	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;2477	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;2478 2479	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /2480					 adev->gfx.config.max_sh_per_se);2481 2482	return (~data) & mask;2483}2484 2485static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)2486{2487	int i, j;2488	u32 data;2489	u32 active_rbs = 0;2490	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /2491					adev->gfx.config.max_sh_per_se;2492 2493	mutex_lock(&adev->grbm_idx_mutex);2494	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {2495		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {2496			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);2497			data = gfx_v9_0_get_rb_active_bitmap(adev);2498			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *2499					       rb_bitmap_width_per_sh);2500		}2501	}2502	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);2503	mutex_unlock(&adev->grbm_idx_mutex);2504 2505	adev->gfx.config.backend_enable_mask = active_rbs;2506	adev->gfx.config.num_rbs = hweight32(active_rbs);2507}2508 2509static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,2510				uint32_t first_vmid,2511				uint32_t last_vmid)2512{2513	uint32_t data;2514	uint32_t trap_config_vmid_mask = 0;2515	int i;2516 2517	/* Calculate trap config vmid mask */2518	for (i = first_vmid; i < last_vmid; i++)2519		trap_config_vmid_mask |= (1 << i);2520 2521	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,2522			VMID_SEL, trap_config_vmid_mask);2523	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,2524			TRAP_EN, 1);2525	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);2526	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);2527 2528	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);2529	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);2530}2531 2532#define DEFAULT_SH_MEM_BASES	(0x6000)2533static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)2534{2535	int i;2536	uint32_t sh_mem_config;2537	uint32_t sh_mem_bases;2538 2539	/*2540	 * Configure apertures:2541	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)2542	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)2543	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)2544	 */2545	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);2546 2547	sh_mem_config = SH_MEM_ADDRESS_MODE_64 |2548			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<2549			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;2550 2551	mutex_lock(&adev->srbm_mutex);2552	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {2553		soc15_grbm_select(adev, 0, 0, 0, i, 0);2554		/* CP and shaders */2555		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);2556		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);2557	}2558	soc15_grbm_select(adev, 0, 0, 0, 0, 0);2559	mutex_unlock(&adev->srbm_mutex);2560 2561	/* Initialize all compute VMIDs to have no GDS, GWS, or OA2562	   access. These should be enabled by FW for target VMIDs. */2563	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {2564		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);2565		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);2566		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);2567		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);2568	}2569}2570 2571static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)2572{2573	int vmid;2574 2575	/*2576	 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA2577	 * access. Compute VMIDs should be enabled by FW for target VMIDs,2578	 * the driver can enable them for graphics. VMID0 should maintain2579	 * access so that HWS firmware can save/restore entries.2580	 */2581	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {2582		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);2583		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);2584		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);2585		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);2586	}2587}2588 2589static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)2590{2591	uint32_t tmp;2592 2593	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {2594	case IP_VERSION(9, 4, 1):2595		tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);2596		tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,2597				!READ_ONCE(adev->barrier_has_auto_waitcnt));2598		WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);2599		break;2600	default:2601		break;2602	}2603}2604 2605static void gfx_v9_0_constants_init(struct amdgpu_device *adev)2606{2607	u32 tmp;2608	int i;2609 2610	WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);2611 2612	gfx_v9_0_tiling_mode_table_init(adev);2613 2614	if (adev->gfx.num_gfx_rings)2615		gfx_v9_0_setup_rb(adev);2616	gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);2617	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);2618 2619	/* XXX SH_MEM regs */2620	/* where to put LDS, scratch, GPUVM in FSA64 space */2621	mutex_lock(&adev->srbm_mutex);2622	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {2623		soc15_grbm_select(adev, 0, 0, 0, i, 0);2624		/* CP and shaders */2625		if (i == 0) {2626			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,2627					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);2628			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,2629					    !!adev->gmc.noretry);2630			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);2631			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);2632		} else {2633			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,2634					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);2635			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,2636					    !!adev->gmc.noretry);2637			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);2638			tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,2639				(adev->gmc.private_aperture_start >> 48));2640			tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,2641				(adev->gmc.shared_aperture_start >> 48));2642			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);2643		}2644	}2645	soc15_grbm_select(adev, 0, 0, 0, 0, 0);2646 2647	mutex_unlock(&adev->srbm_mutex);2648 2649	gfx_v9_0_init_compute_vmid(adev);2650	gfx_v9_0_init_gds_vmid(adev);2651	gfx_v9_0_init_sq_config(adev);2652}2653 2654static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)2655{2656	u32 i, j, k;2657	u32 mask;2658 2659	mutex_lock(&adev->grbm_idx_mutex);2660	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {2661		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {2662			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);2663			for (k = 0; k < adev->usec_timeout; k++) {2664				if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)2665					break;2666				udelay(1);2667			}2668			if (k == adev->usec_timeout) {2669				amdgpu_gfx_select_se_sh(adev, 0xffffffff,2670						      0xffffffff, 0xffffffff, 0);2671				mutex_unlock(&adev->grbm_idx_mutex);2672				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",2673					 i, j);2674				return;2675			}2676		}2677	}2678	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);2679	mutex_unlock(&adev->grbm_idx_mutex);2680 2681	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |2682		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |2683		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |2684		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;2685	for (k = 0; k < adev->usec_timeout; k++) {2686		if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)2687			break;2688		udelay(1);2689	}2690}2691 2692static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,2693					       bool enable)2694{2695	u32 tmp;2696 2697	/* These interrupts should be enabled to drive DS clock */2698 2699	tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);2700 2701	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);2702	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);2703	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);2704	if (adev->gfx.num_gfx_rings)2705		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);2706 2707	WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);2708}2709 2710static void gfx_v9_0_init_csb(struct amdgpu_device *adev)2711{2712	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);2713	/* csib */2714	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),2715			adev->gfx.rlc.clear_state_gpu_addr >> 32);2716	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),2717			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);2718	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),2719			adev->gfx.rlc.clear_state_size);2720}2721 2722static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,2723				int indirect_offset,2724				int list_size,2725				int *unique_indirect_regs,2726				int unique_indirect_reg_count,2727				int *indirect_start_offsets,2728				int *indirect_start_offsets_count,2729				int max_start_offsets_count)2730{2731	int idx;2732 2733	for (; indirect_offset < list_size; indirect_offset++) {2734		WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);2735		indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;2736		*indirect_start_offsets_count = *indirect_start_offsets_count + 1;2737 2738		while (register_list_format[indirect_offset] != 0xFFFFFFFF) {2739			indirect_offset += 2;2740 2741			/* look for the matching indice */2742			for (idx = 0; idx < unique_indirect_reg_count; idx++) {2743				if (unique_indirect_regs[idx] ==2744					register_list_format[indirect_offset] ||2745					!unique_indirect_regs[idx])2746					break;2747			}2748 2749			BUG_ON(idx >= unique_indirect_reg_count);2750 2751			if (!unique_indirect_regs[idx])2752				unique_indirect_regs[idx] = register_list_format[indirect_offset];2753 2754			indirect_offset++;2755		}2756	}2757}2758 2759static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)2760{2761	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};2762	int unique_indirect_reg_count = 0;2763 2764	int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};2765	int indirect_start_offsets_count = 0;2766 2767	int list_size = 0;2768	int i = 0, j = 0;2769	u32 tmp = 0;2770 2771	u32 *register_list_format =2772		kmemdup(adev->gfx.rlc.register_list_format,2773			adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);2774	if (!register_list_format)2775		return -ENOMEM;2776 2777	/* setup unique_indirect_regs array and indirect_start_offsets array */2778	unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);2779	gfx_v9_1_parse_ind_reg_list(register_list_format,2780				    adev->gfx.rlc.reg_list_format_direct_reg_list_length,2781				    adev->gfx.rlc.reg_list_format_size_bytes >> 2,2782				    unique_indirect_regs,2783				    unique_indirect_reg_count,2784				    indirect_start_offsets,2785				    &indirect_start_offsets_count,2786				    ARRAY_SIZE(indirect_start_offsets));2787 2788	/* enable auto inc in case it is disabled */2789	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));2790	tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;2791	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);2792 2793	/* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */2794	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),2795		RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);2796	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)2797		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),2798			adev->gfx.rlc.register_restore[i]);2799 2800	/* load indirect register */2801	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),2802		adev->gfx.rlc.reg_list_format_start);2803 2804	/* direct register portion */2805	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)2806		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),2807			register_list_format[i]);2808 2809	/* indirect register portion */2810	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {2811		if (register_list_format[i] == 0xFFFFFFFF) {2812			WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);2813			continue;2814		}2815 2816		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);2817		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);2818 2819		for (j = 0; j < unique_indirect_reg_count; j++) {2820			if (register_list_format[i] == unique_indirect_regs[j]) {2821				WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);2822				break;2823			}2824		}2825 2826		BUG_ON(j >= unique_indirect_reg_count);2827 2828		i++;2829	}2830 2831	/* set save/restore list size */2832	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;2833	list_size = list_size >> 1;2834	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),2835		adev->gfx.rlc.reg_restore_list_size);2836	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);2837 2838	/* write the starting offsets to RLC scratch ram */2839	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),2840		adev->gfx.rlc.starting_offsets_start);2841	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)2842		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),2843		       indirect_start_offsets[i]);2844 2845	/* load unique indirect regs*/2846	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {2847		if (unique_indirect_regs[i] != 0) {2848			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)2849			       + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],2850			       unique_indirect_regs[i] & 0x3FFFF);2851 2852			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)2853			       + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],2854			       unique_indirect_regs[i] >> 20);2855		}2856	}2857 2858	kfree(register_list_format);2859	return 0;2860}2861 2862static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)2863{2864	WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);2865}2866 2867static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,2868					     bool enable)2869{2870	uint32_t data = 0;2871	uint32_t default_data = 0;2872 2873	default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));2874	if (enable) {2875		/* enable GFXIP control over CGPG */2876		data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;2877		if(default_data != data)2878			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);2879 2880		/* update status */2881		data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;2882		data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);2883		if(default_data != data)2884			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);2885	} else {2886		/* restore GFXIP control over GCPG */2887		data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;2888		if(default_data != data)2889			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);2890	}2891}2892 2893static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)2894{2895	uint32_t data = 0;2896 2897	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |2898			      AMD_PG_SUPPORT_GFX_SMG |2899			      AMD_PG_SUPPORT_GFX_DMG)) {2900		/* init IDLE_POLL_COUNT = 60 */2901		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));2902		data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;2903		data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);2904		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);2905 2906		/* init RLC PG Delay */2907		data = 0;2908		data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);2909		data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);2910		data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);2911		data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);2912		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);2913 2914		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));2915		data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;2916		data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);2917		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);2918 2919		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));2920		data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;2921		data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);2922		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);2923 2924		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));2925		data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;2926 2927		/* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */2928		data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);2929		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);2930		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))2931			pwr_10_0_gfxip_control_over_cgpg(adev, true);2932	}2933}2934 2935static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,2936						bool enable)2937{2938	uint32_t data = 0;2939	uint32_t default_data = 0;2940 2941	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));2942	data = REG_SET_FIELD(data, RLC_PG_CNTL,2943			     SMU_CLK_SLOWDOWN_ON_PU_ENABLE,2944			     enable ? 1 : 0);2945	if (default_data != data)2946		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);2947}2948 2949static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,2950						bool enable)2951{2952	uint32_t data = 0;2953	uint32_t default_data = 0;2954 2955	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));2956	data = REG_SET_FIELD(data, RLC_PG_CNTL,2957			     SMU_CLK_SLOWDOWN_ON_PD_ENABLE,2958			     enable ? 1 : 0);2959	if(default_data != data)2960		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);2961}2962 2963static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,2964					bool enable)2965{2966	uint32_t data = 0;2967	uint32_t default_data = 0;2968 2969	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));2970	data = REG_SET_FIELD(data, RLC_PG_CNTL,2971			     CP_PG_DISABLE,2972			     enable ? 0 : 1);2973	if(default_data != data)2974		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);2975}2976 2977static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,2978						bool enable)2979{2980	uint32_t data, default_data;2981 2982	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));2983	data = REG_SET_FIELD(data, RLC_PG_CNTL,2984			     GFX_POWER_GATING_ENABLE,2985			     enable ? 1 : 0);2986	if(default_data != data)2987		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);2988}2989 2990static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,2991						bool enable)2992{2993	uint32_t data, default_data;2994 2995	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));2996	data = REG_SET_FIELD(data, RLC_PG_CNTL,2997			     GFX_PIPELINE_PG_ENABLE,2998			     enable ? 1 : 0);2999	if(default_data != data)3000		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);3001 3002	if (!enable)3003		/* read any GFX register to wake up GFX */3004		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));3005}3006 3007static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,3008						       bool enable)3009{3010	uint32_t data, default_data;3011 3012	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));3013	data = REG_SET_FIELD(data, RLC_PG_CNTL,3014			     STATIC_PER_CU_PG_ENABLE,3015			     enable ? 1 : 0);3016	if(default_data != data)3017		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);3018}3019 3020static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,3021						bool enable)3022{3023	uint32_t data, default_data;3024 3025	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));3026	data = REG_SET_FIELD(data, RLC_PG_CNTL,3027			     DYN_PER_CU_PG_ENABLE,3028			     enable ? 1 : 0);3029	if(default_data != data)3030		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);3031}3032 3033static void gfx_v9_0_init_pg(struct amdgpu_device *adev)3034{3035	gfx_v9_0_init_csb(adev);3036 3037	/*3038	 * Rlc save restore list is workable since v2_1.3039	 * And it's needed by gfxoff feature.3040	 */3041	if (adev->gfx.rlc.is_rlc_v2_1) {3042		if (amdgpu_ip_version(adev, GC_HWIP, 0) ==3043			    IP_VERSION(9, 2, 1) ||3044		    (adev->apu_flags & AMD_APU_IS_RAVEN2))3045			gfx_v9_1_init_rlc_save_restore_list(adev);3046		gfx_v9_0_enable_save_restore_machine(adev);3047	}3048 3049	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |3050			      AMD_PG_SUPPORT_GFX_SMG |3051			      AMD_PG_SUPPORT_GFX_DMG |3052			      AMD_PG_SUPPORT_CP |3053			      AMD_PG_SUPPORT_GDS |3054			      AMD_PG_SUPPORT_RLC_SMU_HS)) {3055		WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,3056			     adev->gfx.rlc.cp_table_gpu_addr >> 8);3057		gfx_v9_0_init_gfx_power_gating(adev);3058	}3059}3060 3061static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)3062{3063	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);3064	gfx_v9_0_enable_gui_idle_interrupt(adev, false);3065	gfx_v9_0_wait_for_rlc_serdes(adev);3066}3067 3068static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)3069{3070	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);3071	udelay(50);3072	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);3073	udelay(50);3074}3075 3076static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)3077{3078#ifdef AMDGPU_RLC_DEBUG_RETRY3079	u32 rlc_ucode_ver;3080#endif3081 3082	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);3083	udelay(50);3084 3085	/* carrizo do enable cp interrupt after cp inited */3086	if (!(adev->flags & AMD_IS_APU)) {3087		gfx_v9_0_enable_gui_idle_interrupt(adev, true);3088		udelay(50);3089	}3090 3091#ifdef AMDGPU_RLC_DEBUG_RETRY3092	/* RLC_GPM_GENERAL_6 : RLC Ucode version */3093	rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);3094	if(rlc_ucode_ver == 0x108) {3095		DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",3096				rlc_ucode_ver, adev->gfx.rlc_fw_version);3097		/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,3098		 * default is 0x9C4 to create a 100us interval */3099		WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);3100		/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr3101		 * to disable the page fault retry interrupts, default is3102		 * 0x100 (256) */3103		WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);3104	}3105#endif3106}3107 3108static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)3109{3110	const struct rlc_firmware_header_v2_0 *hdr;3111	const __le32 *fw_data;3112	unsigned i, fw_size;3113 3114	if (!adev->gfx.rlc_fw)3115		return -EINVAL;3116 3117	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;3118	amdgpu_ucode_print_rlc_hdr(&hdr->header);3119 3120	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +3121			   le32_to_cpu(hdr->header.ucode_array_offset_bytes));3122	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;3123 3124	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,3125			RLCG_UCODE_LOADING_START_ADDRESS);3126	for (i = 0; i < fw_size; i++)3127		WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));3128	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);3129 3130	return 0;3131}3132 3133static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)3134{3135	int r;3136 3137	if (amdgpu_sriov_vf(adev)) {3138		gfx_v9_0_init_csb(adev);3139		return 0;3140	}3141 3142	adev->gfx.rlc.funcs->stop(adev);3143 3144	/* disable CG */3145	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);3146 3147	gfx_v9_0_init_pg(adev);3148 3149	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {3150		/* legacy rlc firmware loading */3151		r = gfx_v9_0_rlc_load_microcode(adev);3152		if (r)3153			return r;3154	}3155 3156	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {3157	case IP_VERSION(9, 2, 2):3158	case IP_VERSION(9, 1, 0):3159		gfx_v9_0_init_lbpw(adev);3160		if (amdgpu_lbpw == 0)3161			gfx_v9_0_enable_lbpw(adev, false);3162		else3163			gfx_v9_0_enable_lbpw(adev, true);3164		break;3165	case IP_VERSION(9, 4, 0):3166		gfx_v9_4_init_lbpw(adev);3167		if (amdgpu_lbpw > 0)3168			gfx_v9_0_enable_lbpw(adev, true);3169		else3170			gfx_v9_0_enable_lbpw(adev, false);3171		break;3172	default:3173		break;3174	}3175 3176	gfx_v9_0_update_spm_vmid_internal(adev, 0xf);3177 3178	adev->gfx.rlc.funcs->start(adev);3179 3180	return 0;3181}3182 3183static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)3184{3185	u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);3186 3187	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);3188	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);3189	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);3190	WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);3191	udelay(50);3192}3193 3194static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)3195{3196	const struct gfx_firmware_header_v1_0 *pfp_hdr;3197	const struct gfx_firmware_header_v1_0 *ce_hdr;3198	const struct gfx_firmware_header_v1_0 *me_hdr;3199	const __le32 *fw_data;3200	unsigned i, fw_size;3201 3202	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)3203		return -EINVAL;3204 3205	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)3206		adev->gfx.pfp_fw->data;3207	ce_hdr = (const struct gfx_firmware_header_v1_0 *)3208		adev->gfx.ce_fw->data;3209	me_hdr = (const struct gfx_firmware_header_v1_0 *)3210		adev->gfx.me_fw->data;3211 3212	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);3213	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);3214	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);3215 3216	gfx_v9_0_cp_gfx_enable(adev, false);3217 3218	/* PFP */3219	fw_data = (const __le32 *)3220		(adev->gfx.pfp_fw->data +3221		 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));3222	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;3223	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);3224	for (i = 0; i < fw_size; i++)3225		WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));3226	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);3227 3228	/* CE */3229	fw_data = (const __le32 *)3230		(adev->gfx.ce_fw->data +3231		 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));3232	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;3233	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);3234	for (i = 0; i < fw_size; i++)3235		WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));3236	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);3237 3238	/* ME */3239	fw_data = (const __le32 *)3240		(adev->gfx.me_fw->data +3241		 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));3242	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;3243	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);3244	for (i = 0; i < fw_size; i++)3245		WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));3246	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);3247 3248	return 0;3249}3250 3251static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)3252{3253	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];3254	const struct cs_section_def *sect = NULL;3255	const struct cs_extent_def *ext = NULL;3256	int r, i, tmp;3257 3258	/* init the CP */3259	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);3260	WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);3261 3262	gfx_v9_0_cp_gfx_enable(adev, true);3263 3264	/* Now only limit the quirk on the APU gfx9 series and already3265	 * confirmed that the APU gfx10/gfx11 needn't such update.3266	 */3267	if (adev->flags & AMD_IS_APU &&3268			adev->in_s3 && !adev->suspend_complete) {3269		DRM_INFO(" Will skip the CSB packet resubmit\n");3270		return 0;3271	}3272	r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);3273	if (r) {3274		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);3275		return r;3276	}3277 3278	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));3279	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);3280 3281	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));3282	amdgpu_ring_write(ring, 0x80000000);3283	amdgpu_ring_write(ring, 0x80000000);3284 3285	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {3286		for (ext = sect->section; ext->extent != NULL; ++ext) {3287			if (sect->id == SECT_CONTEXT) {3288				amdgpu_ring_write(ring,3289				       PACKET3(PACKET3_SET_CONTEXT_REG,3290					       ext->reg_count));3291				amdgpu_ring_write(ring,3292				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);3293				for (i = 0; i < ext->reg_count; i++)3294					amdgpu_ring_write(ring, ext->extent[i]);3295			}3296		}3297	}3298 3299	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));3300	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);3301 3302	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));3303	amdgpu_ring_write(ring, 0);3304 3305	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));3306	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));3307	amdgpu_ring_write(ring, 0x8000);3308	amdgpu_ring_write(ring, 0x8000);3309 3310	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));3311	tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |3312		(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));3313	amdgpu_ring_write(ring, tmp);3314	amdgpu_ring_write(ring, 0);3315 3316	amdgpu_ring_commit(ring);3317 3318	return 0;3319}3320 3321static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)3322{3323	struct amdgpu_ring *ring;3324	u32 tmp;3325	u32 rb_bufsz;3326	u64 rb_addr, rptr_addr, wptr_gpu_addr;3327 3328	/* Set the write pointer delay */3329	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);3330 3331	/* set the RB to use vmid 0 */3332	WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);3333 3334	/* Set ring buffer size */3335	ring = &adev->gfx.gfx_ring[0];3336	rb_bufsz = order_base_2(ring->ring_size / 8);3337	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);3338	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);3339#ifdef __BIG_ENDIAN3340	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);3341#endif3342	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);3343 3344	/* Initialize the ring buffer's write pointers */3345	ring->wptr = 0;3346	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));3347	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));3348 3349	/* set the wb address wether it's enabled or not */3350	rptr_addr = ring->rptr_gpu_addr;3351	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));3352	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);3353 3354	wptr_gpu_addr = ring->wptr_gpu_addr;3355	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));3356	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));3357 3358	mdelay(1);3359	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);3360 3361	rb_addr = ring->gpu_addr >> 8;3362	WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);3363	WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));3364 3365	tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);3366	if (ring->use_doorbell) {3367		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,3368				    DOORBELL_OFFSET, ring->doorbell_index);3369		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,3370				    DOORBELL_EN, 1);3371	} else {3372		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);3373	}3374	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);3375 3376	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,3377			DOORBELL_RANGE_LOWER, ring->doorbell_index);3378	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);3379 3380	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,3381		       CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);3382 3383 3384	/* start the ring */3385	gfx_v9_0_cp_gfx_start(adev);3386 3387	return 0;3388}3389 3390static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)3391{3392	if (enable) {3393		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);3394	} else {3395		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,3396			(CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));3397		adev->gfx.kiq[0].ring.sched.ready = false;3398	}3399	udelay(50);3400}3401 3402static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)3403{3404	const struct gfx_firmware_header_v1_0 *mec_hdr;3405	const __le32 *fw_data;3406	unsigned i;3407	u32 tmp;3408 3409	if (!adev->gfx.mec_fw)3410		return -EINVAL;3411 3412	gfx_v9_0_cp_compute_enable(adev, false);3413 3414	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;3415	amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);3416 3417	fw_data = (const __le32 *)3418		(adev->gfx.mec_fw->data +3419		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));3420	tmp = 0;3421	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);3422	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);3423	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);3424 3425	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,3426		adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);3427	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,3428		upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));3429 3430	/* MEC1 */3431	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,3432			 mec_hdr->jt_offset);3433	for (i = 0; i < mec_hdr->jt_size; i++)3434		WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,3435			le32_to_cpup(fw_data + mec_hdr->jt_offset + i));3436 3437	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,3438			adev->gfx.mec_fw_version);3439	/* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */3440 3441	return 0;3442}3443 3444/* KIQ functions */3445static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)3446{3447	uint32_t tmp;3448	struct amdgpu_device *adev = ring->adev;3449 3450	/* tell RLC which is KIQ queue */3451	tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);3452	tmp &= 0xffffff00;3453	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);3454	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);3455	tmp |= 0x80;3456	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);3457}3458 3459static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)3460{3461	struct amdgpu_device *adev = ring->adev;3462 3463	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {3464		if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {3465			mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;3466			mqd->cp_hqd_queue_priority =3467				AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;3468		}3469	}3470}3471 3472static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)3473{3474	struct amdgpu_device *adev = ring->adev;3475	struct v9_mqd *mqd = ring->mqd_ptr;3476	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;3477	uint32_t tmp;3478 3479	mqd->header = 0xC0310800;3480	mqd->compute_pipelinestat_enable = 0x00000001;3481	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;3482	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;3483	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;3484	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;3485	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;3486	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;3487	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;3488	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;3489	mqd->compute_misc_reserved = 0x00000003;3490 3491	mqd->dynamic_cu_mask_addr_lo =3492		lower_32_bits(ring->mqd_gpu_addr3493			      + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));3494	mqd->dynamic_cu_mask_addr_hi =3495		upper_32_bits(ring->mqd_gpu_addr3496			      + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));3497 3498	eop_base_addr = ring->eop_gpu_addr >> 8;3499	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;3500	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);3501 3502	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */3503	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);3504	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,3505			(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));3506 3507	mqd->cp_hqd_eop_control = tmp;3508 3509	/* enable doorbell? */3510	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);3511 3512	if (ring->use_doorbell) {3513		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,3514				    DOORBELL_OFFSET, ring->doorbell_index);3515		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,3516				    DOORBELL_EN, 1);3517		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,3518				    DOORBELL_SOURCE, 0);3519		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,3520				    DOORBELL_HIT, 0);3521	} else {3522		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,3523					 DOORBELL_EN, 0);3524	}3525 3526	mqd->cp_hqd_pq_doorbell_control = tmp;3527 3528	/* disable the queue if it's active */3529	ring->wptr = 0;3530	mqd->cp_hqd_dequeue_request = 0;3531	mqd->cp_hqd_pq_rptr = 0;3532	mqd->cp_hqd_pq_wptr_lo = 0;3533	mqd->cp_hqd_pq_wptr_hi = 0;3534 3535	/* set the pointer to the MQD */3536	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;3537	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);3538 3539	/* set MQD vmid to 0 */3540	tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);3541	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);3542	mqd->cp_mqd_control = tmp;3543 3544	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */3545	hqd_gpu_addr = ring->gpu_addr >> 8;3546	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;3547	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);3548 3549	/* set up the HQD, this is similar to CP_RB0_CNTL */3550	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);3551	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,3552			    (order_base_2(ring->ring_size / 4) - 1));3553	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,3554			(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));3555#ifdef __BIG_ENDIAN3556	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);3557#endif3558	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);3559	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);3560	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);3561	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);3562	mqd->cp_hqd_pq_control = tmp;3563 3564	/* set the wb address whether it's enabled or not */3565	wb_gpu_addr = ring->rptr_gpu_addr;3566	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;3567	mqd->cp_hqd_pq_rptr_report_addr_hi =3568		upper_32_bits(wb_gpu_addr) & 0xffff;3569 3570	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */3571	wb_gpu_addr = ring->wptr_gpu_addr;3572	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;3573	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;3574 3575	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */3576	ring->wptr = 0;3577	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);3578 3579	/* set the vmid for the queue */3580	mqd->cp_hqd_vmid = 0;3581 3582	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);3583	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);3584	mqd->cp_hqd_persistent_state = tmp;3585 3586	/* set MIN_IB_AVAIL_SIZE */3587	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);3588	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);3589	mqd->cp_hqd_ib_control = tmp;3590 3591	/* set static priority for a queue/ring */3592	gfx_v9_0_mqd_set_priority(ring, mqd);3593	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);3594 3595	/* map_queues packet doesn't need activate the queue,3596	 * so only kiq need set this field.3597	 */3598	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)3599		mqd->cp_hqd_active = 1;3600 3601	return 0;3602}3603 3604static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)3605{3606	struct amdgpu_device *adev = ring->adev;3607	struct v9_mqd *mqd = ring->mqd_ptr;3608	int j;3609 3610	/* disable wptr polling */3611	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);3612 3613	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,3614	       mqd->cp_hqd_eop_base_addr_lo);3615	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,3616	       mqd->cp_hqd_eop_base_addr_hi);3617 3618	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */3619	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,3620	       mqd->cp_hqd_eop_control);3621 3622	/* enable doorbell? */3623	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,3624	       mqd->cp_hqd_pq_doorbell_control);3625 3626	/* disable the queue if it's active */3627	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {3628		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);3629		for (j = 0; j < adev->usec_timeout; j++) {3630			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))3631				break;3632			udelay(1);3633		}3634		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,3635		       mqd->cp_hqd_dequeue_request);3636		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,3637		       mqd->cp_hqd_pq_rptr);3638		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,3639		       mqd->cp_hqd_pq_wptr_lo);3640		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,3641		       mqd->cp_hqd_pq_wptr_hi);3642	}3643 3644	/* set the pointer to the MQD */3645	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,3646	       mqd->cp_mqd_base_addr_lo);3647	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,3648	       mqd->cp_mqd_base_addr_hi);3649 3650	/* set MQD vmid to 0 */3651	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,3652	       mqd->cp_mqd_control);3653 3654	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */3655	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,3656	       mqd->cp_hqd_pq_base_lo);3657	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,3658	       mqd->cp_hqd_pq_base_hi);3659 3660	/* set up the HQD, this is similar to CP_RB0_CNTL */3661	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,3662	       mqd->cp_hqd_pq_control);3663 3664	/* set the wb address whether it's enabled or not */3665	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,3666				mqd->cp_hqd_pq_rptr_report_addr_lo);3667	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,3668				mqd->cp_hqd_pq_rptr_report_addr_hi);3669 3670	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */3671	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,3672	       mqd->cp_hqd_pq_wptr_poll_addr_lo);3673	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,3674	       mqd->cp_hqd_pq_wptr_poll_addr_hi);3675 3676	/* enable the doorbell if requested */3677	if (ring->use_doorbell) {3678		WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,3679					(adev->doorbell_index.kiq * 2) << 2);3680		/* If GC has entered CGPG, ringing doorbell > first page3681		 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to3682		 * workaround this issue. And this change has to align with firmware3683		 * update.3684		 */3685		if (check_if_enlarge_doorbell_range(adev))3686			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,3687					(adev->doorbell.size - 4));3688		else3689			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,3690					(adev->doorbell_index.userqueue_end * 2) << 2);3691	}3692 3693	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,3694	       mqd->cp_hqd_pq_doorbell_control);3695 3696	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */3697	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,3698	       mqd->cp_hqd_pq_wptr_lo);3699	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,3700	       mqd->cp_hqd_pq_wptr_hi);3701 3702	/* set the vmid for the queue */3703	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);3704 3705	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,3706	       mqd->cp_hqd_persistent_state);3707 3708	/* activate the queue */3709	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,3710	       mqd->cp_hqd_active);3711 3712	if (ring->use_doorbell)3713		WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);3714 3715	return 0;3716}3717 3718static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)3719{3720	struct amdgpu_device *adev = ring->adev;3721	int j;3722 3723	/* disable the queue if it's active */3724	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {3725 3726		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);3727 3728		for (j = 0; j < adev->usec_timeout; j++) {3729			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))3730				break;3731			udelay(1);3732		}3733 3734		if (j == AMDGPU_MAX_USEC_TIMEOUT) {3735			DRM_DEBUG("KIQ dequeue request failed.\n");3736 3737			/* Manual disable if dequeue request times out */3738			WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);3739		}3740 3741		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,3742		      0);3743	}3744 3745	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);3746	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);3747	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);3748	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);3749	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);3750	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);3751	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);3752	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);3753 3754	return 0;3755}3756 3757static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)3758{3759	struct amdgpu_device *adev = ring->adev;3760	struct v9_mqd *mqd = ring->mqd_ptr;3761	struct v9_mqd *tmp_mqd;3762 3763	gfx_v9_0_kiq_setting(ring);3764 3765	/* GPU could be in bad state during probe, driver trigger the reset3766	 * after load the SMU, in this case , the mqd is not be initialized.3767	 * driver need to re-init the mqd.3768	 * check mqd->cp_hqd_pq_control since this value should not be 03769	 */3770	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;3771	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){3772		/* for GPU_RESET case , reset MQD to a clean status */3773		if (adev->gfx.kiq[0].mqd_backup)3774			memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));3775 3776		/* reset ring buffer */3777		ring->wptr = 0;3778		amdgpu_ring_clear_ring(ring);3779 3780		mutex_lock(&adev->srbm_mutex);3781		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);3782		gfx_v9_0_kiq_init_register(ring);3783		soc15_grbm_select(adev, 0, 0, 0, 0, 0);3784		mutex_unlock(&adev->srbm_mutex);3785	} else {3786		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));3787		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;3788		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;3789		if (amdgpu_sriov_vf(adev) && adev->in_suspend)3790			amdgpu_ring_clear_ring(ring);3791		mutex_lock(&adev->srbm_mutex);3792		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);3793		gfx_v9_0_mqd_init(ring);3794		gfx_v9_0_kiq_init_register(ring);3795		soc15_grbm_select(adev, 0, 0, 0, 0, 0);3796		mutex_unlock(&adev->srbm_mutex);3797 3798		if (adev->gfx.kiq[0].mqd_backup)3799			memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));3800	}3801 3802	return 0;3803}3804 3805static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)3806{3807	struct amdgpu_device *adev = ring->adev;3808	struct v9_mqd *mqd = ring->mqd_ptr;3809	int mqd_idx = ring - &adev->gfx.compute_ring[0];3810	struct v9_mqd *tmp_mqd;3811 3812	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control3813	 * is not be initialized before3814	 */3815	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];3816 3817	if (!restore && (!tmp_mqd->cp_hqd_pq_control ||3818	    (!amdgpu_in_reset(adev) && !adev->in_suspend))) {3819		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));3820		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;3821		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;3822		mutex_lock(&adev->srbm_mutex);3823		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);3824		gfx_v9_0_mqd_init(ring);3825		soc15_grbm_select(adev, 0, 0, 0, 0, 0);3826		mutex_unlock(&adev->srbm_mutex);3827 3828		if (adev->gfx.mec.mqd_backup[mqd_idx])3829			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));3830	} else {3831		/* restore MQD to a clean status */3832		if (adev->gfx.mec.mqd_backup[mqd_idx])3833			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));3834		/* reset ring buffer */3835		ring->wptr = 0;3836		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);3837		amdgpu_ring_clear_ring(ring);3838	}3839 3840	return 0;3841}3842 3843static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)3844{3845	struct amdgpu_ring *ring;3846	int r;3847 3848	ring = &adev->gfx.kiq[0].ring;3849 3850	r = amdgpu_bo_reserve(ring->mqd_obj, false);3851	if (unlikely(r != 0))3852		return r;3853 3854	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);3855	if (unlikely(r != 0)) {3856		amdgpu_bo_unreserve(ring->mqd_obj);3857		return r;3858	}3859 3860	gfx_v9_0_kiq_init_queue(ring);3861	amdgpu_bo_kunmap(ring->mqd_obj);3862	ring->mqd_ptr = NULL;3863	amdgpu_bo_unreserve(ring->mqd_obj);3864	return 0;3865}3866 3867static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)3868{3869	struct amdgpu_ring *ring = NULL;3870	int r = 0, i;3871 3872	gfx_v9_0_cp_compute_enable(adev, true);3873 3874	for (i = 0; i < adev->gfx.num_compute_rings; i++) {3875		ring = &adev->gfx.compute_ring[i];3876 3877		r = amdgpu_bo_reserve(ring->mqd_obj, false);3878		if (unlikely(r != 0))3879			goto done;3880		r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);3881		if (!r) {3882			r = gfx_v9_0_kcq_init_queue(ring, false);3883			amdgpu_bo_kunmap(ring->mqd_obj);3884			ring->mqd_ptr = NULL;3885		}3886		amdgpu_bo_unreserve(ring->mqd_obj);3887		if (r)3888			goto done;3889	}3890 3891	r = amdgpu_gfx_enable_kcq(adev, 0);3892done:3893	return r;3894}3895 3896static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)3897{3898	int r, i;3899	struct amdgpu_ring *ring;3900 3901	if (!(adev->flags & AMD_IS_APU))3902		gfx_v9_0_enable_gui_idle_interrupt(adev, false);3903 3904	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {3905		if (adev->gfx.num_gfx_rings) {3906			/* legacy firmware loading */3907			r = gfx_v9_0_cp_gfx_load_microcode(adev);3908			if (r)3909				return r;3910		}3911 3912		r = gfx_v9_0_cp_compute_load_microcode(adev);3913		if (r)3914			return r;3915	}3916 3917	r = gfx_v9_0_kiq_resume(adev);3918	if (r)3919		return r;3920 3921	if (adev->gfx.num_gfx_rings) {3922		r = gfx_v9_0_cp_gfx_resume(adev);3923		if (r)3924			return r;3925	}3926 3927	r = gfx_v9_0_kcq_resume(adev);3928	if (r)3929		return r;3930 3931	if (adev->gfx.num_gfx_rings) {3932		ring = &adev->gfx.gfx_ring[0];3933		r = amdgpu_ring_test_helper(ring);3934		if (r)3935			return r;3936	}3937 3938	for (i = 0; i < adev->gfx.num_compute_rings; i++) {3939		ring = &adev->gfx.compute_ring[i];3940		amdgpu_ring_test_helper(ring);3941	}3942 3943	gfx_v9_0_enable_gui_idle_interrupt(adev, true);3944 3945	return 0;3946}3947 3948static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)3949{3950	u32 tmp;3951 3952	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&3953	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))3954		return;3955 3956	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);3957	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,3958				adev->df.hash_status.hash_64k);3959	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,3960				adev->df.hash_status.hash_2m);3961	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,3962				adev->df.hash_status.hash_1g);3963	WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);3964}3965 3966static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)3967{3968	if (adev->gfx.num_gfx_rings)3969		gfx_v9_0_cp_gfx_enable(adev, enable);3970	gfx_v9_0_cp_compute_enable(adev, enable);3971}3972 3973static int gfx_v9_0_hw_init(void *handle)3974{3975	int r;3976	struct amdgpu_device *adev = (struct amdgpu_device *)handle;3977 3978	amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,3979				       adev->gfx.cleaner_shader_ptr);3980 3981	if (!amdgpu_sriov_vf(adev))3982		gfx_v9_0_init_golden_registers(adev);3983 3984	gfx_v9_0_constants_init(adev);3985 3986	gfx_v9_0_init_tcp_config(adev);3987 3988	r = adev->gfx.rlc.funcs->resume(adev);3989	if (r)3990		return r;3991 3992	r = gfx_v9_0_cp_resume(adev);3993	if (r)3994		return r;3995 3996	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))3997		gfx_v9_4_2_set_power_brake_sequence(adev);3998 3999	return r;4000}4001 4002static int gfx_v9_0_hw_fini(void *handle)4003{4004	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4005 4006	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))4007		amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);4008	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);4009	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);4010	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);4011 4012	/* DF freeze and kcq disable will fail */4013	if (!amdgpu_ras_intr_triggered())4014		/* disable KCQ to avoid CPC touch memory not valid anymore */4015		amdgpu_gfx_disable_kcq(adev, 0);4016 4017	if (amdgpu_sriov_vf(adev)) {4018		gfx_v9_0_cp_gfx_enable(adev, false);4019		/* must disable polling for SRIOV when hw finished, otherwise4020		 * CPC engine may still keep fetching WB address which is already4021		 * invalid after sw finished and trigger DMAR reading error in4022		 * hypervisor side.4023		 */4024		WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);4025		return 0;4026	}4027 4028	/* Use deinitialize sequence from CAIL when unbinding device from driver,4029	 * otherwise KIQ is hanging when binding back4030	 */4031	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {4032		mutex_lock(&adev->srbm_mutex);4033		soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,4034				adev->gfx.kiq[0].ring.pipe,4035				adev->gfx.kiq[0].ring.queue, 0, 0);4036		gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);4037		soc15_grbm_select(adev, 0, 0, 0, 0, 0);4038		mutex_unlock(&adev->srbm_mutex);4039	}4040 4041	gfx_v9_0_cp_enable(adev, false);4042 4043	/* Skip stopping RLC with A+A reset or when RLC controls GFX clock */4044	if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||4045	    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {4046		dev_dbg(adev->dev, "Skipping RLC halt\n");4047		return 0;4048	}4049 4050	adev->gfx.rlc.funcs->stop(adev);4051	return 0;4052}4053 4054static int gfx_v9_0_suspend(void *handle)4055{4056	return gfx_v9_0_hw_fini(handle);4057}4058 4059static int gfx_v9_0_resume(void *handle)4060{4061	return gfx_v9_0_hw_init(handle);4062}4063 4064static bool gfx_v9_0_is_idle(void *handle)4065{4066	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4067 4068	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),4069				GRBM_STATUS, GUI_ACTIVE))4070		return false;4071	else4072		return true;4073}4074 4075static int gfx_v9_0_wait_for_idle(void *handle)4076{4077	unsigned i;4078	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4079 4080	for (i = 0; i < adev->usec_timeout; i++) {4081		if (gfx_v9_0_is_idle(handle))4082			return 0;4083		udelay(1);4084	}4085	return -ETIMEDOUT;4086}4087 4088static int gfx_v9_0_soft_reset(void *handle)4089{4090	u32 grbm_soft_reset = 0;4091	u32 tmp;4092	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4093 4094	/* GRBM_STATUS */4095	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);4096	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |4097		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |4098		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |4099		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |4100		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |4101		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {4102		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4103						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);4104		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4105						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);4106	}4107 4108	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {4109		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4110						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);4111	}4112 4113	/* GRBM_STATUS2 */4114	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);4115	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))4116		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,4117						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);4118 4119 4120	if (grbm_soft_reset) {4121		/* stop the rlc */4122		adev->gfx.rlc.funcs->stop(adev);4123 4124		if (adev->gfx.num_gfx_rings)4125			/* Disable GFX parsing/prefetching */4126			gfx_v9_0_cp_gfx_enable(adev, false);4127 4128		/* Disable MEC parsing/prefetching */4129		gfx_v9_0_cp_compute_enable(adev, false);4130 4131		if (grbm_soft_reset) {4132			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);4133			tmp |= grbm_soft_reset;4134			dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);4135			WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);4136			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);4137 4138			udelay(50);4139 4140			tmp &= ~grbm_soft_reset;4141			WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);4142			tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);4143		}4144 4145		/* Wait a little for things to settle down */4146		udelay(50);4147	}4148	return 0;4149}4150 4151static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)4152{4153	signed long r, cnt = 0;4154	unsigned long flags;4155	uint32_t seq, reg_val_offs = 0;4156	uint64_t value = 0;4157	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];4158	struct amdgpu_ring *ring = &kiq->ring;4159 4160	BUG_ON(!ring->funcs->emit_rreg);4161 4162	spin_lock_irqsave(&kiq->ring_lock, flags);4163	if (amdgpu_device_wb_get(adev, &reg_val_offs)) {4164		pr_err("critical bug! too many kiq readers\n");4165		goto failed_unlock;4166	}4167	amdgpu_ring_alloc(ring, 32);4168	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));4169	amdgpu_ring_write(ring, 9 |	/* src: register*/4170				(5 << 8) |	/* dst: memory */4171				(1 << 16) |	/* count sel */4172				(1 << 20));	/* write confirm */4173	amdgpu_ring_write(ring, 0);4174	amdgpu_ring_write(ring, 0);4175	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +4176				reg_val_offs * 4));4177	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +4178				reg_val_offs * 4));4179	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);4180	if (r)4181		goto failed_undo;4182 4183	amdgpu_ring_commit(ring);4184	spin_unlock_irqrestore(&kiq->ring_lock, flags);4185 4186	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);4187 4188	/* don't wait anymore for gpu reset case because this way may4189	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg4190	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will4191	 * never return if we keep waiting in virt_kiq_rreg, which cause4192	 * gpu_recover() hang there.4193	 *4194	 * also don't wait anymore for IRQ context4195	 * */4196	if (r < 1 && (amdgpu_in_reset(adev)))4197		goto failed_kiq_read;4198 4199	might_sleep();4200	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {4201		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);4202		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);4203	}4204 4205	if (cnt > MAX_KIQ_REG_TRY)4206		goto failed_kiq_read;4207 4208	mb();4209	value = (uint64_t)adev->wb.wb[reg_val_offs] |4210		(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;4211	amdgpu_device_wb_free(adev, reg_val_offs);4212	return value;4213 4214failed_undo:4215	amdgpu_ring_undo(ring);4216failed_unlock:4217	spin_unlock_irqrestore(&kiq->ring_lock, flags);4218failed_kiq_read:4219	if (reg_val_offs)4220		amdgpu_device_wb_free(adev, reg_val_offs);4221	pr_err("failed to read gpu clock\n");4222	return ~0;4223}4224 4225static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)4226{4227	uint64_t clock, clock_lo, clock_hi, hi_check;4228 4229	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {4230	case IP_VERSION(9, 3, 0):4231		preempt_disable();4232		clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);4233		clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);4234		hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);4235		/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over4236		 * roughly every 42 seconds.4237		 */4238		if (hi_check != clock_hi) {4239			clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);4240			clock_hi = hi_check;4241		}4242		preempt_enable();4243		clock = clock_lo | (clock_hi << 32ULL);4244		break;4245	default:4246		amdgpu_gfx_off_ctrl(adev, false);4247		mutex_lock(&adev->gfx.gpu_clock_mutex);4248		if (amdgpu_ip_version(adev, GC_HWIP, 0) ==4249			    IP_VERSION(9, 0, 1) &&4250		    amdgpu_sriov_runtime(adev)) {4251			clock = gfx_v9_0_kiq_read_clock(adev);4252		} else {4253			WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);4254			clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |4255				((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);4256		}4257		mutex_unlock(&adev->gfx.gpu_clock_mutex);4258		amdgpu_gfx_off_ctrl(adev, true);4259		break;4260	}4261	return clock;4262}4263 4264static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,4265					  uint32_t vmid,4266					  uint32_t gds_base, uint32_t gds_size,4267					  uint32_t gws_base, uint32_t gws_size,4268					  uint32_t oa_base, uint32_t oa_size)4269{4270	struct amdgpu_device *adev = ring->adev;4271 4272	/* GDS Base */4273	gfx_v9_0_write_data_to_reg(ring, 0, false,4274				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,4275				   gds_base);4276 4277	/* GDS Size */4278	gfx_v9_0_write_data_to_reg(ring, 0, false,4279				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,4280				   gds_size);4281 4282	/* GWS */4283	gfx_v9_0_write_data_to_reg(ring, 0, false,4284				   SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,4285				   gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);4286 4287	/* OA */4288	gfx_v9_0_write_data_to_reg(ring, 0, false,4289				   SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,4290				   (1 << (oa_size + oa_base)) - (1 << oa_base));4291}4292 4293static const u32 vgpr_init_compute_shader[] =4294{4295	0xb07c0000, 0xbe8000ff,4296	0x000000f8, 0xbf110800,4297	0x7e000280, 0x7e020280,4298	0x7e040280, 0x7e060280,4299	0x7e080280, 0x7e0a0280,4300	0x7e0c0280, 0x7e0e0280,4301	0x80808800, 0xbe803200,4302	0xbf84fff5, 0xbf9c0000,4303	0xd28c0001, 0x0001007f,4304	0xd28d0001, 0x0002027e,4305	0x10020288, 0xb8810904,4306	0xb7814000, 0xd1196a01,4307	0x00000301, 0xbe800087,4308	0xbefc00c1, 0xd89c4000,4309	0x00020201, 0xd89cc080,4310	0x00040401, 0x320202ff,4311	0x00000800, 0x80808100,4312	0xbf84fff8, 0x7e020280,4313	0xbf810000, 0x00000000,4314};4315 4316static const u32 sgpr_init_compute_shader[] =4317{4318	0xb07c0000, 0xbe8000ff,4319	0x0000005f, 0xbee50080,4320	0xbe812c65, 0xbe822c65,4321	0xbe832c65, 0xbe842c65,4322	0xbe852c65, 0xb77c0005,4323	0x80808500, 0xbf84fff8,4324	0xbe800080, 0xbf810000,4325};4326 4327static const u32 vgpr_init_compute_shader_arcturus[] = {4328	0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,4329	0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,4330	0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,4331	0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,4332	0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,4333	0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,4334	0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,4335	0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,4336	0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,4337	0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,4338	0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,4339	0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,4340	0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,4341	0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,4342	0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,4343	0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,4344	0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,4345	0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,4346	0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,4347	0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,4348	0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,4349	0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,4350	0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,4351	0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,4352	0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,4353	0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,4354	0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,4355	0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,4356	0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,4357	0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,4358	0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,4359	0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,4360	0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,4361	0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,4362	0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,4363	0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,4364	0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,4365	0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,4366	0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,4367	0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,4368	0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,4369	0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,4370	0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,4371	0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,4372	0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,4373	0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,4374	0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,4375	0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,4376	0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,4377	0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,4378	0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,4379	0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,4380	0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,4381	0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,4382	0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,4383	0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,4384	0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,4385	0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,4386	0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,4387	0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,4388	0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,4389	0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,4390	0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,4391	0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,4392	0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,4393	0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,4394	0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,4395	0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,4396	0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,4397	0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,4398	0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,4399	0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,4400	0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,4401	0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,4402	0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,4403	0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,4404	0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,4405	0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,4406	0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,4407	0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,4408	0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,4409	0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,4410	0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,4411	0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,4412	0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,4413	0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,4414	0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,4415	0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,4416	0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,4417	0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,4418	0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,4419	0xbf84fff8, 0xbf810000,4420};4421 4422/* When below register arrays changed, please update gpr_reg_size,4423  and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,4424  to cover all gfx9 ASICs */4425static const struct soc15_reg_entry vgpr_init_regs[] = {4426   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },4427   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },4428   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },4429   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },4430   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },4431   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */4432   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },4433   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },4434   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },4435   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },4436   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },4437   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },4438   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },4439   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },4440};4441 4442static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {4443   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },4444   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },4445   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },4446   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },4447   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },4448   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */4449   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },4450   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },4451   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },4452   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },4453   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },4454   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },4455   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },4456   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },4457};4458 4459static const struct soc15_reg_entry sgpr1_init_regs[] = {4460   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },4461   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },4462   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },4463   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },4464   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */4465   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },4466   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },4467   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },4468   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },4469   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },4470   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },4471   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },4472   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },4473   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },4474};4475 4476static const struct soc15_reg_entry sgpr2_init_regs[] = {4477   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },4478   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },4479   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },4480   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },4481   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */4482   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },4483   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },4484   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },4485   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },4486   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },4487   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },4488   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },4489   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },4490   { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },4491};4492 4493static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {4494   { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},4495   { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},4496   { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},4497   { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},4498   { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},4499   { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},4500   { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},4501   { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},4502   { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},4503   { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},4504   { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},4505   { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},4506   { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},4507   { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},4508   { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},4509   { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},4510   { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},4511   { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},4512   { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},4513   { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},4514   { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},4515   { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},4516   { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},4517   { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},4518   { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},4519   { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},4520   { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},4521   { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},4522   { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},4523   { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},4524   { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},4525   { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},4526   { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},4527};4528 4529static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)4530{4531	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];4532	int i, r;4533 4534	/* only support when RAS is enabled */4535	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))4536		return 0;4537 4538	r = amdgpu_ring_alloc(ring, 7);4539	if (r) {4540		DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",4541			ring->name, r);4542		return r;4543	}4544 4545	WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);4546	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);4547 4548	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));4549	amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |4550				PACKET3_DMA_DATA_DST_SEL(1) |4551				PACKET3_DMA_DATA_SRC_SEL(2) |4552				PACKET3_DMA_DATA_ENGINE(0)));4553	amdgpu_ring_write(ring, 0);4554	amdgpu_ring_write(ring, 0);4555	amdgpu_ring_write(ring, 0);4556	amdgpu_ring_write(ring, 0);4557	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |4558				adev->gds.gds_size);4559 4560	amdgpu_ring_commit(ring);4561 4562	for (i = 0; i < adev->usec_timeout; i++) {4563		if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))4564			break;4565		udelay(1);4566	}4567 4568	if (i >= adev->usec_timeout)4569		r = -ETIMEDOUT;4570 4571	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);4572 4573	return r;4574}4575 4576static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)4577{4578	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];4579	struct amdgpu_ib ib;4580	struct dma_fence *f = NULL;4581	int r, i;4582	unsigned total_size, vgpr_offset, sgpr_offset;4583	u64 gpu_addr;4584 4585	int compute_dim_x = adev->gfx.config.max_shader_engines *4586						adev->gfx.config.max_cu_per_sh *4587						adev->gfx.config.max_sh_per_se;4588	int sgpr_work_group_size = 5;4589	int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;4590	int vgpr_init_shader_size;4591	const u32 *vgpr_init_shader_ptr;4592	const struct soc15_reg_entry *vgpr_init_regs_ptr;4593 4594	/* only support when RAS is enabled */4595	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))4596		return 0;4597 4598	/* bail if the compute ring is not ready */4599	if (!ring->sched.ready)4600		return 0;4601 4602	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {4603		vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;4604		vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);4605		vgpr_init_regs_ptr = vgpr_init_regs_arcturus;4606	} else {4607		vgpr_init_shader_ptr = vgpr_init_compute_shader;4608		vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);4609		vgpr_init_regs_ptr = vgpr_init_regs;4610	}4611 4612	total_size =4613		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */4614	total_size +=4615		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */4616	total_size +=4617		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */4618	total_size = ALIGN(total_size, 256);4619	vgpr_offset = total_size;4620	total_size += ALIGN(vgpr_init_shader_size, 256);4621	sgpr_offset = total_size;4622	total_size += sizeof(sgpr_init_compute_shader);4623 4624	/* allocate an indirect buffer to put the commands in */4625	memset(&ib, 0, sizeof(ib));4626	r = amdgpu_ib_get(adev, NULL, total_size,4627					AMDGPU_IB_POOL_DIRECT, &ib);4628	if (r) {4629		DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);4630		return r;4631	}4632 4633	/* load the compute shaders */4634	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)4635		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];4636 4637	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)4638		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];4639 4640	/* init the ib length to 0 */4641	ib.length_dw = 0;4642 4643	/* VGPR */4644	/* write the register state for the compute dispatch */4645	for (i = 0; i < gpr_reg_size; i++) {4646		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);4647		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])4648								- PACKET3_SET_SH_REG_START;4649		ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;4650	}4651	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */4652	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;4653	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);4654	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)4655							- PACKET3_SET_SH_REG_START;4656	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);4657	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);4658 4659	/* write dispatch packet */4660	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);4661	ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */4662	ib.ptr[ib.length_dw++] = 1; /* y */4663	ib.ptr[ib.length_dw++] = 1; /* z */4664	ib.ptr[ib.length_dw++] =4665		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);4666 4667	/* write CS partial flush packet */4668	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);4669	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);4670 4671	/* SGPR1 */4672	/* write the register state for the compute dispatch */4673	for (i = 0; i < gpr_reg_size; i++) {4674		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);4675		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])4676								- PACKET3_SET_SH_REG_START;4677		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;4678	}4679	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */4680	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;4681	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);4682	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)4683							- PACKET3_SET_SH_REG_START;4684	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);4685	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);4686 4687	/* write dispatch packet */4688	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);4689	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */4690	ib.ptr[ib.length_dw++] = 1; /* y */4691	ib.ptr[ib.length_dw++] = 1; /* z */4692	ib.ptr[ib.length_dw++] =4693		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);4694 4695	/* write CS partial flush packet */4696	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);4697	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);4698 4699	/* SGPR2 */4700	/* write the register state for the compute dispatch */4701	for (i = 0; i < gpr_reg_size; i++) {4702		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);4703		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])4704								- PACKET3_SET_SH_REG_START;4705		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;4706	}4707	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */4708	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;4709	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);4710	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)4711							- PACKET3_SET_SH_REG_START;4712	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);4713	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);4714 4715	/* write dispatch packet */4716	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);4717	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */4718	ib.ptr[ib.length_dw++] = 1; /* y */4719	ib.ptr[ib.length_dw++] = 1; /* z */4720	ib.ptr[ib.length_dw++] =4721		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);4722 4723	/* write CS partial flush packet */4724	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);4725	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);4726 4727	/* shedule the ib on the ring */4728	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);4729	if (r) {4730		DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);4731		goto fail;4732	}4733 4734	/* wait for the GPU to finish processing the IB */4735	r = dma_fence_wait(f, false);4736	if (r) {4737		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);4738		goto fail;4739	}4740 4741fail:4742	amdgpu_ib_free(adev, &ib, NULL);4743	dma_fence_put(f);4744 4745	return r;4746}4747 4748static int gfx_v9_0_early_init(void *handle)4749{4750	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4751 4752	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;4753 4754	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||4755	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))4756		adev->gfx.num_gfx_rings = 0;4757	else4758		adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;4759	adev->gfx.xcc_mask = 1;4760	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),4761					  AMDGPU_MAX_COMPUTE_RINGS);4762	gfx_v9_0_set_kiq_pm4_funcs(adev);4763	gfx_v9_0_set_ring_funcs(adev);4764	gfx_v9_0_set_irq_funcs(adev);4765	gfx_v9_0_set_gds_init(adev);4766	gfx_v9_0_set_rlc_funcs(adev);4767 4768	/* init rlcg reg access ctrl */4769	gfx_v9_0_init_rlcg_reg_access_ctrl(adev);4770 4771	return gfx_v9_0_init_microcode(adev);4772}4773 4774static int gfx_v9_0_ecc_late_init(void *handle)4775{4776	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4777	int r;4778 4779	/*4780	 * Temp workaround to fix the issue that CP firmware fails to4781	 * update read pointer when CPDMA is writing clearing operation4782	 * to GDS in suspend/resume sequence on several cards. So just4783	 * limit this operation in cold boot sequence.4784	 */4785	if ((!adev->in_suspend) &&4786	    (adev->gds.gds_size)) {4787		r = gfx_v9_0_do_edc_gds_workarounds(adev);4788		if (r)4789			return r;4790	}4791 4792	/* requires IBs so do in late init after IB pool is initialized */4793	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))4794		r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);4795	else4796		r = gfx_v9_0_do_edc_gpr_workarounds(adev);4797 4798	if (r)4799		return r;4800 4801	if (adev->gfx.ras &&4802	    adev->gfx.ras->enable_watchdog_timer)4803		adev->gfx.ras->enable_watchdog_timer(adev);4804 4805	return 0;4806}4807 4808static int gfx_v9_0_late_init(void *handle)4809{4810	struct amdgpu_device *adev = (struct amdgpu_device *)handle;4811	int r;4812 4813	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);4814	if (r)4815		return r;4816 4817	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);4818	if (r)4819		return r;4820 4821	r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);4822	if (r)4823		return r;4824 4825	r = gfx_v9_0_ecc_late_init(handle);4826	if (r)4827		return r;4828 4829	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))4830		gfx_v9_4_2_debug_trap_config_init(adev,4831			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);4832	else4833		gfx_v9_0_debug_trap_config_init(adev,4834			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);4835 4836	return 0;4837}4838 4839static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)4840{4841	uint32_t rlc_setting;4842 4843	/* if RLC is not enabled, do nothing */4844	rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);4845	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))4846		return false;4847 4848	return true;4849}4850 4851static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)4852{4853	uint32_t data;4854	unsigned i;4855 4856	data = RLC_SAFE_MODE__CMD_MASK;4857	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);4858	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);4859 4860	/* wait for RLC_SAFE_MODE */4861	for (i = 0; i < adev->usec_timeout; i++) {4862		if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))4863			break;4864		udelay(1);4865	}4866}4867 4868static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)4869{4870	uint32_t data;4871 4872	data = RLC_SAFE_MODE__CMD_MASK;4873	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);4874}4875 4876static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,4877						bool enable)4878{4879	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);4880 4881	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {4882		gfx_v9_0_enable_gfx_cg_power_gating(adev, true);4883		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)4884			gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);4885	} else {4886		gfx_v9_0_enable_gfx_cg_power_gating(adev, false);4887		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)4888			gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);4889	}4890 4891	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);4892}4893 4894static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,4895						bool enable)4896{4897	/* TODO: double check if we need to perform under safe mode */4898	/* gfx_v9_0_enter_rlc_safe_mode(adev); */4899 4900	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)4901		gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);4902	else4903		gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);4904 4905	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)4906		gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);4907	else4908		gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);4909 4910	/* gfx_v9_0_exit_rlc_safe_mode(adev); */4911}4912 4913static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,4914						      bool enable)4915{4916	uint32_t data, def;4917 4918	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);4919 4920	/* It is disabled by HW by default */4921	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {4922		/* 1 - RLC_CGTT_MGCG_OVERRIDE */4923		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);4924 4925		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))4926			data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;4927 4928		data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |4929			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |4930			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);4931 4932		/* only for Vega10 & Raven1 */4933		data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;4934 4935		if (def != data)4936			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);4937 4938		/* MGLS is a global flag to control all MGLS in GFX */4939		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {4940			/* 2 - RLC memory Light sleep */4941			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {4942				def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);4943				data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;4944				if (def != data)4945					WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);4946			}4947			/* 3 - CP memory Light sleep */4948			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {4949				def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);4950				data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;4951				if (def != data)4952					WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);4953			}4954		}4955	} else {4956		/* 1 - MGCG_OVERRIDE */4957		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);4958 4959		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))4960			data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;4961 4962		data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |4963			 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |4964			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |4965			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);4966 4967		if (def != data)4968			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);4969 4970		/* 2 - disable MGLS in RLC */4971		data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);4972		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {4973			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;4974			WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);4975		}4976 4977		/* 3 - disable MGLS in CP */4978		data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);4979		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {4980			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;4981			WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);4982		}4983	}4984 4985	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);4986}4987 4988static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,4989					   bool enable)4990{4991	uint32_t data, def;4992 4993	if (!adev->gfx.num_gfx_rings)4994		return;4995 4996	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);4997 4998	/* Enable 3D CGCG/CGLS */4999	if (enable) {5000		/* write cmd to clear cgcg/cgls ov */5001		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);5002		/* unset CGCG override */5003		data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;5004		/* update CGCG and CGLS override bits */5005		if (def != data)5006			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);5007 5008		/* enable 3Dcgcg FSM(0x0000363f) */5009		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);5010 5011		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)5012			data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |5013				RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;5014		else5015			data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;5016 5017		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)5018			data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |5019				RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;5020		if (def != data)5021			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);5022 5023		/* set IDLE_POLL_COUNT(0x00900100) */5024		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);5025		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |5026			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);5027		if (def != data)5028			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);5029	} else {5030		/* Disable CGCG/CGLS */5031		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);5032		/* disable cgcg, cgls should be disabled */5033		data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |5034			  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);5035		/* disable cgcg and cgls in FSM */5036		if (def != data)5037			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);5038	}5039 5040	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5041}5042 5043static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,5044						      bool enable)5045{5046	uint32_t def, data;5047 5048	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);5049 5050	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {5051		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);5052		/* unset CGCG override */5053		data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;5054		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)5055			data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;5056		else5057			data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;5058		/* update CGCG and CGLS override bits */5059		if (def != data)5060			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);5061 5062		/* enable cgcg FSM(0x0000363F) */5063		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);5064 5065		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))5066			data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |5067				RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;5068		else5069			data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |5070				RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;5071		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)5072			data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |5073				RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;5074		if (def != data)5075			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);5076 5077		/* set IDLE_POLL_COUNT(0x00900100) */5078		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);5079		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |5080			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);5081		if (def != data)5082			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);5083	} else {5084		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);5085		/* reset CGCG/CGLS bits */5086		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);5087		/* disable cgcg and cgls in FSM */5088		if (def != data)5089			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);5090	}5091 5092	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5093}5094 5095static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,5096					    bool enable)5097{5098	if (enable) {5099		/* CGCG/CGLS should be enabled after MGCG/MGLS5100		 * ===  MGCG + MGLS ===5101		 */5102		gfx_v9_0_update_medium_grain_clock_gating(adev, enable);5103		/* ===  CGCG /CGLS for GFX 3D Only === */5104		gfx_v9_0_update_3d_clock_gating(adev, enable);5105		/* ===  CGCG + CGLS === */5106		gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);5107	} else {5108		/* CGCG/CGLS should be disabled before MGCG/MGLS5109		 * ===  CGCG + CGLS ===5110		 */5111		gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);5112		/* ===  CGCG /CGLS for GFX 3D Only === */5113		gfx_v9_0_update_3d_clock_gating(adev, enable);5114		/* ===  MGCG + MGLS === */5115		gfx_v9_0_update_medium_grain_clock_gating(adev, enable);5116	}5117	return 0;5118}5119 5120static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,5121					      unsigned int vmid)5122{5123	u32 reg, data;5124 5125	reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);5126	if (amdgpu_sriov_is_pp_one_vf(adev))5127		data = RREG32_NO_KIQ(reg);5128	else5129		data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);5130 5131	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;5132	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;5133 5134	if (amdgpu_sriov_is_pp_one_vf(adev))5135		WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);5136	else5137		WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);5138}5139 5140static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)5141{5142	amdgpu_gfx_off_ctrl(adev, false);5143 5144	gfx_v9_0_update_spm_vmid_internal(adev, vmid);5145 5146	amdgpu_gfx_off_ctrl(adev, true);5147}5148 5149static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,5150					uint32_t offset,5151					struct soc15_reg_rlcg *entries, int arr_size)5152{5153	int i;5154	uint32_t reg;5155 5156	if (!entries)5157		return false;5158 5159	for (i = 0; i < arr_size; i++) {5160		const struct soc15_reg_rlcg *entry;5161 5162		entry = &entries[i];5163		reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;5164		if (offset == reg)5165			return true;5166	}5167 5168	return false;5169}5170 5171static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)5172{5173	return gfx_v9_0_check_rlcg_range(adev, offset,5174					(void *)rlcg_access_gc_9_0,5175					ARRAY_SIZE(rlcg_access_gc_9_0));5176}5177 5178static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {5179	.is_rlc_enabled = gfx_v9_0_is_rlc_enabled,5180	.set_safe_mode = gfx_v9_0_set_safe_mode,5181	.unset_safe_mode = gfx_v9_0_unset_safe_mode,5182	.init = gfx_v9_0_rlc_init,5183	.get_csb_size = gfx_v9_0_get_csb_size,5184	.get_csb_buffer = gfx_v9_0_get_csb_buffer,5185	.get_cp_table_num = gfx_v9_0_cp_jump_table_num,5186	.resume = gfx_v9_0_rlc_resume,5187	.stop = gfx_v9_0_rlc_stop,5188	.reset = gfx_v9_0_rlc_reset,5189	.start = gfx_v9_0_rlc_start,5190	.update_spm_vmid = gfx_v9_0_update_spm_vmid,5191	.is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,5192};5193 5194static int gfx_v9_0_set_powergating_state(void *handle,5195					  enum amd_powergating_state state)5196{5197	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5198	bool enable = (state == AMD_PG_STATE_GATE);5199 5200	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {5201	case IP_VERSION(9, 2, 2):5202	case IP_VERSION(9, 1, 0):5203	case IP_VERSION(9, 3, 0):5204		if (!enable)5205			amdgpu_gfx_off_ctrl(adev, false);5206 5207		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {5208			gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);5209			gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);5210		} else {5211			gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);5212			gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);5213		}5214 5215		if (adev->pg_flags & AMD_PG_SUPPORT_CP)5216			gfx_v9_0_enable_cp_power_gating(adev, true);5217		else5218			gfx_v9_0_enable_cp_power_gating(adev, false);5219 5220		/* update gfx cgpg state */5221		gfx_v9_0_update_gfx_cg_power_gating(adev, enable);5222 5223		/* update mgcg state */5224		gfx_v9_0_update_gfx_mg_power_gating(adev, enable);5225 5226		if (enable)5227			amdgpu_gfx_off_ctrl(adev, true);5228		break;5229	case IP_VERSION(9, 2, 1):5230		amdgpu_gfx_off_ctrl(adev, enable);5231		break;5232	default:5233		break;5234	}5235 5236	return 0;5237}5238 5239static int gfx_v9_0_set_clockgating_state(void *handle,5240					  enum amd_clockgating_state state)5241{5242	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5243 5244	if (amdgpu_sriov_vf(adev))5245		return 0;5246 5247	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {5248	case IP_VERSION(9, 0, 1):5249	case IP_VERSION(9, 2, 1):5250	case IP_VERSION(9, 4, 0):5251	case IP_VERSION(9, 2, 2):5252	case IP_VERSION(9, 1, 0):5253	case IP_VERSION(9, 4, 1):5254	case IP_VERSION(9, 3, 0):5255	case IP_VERSION(9, 4, 2):5256		gfx_v9_0_update_gfx_clock_gating(adev,5257						 state == AMD_CG_STATE_GATE);5258		break;5259	default:5260		break;5261	}5262	return 0;5263}5264 5265static void gfx_v9_0_get_clockgating_state(void *handle, u64 *flags)5266{5267	struct amdgpu_device *adev = (struct amdgpu_device *)handle;5268	int data;5269 5270	if (amdgpu_sriov_vf(adev))5271		*flags = 0;5272 5273	/* AMD_CG_SUPPORT_GFX_MGCG */5274	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));5275	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))5276		*flags |= AMD_CG_SUPPORT_GFX_MGCG;5277 5278	/* AMD_CG_SUPPORT_GFX_CGCG */5279	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));5280	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)5281		*flags |= AMD_CG_SUPPORT_GFX_CGCG;5282 5283	/* AMD_CG_SUPPORT_GFX_CGLS */5284	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)5285		*flags |= AMD_CG_SUPPORT_GFX_CGLS;5286 5287	/* AMD_CG_SUPPORT_GFX_RLC_LS */5288	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));5289	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)5290		*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;5291 5292	/* AMD_CG_SUPPORT_GFX_CP_LS */5293	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));5294	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)5295		*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;5296 5297	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {5298		/* AMD_CG_SUPPORT_GFX_3D_CGCG */5299		data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));5300		if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)5301			*flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;5302 5303		/* AMD_CG_SUPPORT_GFX_3D_CGLS */5304		if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)5305			*flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;5306	}5307}5308 5309static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)5310{5311	return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/5312}5313 5314static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)5315{5316	struct amdgpu_device *adev = ring->adev;5317	u64 wptr;5318 5319	/* XXX check if swapping is necessary on BE */5320	if (ring->use_doorbell) {5321		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);5322	} else {5323		wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);5324		wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;5325	}5326 5327	return wptr;5328}5329 5330static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)5331{5332	struct amdgpu_device *adev = ring->adev;5333 5334	if (ring->use_doorbell) {5335		/* XXX check if swapping is necessary on BE */5336		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);5337		WDOORBELL64(ring->doorbell_index, ring->wptr);5338	} else {5339		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));5340		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));5341	}5342}5343 5344static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)5345{5346	struct amdgpu_device *adev = ring->adev;5347	u32 ref_and_mask, reg_mem_engine;5348	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;5349 5350	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {5351		switch (ring->me) {5352		case 1:5353			ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;5354			break;5355		case 2:5356			ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;5357			break;5358		default:5359			return;5360		}5361		reg_mem_engine = 0;5362	} else {5363		ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;5364		reg_mem_engine = 1; /* pfp */5365	}5366 5367	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,5368			      adev->nbio.funcs->get_hdp_flush_req_offset(adev),5369			      adev->nbio.funcs->get_hdp_flush_done_offset(adev),5370			      ref_and_mask, ref_and_mask, 0x20);5371}5372 5373static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,5374					struct amdgpu_job *job,5375					struct amdgpu_ib *ib,5376					uint32_t flags)5377{5378	unsigned vmid = AMDGPU_JOB_GET_VMID(job);5379	u32 header, control = 0;5380 5381	if (ib->flags & AMDGPU_IB_FLAG_CE)5382		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);5383	else5384		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);5385 5386	control |= ib->length_dw | (vmid << 24);5387 5388	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {5389		control |= INDIRECT_BUFFER_PRE_ENB(1);5390 5391		if (flags & AMDGPU_IB_PREEMPTED)5392			control |= INDIRECT_BUFFER_PRE_RESUME(1);5393 5394		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)5395			gfx_v9_0_ring_emit_de_meta(ring,5396						   (!amdgpu_sriov_vf(ring->adev) &&5397						   flags & AMDGPU_IB_PREEMPTED) ?5398						   true : false,5399						   job->gds_size > 0 && job->gds_base != 0);5400	}5401 5402	amdgpu_ring_write(ring, header);5403	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */5404	amdgpu_ring_write(ring,5405#ifdef __BIG_ENDIAN5406		(2 << 0) |5407#endif5408		lower_32_bits(ib->gpu_addr));5409	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));5410	amdgpu_ring_ib_on_emit_cntl(ring);5411	amdgpu_ring_write(ring, control);5412}5413 5414static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,5415				     unsigned offset)5416{5417	u32 control = ring->ring[offset];5418 5419	control |= INDIRECT_BUFFER_PRE_RESUME(1);5420	ring->ring[offset] = control;5421}5422 5423static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,5424					unsigned offset)5425{5426	struct amdgpu_device *adev = ring->adev;5427	void *ce_payload_cpu_addr;5428	uint64_t payload_offset, payload_size;5429 5430	payload_size = sizeof(struct v9_ce_ib_state);5431 5432	if (ring->is_mes_queue) {5433		payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,5434					  gfx[0].gfx_meta_data) +5435			offsetof(struct v9_gfx_meta_data, ce_payload);5436		ce_payload_cpu_addr =5437			amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);5438	} else {5439		payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);5440		ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;5441	}5442 5443	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {5444		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);5445	} else {5446		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,5447		       (ring->buf_mask + 1 - offset) << 2);5448		payload_size -= (ring->buf_mask + 1 - offset) << 2;5449		memcpy((void *)&ring->ring[0],5450		       ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),5451		       payload_size);5452	}5453}5454 5455static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,5456					unsigned offset)5457{5458	struct amdgpu_device *adev = ring->adev;5459	void *de_payload_cpu_addr;5460	uint64_t payload_offset, payload_size;5461 5462	payload_size = sizeof(struct v9_de_ib_state);5463 5464	if (ring->is_mes_queue) {5465		payload_offset = offsetof(struct amdgpu_mes_ctx_meta_data,5466					  gfx[0].gfx_meta_data) +5467			offsetof(struct v9_gfx_meta_data, de_payload);5468		de_payload_cpu_addr =5469			amdgpu_mes_ctx_get_offs_cpu_addr(ring, payload_offset);5470	} else {5471		payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);5472		de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;5473	}5474 5475	((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =5476		IB_COMPLETION_STATUS_PREEMPTED;5477 5478	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {5479		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);5480	} else {5481		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,5482		       (ring->buf_mask + 1 - offset) << 2);5483		payload_size -= (ring->buf_mask + 1 - offset) << 2;5484		memcpy((void *)&ring->ring[0],5485		       de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),5486		       payload_size);5487	}5488}5489 5490static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,5491					  struct amdgpu_job *job,5492					  struct amdgpu_ib *ib,5493					  uint32_t flags)5494{5495	unsigned vmid = AMDGPU_JOB_GET_VMID(job);5496	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);5497 5498	/* Currently, there is a high possibility to get wave ID mismatch5499	 * between ME and GDS, leading to a hw deadlock, because ME generates5500	 * different wave IDs than the GDS expects. This situation happens5501	 * randomly when at least 5 compute pipes use GDS ordered append.5502	 * The wave IDs generated by ME are also wrong after suspend/resume.5503	 * Those are probably bugs somewhere else in the kernel driver.5504	 *5505	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and5506	 * GDS to 0 for this ring (me/pipe).5507	 */5508	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {5509		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));5510		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);5511		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);5512	}5513 5514	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));5515	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */5516	amdgpu_ring_write(ring,5517#ifdef __BIG_ENDIAN5518				(2 << 0) |5519#endif5520				lower_32_bits(ib->gpu_addr));5521	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));5522	amdgpu_ring_write(ring, control);5523}5524 5525static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,5526				     u64 seq, unsigned flags)5527{5528	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;5529	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;5530	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;5531	bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;5532	uint32_t dw2 = 0;5533 5534	/* RELEASE_MEM - flush caches, send int */5535	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));5536 5537	if (writeback) {5538		dw2 = EOP_TC_NC_ACTION_EN;5539	} else {5540		dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |5541				EOP_TC_MD_ACTION_EN;5542	}5543	dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |5544				EVENT_INDEX(5);5545	if (exec)5546		dw2 |= EOP_EXEC;5547 5548	amdgpu_ring_write(ring, dw2);5549	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));5550 5551	/*5552	 * the address should be Qword aligned if 64bit write, Dword5553	 * aligned if only send 32bit data low (discard data high)5554	 */5555	if (write64bit)5556		BUG_ON(addr & 0x7);5557	else5558		BUG_ON(addr & 0x3);5559	amdgpu_ring_write(ring, lower_32_bits(addr));5560	amdgpu_ring_write(ring, upper_32_bits(addr));5561	amdgpu_ring_write(ring, lower_32_bits(seq));5562	amdgpu_ring_write(ring, upper_32_bits(seq));5563	amdgpu_ring_write(ring, 0);5564}5565 5566static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)5567{5568	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);5569	uint32_t seq = ring->fence_drv.sync_seq;5570	uint64_t addr = ring->fence_drv.gpu_addr;5571 5572	gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,5573			      lower_32_bits(addr), upper_32_bits(addr),5574			      seq, 0xffffffff, 4);5575}5576 5577static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,5578					unsigned vmid, uint64_t pd_addr)5579{5580	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);5581 5582	/* compute doesn't have PFP */5583	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {5584		/* sync PFP to ME, otherwise we might get invalid PFP reads */5585		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));5586		amdgpu_ring_write(ring, 0x0);5587	}5588}5589 5590static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)5591{5592	return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */5593}5594 5595static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)5596{5597	u64 wptr;5598 5599	/* XXX check if swapping is necessary on BE */5600	if (ring->use_doorbell)5601		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);5602	else5603		BUG();5604	return wptr;5605}5606 5607static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)5608{5609	struct amdgpu_device *adev = ring->adev;5610 5611	/* XXX check if swapping is necessary on BE */5612	if (ring->use_doorbell) {5613		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);5614		WDOORBELL64(ring->doorbell_index, ring->wptr);5615	} else{5616		BUG(); /* only DOORBELL method supported on gfx9 now */5617	}5618}5619 5620static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,5621					 u64 seq, unsigned int flags)5622{5623	struct amdgpu_device *adev = ring->adev;5624 5625	/* we only allocate 32bit for each seq wb address */5626	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);5627 5628	/* write fence seq to the "addr" */5629	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5630	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5631				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));5632	amdgpu_ring_write(ring, lower_32_bits(addr));5633	amdgpu_ring_write(ring, upper_32_bits(addr));5634	amdgpu_ring_write(ring, lower_32_bits(seq));5635 5636	if (flags & AMDGPU_FENCE_FLAG_INT) {5637		/* set register to trigger INT */5638		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5639		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |5640					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));5641		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));5642		amdgpu_ring_write(ring, 0);5643		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */5644	}5645}5646 5647static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)5648{5649	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));5650	amdgpu_ring_write(ring, 0);5651}5652 5653static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)5654{5655	struct amdgpu_device *adev = ring->adev;5656	struct v9_ce_ib_state ce_payload = {0};5657	uint64_t offset, ce_payload_gpu_addr;5658	void *ce_payload_cpu_addr;5659	int cnt;5660 5661	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;5662 5663	if (ring->is_mes_queue) {5664		offset = offsetof(struct amdgpu_mes_ctx_meta_data,5665				  gfx[0].gfx_meta_data) +5666			offsetof(struct v9_gfx_meta_data, ce_payload);5667		ce_payload_gpu_addr =5668			amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);5669		ce_payload_cpu_addr =5670			amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);5671	} else {5672		offset = offsetof(struct v9_gfx_meta_data, ce_payload);5673		ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;5674		ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;5675	}5676 5677	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));5678	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |5679				 WRITE_DATA_DST_SEL(8) |5680				 WR_CONFIRM) |5681				 WRITE_DATA_CACHE_POLICY(0));5682	amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));5683	amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));5684 5685	amdgpu_ring_ib_on_emit_ce(ring);5686 5687	if (resume)5688		amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,5689					   sizeof(ce_payload) >> 2);5690	else5691		amdgpu_ring_write_multiple(ring, (void *)&ce_payload,5692					   sizeof(ce_payload) >> 2);5693}5694 5695static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)5696{5697	int i, r = 0;5698	struct amdgpu_device *adev = ring->adev;5699	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];5700	struct amdgpu_ring *kiq_ring = &kiq->ring;5701	unsigned long flags;5702 5703	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)5704		return -EINVAL;5705 5706	spin_lock_irqsave(&kiq->ring_lock, flags);5707 5708	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {5709		spin_unlock_irqrestore(&kiq->ring_lock, flags);5710		return -ENOMEM;5711	}5712 5713	/* assert preemption condition */5714	amdgpu_ring_set_preempt_cond_exec(ring, false);5715 5716	ring->trail_seq += 1;5717	amdgpu_ring_alloc(ring, 13);5718	gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,5719				 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);5720 5721	/* assert IB preemption, emit the trailing fence */5722	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,5723				   ring->trail_fence_gpu_addr,5724				   ring->trail_seq);5725 5726	amdgpu_ring_commit(kiq_ring);5727	spin_unlock_irqrestore(&kiq->ring_lock, flags);5728 5729	/* poll the trailing fence */5730	for (i = 0; i < adev->usec_timeout; i++) {5731		if (ring->trail_seq ==5732			le32_to_cpu(*ring->trail_fence_cpu_addr))5733			break;5734		udelay(1);5735	}5736 5737	if (i >= adev->usec_timeout) {5738		r = -EINVAL;5739		DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);5740	}5741 5742	/*reset the CP_VMID_PREEMPT after trailing fence*/5743	amdgpu_ring_emit_wreg(ring,5744			      SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),5745			      0x0);5746	amdgpu_ring_commit(ring);5747 5748	/* deassert preemption condition */5749	amdgpu_ring_set_preempt_cond_exec(ring, true);5750	return r;5751}5752 5753static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)5754{5755	struct amdgpu_device *adev = ring->adev;5756	struct v9_de_ib_state de_payload = {0};5757	uint64_t offset, gds_addr, de_payload_gpu_addr;5758	void *de_payload_cpu_addr;5759	int cnt;5760 5761	if (ring->is_mes_queue) {5762		offset = offsetof(struct amdgpu_mes_ctx_meta_data,5763				  gfx[0].gfx_meta_data) +5764			offsetof(struct v9_gfx_meta_data, de_payload);5765		de_payload_gpu_addr =5766			amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);5767		de_payload_cpu_addr =5768			amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);5769 5770		offset = offsetof(struct amdgpu_mes_ctx_meta_data,5771				  gfx[0].gds_backup) +5772			offsetof(struct v9_gfx_meta_data, de_payload);5773		gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);5774	} else {5775		offset = offsetof(struct v9_gfx_meta_data, de_payload);5776		de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;5777		de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;5778 5779		gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +5780				 AMDGPU_CSA_SIZE - adev->gds.gds_size,5781				 PAGE_SIZE);5782	}5783 5784	if (usegds) {5785		de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);5786		de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);5787	}5788 5789	cnt = (sizeof(de_payload) >> 2) + 4 - 2;5790	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));5791	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |5792				 WRITE_DATA_DST_SEL(8) |5793				 WR_CONFIRM) |5794				 WRITE_DATA_CACHE_POLICY(0));5795	amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));5796	amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));5797 5798	amdgpu_ring_ib_on_emit_de(ring);5799	if (resume)5800		amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,5801					   sizeof(de_payload) >> 2);5802	else5803		amdgpu_ring_write_multiple(ring, (void *)&de_payload,5804					   sizeof(de_payload) >> 2);5805}5806 5807static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,5808				   bool secure)5809{5810	uint32_t v = secure ? FRAME_TMZ : 0;5811 5812	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));5813	amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));5814}5815 5816static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)5817{5818	uint32_t dw2 = 0;5819 5820	gfx_v9_0_ring_emit_ce_meta(ring,5821				   (!amdgpu_sriov_vf(ring->adev) &&5822				   flags & AMDGPU_IB_PREEMPTED) ? true : false);5823 5824	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */5825	if (flags & AMDGPU_HAVE_CTX_SWITCH) {5826		/* set load_global_config & load_global_uconfig */5827		dw2 |= 0x8001;5828		/* set load_cs_sh_regs */5829		dw2 |= 0x01000000;5830		/* set load_per_context_state & load_gfx_sh_regs for GFX */5831		dw2 |= 0x10002;5832 5833		/* set load_ce_ram if preamble presented */5834		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)5835			dw2 |= 0x10000000;5836	} else {5837		/* still load_ce_ram if this is the first time preamble presented5838		 * although there is no context switch happens.5839		 */5840		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)5841			dw2 |= 0x10000000;5842	}5843 5844	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));5845	amdgpu_ring_write(ring, dw2);5846	amdgpu_ring_write(ring, 0);5847}5848 5849static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,5850						  uint64_t addr)5851{5852	unsigned ret;5853	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));5854	amdgpu_ring_write(ring, lower_32_bits(addr));5855	amdgpu_ring_write(ring, upper_32_bits(addr));5856	/* discard following DWs if *cond_exec_gpu_addr==0 */5857	amdgpu_ring_write(ring, 0);5858	ret = ring->wptr & ring->buf_mask;5859	/* patch dummy value later */5860	amdgpu_ring_write(ring, 0);5861	return ret;5862}5863 5864static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,5865				    uint32_t reg_val_offs)5866{5867	struct amdgpu_device *adev = ring->adev;5868 5869	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));5870	amdgpu_ring_write(ring, 0 |	/* src: register*/5871				(5 << 8) |	/* dst: memory */5872				(1 << 20));	/* write confirm */5873	amdgpu_ring_write(ring, reg);5874	amdgpu_ring_write(ring, 0);5875	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +5876				reg_val_offs * 4));5877	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +5878				reg_val_offs * 4));5879}5880 5881static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,5882				    uint32_t val)5883{5884	uint32_t cmd = 0;5885 5886	switch (ring->funcs->type) {5887	case AMDGPU_RING_TYPE_GFX:5888		cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;5889		break;5890	case AMDGPU_RING_TYPE_KIQ:5891		cmd = (1 << 16); /* no inc addr */5892		break;5893	default:5894		cmd = WR_CONFIRM;5895		break;5896	}5897	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));5898	amdgpu_ring_write(ring, cmd);5899	amdgpu_ring_write(ring, reg);5900	amdgpu_ring_write(ring, 0);5901	amdgpu_ring_write(ring, val);5902}5903 5904static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,5905					uint32_t val, uint32_t mask)5906{5907	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);5908}5909 5910static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,5911						  uint32_t reg0, uint32_t reg1,5912						  uint32_t ref, uint32_t mask)5913{5914	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);5915	struct amdgpu_device *adev = ring->adev;5916	bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?5917		adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;5918 5919	if (fw_version_ok)5920		gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,5921				      ref, mask, 0x20);5922	else5923		amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,5924							   ref, mask);5925}5926 5927static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)5928{5929	struct amdgpu_device *adev = ring->adev;5930	uint32_t value = 0;5931 5932	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);5933	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);5934	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);5935	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);5936	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);5937	WREG32_SOC15(GC, 0, mmSQ_CMD, value);5938	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);5939}5940 5941static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,5942						 enum amdgpu_interrupt_state state)5943{5944	switch (state) {5945	case AMDGPU_IRQ_STATE_DISABLE:5946	case AMDGPU_IRQ_STATE_ENABLE:5947		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,5948			       TIME_STAMP_INT_ENABLE,5949			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);5950		break;5951	default:5952		break;5953	}5954}5955 5956static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,5957						     int me, int pipe,5958						     enum amdgpu_interrupt_state state)5959{5960	u32 mec_int_cntl, mec_int_cntl_reg;5961 5962	/*5963	 * amdgpu controls only the first MEC. That's why this function only5964	 * handles the setting of interrupts for this specific MEC. All other5965	 * pipes' interrupts are set by amdkfd.5966	 */5967 5968	if (me == 1) {5969		switch (pipe) {5970		case 0:5971			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);5972			break;5973		case 1:5974			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);5975			break;5976		case 2:5977			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);5978			break;5979		case 3:5980			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);5981			break;5982		default:5983			DRM_DEBUG("invalid pipe %d\n", pipe);5984			return;5985		}5986	} else {5987		DRM_DEBUG("invalid me %d\n", me);5988		return;5989	}5990 5991	switch (state) {5992	case AMDGPU_IRQ_STATE_DISABLE:5993		mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);5994		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,5995					     TIME_STAMP_INT_ENABLE, 0);5996		WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);5997		break;5998	case AMDGPU_IRQ_STATE_ENABLE:5999		mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);6000		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,6001					     TIME_STAMP_INT_ENABLE, 1);6002		WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);6003		break;6004	default:6005		break;6006	}6007}6008 6009static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,6010				     int me, int pipe)6011{6012	/*6013	 * amdgpu controls only the first MEC. That's why this function only6014	 * handles the setting of interrupts for this specific MEC. All other6015	 * pipes' interrupts are set by amdkfd.6016	 */6017	if (me != 1)6018		return 0;6019 6020	switch (pipe) {6021	case 0:6022		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);6023	case 1:6024		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);6025	case 2:6026		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);6027	case 3:6028		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);6029	default:6030		return 0;6031	}6032}6033 6034static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,6035					     struct amdgpu_irq_src *source,6036					     unsigned type,6037					     enum amdgpu_interrupt_state state)6038{6039	u32 cp_int_cntl_reg, cp_int_cntl;6040	int i, j;6041 6042	switch (state) {6043	case AMDGPU_IRQ_STATE_DISABLE:6044	case AMDGPU_IRQ_STATE_ENABLE:6045		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,6046			       PRIV_REG_INT_ENABLE,6047			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);6048		for (i = 0; i < adev->gfx.mec.num_mec; i++) {6049			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {6050				/* MECs start at 1 */6051				cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);6052 6053				if (cp_int_cntl_reg) {6054					cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);6055					cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,6056								    PRIV_REG_INT_ENABLE,6057								    state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);6058					WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);6059				}6060			}6061		}6062		break;6063	default:6064		break;6065	}6066 6067	return 0;6068}6069 6070static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,6071					   struct amdgpu_irq_src *source,6072					   unsigned type,6073					   enum amdgpu_interrupt_state state)6074{6075	u32 cp_int_cntl_reg, cp_int_cntl;6076	int i, j;6077 6078	switch (state) {6079	case AMDGPU_IRQ_STATE_DISABLE:6080	case AMDGPU_IRQ_STATE_ENABLE:6081		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,6082			       OPCODE_ERROR_INT_ENABLE,6083			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);6084		for (i = 0; i < adev->gfx.mec.num_mec; i++) {6085			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {6086				/* MECs start at 1 */6087				cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);6088 6089				if (cp_int_cntl_reg) {6090					cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);6091					cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,6092								    OPCODE_ERROR_INT_ENABLE,6093								    state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);6094					WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);6095				}6096			}6097		}6098		break;6099	default:6100		break;6101	}6102 6103	return 0;6104}6105 6106static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,6107					      struct amdgpu_irq_src *source,6108					      unsigned type,6109					      enum amdgpu_interrupt_state state)6110{6111	switch (state) {6112	case AMDGPU_IRQ_STATE_DISABLE:6113	case AMDGPU_IRQ_STATE_ENABLE:6114		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,6115			       PRIV_INSTR_INT_ENABLE,6116			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);6117		break;6118	default:6119		break;6120	}6121 6122	return 0;6123}6124 6125#define ENABLE_ECC_ON_ME_PIPE(me, pipe)				\6126	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\6127			CP_ECC_ERROR_INT_ENABLE, 1)6128 6129#define DISABLE_ECC_ON_ME_PIPE(me, pipe)			\6130	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\6131			CP_ECC_ERROR_INT_ENABLE, 0)6132 6133static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,6134					      struct amdgpu_irq_src *source,6135					      unsigned type,6136					      enum amdgpu_interrupt_state state)6137{6138	switch (state) {6139	case AMDGPU_IRQ_STATE_DISABLE:6140		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,6141				CP_ECC_ERROR_INT_ENABLE, 0);6142		DISABLE_ECC_ON_ME_PIPE(1, 0);6143		DISABLE_ECC_ON_ME_PIPE(1, 1);6144		DISABLE_ECC_ON_ME_PIPE(1, 2);6145		DISABLE_ECC_ON_ME_PIPE(1, 3);6146		break;6147 6148	case AMDGPU_IRQ_STATE_ENABLE:6149		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,6150				CP_ECC_ERROR_INT_ENABLE, 1);6151		ENABLE_ECC_ON_ME_PIPE(1, 0);6152		ENABLE_ECC_ON_ME_PIPE(1, 1);6153		ENABLE_ECC_ON_ME_PIPE(1, 2);6154		ENABLE_ECC_ON_ME_PIPE(1, 3);6155		break;6156	default:6157		break;6158	}6159 6160	return 0;6161}6162 6163 6164static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,6165					    struct amdgpu_irq_src *src,6166					    unsigned type,6167					    enum amdgpu_interrupt_state state)6168{6169	switch (type) {6170	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:6171		gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);6172		break;6173	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:6174		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);6175		break;6176	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:6177		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);6178		break;6179	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:6180		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);6181		break;6182	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:6183		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);6184		break;6185	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:6186		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);6187		break;6188	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:6189		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);6190		break;6191	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:6192		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);6193		break;6194	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:6195		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);6196		break;6197	default:6198		break;6199	}6200	return 0;6201}6202 6203static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,6204			    struct amdgpu_irq_src *source,6205			    struct amdgpu_iv_entry *entry)6206{6207	int i;6208	u8 me_id, pipe_id, queue_id;6209	struct amdgpu_ring *ring;6210 6211	DRM_DEBUG("IH: CP EOP\n");6212	me_id = (entry->ring_id & 0x0c) >> 2;6213	pipe_id = (entry->ring_id & 0x03) >> 0;6214	queue_id = (entry->ring_id & 0x70) >> 4;6215 6216	switch (me_id) {6217	case 0:6218		if (adev->gfx.num_gfx_rings) {6219			if (!adev->gfx.mcbp) {6220				amdgpu_fence_process(&adev->gfx.gfx_ring[0]);6221			} else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {6222				/* Fence signals are handled on the software rings*/6223				for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)6224					amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);6225			}6226		}6227		break;6228	case 1:6229	case 2:6230		for (i = 0; i < adev->gfx.num_compute_rings; i++) {6231			ring = &adev->gfx.compute_ring[i];6232			/* Per-queue interrupt is supported for MEC starting from VI.6233			  * The interrupt can only be enabled/disabled per pipe instead of per queue.6234			  */6235			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))6236				amdgpu_fence_process(ring);6237		}6238		break;6239	}6240	return 0;6241}6242 6243static void gfx_v9_0_fault(struct amdgpu_device *adev,6244			   struct amdgpu_iv_entry *entry)6245{6246	u8 me_id, pipe_id, queue_id;6247	struct amdgpu_ring *ring;6248	int i;6249 6250	me_id = (entry->ring_id & 0x0c) >> 2;6251	pipe_id = (entry->ring_id & 0x03) >> 0;6252	queue_id = (entry->ring_id & 0x70) >> 4;6253 6254	switch (me_id) {6255	case 0:6256		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);6257		break;6258	case 1:6259	case 2:6260		for (i = 0; i < adev->gfx.num_compute_rings; i++) {6261			ring = &adev->gfx.compute_ring[i];6262			if (ring->me == me_id && ring->pipe == pipe_id &&6263			    ring->queue == queue_id)6264				drm_sched_fault(&ring->sched);6265		}6266		break;6267	}6268}6269 6270static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,6271				 struct amdgpu_irq_src *source,6272				 struct amdgpu_iv_entry *entry)6273{6274	DRM_ERROR("Illegal register access in command stream\n");6275	gfx_v9_0_fault(adev, entry);6276	return 0;6277}6278 6279static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,6280			       struct amdgpu_irq_src *source,6281			       struct amdgpu_iv_entry *entry)6282{6283	DRM_ERROR("Illegal opcode in command stream\n");6284	gfx_v9_0_fault(adev, entry);6285	return 0;6286}6287 6288static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,6289				  struct amdgpu_irq_src *source,6290				  struct amdgpu_iv_entry *entry)6291{6292	DRM_ERROR("Illegal instruction in command stream\n");6293	gfx_v9_0_fault(adev, entry);6294	return 0;6295}6296 6297 6298static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {6299	{ "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),6300	  SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),6301	  SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)6302	},6303	{ "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),6304	  SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),6305	  SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)6306	},6307	{ "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),6308	  SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),6309	  0, 06310	},6311	{ "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),6312	  SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),6313	  0, 06314	},6315	{ "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),6316	  SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),6317	  SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)6318	},6319	{ "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),6320	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),6321	  0, 06322	},6323	{ "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),6324	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),6325	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)6326	},6327	{ "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),6328	  SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),6329	  SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)6330	},6331	{ "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),6332	  SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),6333	  0, 06334	},6335	{ "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),6336	  SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),6337	  0, 06338	},6339	{ "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),6340	  SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),6341	  0, 06342	},6343	{ "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),6344	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),6345	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)6346	},6347	{ "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),6348	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),6349	  0, 06350	},6351	{ "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),6352	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),6353	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)6354	},6355	{ "GDS_OA_PHY_PHY_CMD_RAM_MEM",6356	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),6357	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),6358	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)6359	},6360	{ "GDS_OA_PHY_PHY_DATA_RAM_MEM",6361	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),6362	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),6363	  0, 06364	},6365	{ "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",6366	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),6367	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),6368	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)6369	},6370	{ "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",6371	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),6372	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),6373	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)6374	},6375	{ "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",6376	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),6377	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),6378	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)6379	},6380	{ "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",6381	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),6382	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),6383	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)6384	},6385	{ "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),6386	  SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),6387	  0, 06388	},6389	{ "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),6390	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),6391	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)6392	},6393	{ "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),6394	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),6395	  0, 06396	},6397	{ "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),6398	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),6399	  0, 06400	},6401	{ "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),6402	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),6403	  0, 06404	},6405	{ "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),6406	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),6407	  0, 06408	},6409	{ "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),6410	  SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),6411	  0, 06412	},6413	{ "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),6414	  SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),6415	  0, 06416	},6417	{ "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6418	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),6419	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)6420	},6421	{ "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6422	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),6423	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)6424	},6425	{ "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6426	  SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),6427	  SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)6428	},6429	{ "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6430	  SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),6431	  SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)6432	},6433	{ "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6434	  SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),6435	  SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)6436	},6437	{ "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6438	  SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),6439	  0, 06440	},6441	{ "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6442	  SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),6443	  0, 06444	},6445	{ "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6446	  SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),6447	  0, 06448	},6449	{ "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6450	  SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),6451	  0, 06452	},6453	{ "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6454	  SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),6455	  0, 06456	},6457	{ "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),6458	  SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),6459	  0, 06460	},6461	{ "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6462	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),6463	  0, 06464	},6465	{ "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6466	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),6467	  0, 06468	},6469	{ "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6470	  SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),6471	  0, 06472	},6473	{ "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6474	  SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),6475	  0, 06476	},6477	{ "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6478	  SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),6479	  0, 06480	},6481	{ "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6482	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),6483	  0, 06484	},6485	{ "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),6486	  SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),6487	  0, 06488	},6489	{ "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),6490	  SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),6491	  0, 06492	},6493	{ "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6494	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),6495	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)6496	},6497	{ "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6498	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),6499	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)6500	},6501	{ "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6502	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),6503	  0, 06504	},6505	{ "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6506	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),6507	  0, 06508	},6509	{ "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6510	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),6511	  0, 06512	},6513	{ "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6514	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),6515	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)6516	},6517	{ "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),6518	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),6519	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)6520	},6521	{ "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),6522	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),6523	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)6524	},6525	{ "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),6526	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),6527	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)6528	},6529	{ "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),6530	  SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),6531	  0, 06532	},6533	{ "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6534	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),6535	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)6536	},6537	{ "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6538	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),6539	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)6540	},6541	{ "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6542	  SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),6543	  SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)6544	},6545	{ "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6546	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),6547	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)6548	},6549	{ "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6550	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),6551	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)6552	},6553	{ "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6554	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),6555	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)6556	},6557	{ "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),6558	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),6559	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)6560	},6561	{ "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6562	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),6563	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)6564	},6565	{ "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6566	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),6567	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)6568	},6569	{ "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6570	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),6571	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)6572	},6573	{ "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6574	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),6575	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)6576	},6577	{ "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6578	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),6579	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)6580	},6581	{ "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),6582	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),6583	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)6584	},6585	{ "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6586	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),6587	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)6588	},6589	{ "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6590	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),6591	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)6592	},6593	{ "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6594	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),6595	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)6596	},6597	{ "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6598	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),6599	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)6600	},6601	{ "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6602	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),6603	  0, 06604	},6605	{ "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6606	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),6607	  0, 06608	},6609	{ "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6610	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),6611	  0, 06612	},6613	{ "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6614	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),6615	  0, 06616	},6617	{ "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6618	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),6619	  0, 06620	},6621	{ "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),6622	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),6623	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)6624	},6625	{ "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6626	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),6627	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)6628	},6629	{ "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6630	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),6631	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)6632	},6633	{ "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6634	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),6635	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)6636	},6637	{ "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6638	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),6639	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)6640	},6641	{ "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6642	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),6643	  0, 06644	},6645	{ "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6646	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),6647	  0, 06648	},6649	{ "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6650	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),6651	  0, 06652	},6653	{ "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6654	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),6655	  0, 06656	},6657	{ "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),6658	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),6659	  0, 06660	},6661	{ "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6662	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),6663	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)6664	},6665	{ "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6666	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),6667	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)6668	},6669	{ "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6670	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),6671	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)6672	},6673	{ "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6674	  SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),6675	  SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)6676	},6677	{ "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6678	  SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),6679	  SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)6680	},6681	{ "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6682	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),6683	  0, 06684	},6685	{ "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6686	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),6687	  0, 06688	},6689	{ "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6690	  SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),6691	  0, 06692	},6693	{ "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6694	  SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),6695	  0, 06696	},6697	{ "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),6698	  SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),6699	  0, 06700	},6701	{ "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6702	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),6703	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)6704	},6705	{ "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6706	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),6707	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)6708	},6709	{ "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6710	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),6711	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)6712	},6713	{ "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6714	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),6715	  0, 06716	},6717	{ "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6718	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),6719	  0, 06720	},6721	{ "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6722	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),6723	  0, 06724	},6725	{ "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6726	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),6727	  0, 06728	},6729	{ "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6730	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),6731	  0, 06732	},6733	{ "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),6734	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),6735	  0, 06736	}6737};6738 6739static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,6740				     void *inject_if, uint32_t instance_mask)6741{6742	struct ras_inject_if *info = (struct ras_inject_if *)inject_if;6743	int ret;6744	struct ta_ras_trigger_error_input block_info = { 0 };6745 6746	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))6747		return -EINVAL;6748 6749	if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))6750		return -EINVAL;6751 6752	if (!ras_gfx_subblocks[info->head.sub_block_index].name)6753		return -EPERM;6754 6755	if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &6756	      info->head.type)) {6757		DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",6758			ras_gfx_subblocks[info->head.sub_block_index].name,6759			info->head.type);6760		return -EPERM;6761	}6762 6763	if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &6764	      info->head.type)) {6765		DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",6766			ras_gfx_subblocks[info->head.sub_block_index].name,6767			info->head.type);6768		return -EPERM;6769	}6770 6771	block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);6772	block_info.sub_block_index =6773		ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;6774	block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);6775	block_info.address = info->address;6776	block_info.value = info->value;6777 6778	mutex_lock(&adev->grbm_idx_mutex);6779	ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);6780	mutex_unlock(&adev->grbm_idx_mutex);6781 6782	return ret;6783}6784 6785static const char * const vml2_mems[] = {6786	"UTC_VML2_BANK_CACHE_0_BIGK_MEM0",6787	"UTC_VML2_BANK_CACHE_0_BIGK_MEM1",6788	"UTC_VML2_BANK_CACHE_0_4K_MEM0",6789	"UTC_VML2_BANK_CACHE_0_4K_MEM1",6790	"UTC_VML2_BANK_CACHE_1_BIGK_MEM0",6791	"UTC_VML2_BANK_CACHE_1_BIGK_MEM1",6792	"UTC_VML2_BANK_CACHE_1_4K_MEM0",6793	"UTC_VML2_BANK_CACHE_1_4K_MEM1",6794	"UTC_VML2_BANK_CACHE_2_BIGK_MEM0",6795	"UTC_VML2_BANK_CACHE_2_BIGK_MEM1",6796	"UTC_VML2_BANK_CACHE_2_4K_MEM0",6797	"UTC_VML2_BANK_CACHE_2_4K_MEM1",6798	"UTC_VML2_BANK_CACHE_3_BIGK_MEM0",6799	"UTC_VML2_BANK_CACHE_3_BIGK_MEM1",6800	"UTC_VML2_BANK_CACHE_3_4K_MEM0",6801	"UTC_VML2_BANK_CACHE_3_4K_MEM1",6802};6803 6804static const char * const vml2_walker_mems[] = {6805	"UTC_VML2_CACHE_PDE0_MEM0",6806	"UTC_VML2_CACHE_PDE0_MEM1",6807	"UTC_VML2_CACHE_PDE1_MEM0",6808	"UTC_VML2_CACHE_PDE1_MEM1",6809	"UTC_VML2_CACHE_PDE2_MEM0",6810	"UTC_VML2_CACHE_PDE2_MEM1",6811	"UTC_VML2_RDIF_LOG_FIFO",6812};6813 6814static const char * const atc_l2_cache_2m_mems[] = {6815	"UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",6816	"UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",6817	"UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",6818	"UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",6819};6820 6821static const char *atc_l2_cache_4k_mems[] = {6822	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",6823	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",6824	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",6825	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",6826	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",6827	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",6828	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",6829	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",6830	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",6831	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",6832	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",6833	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",6834	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",6835	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",6836	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",6837	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",6838	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",6839	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",6840	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",6841	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",6842	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",6843	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",6844	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",6845	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",6846	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",6847	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",6848	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",6849	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",6850	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",6851	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",6852	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",6853	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",6854};6855 6856static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,6857					 struct ras_err_data *err_data)6858{6859	uint32_t i, data;6860	uint32_t sec_count, ded_count;6861 6862	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);6863	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);6864	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);6865	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);6866	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);6867	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);6868	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);6869	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);6870 6871	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {6872		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);6873		data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);6874 6875		sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);6876		if (sec_count) {6877			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6878				"SEC %d\n", i, vml2_mems[i], sec_count);6879			err_data->ce_count += sec_count;6880		}6881 6882		ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);6883		if (ded_count) {6884			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6885				"DED %d\n", i, vml2_mems[i], ded_count);6886			err_data->ue_count += ded_count;6887		}6888	}6889 6890	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {6891		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);6892		data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);6893 6894		sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,6895						SEC_COUNT);6896		if (sec_count) {6897			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6898				"SEC %d\n", i, vml2_walker_mems[i], sec_count);6899			err_data->ce_count += sec_count;6900		}6901 6902		ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,6903						DED_COUNT);6904		if (ded_count) {6905			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6906				"DED %d\n", i, vml2_walker_mems[i], ded_count);6907			err_data->ue_count += ded_count;6908		}6909	}6910 6911	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {6912		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);6913		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);6914 6915		sec_count = (data & 0x00006000L) >> 0xd;6916		if (sec_count) {6917			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6918				"SEC %d\n", i, atc_l2_cache_2m_mems[i],6919				sec_count);6920			err_data->ce_count += sec_count;6921		}6922	}6923 6924	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {6925		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);6926		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);6927 6928		sec_count = (data & 0x00006000L) >> 0xd;6929		if (sec_count) {6930			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6931				"SEC %d\n", i, atc_l2_cache_4k_mems[i],6932				sec_count);6933			err_data->ce_count += sec_count;6934		}6935 6936		ded_count = (data & 0x00018000L) >> 0xf;6937		if (ded_count) {6938			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "6939				"DED %d\n", i, atc_l2_cache_4k_mems[i],6940				ded_count);6941			err_data->ue_count += ded_count;6942		}6943	}6944 6945	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);6946	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);6947	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);6948	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);6949 6950	return 0;6951}6952 6953static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,6954	const struct soc15_reg_entry *reg,6955	uint32_t se_id, uint32_t inst_id, uint32_t value,6956	uint32_t *sec_count, uint32_t *ded_count)6957{6958	uint32_t i;6959	uint32_t sec_cnt, ded_cnt;6960 6961	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {6962		if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||6963			gfx_v9_0_ras_fields[i].seg != reg->seg ||6964			gfx_v9_0_ras_fields[i].inst != reg->inst)6965			continue;6966 6967		sec_cnt = (value &6968				gfx_v9_0_ras_fields[i].sec_count_mask) >>6969				gfx_v9_0_ras_fields[i].sec_count_shift;6970		if (sec_cnt) {6971			dev_info(adev->dev, "GFX SubBlock %s, "6972				"Instance[%d][%d], SEC %d\n",6973				gfx_v9_0_ras_fields[i].name,6974				se_id, inst_id,6975				sec_cnt);6976			*sec_count += sec_cnt;6977		}6978 6979		ded_cnt = (value &6980				gfx_v9_0_ras_fields[i].ded_count_mask) >>6981				gfx_v9_0_ras_fields[i].ded_count_shift;6982		if (ded_cnt) {6983			dev_info(adev->dev, "GFX SubBlock %s, "6984				"Instance[%d][%d], DED %d\n",6985				gfx_v9_0_ras_fields[i].name,6986				se_id, inst_id,6987				ded_cnt);6988			*ded_count += ded_cnt;6989		}6990	}6991 6992	return 0;6993}6994 6995static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)6996{6997	int i, j, k;6998 6999	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))7000		return;7001 7002	/* read back registers to clear the counters */7003	mutex_lock(&adev->grbm_idx_mutex);7004	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {7005		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {7006			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {7007				amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);7008				RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));7009			}7010		}7011	}7012	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);7013	mutex_unlock(&adev->grbm_idx_mutex);7014 7015	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);7016	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);7017	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);7018	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);7019	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);7020	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);7021	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);7022	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);7023 7024	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {7025		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);7026		RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);7027	}7028 7029	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {7030		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);7031		RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);7032	}7033 7034	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {7035		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);7036		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);7037	}7038 7039	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {7040		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);7041		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);7042	}7043 7044	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);7045	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);7046	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);7047	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);7048}7049 7050static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,7051					  void *ras_error_status)7052{7053	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;7054	uint32_t sec_count = 0, ded_count = 0;7055	uint32_t i, j, k;7056	uint32_t reg_value;7057 7058	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))7059		return;7060 7061	err_data->ue_count = 0;7062	err_data->ce_count = 0;7063 7064	mutex_lock(&adev->grbm_idx_mutex);7065 7066	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {7067		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {7068			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {7069				amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);7070				reg_value =7071					RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));7072				if (reg_value)7073					gfx_v9_0_ras_error_count(adev,7074						&gfx_v9_0_edc_counter_regs[i],7075						j, k, reg_value,7076						&sec_count, &ded_count);7077			}7078		}7079	}7080 7081	err_data->ce_count += sec_count;7082	err_data->ue_count += ded_count;7083 7084	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);7085	mutex_unlock(&adev->grbm_idx_mutex);7086 7087	gfx_v9_0_query_utc_edc_status(adev, err_data);7088}7089 7090static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)7091{7092	const unsigned int cp_coher_cntl =7093			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |7094			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |7095			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |7096			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |7097			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);7098 7099	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */7100	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));7101	amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */7102	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */7103	amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */7104	amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */7105	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */7106	amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */7107}7108 7109static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,7110					uint32_t pipe, bool enable)7111{7112	struct amdgpu_device *adev = ring->adev;7113	uint32_t val;7114	uint32_t wcl_cs_reg;7115 7116	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */7117	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;7118 7119	switch (pipe) {7120	case 0:7121		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);7122		break;7123	case 1:7124		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);7125		break;7126	case 2:7127		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);7128		break;7129	case 3:7130		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);7131		break;7132	default:7133		DRM_DEBUG("invalid pipe %d\n", pipe);7134		return;7135	}7136 7137	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);7138 7139}7140static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)7141{7142	struct amdgpu_device *adev = ring->adev;7143	uint32_t val;7144	int i;7145 7146 7147	/* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit7148	 * number of gfx waves. Setting 5 bit will make sure gfx only gets7149	 * around 25% of gpu resources.7150	 */7151	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;7152	amdgpu_ring_emit_wreg(ring,7153			      SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),7154			      val);7155 7156	/* Restrict waves for normal/low priority compute queues as well7157	 * to get best QoS for high priority compute jobs.7158	 *7159	 * amdgpu controls only 1st ME(0-3 CS pipes).7160	 */7161	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {7162		if (i != ring->pipe)7163			gfx_v9_0_emit_wave_limit_cs(ring, i, enable);7164 7165	}7166}7167 7168static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)7169{7170	int i;7171 7172	/* Header itself is a NOP packet */7173	if (num_nop == 1) {7174		amdgpu_ring_write(ring, ring->funcs->nop);7175		return;7176	}7177 7178	/* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/7179	amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));7180 7181	/* Header is at index 0, followed by num_nops - 1 NOP packet's */7182	for (i = 1; i < num_nop; i++)7183		amdgpu_ring_write(ring, ring->funcs->nop);7184}7185 7186static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)7187{7188	struct amdgpu_device *adev = ring->adev;7189	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];7190	struct amdgpu_ring *kiq_ring = &kiq->ring;7191	unsigned long flags;7192	u32 tmp;7193	int r;7194 7195	if (amdgpu_sriov_vf(adev))7196		return -EINVAL;7197 7198	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)7199		return -EINVAL;7200 7201	spin_lock_irqsave(&kiq->ring_lock, flags);7202 7203	if (amdgpu_ring_alloc(kiq_ring, 5)) {7204		spin_unlock_irqrestore(&kiq->ring_lock, flags);7205		return -ENOMEM;7206	}7207 7208	tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);7209	gfx_v9_0_ring_emit_wreg(kiq_ring,7210				 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);7211	amdgpu_ring_commit(kiq_ring);7212 7213	spin_unlock_irqrestore(&kiq->ring_lock, flags);7214 7215	r = amdgpu_ring_test_ring(kiq_ring);7216	if (r)7217		return r;7218 7219	if (amdgpu_ring_alloc(ring, 7 + 7 + 5))7220		return -ENOMEM;7221	gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,7222				 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);7223	gfx_v9_0_ring_emit_reg_wait(ring,7224				    SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff);7225	gfx_v9_0_ring_emit_wreg(ring,7226				SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);7227 7228	return amdgpu_ring_test_ring(ring);7229}7230 7231static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,7232			      unsigned int vmid)7233{7234	struct amdgpu_device *adev = ring->adev;7235	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];7236	struct amdgpu_ring *kiq_ring = &kiq->ring;7237	unsigned long flags;7238	int i, r;7239 7240	if (!adev->debug_exp_resets &&7241	    !adev->gfx.num_gfx_rings)7242		return -EINVAL;7243 7244	if (amdgpu_sriov_vf(adev))7245		return -EINVAL;7246 7247	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)7248		return -EINVAL;7249 7250	spin_lock_irqsave(&kiq->ring_lock, flags);7251 7252	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {7253		spin_unlock_irqrestore(&kiq->ring_lock, flags);7254		return -ENOMEM;7255	}7256 7257	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,7258				   0, 0);7259	amdgpu_ring_commit(kiq_ring);7260 7261	spin_unlock_irqrestore(&kiq->ring_lock, flags);7262 7263	r = amdgpu_ring_test_ring(kiq_ring);7264	if (r)7265		return r;7266 7267	/* make sure dequeue is complete*/7268	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);7269	mutex_lock(&adev->srbm_mutex);7270	soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);7271	for (i = 0; i < adev->usec_timeout; i++) {7272		if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))7273			break;7274		udelay(1);7275	}7276	if (i >= adev->usec_timeout)7277		r = -ETIMEDOUT;7278	soc15_grbm_select(adev, 0, 0, 0, 0, 0);7279	mutex_unlock(&adev->srbm_mutex);7280	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);7281	if (r) {7282		dev_err(adev->dev, "fail to wait on hqd deactive\n");7283		return r;7284	}7285 7286	r = amdgpu_bo_reserve(ring->mqd_obj, false);7287	if (unlikely(r != 0)){7288		dev_err(adev->dev, "fail to resv mqd_obj\n");7289		return r;7290	}7291	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);7292	if (!r) {7293		r = gfx_v9_0_kcq_init_queue(ring, true);7294		amdgpu_bo_kunmap(ring->mqd_obj);7295		ring->mqd_ptr = NULL;7296	}7297	amdgpu_bo_unreserve(ring->mqd_obj);7298	if (r) {7299		dev_err(adev->dev, "fail to unresv mqd_obj\n");7300		return r;7301	}7302	spin_lock_irqsave(&kiq->ring_lock, flags);7303	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);7304	if (r) {7305		spin_unlock_irqrestore(&kiq->ring_lock, flags);7306		return -ENOMEM;7307	}7308	kiq->pmf->kiq_map_queues(kiq_ring, ring);7309	amdgpu_ring_commit(kiq_ring);7310	spin_unlock_irqrestore(&kiq->ring_lock, flags);7311	r = amdgpu_ring_test_ring(kiq_ring);7312	if (r) {7313		DRM_ERROR("fail to remap queue\n");7314		return r;7315	}7316	return amdgpu_ring_test_ring(ring);7317}7318 7319static void gfx_v9_ip_print(void *handle, struct drm_printer *p)7320{7321	struct amdgpu_device *adev = (struct amdgpu_device *)handle;7322	uint32_t i, j, k, reg, index = 0;7323	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);7324 7325	if (!adev->gfx.ip_dump_core)7326		return;7327 7328	for (i = 0; i < reg_count; i++)7329		drm_printf(p, "%-50s \t 0x%08x\n",7330			   gc_reg_list_9[i].reg_name,7331			   adev->gfx.ip_dump_core[i]);7332 7333	/* print compute queue registers for all instances */7334	if (!adev->gfx.ip_dump_compute_queues)7335		return;7336 7337	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);7338	drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",7339		   adev->gfx.mec.num_mec,7340		   adev->gfx.mec.num_pipe_per_mec,7341		   adev->gfx.mec.num_queue_per_pipe);7342 7343	for (i = 0; i < adev->gfx.mec.num_mec; i++) {7344		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {7345			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {7346				drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);7347				for (reg = 0; reg < reg_count; reg++) {7348					drm_printf(p, "%-50s \t 0x%08x\n",7349						   gc_cp_reg_list_9[reg].reg_name,7350						   adev->gfx.ip_dump_compute_queues[index + reg]);7351				}7352				index += reg_count;7353			}7354		}7355	}7356 7357}7358 7359static void gfx_v9_ip_dump(void *handle)7360{7361	struct amdgpu_device *adev = (struct amdgpu_device *)handle;7362	uint32_t i, j, k, reg, index = 0;7363	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);7364 7365	if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)7366		return;7367 7368	amdgpu_gfx_off_ctrl(adev, false);7369	for (i = 0; i < reg_count; i++)7370		adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));7371	amdgpu_gfx_off_ctrl(adev, true);7372 7373	/* dump compute queue registers for all instances */7374	if (!adev->gfx.ip_dump_compute_queues)7375		return;7376 7377	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);7378	amdgpu_gfx_off_ctrl(adev, false);7379	mutex_lock(&adev->srbm_mutex);7380	for (i = 0; i < adev->gfx.mec.num_mec; i++) {7381		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {7382			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {7383				/* ME0 is for GFX so start from 1 for CP */7384				soc15_grbm_select(adev, 1 + i, j, k, 0, 0);7385 7386				for (reg = 0; reg < reg_count; reg++) {7387					adev->gfx.ip_dump_compute_queues[index + reg] =7388						RREG32(SOC15_REG_ENTRY_OFFSET(7389							gc_cp_reg_list_9[reg]));7390				}7391				index += reg_count;7392			}7393		}7394	}7395	soc15_grbm_select(adev, 0, 0, 0, 0, 0);7396	mutex_unlock(&adev->srbm_mutex);7397	amdgpu_gfx_off_ctrl(adev, true);7398 7399}7400 7401static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)7402{7403	/* Emit the cleaner shader */7404	amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));7405	amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */7406}7407 7408static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {7409	.name = "gfx_v9_0",7410	.early_init = gfx_v9_0_early_init,7411	.late_init = gfx_v9_0_late_init,7412	.sw_init = gfx_v9_0_sw_init,7413	.sw_fini = gfx_v9_0_sw_fini,7414	.hw_init = gfx_v9_0_hw_init,7415	.hw_fini = gfx_v9_0_hw_fini,7416	.suspend = gfx_v9_0_suspend,7417	.resume = gfx_v9_0_resume,7418	.is_idle = gfx_v9_0_is_idle,7419	.wait_for_idle = gfx_v9_0_wait_for_idle,7420	.soft_reset = gfx_v9_0_soft_reset,7421	.set_clockgating_state = gfx_v9_0_set_clockgating_state,7422	.set_powergating_state = gfx_v9_0_set_powergating_state,7423	.get_clockgating_state = gfx_v9_0_get_clockgating_state,7424	.dump_ip_state = gfx_v9_ip_dump,7425	.print_ip_state = gfx_v9_ip_print,7426};7427 7428static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {7429	.type = AMDGPU_RING_TYPE_GFX,7430	.align_mask = 0xff,7431	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7432	.support_64bit_ptrs = true,7433	.secure_submission_supported = true,7434	.get_rptr = gfx_v9_0_ring_get_rptr_gfx,7435	.get_wptr = gfx_v9_0_ring_get_wptr_gfx,7436	.set_wptr = gfx_v9_0_ring_set_wptr_gfx,7437	.emit_frame_size = /* totally 242 maximum if 16 IBs */7438		5 +  /* COND_EXEC */7439		7 +  /* PIPELINE_SYNC */7440		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +7441		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +7442		2 + /* VM_FLUSH */7443		8 +  /* FENCE for VM_FLUSH */7444		20 + /* GDS switch */7445		4 + /* double SWITCH_BUFFER,7446		       the first COND_EXEC jump to the place just7447			   prior to this double SWITCH_BUFFER  */7448		5 + /* COND_EXEC */7449		7 +	 /*	HDP_flush */7450		4 +	 /*	VGT_flush */7451		14 + /*	CE_META */7452		31 + /*	DE_META */7453		3 + /* CNTX_CTRL */7454		5 + /* HDP_INVL */7455		8 + 8 + /* FENCE x2 */7456		2 + /* SWITCH_BUFFER */7457		7 + /* gfx_v9_0_emit_mem_sync */7458		2, /* gfx_v9_0_ring_emit_cleaner_shader */7459	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */7460	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,7461	.emit_fence = gfx_v9_0_ring_emit_fence,7462	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,7463	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,7464	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,7465	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,7466	.test_ring = gfx_v9_0_ring_test_ring,7467	.insert_nop = gfx_v9_ring_insert_nop,7468	.pad_ib = amdgpu_ring_generic_pad_ib,7469	.emit_switch_buffer = gfx_v9_ring_emit_sb,7470	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,7471	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,7472	.preempt_ib = gfx_v9_0_ring_preempt_ib,7473	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,7474	.emit_wreg = gfx_v9_0_ring_emit_wreg,7475	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,7476	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,7477	.soft_recovery = gfx_v9_0_ring_soft_recovery,7478	.emit_mem_sync = gfx_v9_0_emit_mem_sync,7479	.reset = gfx_v9_0_reset_kgq,7480	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,7481	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,7482	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,7483};7484 7485static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {7486	.type = AMDGPU_RING_TYPE_GFX,7487	.align_mask = 0xff,7488	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7489	.support_64bit_ptrs = true,7490	.secure_submission_supported = true,7491	.get_rptr = amdgpu_sw_ring_get_rptr_gfx,7492	.get_wptr = amdgpu_sw_ring_get_wptr_gfx,7493	.set_wptr = amdgpu_sw_ring_set_wptr_gfx,7494	.emit_frame_size = /* totally 242 maximum if 16 IBs */7495		5 +  /* COND_EXEC */7496		7 +  /* PIPELINE_SYNC */7497		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +7498		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +7499		2 + /* VM_FLUSH */7500		8 +  /* FENCE for VM_FLUSH */7501		20 + /* GDS switch */7502		4 + /* double SWITCH_BUFFER,7503		     * the first COND_EXEC jump to the place just7504		     * prior to this double SWITCH_BUFFER7505		     */7506		5 + /* COND_EXEC */7507		7 +	 /*	HDP_flush */7508		4 +	 /*	VGT_flush */7509		14 + /*	CE_META */7510		31 + /*	DE_META */7511		3 + /* CNTX_CTRL */7512		5 + /* HDP_INVL */7513		8 + 8 + /* FENCE x2 */7514		2 + /* SWITCH_BUFFER */7515		7 + /* gfx_v9_0_emit_mem_sync */7516		2, /* gfx_v9_0_ring_emit_cleaner_shader */7517	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */7518	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,7519	.emit_fence = gfx_v9_0_ring_emit_fence,7520	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,7521	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,7522	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,7523	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,7524	.test_ring = gfx_v9_0_ring_test_ring,7525	.test_ib = gfx_v9_0_ring_test_ib,7526	.insert_nop = gfx_v9_ring_insert_nop,7527	.pad_ib = amdgpu_ring_generic_pad_ib,7528	.emit_switch_buffer = gfx_v9_ring_emit_sb,7529	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,7530	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,7531	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,7532	.emit_wreg = gfx_v9_0_ring_emit_wreg,7533	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,7534	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,7535	.soft_recovery = gfx_v9_0_ring_soft_recovery,7536	.emit_mem_sync = gfx_v9_0_emit_mem_sync,7537	.patch_cntl = gfx_v9_0_ring_patch_cntl,7538	.patch_de = gfx_v9_0_ring_patch_de_meta,7539	.patch_ce = gfx_v9_0_ring_patch_ce_meta,7540	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,7541	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,7542	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,7543};7544 7545static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {7546	.type = AMDGPU_RING_TYPE_COMPUTE,7547	.align_mask = 0xff,7548	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7549	.support_64bit_ptrs = true,7550	.get_rptr = gfx_v9_0_ring_get_rptr_compute,7551	.get_wptr = gfx_v9_0_ring_get_wptr_compute,7552	.set_wptr = gfx_v9_0_ring_set_wptr_compute,7553	.emit_frame_size =7554		20 + /* gfx_v9_0_ring_emit_gds_switch */7555		7 + /* gfx_v9_0_ring_emit_hdp_flush */7556		5 + /* hdp invalidate */7557		7 + /* gfx_v9_0_ring_emit_pipeline_sync */7558		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +7559		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +7560		8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */7561		7 + /* gfx_v9_0_emit_mem_sync */7562		5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */7563		15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */7564		2, /* gfx_v9_0_ring_emit_cleaner_shader */7565	.emit_ib_size =	7, /* gfx_v9_0_ring_emit_ib_compute */7566	.emit_ib = gfx_v9_0_ring_emit_ib_compute,7567	.emit_fence = gfx_v9_0_ring_emit_fence,7568	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,7569	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,7570	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,7571	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,7572	.test_ring = gfx_v9_0_ring_test_ring,7573	.test_ib = gfx_v9_0_ring_test_ib,7574	.insert_nop = gfx_v9_ring_insert_nop,7575	.pad_ib = amdgpu_ring_generic_pad_ib,7576	.emit_wreg = gfx_v9_0_ring_emit_wreg,7577	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,7578	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,7579	.soft_recovery = gfx_v9_0_ring_soft_recovery,7580	.emit_mem_sync = gfx_v9_0_emit_mem_sync,7581	.emit_wave_limit = gfx_v9_0_emit_wave_limit,7582	.reset = gfx_v9_0_reset_kcq,7583	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,7584	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,7585	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,7586};7587 7588static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {7589	.type = AMDGPU_RING_TYPE_KIQ,7590	.align_mask = 0xff,7591	.nop = PACKET3(PACKET3_NOP, 0x3FFF),7592	.support_64bit_ptrs = true,7593	.get_rptr = gfx_v9_0_ring_get_rptr_compute,7594	.get_wptr = gfx_v9_0_ring_get_wptr_compute,7595	.set_wptr = gfx_v9_0_ring_set_wptr_compute,7596	.emit_frame_size =7597		20 + /* gfx_v9_0_ring_emit_gds_switch */7598		7 + /* gfx_v9_0_ring_emit_hdp_flush */7599		5 + /* hdp invalidate */7600		7 + /* gfx_v9_0_ring_emit_pipeline_sync */7601		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +7602		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +7603		8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */7604	.emit_ib_size =	7, /* gfx_v9_0_ring_emit_ib_compute */7605	.emit_fence = gfx_v9_0_ring_emit_fence_kiq,7606	.test_ring = gfx_v9_0_ring_test_ring,7607	.insert_nop = amdgpu_ring_insert_nop,7608	.pad_ib = amdgpu_ring_generic_pad_ib,7609	.emit_rreg = gfx_v9_0_ring_emit_rreg,7610	.emit_wreg = gfx_v9_0_ring_emit_wreg,7611	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,7612	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,7613};7614 7615static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)7616{7617	int i;7618 7619	adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;7620 7621	for (i = 0; i < adev->gfx.num_gfx_rings; i++)7622		adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;7623 7624	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {7625		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)7626			adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;7627	}7628 7629	for (i = 0; i < adev->gfx.num_compute_rings; i++)7630		adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;7631}7632 7633static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {7634	.set = gfx_v9_0_set_eop_interrupt_state,7635	.process = gfx_v9_0_eop_irq,7636};7637 7638static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {7639	.set = gfx_v9_0_set_priv_reg_fault_state,7640	.process = gfx_v9_0_priv_reg_irq,7641};7642 7643static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {7644	.set = gfx_v9_0_set_bad_op_fault_state,7645	.process = gfx_v9_0_bad_op_irq,7646};7647 7648static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {7649	.set = gfx_v9_0_set_priv_inst_fault_state,7650	.process = gfx_v9_0_priv_inst_irq,7651};7652 7653static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {7654	.set = gfx_v9_0_set_cp_ecc_error_state,7655	.process = amdgpu_gfx_cp_ecc_error_irq,7656};7657 7658 7659static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)7660{7661	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;7662	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;7663 7664	adev->gfx.priv_reg_irq.num_types = 1;7665	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;7666 7667	adev->gfx.bad_op_irq.num_types = 1;7668	adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;7669 7670	adev->gfx.priv_inst_irq.num_types = 1;7671	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;7672 7673	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/7674	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;7675}7676 7677static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)7678{7679	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {7680	case IP_VERSION(9, 0, 1):7681	case IP_VERSION(9, 2, 1):7682	case IP_VERSION(9, 4, 0):7683	case IP_VERSION(9, 2, 2):7684	case IP_VERSION(9, 1, 0):7685	case IP_VERSION(9, 4, 1):7686	case IP_VERSION(9, 3, 0):7687	case IP_VERSION(9, 4, 2):7688		adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;7689		break;7690	default:7691		break;7692	}7693}7694 7695static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)7696{7697	/* init asci gds info */7698	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {7699	case IP_VERSION(9, 0, 1):7700	case IP_VERSION(9, 2, 1):7701	case IP_VERSION(9, 4, 0):7702		adev->gds.gds_size = 0x10000;7703		break;7704	case IP_VERSION(9, 2, 2):7705	case IP_VERSION(9, 1, 0):7706	case IP_VERSION(9, 4, 1):7707		adev->gds.gds_size = 0x1000;7708		break;7709	case IP_VERSION(9, 4, 2):7710		/* aldebaran removed all the GDS internal memory,7711		 * only support GWS opcode in kernel, like barrier7712		 * semaphore.etc */7713		adev->gds.gds_size = 0;7714		break;7715	default:7716		adev->gds.gds_size = 0x10000;7717		break;7718	}7719 7720	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {7721	case IP_VERSION(9, 0, 1):7722	case IP_VERSION(9, 4, 0):7723		adev->gds.gds_compute_max_wave_id = 0x7ff;7724		break;7725	case IP_VERSION(9, 2, 1):7726		adev->gds.gds_compute_max_wave_id = 0x27f;7727		break;7728	case IP_VERSION(9, 2, 2):7729	case IP_VERSION(9, 1, 0):7730		if (adev->apu_flags & AMD_APU_IS_RAVEN2)7731			adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */7732		else7733			adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */7734		break;7735	case IP_VERSION(9, 4, 1):7736		adev->gds.gds_compute_max_wave_id = 0xfff;7737		break;7738	case IP_VERSION(9, 4, 2):7739		/* deprecated for Aldebaran, no usage at all */7740		adev->gds.gds_compute_max_wave_id = 0;7741		break;7742	default:7743		/* this really depends on the chip */7744		adev->gds.gds_compute_max_wave_id = 0x7ff;7745		break;7746	}7747 7748	adev->gds.gws_size = 64;7749	adev->gds.oa_size = 16;7750}7751 7752static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,7753						 u32 bitmap)7754{7755	u32 data;7756 7757	if (!bitmap)7758		return;7759 7760	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;7761	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;7762 7763	WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);7764}7765 7766static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)7767{7768	u32 data, mask;7769 7770	data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);7771	data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);7772 7773	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;7774	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;7775 7776	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);7777 7778	return (~data) & mask;7779}7780 7781static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,7782				 struct amdgpu_cu_info *cu_info)7783{7784	int i, j, k, counter, active_cu_number = 0;7785	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;7786	unsigned disable_masks[4 * 4];7787 7788	if (!adev || !cu_info)7789		return -EINVAL;7790 7791	/*7792	 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs7793	 */7794	if (adev->gfx.config.max_shader_engines *7795		adev->gfx.config.max_sh_per_se > 16)7796		return -EINVAL;7797 7798	amdgpu_gfx_parse_disable_cu(disable_masks,7799				    adev->gfx.config.max_shader_engines,7800				    adev->gfx.config.max_sh_per_se);7801 7802	mutex_lock(&adev->grbm_idx_mutex);7803	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {7804		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {7805			mask = 1;7806			ao_bitmap = 0;7807			counter = 0;7808			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);7809			gfx_v9_0_set_user_cu_inactive_bitmap(7810				adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);7811			bitmap = gfx_v9_0_get_cu_active_bitmap(adev);7812 7813			/*7814			 * The bitmap(and ao_cu_bitmap) in cu_info structure is7815			 * 4x4 size array, and it's usually suitable for Vega7816			 * ASICs which has 4*2 SE/SH layout.7817			 * But for Arcturus, SE/SH layout is changed to 8*1.7818			 * To mostly reduce the impact, we make it compatible7819			 * with current bitmap array as below:7820			 *    SE4,SH0 --> bitmap[0][1]7821			 *    SE5,SH0 --> bitmap[1][1]7822			 *    SE6,SH0 --> bitmap[2][1]7823			 *    SE7,SH0 --> bitmap[3][1]7824			 */7825			cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;7826 7827			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {7828				if (bitmap & mask) {7829					if (counter < adev->gfx.config.max_cu_per_sh)7830						ao_bitmap |= mask;7831					counter ++;7832				}7833				mask <<= 1;7834			}7835			active_cu_number += counter;7836			if (i < 2 && j < 2)7837				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));7838			cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;7839		}7840	}7841	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);7842	mutex_unlock(&adev->grbm_idx_mutex);7843 7844	cu_info->number = active_cu_number;7845	cu_info->ao_cu_mask = ao_cu_mask;7846	cu_info->simd_per_cu = NUM_SIMD_PER_CU;7847 7848	return 0;7849}7850 7851const struct amdgpu_ip_block_version gfx_v9_0_ip_block =7852{7853	.type = AMD_IP_BLOCK_TYPE_GFX,7854	.major = 9,7855	.minor = 0,7856	.rev = 0,7857	.funcs = &gfx_v9_0_ip_funcs,7858};7859