brintos / linux-shallow public Read only

1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/delay.h>25#include <linux/firmware.h>26#include <linux/module.h>27#include <linux/pci.h>28 29#include "amdgpu.h"30#include "amdgpu_ucode.h"31#include "amdgpu_trace.h"32 33#include "sdma0/sdma0_4_2_offset.h"34#include "sdma0/sdma0_4_2_sh_mask.h"35#include "sdma1/sdma1_4_2_offset.h"36#include "sdma1/sdma1_4_2_sh_mask.h"37#include "sdma2/sdma2_4_2_2_offset.h"38#include "sdma2/sdma2_4_2_2_sh_mask.h"39#include "sdma3/sdma3_4_2_2_offset.h"40#include "sdma3/sdma3_4_2_2_sh_mask.h"41#include "sdma4/sdma4_4_2_2_offset.h"42#include "sdma4/sdma4_4_2_2_sh_mask.h"43#include "sdma5/sdma5_4_2_2_offset.h"44#include "sdma5/sdma5_4_2_2_sh_mask.h"45#include "sdma6/sdma6_4_2_2_offset.h"46#include "sdma6/sdma6_4_2_2_sh_mask.h"47#include "sdma7/sdma7_4_2_2_offset.h"48#include "sdma7/sdma7_4_2_2_sh_mask.h"49#include "sdma0/sdma0_4_1_default.h"50 51#include "soc15_common.h"52#include "soc15.h"53#include "vega10_sdma_pkt_open.h"54 55#include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"56#include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"57 58#include "amdgpu_ras.h"59#include "sdma_v4_4.h"60 61MODULE_FIRMWARE("amdgpu/vega10_sdma.bin");62MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin");63MODULE_FIRMWARE("amdgpu/vega12_sdma.bin");64MODULE_FIRMWARE("amdgpu/vega12_sdma1.bin");65MODULE_FIRMWARE("amdgpu/vega20_sdma.bin");66MODULE_FIRMWARE("amdgpu/vega20_sdma1.bin");67MODULE_FIRMWARE("amdgpu/raven_sdma.bin");68MODULE_FIRMWARE("amdgpu/picasso_sdma.bin");69MODULE_FIRMWARE("amdgpu/raven2_sdma.bin");70MODULE_FIRMWARE("amdgpu/arcturus_sdma.bin");71MODULE_FIRMWARE("amdgpu/renoir_sdma.bin");72MODULE_FIRMWARE("amdgpu/green_sardine_sdma.bin");73MODULE_FIRMWARE("amdgpu/aldebaran_sdma.bin");74 75static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_0[] = {76	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG),77	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG),78	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG),79	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG),80	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM),81	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI),82	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH),83	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS),84	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS),85	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0),86	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1),87	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0),88	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1),89	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL),90	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR),91	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI),92	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR),93	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI),94	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET),95	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO),96	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI),97	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL),98	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR),99	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN),100	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG),101	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL),102	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR),103	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI),104	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR),105	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI),106	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET),107	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO),108	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI),109	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG),110	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL),111	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR),112	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI),113	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR),114	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI),115	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET),116	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO),117	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI),118	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG),119	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL)120};121 122#define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK  0x000000F8L123#define SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK 0xFC000000L124 125#define WREG32_SDMA(instance, offset, value) \126	WREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)), value)127#define RREG32_SDMA(instance, offset) \128	RREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)))129 130static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev);131static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev);132static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);133static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);134static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev);135 136static const struct soc15_reg_golden golden_settings_sdma_4[] = {137	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),138	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xff000ff0, 0x3f000100),139	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0100, 0x00000100),140	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),141	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_IB_CNTL, 0x800f0100, 0x00000100),142	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),143	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003ff006, 0x0003c000),144	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0100, 0x00000100),145	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),146	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0100, 0x00000100),147	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),148	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),149	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000),150	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),151	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_IB_CNTL, 0x800f0100, 0x00000100),152	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),153	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_IB_CNTL, 0x800f0100, 0x00000100),154	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),155	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_POWER_CNTL, 0x003ff000, 0x0003c000),156	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_IB_CNTL, 0x800f0100, 0x00000100),157	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),158	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_IB_CNTL, 0x800f0100, 0x00000100),159	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),160	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0),161	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_WATERMK, 0xfc000000, 0x00000000)162};163 164static const struct soc15_reg_golden golden_settings_sdma_vg10[] = {165	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),166	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),167	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),168	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07),169	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),170	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),171	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),172};173 174static const struct soc15_reg_golden golden_settings_sdma_vg12[] = {175	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104001),176	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001),177	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),178	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07),179	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104001),180	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001),181	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),182};183 184static const struct soc15_reg_golden golden_settings_sdma_4_1[] = {185	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),186	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),187	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100),188	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),189	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0xfc3fffff, 0x40000051),190	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100),191	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),192	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100),193	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),194	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0),195	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000)196};197 198static const struct soc15_reg_golden golden_settings_sdma0_4_2_init[] = {199	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),200};201 202static const struct soc15_reg_golden golden_settings_sdma0_4_2[] =203{204	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),205	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),206	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),207	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),208	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),209	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),210	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),211	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),212	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RD_BURST_CNTL, 0x0000000f, 0x00000003),213	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),214	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),215	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),216	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),217	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),218	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),219	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),220	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),221	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),222	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),223	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),224	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),225	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),226	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),227	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),228	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),229	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),230	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),231};232 233static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = {234	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07),235	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),236	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),237	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),238	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),239	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),240	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),241	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),242	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RD_BURST_CNTL, 0x0000000f, 0x00000003),243	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),244	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),245	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),246	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),247	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),248	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),249	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),250	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),251	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),252	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),253	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),254	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),255	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),256	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),257	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),258	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),259	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0),260	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),261};262 263static const struct soc15_reg_golden golden_settings_sdma_rv1[] =264{265	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),266	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002)267};268 269static const struct soc15_reg_golden golden_settings_sdma_rv2[] =270{271	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00003001),272	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00003001)273};274 275static const struct soc15_reg_golden golden_settings_sdma_arct[] =276{277	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),278	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),279	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),280	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),281	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07),282	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),283	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),284	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),285	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_CHICKEN_BITS, 0xfe931f07, 0x02831f07),286	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),287	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),288	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),289	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_CHICKEN_BITS, 0xfe931f07, 0x02831f07),290	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),291	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),292	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),293	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_CHICKEN_BITS, 0xfe931f07, 0x02831f07),294	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),295	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),296	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),297	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_CHICKEN_BITS, 0xfe931f07, 0x02831f07),298	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),299	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),300	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),301	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_CHICKEN_BITS, 0xfe931f07, 0x02831f07),302	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),303	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),304	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),305	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_CHICKEN_BITS, 0xfe931f07, 0x02831f07),306	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),307	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),308	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_UTCL1_TIMEOUT, 0xffffffff, 0x00010001)309};310 311static const struct soc15_reg_golden golden_settings_sdma_aldebaran[] = {312	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),313	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),314	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),315	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),316	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),317	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),318	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),319	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),320	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA2_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),321	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),322	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),323	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),324	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),325	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),326	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),327};328 329static const struct soc15_reg_golden golden_settings_sdma_4_3[] = {330	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),331	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),332	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),333	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002),334	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),335	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003fff07, 0x40000051),336	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),337	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),338	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0),339	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x03fbe1fe)340};341 342static const struct soc15_ras_field_entry sdma_v4_0_ras_fields[] = {343	{ "SDMA_UCODE_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),344	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UCODE_BUF_SED),345	0, 0,346	},347	{ "SDMA_RB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),348	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_RB_CMD_BUF_SED),349	0, 0,350	},351	{ "SDMA_IB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),352	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_IB_CMD_BUF_SED),353	0, 0,354	},355	{ "SDMA_UTCL1_RD_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),356	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RD_FIFO_SED),357	0, 0,358	},359	{ "SDMA_UTCL1_RDBST_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),360	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RDBST_FIFO_SED),361	0, 0,362	},363	{ "SDMA_DATA_LUT_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),364	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_DATA_LUT_FIFO_SED),365	0, 0,366	},367	{ "SDMA_MBANK_DATA_BUF0_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),368	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF0_SED),369	0, 0,370	},371	{ "SDMA_MBANK_DATA_BUF1_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),372	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF1_SED),373	0, 0,374	},375	{ "SDMA_MBANK_DATA_BUF2_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),376	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF2_SED),377	0, 0,378	},379	{ "SDMA_MBANK_DATA_BUF3_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),380	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF3_SED),381	0, 0,382	},383	{ "SDMA_MBANK_DATA_BUF4_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),384	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF4_SED),385	0, 0,386	},387	{ "SDMA_MBANK_DATA_BUF5_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),388	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF5_SED),389	0, 0,390	},391	{ "SDMA_MBANK_DATA_BUF6_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),392	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF6_SED),393	0, 0,394	},395	{ "SDMA_MBANK_DATA_BUF7_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),396	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF7_SED),397	0, 0,398	},399	{ "SDMA_MBANK_DATA_BUF8_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),400	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF8_SED),401	0, 0,402	},403	{ "SDMA_MBANK_DATA_BUF9_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),404	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF9_SED),405	0, 0,406	},407	{ "SDMA_MBANK_DATA_BUF10_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),408	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF10_SED),409	0, 0,410	},411	{ "SDMA_MBANK_DATA_BUF11_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),412	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF11_SED),413	0, 0,414	},415	{ "SDMA_MBANK_DATA_BUF12_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),416	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF12_SED),417	0, 0,418	},419	{ "SDMA_MBANK_DATA_BUF13_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),420	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF13_SED),421	0, 0,422	},423	{ "SDMA_MBANK_DATA_BUF14_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),424	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF14_SED),425	0, 0,426	},427	{ "SDMA_MBANK_DATA_BUF15_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),428	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF15_SED),429	0, 0,430	},431	{ "SDMA_SPLIT_DAT_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),432	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_SPLIT_DAT_BUF_SED),433	0, 0,434	},435	{ "SDMA_MC_WR_ADDR_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),436	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MC_WR_ADDR_FIFO_SED),437	0, 0,438	},439};440 441static u32 sdma_v4_0_get_reg_offset(struct amdgpu_device *adev,442		u32 instance, u32 offset)443{444	switch (instance) {445	case 0:446		return (adev->reg_offset[SDMA0_HWIP][0][0] + offset);447	case 1:448		return (adev->reg_offset[SDMA1_HWIP][0][0] + offset);449	case 2:450		return (adev->reg_offset[SDMA2_HWIP][0][1] + offset);451	case 3:452		return (adev->reg_offset[SDMA3_HWIP][0][1] + offset);453	case 4:454		return (adev->reg_offset[SDMA4_HWIP][0][1] + offset);455	case 5:456		return (adev->reg_offset[SDMA5_HWIP][0][1] + offset);457	case 6:458		return (adev->reg_offset[SDMA6_HWIP][0][1] + offset);459	case 7:460		return (adev->reg_offset[SDMA7_HWIP][0][1] + offset);461	default:462		break;463	}464	return 0;465}466 467static unsigned sdma_v4_0_seq_to_irq_id(int seq_num)468{469	switch (seq_num) {470	case 0:471		return SOC15_IH_CLIENTID_SDMA0;472	case 1:473		return SOC15_IH_CLIENTID_SDMA1;474	case 2:475		return SOC15_IH_CLIENTID_SDMA2;476	case 3:477		return SOC15_IH_CLIENTID_SDMA3;478	case 4:479		return SOC15_IH_CLIENTID_SDMA4;480	case 5:481		return SOC15_IH_CLIENTID_SDMA5;482	case 6:483		return SOC15_IH_CLIENTID_SDMA6;484	case 7:485		return SOC15_IH_CLIENTID_SDMA7;486	default:487		break;488	}489	return -EINVAL;490}491 492static int sdma_v4_0_irq_id_to_seq(unsigned client_id)493{494	switch (client_id) {495	case SOC15_IH_CLIENTID_SDMA0:496		return 0;497	case SOC15_IH_CLIENTID_SDMA1:498		return 1;499	case SOC15_IH_CLIENTID_SDMA2:500		return 2;501	case SOC15_IH_CLIENTID_SDMA3:502		return 3;503	case SOC15_IH_CLIENTID_SDMA4:504		return 4;505	case SOC15_IH_CLIENTID_SDMA5:506		return 5;507	case SOC15_IH_CLIENTID_SDMA6:508		return 6;509	case SOC15_IH_CLIENTID_SDMA7:510		return 7;511	default:512		break;513	}514	return -EINVAL;515}516 517static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)518{519	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {520	case IP_VERSION(4, 0, 0):521		soc15_program_register_sequence(adev,522						golden_settings_sdma_4,523						ARRAY_SIZE(golden_settings_sdma_4));524		soc15_program_register_sequence(adev,525						golden_settings_sdma_vg10,526						ARRAY_SIZE(golden_settings_sdma_vg10));527		break;528	case IP_VERSION(4, 0, 1):529		soc15_program_register_sequence(adev,530						golden_settings_sdma_4,531						ARRAY_SIZE(golden_settings_sdma_4));532		soc15_program_register_sequence(adev,533						golden_settings_sdma_vg12,534						ARRAY_SIZE(golden_settings_sdma_vg12));535		break;536	case IP_VERSION(4, 2, 0):537		soc15_program_register_sequence(adev,538						golden_settings_sdma0_4_2_init,539						ARRAY_SIZE(golden_settings_sdma0_4_2_init));540		soc15_program_register_sequence(adev,541						golden_settings_sdma0_4_2,542						ARRAY_SIZE(golden_settings_sdma0_4_2));543		soc15_program_register_sequence(adev,544						golden_settings_sdma1_4_2,545						ARRAY_SIZE(golden_settings_sdma1_4_2));546		break;547	case IP_VERSION(4, 2, 2):548		soc15_program_register_sequence(adev,549						golden_settings_sdma_arct,550						ARRAY_SIZE(golden_settings_sdma_arct));551		break;552	case IP_VERSION(4, 4, 0):553		soc15_program_register_sequence(adev,554						golden_settings_sdma_aldebaran,555						ARRAY_SIZE(golden_settings_sdma_aldebaran));556		break;557	case IP_VERSION(4, 1, 0):558	case IP_VERSION(4, 1, 1):559		soc15_program_register_sequence(adev,560						golden_settings_sdma_4_1,561						ARRAY_SIZE(golden_settings_sdma_4_1));562		if (adev->apu_flags & AMD_APU_IS_RAVEN2)563			soc15_program_register_sequence(adev,564							golden_settings_sdma_rv2,565							ARRAY_SIZE(golden_settings_sdma_rv2));566		else567			soc15_program_register_sequence(adev,568							golden_settings_sdma_rv1,569							ARRAY_SIZE(golden_settings_sdma_rv1));570		break;571	case IP_VERSION(4, 1, 2):572		soc15_program_register_sequence(adev,573						golden_settings_sdma_4_3,574						ARRAY_SIZE(golden_settings_sdma_4_3));575		break;576	default:577		break;578	}579}580 581static void sdma_v4_0_setup_ulv(struct amdgpu_device *adev)582{583	int i;584 585	/*586	 * The only chips with SDMAv4 and ULV are VG10 and VG20.587	 * Server SKUs take a different hysteresis setting from other SKUs.588	 */589	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {590	case IP_VERSION(4, 0, 0):591		if (adev->pdev->device == 0x6860)592			break;593		return;594	case IP_VERSION(4, 2, 0):595		if (adev->pdev->device == 0x66a1)596			break;597		return;598	default:599		return;600	}601 602	for (i = 0; i < adev->sdma.num_instances; i++) {603		uint32_t temp;604 605		temp = RREG32_SDMA(i, mmSDMA0_ULV_CNTL);606		temp = REG_SET_FIELD(temp, SDMA0_ULV_CNTL, HYSTERESIS, 0x0);607		WREG32_SDMA(i, mmSDMA0_ULV_CNTL, temp);608	}609}610 611/**612 * sdma_v4_0_init_microcode - load ucode images from disk613 *614 * @adev: amdgpu_device pointer615 *616 * Use the firmware interface to load the ucode images into617 * the driver (not loaded into hw).618 * Returns 0 on success, error on failure.619 */620 621// emulation only, won't work on real chip622// vega10 real chip need to use PSP to load firmware623static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)624{625	int ret, i;626 627	for (i = 0; i < adev->sdma.num_instances; i++) {628		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==629			    IP_VERSION(4, 2, 2) ||630		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==631			    IP_VERSION(4, 4, 0)) {632			/* Acturus & Aldebaran will leverage the same FW memory633			   for every SDMA instance */634			ret = amdgpu_sdma_init_microcode(adev, 0, true);635			break;636		} else {637			ret = amdgpu_sdma_init_microcode(adev, i, false);638			if (ret)639				return ret;640		}641	}642 643	return ret;644}645 646/**647 * sdma_v4_0_ring_get_rptr - get the current read pointer648 *649 * @ring: amdgpu ring pointer650 *651 * Get the current rptr from the hardware (VEGA10+).652 */653static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring)654{655	u64 *rptr;656 657	/* XXX check if swapping is necessary on BE */658	rptr = ((u64 *)ring->rptr_cpu_addr);659 660	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);661	return ((*rptr) >> 2);662}663 664/**665 * sdma_v4_0_ring_get_wptr - get the current write pointer666 *667 * @ring: amdgpu ring pointer668 *669 * Get the current wptr from the hardware (VEGA10+).670 */671static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring)672{673	struct amdgpu_device *adev = ring->adev;674	u64 wptr;675 676	if (ring->use_doorbell) {677		/* XXX check if swapping is necessary on BE */678		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));679		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);680	} else {681		wptr = RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI);682		wptr = wptr << 32;683		wptr |= RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR);684		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",685				ring->me, wptr);686	}687 688	return wptr >> 2;689}690 691/**692 * sdma_v4_0_ring_set_wptr - commit the write pointer693 *694 * @ring: amdgpu ring pointer695 *696 * Write the wptr back to the hardware (VEGA10+).697 */698static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)699{700	struct amdgpu_device *adev = ring->adev;701 702	DRM_DEBUG("Setting write pointer\n");703	if (ring->use_doorbell) {704		u64 *wb = (u64 *)ring->wptr_cpu_addr;705 706		DRM_DEBUG("Using doorbell -- "707				"wptr_offs == 0x%08x "708				"lower_32_bits(ring->wptr << 2) == 0x%08x "709				"upper_32_bits(ring->wptr << 2) == 0x%08x\n",710				ring->wptr_offs,711				lower_32_bits(ring->wptr << 2),712				upper_32_bits(ring->wptr << 2));713		/* XXX check if swapping is necessary on BE */714		WRITE_ONCE(*wb, (ring->wptr << 2));715		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",716				ring->doorbell_index, ring->wptr << 2);717		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);718	} else {719		DRM_DEBUG("Not using doorbell -- "720				"mmSDMA%i_GFX_RB_WPTR == 0x%08x "721				"mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",722				ring->me,723				lower_32_bits(ring->wptr << 2),724				ring->me,725				upper_32_bits(ring->wptr << 2));726		WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR,727			    lower_32_bits(ring->wptr << 2));728		WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI,729			    upper_32_bits(ring->wptr << 2));730	}731}732 733/**734 * sdma_v4_0_page_ring_get_wptr - get the current write pointer735 *736 * @ring: amdgpu ring pointer737 *738 * Get the current wptr from the hardware (VEGA10+).739 */740static uint64_t sdma_v4_0_page_ring_get_wptr(struct amdgpu_ring *ring)741{742	struct amdgpu_device *adev = ring->adev;743	u64 wptr;744 745	if (ring->use_doorbell) {746		/* XXX check if swapping is necessary on BE */747		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));748	} else {749		wptr = RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI);750		wptr = wptr << 32;751		wptr |= RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR);752	}753 754	return wptr >> 2;755}756 757/**758 * sdma_v4_0_page_ring_set_wptr - commit the write pointer759 *760 * @ring: amdgpu ring pointer761 *762 * Write the wptr back to the hardware (VEGA10+).763 */764static void sdma_v4_0_page_ring_set_wptr(struct amdgpu_ring *ring)765{766	struct amdgpu_device *adev = ring->adev;767 768	if (ring->use_doorbell) {769		u64 *wb = (u64 *)ring->wptr_cpu_addr;770 771		/* XXX check if swapping is necessary on BE */772		WRITE_ONCE(*wb, (ring->wptr << 2));773		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);774	} else {775		uint64_t wptr = ring->wptr << 2;776 777		WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR,778			    lower_32_bits(wptr));779		WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI,780			    upper_32_bits(wptr));781	}782}783 784static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)785{786	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);787	int i;788 789	for (i = 0; i < count; i++)790		if (sdma && sdma->burst_nop && (i == 0))791			amdgpu_ring_write(ring, ring->funcs->nop |792				SDMA_PKT_NOP_HEADER_COUNT(count - 1));793		else794			amdgpu_ring_write(ring, ring->funcs->nop);795}796 797/**798 * sdma_v4_0_ring_emit_ib - Schedule an IB on the DMA engine799 *800 * @ring: amdgpu ring pointer801 * @job: job to retrieve vmid from802 * @ib: IB object to schedule803 * @flags: unused804 *805 * Schedule an IB in the DMA ring (VEGA10).806 */807static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,808				   struct amdgpu_job *job,809				   struct amdgpu_ib *ib,810				   uint32_t flags)811{812	unsigned vmid = AMDGPU_JOB_GET_VMID(job);813 814	/* IB packet must end on a 8 DW boundary */815	sdma_v4_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);816 817	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |818			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));819	/* base must be 32 byte aligned */820	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);821	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));822	amdgpu_ring_write(ring, ib->length_dw);823	amdgpu_ring_write(ring, 0);824	amdgpu_ring_write(ring, 0);825 826}827 828static void sdma_v4_0_wait_reg_mem(struct amdgpu_ring *ring,829				   int mem_space, int hdp,830				   uint32_t addr0, uint32_t addr1,831				   uint32_t ref, uint32_t mask,832				   uint32_t inv)833{834	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |835			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |836			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |837			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */838	if (mem_space) {839		/* memory */840		amdgpu_ring_write(ring, addr0);841		amdgpu_ring_write(ring, addr1);842	} else {843		/* registers */844		amdgpu_ring_write(ring, addr0 << 2);845		amdgpu_ring_write(ring, addr1 << 2);846	}847	amdgpu_ring_write(ring, ref); /* reference */848	amdgpu_ring_write(ring, mask); /* mask */849	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |850			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */851}852 853/**854 * sdma_v4_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring855 *856 * @ring: amdgpu ring pointer857 *858 * Emit an hdp flush packet on the requested DMA ring.859 */860static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)861{862	struct amdgpu_device *adev = ring->adev;863	u32 ref_and_mask = 0;864	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;865 866	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;867 868	sdma_v4_0_wait_reg_mem(ring, 0, 1,869			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),870			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),871			       ref_and_mask, ref_and_mask, 10);872}873 874/**875 * sdma_v4_0_ring_emit_fence - emit a fence on the DMA ring876 *877 * @ring: amdgpu ring pointer878 * @addr: address879 * @seq: sequence number880 * @flags: fence related flags881 *882 * Add a DMA fence packet to the ring to write883 * the fence seq number and DMA trap packet to generate884 * an interrupt if needed (VEGA10).885 */886static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,887				      unsigned flags)888{889	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;890	/* write the fence */891	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));892	/* zero in first two bits */893	BUG_ON(addr & 0x3);894	amdgpu_ring_write(ring, lower_32_bits(addr));895	amdgpu_ring_write(ring, upper_32_bits(addr));896	amdgpu_ring_write(ring, lower_32_bits(seq));897 898	/* optionally write high bits as well */899	if (write64bit) {900		addr += 4;901		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));902		/* zero in first two bits */903		BUG_ON(addr & 0x3);904		amdgpu_ring_write(ring, lower_32_bits(addr));905		amdgpu_ring_write(ring, upper_32_bits(addr));906		amdgpu_ring_write(ring, upper_32_bits(seq));907	}908 909	/* generate an interrupt */910	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));911	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));912}913 914 915/**916 * sdma_v4_0_gfx_enable - enable the gfx async dma engines917 *918 * @adev: amdgpu_device pointer919 * @enable: enable SDMA RB/IB920 * control the gfx async dma ring buffers (VEGA10).921 */922static void sdma_v4_0_gfx_enable(struct amdgpu_device *adev, bool enable)923{924	u32 rb_cntl, ib_cntl;925	int i;926 927	for (i = 0; i < adev->sdma.num_instances; i++) {928		rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL);929		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, enable ? 1 : 0);930		WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);931		ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL);932		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, enable ? 1 : 0);933		WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl);934	}935}936 937/**938 * sdma_v4_0_rlc_stop - stop the compute async dma engines939 *940 * @adev: amdgpu_device pointer941 *942 * Stop the compute async dma queues (VEGA10).943 */944static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev)945{946	/* XXX todo */947}948 949/**950 * sdma_v4_0_page_stop - stop the page async dma engines951 *952 * @adev: amdgpu_device pointer953 *954 * Stop the page async dma ring buffers (VEGA10).955 */956static void sdma_v4_0_page_stop(struct amdgpu_device *adev)957{958	u32 rb_cntl, ib_cntl;959	int i;960 961	for (i = 0; i < adev->sdma.num_instances; i++) {962		rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL);963		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,964					RB_ENABLE, 0);965		WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);966		ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL);967		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL,968					IB_ENABLE, 0);969		WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl);970	}971}972 973/**974 * sdma_v4_0_ctx_switch_enable - stop the async dma engines context switch975 *976 * @adev: amdgpu_device pointer977 * @enable: enable/disable the DMA MEs context switch.978 *979 * Halt or unhalt the async dma engines context switch (VEGA10).980 */981static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)982{983	u32 f32_cntl, phase_quantum = 0;984	int i;985 986	if (amdgpu_sdma_phase_quantum) {987		unsigned value = amdgpu_sdma_phase_quantum;988		unsigned unit = 0;989 990		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>991				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {992			value = (value + 1) >> 1;993			unit++;994		}995		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>996			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {997			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>998				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);999			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>1000				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);1001			WARN_ONCE(1,1002			"clamping sdma_phase_quantum to %uK clock cycles\n",1003				  value << unit);1004		}1005		phase_quantum =1006			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |1007			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;1008	}1009 1010	for (i = 0; i < adev->sdma.num_instances; i++) {1011		f32_cntl = RREG32_SDMA(i, mmSDMA0_CNTL);1012		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,1013				AUTO_CTXSW_ENABLE, enable ? 1 : 0);1014		if (enable && amdgpu_sdma_phase_quantum) {1015			WREG32_SDMA(i, mmSDMA0_PHASE0_QUANTUM, phase_quantum);1016			WREG32_SDMA(i, mmSDMA0_PHASE1_QUANTUM, phase_quantum);1017			WREG32_SDMA(i, mmSDMA0_PHASE2_QUANTUM, phase_quantum);1018		}1019		WREG32_SDMA(i, mmSDMA0_CNTL, f32_cntl);1020 1021		/*1022		 * Enable SDMA utilization. Its only supported on1023		 * Arcturus for the moment and firmware version 141024		 * and above.1025		 */1026		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==1027			    IP_VERSION(4, 2, 2) &&1028		    adev->sdma.instance[i].fw_version >= 14)1029			WREG32_SDMA(i, mmSDMA0_PUB_DUMMY_REG2, enable);1030		/* Extend page fault timeout to avoid interrupt storm */1031		WREG32_SDMA(i, mmSDMA0_UTCL1_TIMEOUT, 0x00800080);1032	}1033 1034}1035 1036/**1037 * sdma_v4_0_enable - stop the async dma engines1038 *1039 * @adev: amdgpu_device pointer1040 * @enable: enable/disable the DMA MEs.1041 *1042 * Halt or unhalt the async dma engines (VEGA10).1043 */1044static void sdma_v4_0_enable(struct amdgpu_device *adev, bool enable)1045{1046	u32 f32_cntl;1047	int i;1048 1049	if (!enable) {1050		sdma_v4_0_gfx_enable(adev, enable);1051		sdma_v4_0_rlc_stop(adev);1052		if (adev->sdma.has_page_queue)1053			sdma_v4_0_page_stop(adev);1054	}1055 1056	for (i = 0; i < adev->sdma.num_instances; i++) {1057		f32_cntl = RREG32_SDMA(i, mmSDMA0_F32_CNTL);1058		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);1059		WREG32_SDMA(i, mmSDMA0_F32_CNTL, f32_cntl);1060	}1061}1062 1063/*1064 * sdma_v4_0_rb_cntl - get parameters for rb_cntl1065 */1066static uint32_t sdma_v4_0_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)1067{1068	/* Set ring buffer size in dwords */1069	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);1070 1071	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);1072#ifdef __BIG_ENDIAN1073	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);1074	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,1075				RPTR_WRITEBACK_SWAP_ENABLE, 1);1076#endif1077	return rb_cntl;1078}1079 1080/**1081 * sdma_v4_0_gfx_resume - setup and start the async dma engines1082 *1083 * @adev: amdgpu_device pointer1084 * @i: instance to resume1085 *1086 * Set up the gfx DMA ring buffers and enable them (VEGA10).1087 * Returns 0 for success, error for failure.1088 */1089static void sdma_v4_0_gfx_resume(struct amdgpu_device *adev, unsigned int i)1090{1091	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;1092	u32 rb_cntl, ib_cntl, wptr_poll_cntl;1093	u32 doorbell;1094	u32 doorbell_offset;1095	u64 wptr_gpu_addr;1096 1097	rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL);1098	rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl);1099	WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);1100 1101	/* Initialize the ring buffer's read and write pointers */1102	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR, 0);1103	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_HI, 0);1104	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR, 0);1105	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_HI, 0);1106 1107	/* set the wb address whether it's enabled or not */1108	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_HI,1109	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);1110	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_LO,1111	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);1112 1113	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,1114				RPTR_WRITEBACK_ENABLE, 1);1115 1116	WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE, ring->gpu_addr >> 8);1117	WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE_HI, ring->gpu_addr >> 40);1118 1119	ring->wptr = 0;1120 1121	/* before programing wptr to a less value, need set minor_ptr_update first */1122	WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 1);1123 1124	doorbell = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL);1125	doorbell_offset = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET);1126 1127	doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE,1128				 ring->use_doorbell);1129	doorbell_offset = REG_SET_FIELD(doorbell_offset,1130					SDMA0_GFX_DOORBELL_OFFSET,1131					OFFSET, ring->doorbell_index);1132	WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL, doorbell);1133	WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET, doorbell_offset);1134 1135	sdma_v4_0_ring_set_wptr(ring);1136 1137	/* set minor_ptr_update to 0 after wptr programed */1138	WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 0);1139 1140	/* setup the wptr shadow polling */1141	wptr_gpu_addr = ring->wptr_gpu_addr;1142	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO,1143		    lower_32_bits(wptr_gpu_addr));1144	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI,1145		    upper_32_bits(wptr_gpu_addr));1146	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL);1147	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,1148				       SDMA0_GFX_RB_WPTR_POLL_CNTL,1149				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);1150	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);1151 1152	/* enable DMA RB */1153	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);1154	WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);1155 1156	ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL);1157	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);1158#ifdef __BIG_ENDIAN1159	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);1160#endif1161	/* enable DMA IBs */1162	WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl);1163}1164 1165/**1166 * sdma_v4_0_page_resume - setup and start the async dma engines1167 *1168 * @adev: amdgpu_device pointer1169 * @i: instance to resume1170 *1171 * Set up the page DMA ring buffers and enable them (VEGA10).1172 * Returns 0 for success, error for failure.1173 */1174static void sdma_v4_0_page_resume(struct amdgpu_device *adev, unsigned int i)1175{1176	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;1177	u32 rb_cntl, ib_cntl, wptr_poll_cntl;1178	u32 doorbell;1179	u32 doorbell_offset;1180	u64 wptr_gpu_addr;1181 1182	rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL);1183	rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl);1184	WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);1185 1186	/* Initialize the ring buffer's read and write pointers */1187	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR, 0);1188	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_HI, 0);1189	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR, 0);1190	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_HI, 0);1191 1192	/* set the wb address whether it's enabled or not */1193	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_HI,1194	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);1195	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_LO,1196	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);1197 1198	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,1199				RPTR_WRITEBACK_ENABLE, 1);1200 1201	WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE, ring->gpu_addr >> 8);1202	WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);1203 1204	ring->wptr = 0;1205 1206	/* before programing wptr to a less value, need set minor_ptr_update first */1207	WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 1);1208 1209	doorbell = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL);1210	doorbell_offset = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET);1211 1212	doorbell = REG_SET_FIELD(doorbell, SDMA0_PAGE_DOORBELL, ENABLE,1213				 ring->use_doorbell);1214	doorbell_offset = REG_SET_FIELD(doorbell_offset,1215					SDMA0_PAGE_DOORBELL_OFFSET,1216					OFFSET, ring->doorbell_index);1217	WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL, doorbell);1218	WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET, doorbell_offset);1219 1220	/* paging queue doorbell range is setup at sdma_v4_0_gfx_resume */1221	sdma_v4_0_page_ring_set_wptr(ring);1222 1223	/* set minor_ptr_update to 0 after wptr programed */1224	WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 0);1225 1226	/* setup the wptr shadow polling */1227	wptr_gpu_addr = ring->wptr_gpu_addr;1228	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO,1229		    lower_32_bits(wptr_gpu_addr));1230	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI,1231		    upper_32_bits(wptr_gpu_addr));1232	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL);1233	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,1234				       SDMA0_PAGE_RB_WPTR_POLL_CNTL,1235				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);1236	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);1237 1238	/* enable DMA RB */1239	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL, RB_ENABLE, 1);1240	WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);1241 1242	ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL);1243	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_ENABLE, 1);1244#ifdef __BIG_ENDIAN1245	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);1246#endif1247	/* enable DMA IBs */1248	WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl);1249}1250 1251static void1252sdma_v4_1_update_power_gating(struct amdgpu_device *adev, bool enable)1253{1254	uint32_t def, data;1255 1256	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_SDMA)) {1257		/* enable idle interrupt */1258		def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));1259		data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;1260 1261		if (data != def)1262			WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);1263	} else {1264		/* disable idle interrupt */1265		def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));1266		data &= ~SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;1267		if (data != def)1268			WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);1269	}1270}1271 1272static void sdma_v4_1_init_power_gating(struct amdgpu_device *adev)1273{1274	uint32_t def, data;1275 1276	/* Enable HW based PG. */1277	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));1278	data |= SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK;1279	if (data != def)1280		WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data);1281 1282	/* enable interrupt */1283	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));1284	data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;1285	if (data != def)1286		WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);1287 1288	/* Configure hold time to filter in-valid power on/off request. Use default right now */1289	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));1290	data &= ~SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK;1291	data |= (mmSDMA0_POWER_CNTL_DEFAULT & SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK);1292	/* Configure switch time for hysteresis purpose. Use default right now */1293	data &= ~SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK;1294	data |= (mmSDMA0_POWER_CNTL_DEFAULT & SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK);1295	if(data != def)1296		WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data);1297}1298 1299static void sdma_v4_0_init_pg(struct amdgpu_device *adev)1300{1301	if (!(adev->pg_flags & AMD_PG_SUPPORT_SDMA))1302		return;1303 1304	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {1305	case IP_VERSION(4, 1, 0):1306        case IP_VERSION(4, 1, 1):1307	case IP_VERSION(4, 1, 2):1308		sdma_v4_1_init_power_gating(adev);1309		sdma_v4_1_update_power_gating(adev, true);1310		break;1311	default:1312		break;1313	}1314}1315 1316/**1317 * sdma_v4_0_rlc_resume - setup and start the async dma engines1318 *1319 * @adev: amdgpu_device pointer1320 *1321 * Set up the compute DMA queues and enable them (VEGA10).1322 * Returns 0 for success, error for failure.1323 */1324static int sdma_v4_0_rlc_resume(struct amdgpu_device *adev)1325{1326	sdma_v4_0_init_pg(adev);1327 1328	return 0;1329}1330 1331/**1332 * sdma_v4_0_load_microcode - load the sDMA ME ucode1333 *1334 * @adev: amdgpu_device pointer1335 *1336 * Loads the sDMA0/1 ucode.1337 * Returns 0 for success, -EINVAL if the ucode is not available.1338 */1339static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)1340{1341	const struct sdma_firmware_header_v1_0 *hdr;1342	const __le32 *fw_data;1343	u32 fw_size;1344	int i, j;1345 1346	/* halt the MEs */1347	sdma_v4_0_enable(adev, false);1348 1349	for (i = 0; i < adev->sdma.num_instances; i++) {1350		if (!adev->sdma.instance[i].fw)1351			return -EINVAL;1352 1353		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;1354		amdgpu_ucode_print_sdma_hdr(&hdr->header);1355		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;1356 1357		fw_data = (const __le32 *)1358			(adev->sdma.instance[i].fw->data +1359				le32_to_cpu(hdr->header.ucode_array_offset_bytes));1360 1361		WREG32_SDMA(i, mmSDMA0_UCODE_ADDR, 0);1362 1363		for (j = 0; j < fw_size; j++)1364			WREG32_SDMA(i, mmSDMA0_UCODE_DATA,1365				    le32_to_cpup(fw_data++));1366 1367		WREG32_SDMA(i, mmSDMA0_UCODE_ADDR,1368			    adev->sdma.instance[i].fw_version);1369	}1370 1371	return 0;1372}1373 1374/**1375 * sdma_v4_0_start - setup and start the async dma engines1376 *1377 * @adev: amdgpu_device pointer1378 *1379 * Set up the DMA engines and enable them (VEGA10).1380 * Returns 0 for success, error for failure.1381 */1382static int sdma_v4_0_start(struct amdgpu_device *adev)1383{1384	struct amdgpu_ring *ring;1385	int i, r = 0;1386 1387	if (amdgpu_sriov_vf(adev)) {1388		sdma_v4_0_ctx_switch_enable(adev, false);1389		sdma_v4_0_enable(adev, false);1390	} else {1391 1392		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {1393			r = sdma_v4_0_load_microcode(adev);1394			if (r)1395				return r;1396		}1397 1398		/* unhalt the MEs */1399		sdma_v4_0_enable(adev, true);1400		/* enable sdma ring preemption */1401		sdma_v4_0_ctx_switch_enable(adev, true);1402	}1403 1404	/* start the gfx rings and rlc compute queues */1405	for (i = 0; i < adev->sdma.num_instances; i++) {1406		uint32_t temp;1407 1408		WREG32_SDMA(i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL, 0);1409		sdma_v4_0_gfx_resume(adev, i);1410		if (adev->sdma.has_page_queue)1411			sdma_v4_0_page_resume(adev, i);1412 1413		/* set utc l1 enable flag always to 1 */1414		temp = RREG32_SDMA(i, mmSDMA0_CNTL);1415		temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);1416		WREG32_SDMA(i, mmSDMA0_CNTL, temp);1417 1418		if (!amdgpu_sriov_vf(adev)) {1419			/* unhalt engine */1420			temp = RREG32_SDMA(i, mmSDMA0_F32_CNTL);1421			temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);1422			WREG32_SDMA(i, mmSDMA0_F32_CNTL, temp);1423		}1424	}1425 1426	if (amdgpu_sriov_vf(adev)) {1427		sdma_v4_0_ctx_switch_enable(adev, true);1428		sdma_v4_0_enable(adev, true);1429	} else {1430		r = sdma_v4_0_rlc_resume(adev);1431		if (r)1432			return r;1433	}1434 1435	for (i = 0; i < adev->sdma.num_instances; i++) {1436		ring = &adev->sdma.instance[i].ring;1437 1438		r = amdgpu_ring_test_helper(ring);1439		if (r)1440			return r;1441 1442		if (adev->sdma.has_page_queue) {1443			struct amdgpu_ring *page = &adev->sdma.instance[i].page;1444 1445			r = amdgpu_ring_test_helper(page);1446			if (r)1447				return r;1448		}1449	}1450 1451	return r;1452}1453 1454/**1455 * sdma_v4_0_ring_test_ring - simple async dma engine test1456 *1457 * @ring: amdgpu_ring structure holding ring information1458 *1459 * Test the DMA engine by writing using it to write an1460 * value to memory. (VEGA10).1461 * Returns 0 for success, error for failure.1462 */1463static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)1464{1465	struct amdgpu_device *adev = ring->adev;1466	unsigned i;1467	unsigned index;1468	int r;1469	u32 tmp;1470	u64 gpu_addr;1471 1472	r = amdgpu_device_wb_get(adev, &index);1473	if (r)1474		return r;1475 1476	gpu_addr = adev->wb.gpu_addr + (index * 4);1477	tmp = 0xCAFEDEAD;1478	adev->wb.wb[index] = cpu_to_le32(tmp);1479 1480	r = amdgpu_ring_alloc(ring, 5);1481	if (r)1482		goto error_free_wb;1483 1484	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |1485			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));1486	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));1487	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));1488	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));1489	amdgpu_ring_write(ring, 0xDEADBEEF);1490	amdgpu_ring_commit(ring);1491 1492	for (i = 0; i < adev->usec_timeout; i++) {1493		tmp = le32_to_cpu(adev->wb.wb[index]);1494		if (tmp == 0xDEADBEEF)1495			break;1496		udelay(1);1497	}1498 1499	if (i >= adev->usec_timeout)1500		r = -ETIMEDOUT;1501 1502error_free_wb:1503	amdgpu_device_wb_free(adev, index);1504	return r;1505}1506 1507/**1508 * sdma_v4_0_ring_test_ib - test an IB on the DMA engine1509 *1510 * @ring: amdgpu_ring structure holding ring information1511 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT1512 *1513 * Test a simple IB in the DMA ring (VEGA10).1514 * Returns 0 on success, error on failure.1515 */1516static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)1517{1518	struct amdgpu_device *adev = ring->adev;1519	struct amdgpu_ib ib;1520	struct dma_fence *f = NULL;1521	unsigned index;1522	long r;1523	u32 tmp = 0;1524	u64 gpu_addr;1525 1526	r = amdgpu_device_wb_get(adev, &index);1527	if (r)1528		return r;1529 1530	gpu_addr = adev->wb.gpu_addr + (index * 4);1531	tmp = 0xCAFEDEAD;1532	adev->wb.wb[index] = cpu_to_le32(tmp);1533	memset(&ib, 0, sizeof(ib));1534	r = amdgpu_ib_get(adev, NULL, 256,1535					AMDGPU_IB_POOL_DIRECT, &ib);1536	if (r)1537		goto err0;1538 1539	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |1540		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);1541	ib.ptr[1] = lower_32_bits(gpu_addr);1542	ib.ptr[2] = upper_32_bits(gpu_addr);1543	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);1544	ib.ptr[4] = 0xDEADBEEF;1545	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);1546	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);1547	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);1548	ib.length_dw = 8;1549 1550	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);1551	if (r)1552		goto err1;1553 1554	r = dma_fence_wait_timeout(f, false, timeout);1555	if (r == 0) {1556		r = -ETIMEDOUT;1557		goto err1;1558	} else if (r < 0) {1559		goto err1;1560	}1561	tmp = le32_to_cpu(adev->wb.wb[index]);1562	if (tmp == 0xDEADBEEF)1563		r = 0;1564	else1565		r = -EINVAL;1566 1567err1:1568	amdgpu_ib_free(adev, &ib, NULL);1569	dma_fence_put(f);1570err0:1571	amdgpu_device_wb_free(adev, index);1572	return r;1573}1574 1575 1576/**1577 * sdma_v4_0_vm_copy_pte - update PTEs by copying them from the GART1578 *1579 * @ib: indirect buffer to fill with commands1580 * @pe: addr of the page entry1581 * @src: src addr to copy from1582 * @count: number of page entries to update1583 *1584 * Update PTEs by copying them from the GART using sDMA (VEGA10).1585 */1586static void sdma_v4_0_vm_copy_pte(struct amdgpu_ib *ib,1587				  uint64_t pe, uint64_t src,1588				  unsigned count)1589{1590	unsigned bytes = count * 8;1591 1592	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |1593		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);1594	ib->ptr[ib->length_dw++] = bytes - 1;1595	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */1596	ib->ptr[ib->length_dw++] = lower_32_bits(src);1597	ib->ptr[ib->length_dw++] = upper_32_bits(src);1598	ib->ptr[ib->length_dw++] = lower_32_bits(pe);1599	ib->ptr[ib->length_dw++] = upper_32_bits(pe);1600 1601}1602 1603/**1604 * sdma_v4_0_vm_write_pte - update PTEs by writing them manually1605 *1606 * @ib: indirect buffer to fill with commands1607 * @pe: addr of the page entry1608 * @value: dst addr to write into pe1609 * @count: number of page entries to update1610 * @incr: increase next addr by incr bytes1611 *1612 * Update PTEs by writing them manually using sDMA (VEGA10).1613 */1614static void sdma_v4_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,1615				   uint64_t value, unsigned count,1616				   uint32_t incr)1617{1618	unsigned ndw = count * 2;1619 1620	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |1621		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);1622	ib->ptr[ib->length_dw++] = lower_32_bits(pe);1623	ib->ptr[ib->length_dw++] = upper_32_bits(pe);1624	ib->ptr[ib->length_dw++] = ndw - 1;1625	for (; ndw > 0; ndw -= 2) {1626		ib->ptr[ib->length_dw++] = lower_32_bits(value);1627		ib->ptr[ib->length_dw++] = upper_32_bits(value);1628		value += incr;1629	}1630}1631 1632/**1633 * sdma_v4_0_vm_set_pte_pde - update the page tables using sDMA1634 *1635 * @ib: indirect buffer to fill with commands1636 * @pe: addr of the page entry1637 * @addr: dst addr to write into pe1638 * @count: number of page entries to update1639 * @incr: increase next addr by incr bytes1640 * @flags: access flags1641 *1642 * Update the page tables using sDMA (VEGA10).1643 */1644static void sdma_v4_0_vm_set_pte_pde(struct amdgpu_ib *ib,1645				     uint64_t pe,1646				     uint64_t addr, unsigned count,1647				     uint32_t incr, uint64_t flags)1648{1649	/* for physically contiguous pages (vram) */1650	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);1651	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */1652	ib->ptr[ib->length_dw++] = upper_32_bits(pe);1653	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */1654	ib->ptr[ib->length_dw++] = upper_32_bits(flags);1655	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */1656	ib->ptr[ib->length_dw++] = upper_32_bits(addr);1657	ib->ptr[ib->length_dw++] = incr; /* increment size */1658	ib->ptr[ib->length_dw++] = 0;1659	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */1660}1661 1662/**1663 * sdma_v4_0_ring_pad_ib - pad the IB to the required number of dw1664 *1665 * @ring: amdgpu_ring structure holding ring information1666 * @ib: indirect buffer to fill with padding1667 */1668static void sdma_v4_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)1669{1670	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);1671	u32 pad_count;1672	int i;1673 1674	pad_count = (-ib->length_dw) & 7;1675	for (i = 0; i < pad_count; i++)1676		if (sdma && sdma->burst_nop && (i == 0))1677			ib->ptr[ib->length_dw++] =1678				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |1679				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);1680		else1681			ib->ptr[ib->length_dw++] =1682				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);1683}1684 1685 1686/**1687 * sdma_v4_0_ring_emit_pipeline_sync - sync the pipeline1688 *1689 * @ring: amdgpu_ring pointer1690 *1691 * Make sure all previous operations are completed (CIK).1692 */1693static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)1694{1695	uint32_t seq = ring->fence_drv.sync_seq;1696	uint64_t addr = ring->fence_drv.gpu_addr;1697 1698	/* wait for idle */1699	sdma_v4_0_wait_reg_mem(ring, 1, 0,1700			       addr & 0xfffffffc,1701			       upper_32_bits(addr) & 0xffffffff,1702			       seq, 0xffffffff, 4);1703}1704 1705 1706/**1707 * sdma_v4_0_ring_emit_vm_flush - vm flush using sDMA1708 *1709 * @ring: amdgpu_ring pointer1710 * @vmid: vmid number to use1711 * @pd_addr: address1712 *1713 * Update the page table base and flush the VM TLB1714 * using sDMA (VEGA10).1715 */1716static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,1717					 unsigned vmid, uint64_t pd_addr)1718{1719	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);1720}1721 1722static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring,1723				     uint32_t reg, uint32_t val)1724{1725	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |1726			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));1727	amdgpu_ring_write(ring, reg);1728	amdgpu_ring_write(ring, val);1729}1730 1731static void sdma_v4_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,1732					 uint32_t val, uint32_t mask)1733{1734	sdma_v4_0_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);1735}1736 1737static bool sdma_v4_0_fw_support_paging_queue(struct amdgpu_device *adev)1738{1739	uint fw_version = adev->sdma.instance[0].fw_version;1740 1741	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {1742	case IP_VERSION(4, 0, 0):1743		return fw_version >= 430;1744	case IP_VERSION(4, 0, 1):1745		/*return fw_version >= 31;*/1746		return false;1747	case IP_VERSION(4, 2, 0):1748		return fw_version >= 123;1749	default:1750		return false;1751	}1752}1753 1754static int sdma_v4_0_early_init(void *handle)1755{1756	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1757	int r;1758 1759	r = sdma_v4_0_init_microcode(adev);1760	if (r)1761		return r;1762 1763	/* TODO: Page queue breaks driver reload under SRIOV */1764	if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 0, 0)) &&1765	    amdgpu_sriov_vf((adev)))1766		adev->sdma.has_page_queue = false;1767	else if (sdma_v4_0_fw_support_paging_queue(adev))1768		adev->sdma.has_page_queue = true;1769 1770	sdma_v4_0_set_ring_funcs(adev);1771	sdma_v4_0_set_buffer_funcs(adev);1772	sdma_v4_0_set_vm_pte_funcs(adev);1773	sdma_v4_0_set_irq_funcs(adev);1774	sdma_v4_0_set_ras_funcs(adev);1775 1776	return 0;1777}1778 1779static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,1780		void *err_data,1781		struct amdgpu_iv_entry *entry);1782 1783static int sdma_v4_0_late_init(void *handle)1784{1785	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1786 1787	sdma_v4_0_setup_ulv(adev);1788 1789	if (!amdgpu_persistent_edc_harvesting_supported(adev))1790		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__SDMA);1791 1792	return 0;1793}1794 1795static int sdma_v4_0_sw_init(void *handle)1796{1797	struct amdgpu_ring *ring;1798	int r, i;1799	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1800	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);1801	uint32_t *ptr;1802 1803	/* SDMA trap event */1804	for (i = 0; i < adev->sdma.num_instances; i++) {1805		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1806				      SDMA0_4_0__SRCID__SDMA_TRAP,1807				      &adev->sdma.trap_irq);1808		if (r)1809			return r;1810	}1811 1812	/* SDMA SRAM ECC event */1813	for (i = 0; i < adev->sdma.num_instances; i++) {1814		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1815				      SDMA0_4_0__SRCID__SDMA_SRAM_ECC,1816				      &adev->sdma.ecc_irq);1817		if (r)1818			return r;1819	}1820 1821	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/1822	for (i = 0; i < adev->sdma.num_instances; i++) {1823		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1824				      SDMA0_4_0__SRCID__SDMA_VM_HOLE,1825				      &adev->sdma.vm_hole_irq);1826		if (r)1827			return r;1828 1829		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1830				      SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,1831				      &adev->sdma.doorbell_invalid_irq);1832		if (r)1833			return r;1834 1835		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1836				      SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,1837				      &adev->sdma.pool_timeout_irq);1838		if (r)1839			return r;1840 1841		r = amdgpu_irq_add_id(adev, sdma_v4_0_seq_to_irq_id(i),1842				      SDMA0_4_0__SRCID__SDMA_SRBMWRITE,1843				      &adev->sdma.srbm_write_irq);1844		if (r)1845			return r;1846	}1847 1848	for (i = 0; i < adev->sdma.num_instances; i++) {1849		ring = &adev->sdma.instance[i].ring;1850		ring->ring_obj = NULL;1851		ring->use_doorbell = true;1852 1853		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,1854				ring->use_doorbell?"true":"false");1855 1856		/* doorbell size is 2 dwords, get DWORD offset */1857		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;1858 1859		/*1860		 * On Arcturus, SDMA instance 5~7 has a different vmhub1861		 * type(AMDGPU_MMHUB1).1862		 */1863		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==1864			    IP_VERSION(4, 2, 2) &&1865		    i >= 5)1866			ring->vm_hub = AMDGPU_MMHUB1(0);1867		else1868			ring->vm_hub = AMDGPU_MMHUB0(0);1869 1870		sprintf(ring->name, "sdma%d", i);1871		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,1872				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,1873				     AMDGPU_RING_PRIO_DEFAULT, NULL);1874		if (r)1875			return r;1876 1877		if (adev->sdma.has_page_queue) {1878			ring = &adev->sdma.instance[i].page;1879			ring->ring_obj = NULL;1880			ring->use_doorbell = true;1881 1882			/* paging queue use same doorbell index/routing as gfx queue1883			 * with 0x400 (4096 dwords) offset on second doorbell page1884			 */1885			if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=1886				    IP_VERSION(4, 0, 0) &&1887			    amdgpu_ip_version(adev, SDMA0_HWIP, 0) <1888				    IP_VERSION(4, 2, 0)) {1889				ring->doorbell_index =1890					adev->doorbell_index.sdma_engine[i] << 1;1891				ring->doorbell_index += 0x400;1892			} else {1893				/* From vega20, the sdma_doorbell_range in 1st1894				 * doorbell page is reserved for page queue.1895				 */1896				ring->doorbell_index =1897					(adev->doorbell_index.sdma_engine[i] + 1) << 1;1898			}1899 1900			if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==1901				    IP_VERSION(4, 2, 2) &&1902			    i >= 5)1903				ring->vm_hub = AMDGPU_MMHUB1(0);1904			else1905				ring->vm_hub = AMDGPU_MMHUB0(0);1906 1907			sprintf(ring->name, "page%d", i);1908			r = amdgpu_ring_init(adev, ring, 1024,1909					     &adev->sdma.trap_irq,1910					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,1911					     AMDGPU_RING_PRIO_DEFAULT, NULL);1912			if (r)1913				return r;1914		}1915	}1916 1917	if (amdgpu_sdma_ras_sw_init(adev)) {1918		dev_err(adev->dev, "Failed to initialize sdma ras block!\n");1919		return -EINVAL;1920	}1921 1922	/* Allocate memory for SDMA IP Dump buffer */1923	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);1924	if (ptr)1925		adev->sdma.ip_dump = ptr;1926	else1927		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");1928 1929	return r;1930}1931 1932static int sdma_v4_0_sw_fini(void *handle)1933{1934	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1935	int i;1936 1937	for (i = 0; i < adev->sdma.num_instances; i++) {1938		amdgpu_ring_fini(&adev->sdma.instance[i].ring);1939		if (adev->sdma.has_page_queue)1940			amdgpu_ring_fini(&adev->sdma.instance[i].page);1941	}1942 1943	if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 2, 2) ||1944	    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 0))1945		amdgpu_sdma_destroy_inst_ctx(adev, true);1946	else1947		amdgpu_sdma_destroy_inst_ctx(adev, false);1948 1949	kfree(adev->sdma.ip_dump);1950 1951	return 0;1952}1953 1954static int sdma_v4_0_hw_init(void *handle)1955{1956	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1957 1958	if (adev->flags & AMD_IS_APU)1959		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false);1960 1961	if (!amdgpu_sriov_vf(adev))1962		sdma_v4_0_init_golden_registers(adev);1963 1964	return sdma_v4_0_start(adev);1965}1966 1967static int sdma_v4_0_hw_fini(void *handle)1968{1969	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1970	int i;1971 1972	if (amdgpu_sriov_vf(adev))1973		return 0;1974 1975	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {1976		for (i = 0; i < adev->sdma.num_instances; i++) {1977			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,1978				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);1979		}1980	}1981 1982	sdma_v4_0_ctx_switch_enable(adev, false);1983	sdma_v4_0_enable(adev, false);1984 1985	if (adev->flags & AMD_IS_APU)1986		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, true);1987 1988	return 0;1989}1990 1991static int sdma_v4_0_suspend(void *handle)1992{1993	struct amdgpu_device *adev = (struct amdgpu_device *)handle;1994 1995	/* SMU saves SDMA state for us */1996	if (adev->in_s0ix) {1997		sdma_v4_0_gfx_enable(adev, false);1998		return 0;1999	}2000 2001	return sdma_v4_0_hw_fini(adev);2002}2003 2004static int sdma_v4_0_resume(void *handle)2005{2006	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2007 2008	/* SMU restores SDMA state for us */2009	if (adev->in_s0ix) {2010		sdma_v4_0_enable(adev, true);2011		sdma_v4_0_gfx_enable(adev, true);2012		return 0;2013	}2014 2015	return sdma_v4_0_hw_init(adev);2016}2017 2018static bool sdma_v4_0_is_idle(void *handle)2019{2020	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2021	u32 i;2022 2023	for (i = 0; i < adev->sdma.num_instances; i++) {2024		u32 tmp = RREG32_SDMA(i, mmSDMA0_STATUS_REG);2025 2026		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))2027			return false;2028	}2029 2030	return true;2031}2032 2033static int sdma_v4_0_wait_for_idle(void *handle)2034{2035	unsigned i, j;2036	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];2037	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2038 2039	for (i = 0; i < adev->usec_timeout; i++) {2040		for (j = 0; j < adev->sdma.num_instances; j++) {2041			sdma[j] = RREG32_SDMA(j, mmSDMA0_STATUS_REG);2042			if (!(sdma[j] & SDMA0_STATUS_REG__IDLE_MASK))2043				break;2044		}2045		if (j == adev->sdma.num_instances)2046			return 0;2047		udelay(1);2048	}2049	return -ETIMEDOUT;2050}2051 2052static int sdma_v4_0_soft_reset(void *handle)2053{2054	/* todo */2055 2056	return 0;2057}2058 2059static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,2060					struct amdgpu_irq_src *source,2061					unsigned type,2062					enum amdgpu_interrupt_state state)2063{2064	u32 sdma_cntl;2065 2066	sdma_cntl = RREG32_SDMA(type, mmSDMA0_CNTL);2067	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,2068		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);2069	WREG32_SDMA(type, mmSDMA0_CNTL, sdma_cntl);2070 2071	return 0;2072}2073 2074static int sdma_v4_0_process_trap_irq(struct amdgpu_device *adev,2075				      struct amdgpu_irq_src *source,2076				      struct amdgpu_iv_entry *entry)2077{2078	int instance;2079 2080	DRM_DEBUG("IH: SDMA trap\n");2081	instance = sdma_v4_0_irq_id_to_seq(entry->client_id);2082	if (instance < 0)2083		return instance;2084 2085	switch (entry->ring_id) {2086	case 0:2087		amdgpu_fence_process(&adev->sdma.instance[instance].ring);2088		break;2089	case 1:2090		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==2091		    IP_VERSION(4, 2, 0))2092			amdgpu_fence_process(&adev->sdma.instance[instance].page);2093		break;2094	case 2:2095		/* XXX compute */2096		break;2097	case 3:2098		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) !=2099		    IP_VERSION(4, 2, 0))2100			amdgpu_fence_process(&adev->sdma.instance[instance].page);2101		break;2102	}2103	return 0;2104}2105 2106static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,2107		void *err_data,2108		struct amdgpu_iv_entry *entry)2109{2110	int instance;2111 2112	/* When “Full RAS” is enabled, the per-IP interrupt sources should2113	 * be disabled and the driver should only look for the aggregated2114	 * interrupt via sync flood2115	 */2116	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))2117		goto out;2118 2119	instance = sdma_v4_0_irq_id_to_seq(entry->client_id);2120	if (instance < 0)2121		goto out;2122 2123	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);2124 2125out:2126	return AMDGPU_RAS_SUCCESS;2127}2128 2129static int sdma_v4_0_process_illegal_inst_irq(struct amdgpu_device *adev,2130					      struct amdgpu_irq_src *source,2131					      struct amdgpu_iv_entry *entry)2132{2133	int instance;2134 2135	DRM_ERROR("Illegal instruction in SDMA command stream\n");2136 2137	instance = sdma_v4_0_irq_id_to_seq(entry->client_id);2138	if (instance < 0)2139		return 0;2140 2141	switch (entry->ring_id) {2142	case 0:2143		drm_sched_fault(&adev->sdma.instance[instance].ring.sched);2144		break;2145	}2146	return 0;2147}2148 2149static int sdma_v4_0_set_ecc_irq_state(struct amdgpu_device *adev,2150					struct amdgpu_irq_src *source,2151					unsigned type,2152					enum amdgpu_interrupt_state state)2153{2154	u32 sdma_edc_config;2155 2156	sdma_edc_config = RREG32_SDMA(type, mmSDMA0_EDC_CONFIG);2157	sdma_edc_config = REG_SET_FIELD(sdma_edc_config, SDMA0_EDC_CONFIG, ECC_INT_ENABLE,2158		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);2159	WREG32_SDMA(type, mmSDMA0_EDC_CONFIG, sdma_edc_config);2160 2161	return 0;2162}2163 2164static int sdma_v4_0_print_iv_entry(struct amdgpu_device *adev,2165					      struct amdgpu_iv_entry *entry)2166{2167	int instance;2168	struct amdgpu_task_info *task_info;2169	u64 addr;2170 2171	instance = sdma_v4_0_irq_id_to_seq(entry->client_id);2172	if (instance < 0 || instance >= adev->sdma.num_instances) {2173		dev_err(adev->dev, "sdma instance invalid %d\n", instance);2174		return -EINVAL;2175	}2176 2177	addr = (u64)entry->src_data[0] << 12;2178	addr |= ((u64)entry->src_data[1] & 0xf) << 44;2179 2180	dev_dbg_ratelimited(adev->dev,2181			   "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n",2182			   instance, addr, entry->src_id, entry->ring_id, entry->vmid,2183			   entry->pasid);2184 2185	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);2186	if (task_info) {2187		dev_dbg_ratelimited(adev->dev,2188				    " for process %s pid %d thread %s pid %d\n",2189				    task_info->process_name, task_info->tgid,2190				    task_info->task_name, task_info->pid);2191		amdgpu_vm_put_task_info(task_info);2192	}2193 2194	return 0;2195}2196 2197static int sdma_v4_0_process_vm_hole_irq(struct amdgpu_device *adev,2198					      struct amdgpu_irq_src *source,2199					      struct amdgpu_iv_entry *entry)2200{2201	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");2202	sdma_v4_0_print_iv_entry(adev, entry);2203	return 0;2204}2205 2206static int sdma_v4_0_process_doorbell_invalid_irq(struct amdgpu_device *adev,2207					      struct amdgpu_irq_src *source,2208					      struct amdgpu_iv_entry *entry)2209{2210	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");2211	sdma_v4_0_print_iv_entry(adev, entry);2212	return 0;2213}2214 2215static int sdma_v4_0_process_pool_timeout_irq(struct amdgpu_device *adev,2216					      struct amdgpu_irq_src *source,2217					      struct amdgpu_iv_entry *entry)2218{2219	dev_dbg_ratelimited(adev->dev,2220		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");2221	sdma_v4_0_print_iv_entry(adev, entry);2222	return 0;2223}2224 2225static int sdma_v4_0_process_srbm_write_irq(struct amdgpu_device *adev,2226					      struct amdgpu_irq_src *source,2227					      struct amdgpu_iv_entry *entry)2228{2229	dev_dbg_ratelimited(adev->dev,2230		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");2231	sdma_v4_0_print_iv_entry(adev, entry);2232	return 0;2233}2234 2235static void sdma_v4_0_update_medium_grain_clock_gating(2236		struct amdgpu_device *adev,2237		bool enable)2238{2239	uint32_t data, def;2240	int i;2241 2242	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {2243		for (i = 0; i < adev->sdma.num_instances; i++) {2244			def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL);2245			data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |2246				  SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |2247				  SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |2248				  SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |2249				  SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |2250				  SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |2251				  SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |2252				  SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);2253			if (def != data)2254				WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data);2255		}2256	} else {2257		for (i = 0; i < adev->sdma.num_instances; i++) {2258			def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL);2259			data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |2260				 SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |2261				 SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |2262				 SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |2263				 SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |2264				 SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |2265				 SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |2266				 SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);2267			if (def != data)2268				WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data);2269		}2270	}2271}2272 2273 2274static void sdma_v4_0_update_medium_grain_light_sleep(2275		struct amdgpu_device *adev,2276		bool enable)2277{2278	uint32_t data, def;2279	int i;2280 2281	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {2282		for (i = 0; i < adev->sdma.num_instances; i++) {2283			/* 1-not override: enable sdma mem light sleep */2284			def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL);2285			data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;2286			if (def != data)2287				WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data);2288		}2289	} else {2290		for (i = 0; i < adev->sdma.num_instances; i++) {2291		/* 0-override:disable sdma mem light sleep */2292			def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL);2293			data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;2294			if (def != data)2295				WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data);2296		}2297	}2298}2299 2300static int sdma_v4_0_set_clockgating_state(void *handle,2301					  enum amd_clockgating_state state)2302{2303	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2304 2305	if (amdgpu_sriov_vf(adev))2306		return 0;2307 2308	sdma_v4_0_update_medium_grain_clock_gating(adev,2309			state == AMD_CG_STATE_GATE);2310	sdma_v4_0_update_medium_grain_light_sleep(adev,2311			state == AMD_CG_STATE_GATE);2312	return 0;2313}2314 2315static int sdma_v4_0_set_powergating_state(void *handle,2316					  enum amd_powergating_state state)2317{2318	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2319 2320	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {2321	case IP_VERSION(4, 1, 0):2322	case IP_VERSION(4, 1, 1):2323	case IP_VERSION(4, 1, 2):2324		sdma_v4_1_update_power_gating(adev,2325				state == AMD_PG_STATE_GATE);2326		break;2327	default:2328		break;2329	}2330 2331	return 0;2332}2333 2334static void sdma_v4_0_get_clockgating_state(void *handle, u64 *flags)2335{2336	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2337	int data;2338 2339	if (amdgpu_sriov_vf(adev))2340		*flags = 0;2341 2342	/* AMD_CG_SUPPORT_SDMA_MGCG */2343	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL));2344	if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK))2345		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;2346 2347	/* AMD_CG_SUPPORT_SDMA_LS */2348	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));2349	if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)2350		*flags |= AMD_CG_SUPPORT_SDMA_LS;2351}2352 2353static void sdma_v4_0_print_ip_state(void *handle, struct drm_printer *p)2354{2355	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2356	int i, j;2357	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);2358	uint32_t instance_offset;2359 2360	if (!adev->sdma.ip_dump)2361		return;2362 2363	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);2364	for (i = 0; i < adev->sdma.num_instances; i++) {2365		instance_offset = i * reg_count;2366		drm_printf(p, "\nInstance:%d\n", i);2367 2368		for (j = 0; j < reg_count; j++)2369			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_4_0[j].reg_name,2370				   adev->sdma.ip_dump[instance_offset + j]);2371	}2372}2373 2374static void sdma_v4_0_dump_ip_state(void *handle)2375{2376	struct amdgpu_device *adev = (struct amdgpu_device *)handle;2377	int i, j;2378	uint32_t instance_offset;2379	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);2380 2381	if (!adev->sdma.ip_dump)2382		return;2383 2384	amdgpu_gfx_off_ctrl(adev, false);2385	for (i = 0; i < adev->sdma.num_instances; i++) {2386		instance_offset = i * reg_count;2387		for (j = 0; j < reg_count; j++)2388			adev->sdma.ip_dump[instance_offset + j] =2389				RREG32(sdma_v4_0_get_reg_offset(adev, i,2390				       sdma_reg_list_4_0[j].reg_offset));2391	}2392	amdgpu_gfx_off_ctrl(adev, true);2393}2394 2395const struct amd_ip_funcs sdma_v4_0_ip_funcs = {2396	.name = "sdma_v4_0",2397	.early_init = sdma_v4_0_early_init,2398	.late_init = sdma_v4_0_late_init,2399	.sw_init = sdma_v4_0_sw_init,2400	.sw_fini = sdma_v4_0_sw_fini,2401	.hw_init = sdma_v4_0_hw_init,2402	.hw_fini = sdma_v4_0_hw_fini,2403	.suspend = sdma_v4_0_suspend,2404	.resume = sdma_v4_0_resume,2405	.is_idle = sdma_v4_0_is_idle,2406	.wait_for_idle = sdma_v4_0_wait_for_idle,2407	.soft_reset = sdma_v4_0_soft_reset,2408	.set_clockgating_state = sdma_v4_0_set_clockgating_state,2409	.set_powergating_state = sdma_v4_0_set_powergating_state,2410	.get_clockgating_state = sdma_v4_0_get_clockgating_state,2411	.dump_ip_state = sdma_v4_0_dump_ip_state,2412	.print_ip_state = sdma_v4_0_print_ip_state,2413};2414 2415static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {2416	.type = AMDGPU_RING_TYPE_SDMA,2417	.align_mask = 0xff,2418	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),2419	.support_64bit_ptrs = true,2420	.secure_submission_supported = true,2421	.get_rptr = sdma_v4_0_ring_get_rptr,2422	.get_wptr = sdma_v4_0_ring_get_wptr,2423	.set_wptr = sdma_v4_0_ring_set_wptr,2424	.emit_frame_size =2425		6 + /* sdma_v4_0_ring_emit_hdp_flush */2426		3 + /* hdp invalidate */2427		6 + /* sdma_v4_0_ring_emit_pipeline_sync */2428		/* sdma_v4_0_ring_emit_vm_flush */2429		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +2430		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +2431		10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */2432	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */2433	.emit_ib = sdma_v4_0_ring_emit_ib,2434	.emit_fence = sdma_v4_0_ring_emit_fence,2435	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,2436	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,2437	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,2438	.test_ring = sdma_v4_0_ring_test_ring,2439	.test_ib = sdma_v4_0_ring_test_ib,2440	.insert_nop = sdma_v4_0_ring_insert_nop,2441	.pad_ib = sdma_v4_0_ring_pad_ib,2442	.emit_wreg = sdma_v4_0_ring_emit_wreg,2443	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,2444	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,2445};2446 2447static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs = {2448	.type = AMDGPU_RING_TYPE_SDMA,2449	.align_mask = 0xff,2450	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),2451	.support_64bit_ptrs = true,2452	.secure_submission_supported = true,2453	.get_rptr = sdma_v4_0_ring_get_rptr,2454	.get_wptr = sdma_v4_0_page_ring_get_wptr,2455	.set_wptr = sdma_v4_0_page_ring_set_wptr,2456	.emit_frame_size =2457		6 + /* sdma_v4_0_ring_emit_hdp_flush */2458		3 + /* hdp invalidate */2459		6 + /* sdma_v4_0_ring_emit_pipeline_sync */2460		/* sdma_v4_0_ring_emit_vm_flush */2461		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +2462		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +2463		10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */2464	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */2465	.emit_ib = sdma_v4_0_ring_emit_ib,2466	.emit_fence = sdma_v4_0_ring_emit_fence,2467	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,2468	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,2469	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,2470	.test_ring = sdma_v4_0_ring_test_ring,2471	.test_ib = sdma_v4_0_ring_test_ib,2472	.insert_nop = sdma_v4_0_ring_insert_nop,2473	.pad_ib = sdma_v4_0_ring_pad_ib,2474	.emit_wreg = sdma_v4_0_ring_emit_wreg,2475	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,2476	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,2477};2478 2479static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev)2480{2481	int i;2482 2483	for (i = 0; i < adev->sdma.num_instances; i++) {2484		adev->sdma.instance[i].ring.funcs = &sdma_v4_0_ring_funcs;2485		adev->sdma.instance[i].ring.me = i;2486		if (adev->sdma.has_page_queue) {2487			adev->sdma.instance[i].page.funcs =2488					&sdma_v4_0_page_ring_funcs;2489			adev->sdma.instance[i].page.me = i;2490		}2491	}2492}2493 2494static const struct amdgpu_irq_src_funcs sdma_v4_0_trap_irq_funcs = {2495	.set = sdma_v4_0_set_trap_irq_state,2496	.process = sdma_v4_0_process_trap_irq,2497};2498 2499static const struct amdgpu_irq_src_funcs sdma_v4_0_illegal_inst_irq_funcs = {2500	.process = sdma_v4_0_process_illegal_inst_irq,2501};2502 2503static const struct amdgpu_irq_src_funcs sdma_v4_0_ecc_irq_funcs = {2504	.set = sdma_v4_0_set_ecc_irq_state,2505	.process = amdgpu_sdma_process_ecc_irq,2506};2507 2508static const struct amdgpu_irq_src_funcs sdma_v4_0_vm_hole_irq_funcs = {2509	.process = sdma_v4_0_process_vm_hole_irq,2510};2511 2512static const struct amdgpu_irq_src_funcs sdma_v4_0_doorbell_invalid_irq_funcs = {2513	.process = sdma_v4_0_process_doorbell_invalid_irq,2514};2515 2516static const struct amdgpu_irq_src_funcs sdma_v4_0_pool_timeout_irq_funcs = {2517	.process = sdma_v4_0_process_pool_timeout_irq,2518};2519 2520static const struct amdgpu_irq_src_funcs sdma_v4_0_srbm_write_irq_funcs = {2521	.process = sdma_v4_0_process_srbm_write_irq,2522};2523 2524static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)2525{2526	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;2527	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;2528	/*For Arcturus and Aldebaran, add another 4 irq handler*/2529	switch (adev->sdma.num_instances) {2530	case 5:2531	case 8:2532		adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;2533		adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;2534		adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;2535		adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;2536		break;2537	default:2538		break;2539	}2540	adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs;2541	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_0_illegal_inst_irq_funcs;2542	adev->sdma.ecc_irq.funcs = &sdma_v4_0_ecc_irq_funcs;2543	adev->sdma.vm_hole_irq.funcs = &sdma_v4_0_vm_hole_irq_funcs;2544	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_0_doorbell_invalid_irq_funcs;2545	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_0_pool_timeout_irq_funcs;2546	adev->sdma.srbm_write_irq.funcs = &sdma_v4_0_srbm_write_irq_funcs;2547}2548 2549/**2550 * sdma_v4_0_emit_copy_buffer - copy buffer using the sDMA engine2551 *2552 * @ib: indirect buffer to copy to2553 * @src_offset: src GPU address2554 * @dst_offset: dst GPU address2555 * @byte_count: number of bytes to xfer2556 * @copy_flags: copy flags for the buffers2557 *2558 * Copy GPU buffers using the DMA engine (VEGA10/12).2559 * Used by the amdgpu ttm implementation to move pages if2560 * registered as the asic copy callback.2561 */2562static void sdma_v4_0_emit_copy_buffer(struct amdgpu_ib *ib,2563				       uint64_t src_offset,2564				       uint64_t dst_offset,2565				       uint32_t byte_count,2566				       uint32_t copy_flags)2567{2568	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |2569		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |2570		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);2571	ib->ptr[ib->length_dw++] = byte_count - 1;2572	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */2573	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);2574	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);2575	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);2576	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);2577}2578 2579/**2580 * sdma_v4_0_emit_fill_buffer - fill buffer using the sDMA engine2581 *2582 * @ib: indirect buffer to copy to2583 * @src_data: value to write to buffer2584 * @dst_offset: dst GPU address2585 * @byte_count: number of bytes to xfer2586 *2587 * Fill GPU buffers using the DMA engine (VEGA10/12).2588 */2589static void sdma_v4_0_emit_fill_buffer(struct amdgpu_ib *ib,2590				       uint32_t src_data,2591				       uint64_t dst_offset,2592				       uint32_t byte_count)2593{2594	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);2595	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);2596	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);2597	ib->ptr[ib->length_dw++] = src_data;2598	ib->ptr[ib->length_dw++] = byte_count - 1;2599}2600 2601static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = {2602	.copy_max_bytes = 0x400000,2603	.copy_num_dw = 7,2604	.emit_copy_buffer = sdma_v4_0_emit_copy_buffer,2605 2606	.fill_max_bytes = 0x400000,2607	.fill_num_dw = 5,2608	.emit_fill_buffer = sdma_v4_0_emit_fill_buffer,2609};2610 2611static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)2612{2613	adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;2614	if (adev->sdma.has_page_queue)2615		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;2616	else2617		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;2618}2619 2620static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {2621	.copy_pte_num_dw = 7,2622	.copy_pte = sdma_v4_0_vm_copy_pte,2623 2624	.write_pte = sdma_v4_0_vm_write_pte,2625	.set_pte_pde = sdma_v4_0_vm_set_pte_pde,2626};2627 2628static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev)2629{2630	struct drm_gpu_scheduler *sched;2631	unsigned i;2632 2633	adev->vm_manager.vm_pte_funcs = &sdma_v4_0_vm_pte_funcs;2634	for (i = 0; i < adev->sdma.num_instances; i++) {2635		if (adev->sdma.has_page_queue)2636			sched = &adev->sdma.instance[i].page.sched;2637		else2638			sched = &adev->sdma.instance[i].ring.sched;2639		adev->vm_manager.vm_pte_scheds[i] = sched;2640	}2641	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;2642}2643 2644static void sdma_v4_0_get_ras_error_count(uint32_t value,2645					uint32_t instance,2646					uint32_t *sec_count)2647{2648	uint32_t i;2649	uint32_t sec_cnt;2650 2651	/* double bits error (multiple bits) error detection is not supported */2652	for (i = 0; i < ARRAY_SIZE(sdma_v4_0_ras_fields); i++) {2653		/* the SDMA_EDC_COUNTER register in each sdma instance2654		 * shares the same sed shift_mask2655		 * */2656		sec_cnt = (value &2657			sdma_v4_0_ras_fields[i].sec_count_mask) >>2658			sdma_v4_0_ras_fields[i].sec_count_shift;2659		if (sec_cnt) {2660			DRM_INFO("Detected %s in SDMA%d, SED %d\n",2661				sdma_v4_0_ras_fields[i].name,2662				instance, sec_cnt);2663			*sec_count += sec_cnt;2664		}2665	}2666}2667 2668static int sdma_v4_0_query_ras_error_count_by_instance(struct amdgpu_device *adev,2669			uint32_t instance, void *ras_error_status)2670{2671	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;2672	uint32_t sec_count = 0;2673	uint32_t reg_value = 0;2674 2675	reg_value = RREG32_SDMA(instance, mmSDMA0_EDC_COUNTER);2676	/* double bit error is not supported */2677	if (reg_value)2678		sdma_v4_0_get_ras_error_count(reg_value,2679				instance, &sec_count);2680	/* err_data->ce_count should be initialized to 02681	 * before calling into this function */2682	err_data->ce_count += sec_count;2683	/* double bit error is not supported2684	 * set ue count to 0 */2685	err_data->ue_count = 0;2686 2687	return 0;2688};2689 2690static void sdma_v4_0_query_ras_error_count(struct amdgpu_device *adev,  void *ras_error_status)2691{2692	int i = 0;2693 2694	for (i = 0; i < adev->sdma.num_instances; i++) {2695		if (sdma_v4_0_query_ras_error_count_by_instance(adev, i, ras_error_status)) {2696			dev_err(adev->dev, "Query ras error count failed in SDMA%d\n", i);2697			return;2698		}2699	}2700}2701 2702static void sdma_v4_0_reset_ras_error_count(struct amdgpu_device *adev)2703{2704	int i;2705 2706	/* read back edc counter registers to clear the counters */2707	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {2708		for (i = 0; i < adev->sdma.num_instances; i++)2709			RREG32_SDMA(i, mmSDMA0_EDC_COUNTER);2710	}2711}2712 2713const struct amdgpu_ras_block_hw_ops sdma_v4_0_ras_hw_ops = {2714	.query_ras_error_count = sdma_v4_0_query_ras_error_count,2715	.reset_ras_error_count = sdma_v4_0_reset_ras_error_count,2716};2717 2718static struct amdgpu_sdma_ras sdma_v4_0_ras = {2719	.ras_block = {2720		.hw_ops = &sdma_v4_0_ras_hw_ops,2721		.ras_cb = sdma_v4_0_process_ras_data_cb,2722	},2723};2724 2725static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev)2726{2727	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {2728	case IP_VERSION(4, 2, 0):2729	case IP_VERSION(4, 2, 2):2730		adev->sdma.ras = &sdma_v4_0_ras;2731		break;2732	case IP_VERSION(4, 4, 0):2733		adev->sdma.ras = &sdma_v4_4_ras;2734		break;2735	default:2736		break;2737	}2738}2739 2740const struct amdgpu_ip_block_version sdma_v4_0_ip_block = {2741	.type = AMD_IP_BLOCK_TYPE_SDMA,2742	.major = 4,2743	.minor = 0,2744	.rev = 0,2745	.funcs = &sdma_v4_0_ip_funcs,2746};2747