1100 lines · plain
1/*2 * Copyright 2016 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22 23/* To compile this assembly code:24 *25 * gfx9:26 * cpp -DASIC_FAMILY=CHIP_VEGAM cwsr_trap_handler_gfx9.asm -P -o gfx9.sp327 * sp3 gfx9.sp3 -hex gfx9.hex28 *29 * arcturus:30 * cpp -DASIC_FAMILY=CHIP_ARCTURUS cwsr_trap_handler_gfx9.asm -P -o arcturus.sp331 * sp3 arcturus.sp3 -hex arcturus.hex32 *33 * aldebaran:34 * cpp -DASIC_FAMILY=CHIP_ALDEBARAN cwsr_trap_handler_gfx9.asm -P -o aldebaran.sp335 * sp3 aldebaran.sp3 -hex aldebaran.hex36 *37 * gc_9_4_3:38 * cpp -DASIC_FAMILY=GC_9_4_3 cwsr_trap_handler_gfx9.asm -P -o gc_9_4_3.sp339 * sp3 gc_9_4_3.sp3 -hex gc_9_4_3.hex40 */41 42#define CHIP_VEGAM 1843#define CHIP_ARCTURUS 2344#define CHIP_ALDEBARAN 2545#define CHIP_GC_9_4_3 2646 47var ACK_SQC_STORE = 1 //workaround for suspected SQC store bug causing incorrect stores under concurrency48var SAVE_AFTER_XNACK_ERROR = 1 //workaround for TCP store failure after XNACK error when ALLOW_REPLAY=0, for debugger49var SINGLE_STEP_MISSED_WORKAROUND = (ASIC_FAMILY <= CHIP_ALDEBARAN) //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised50 51/**************************************************************************/52/* variables */53/**************************************************************************/54var SQ_WAVE_STATUS_SPI_PRIO_SHIFT = 155var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x0000000656var SQ_WAVE_STATUS_HALT_MASK = 0x200057var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT = 058var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE = 159var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT = 360var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE = 2961var SQ_WAVE_STATUS_ALLOW_REPLAY_MASK = 0x40000062var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x2000063 64var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 1265var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 966var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 667var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE = 3 //FIXME sq.blk still has 4 bits at this time while SQ programming guide has 3 bits68var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT = 2469 70#if ASIC_FAMILY >= CHIP_ALDEBARAN71var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 672var SQ_WAVE_GPR_ALLOC_ACCV_OFFSET_SHIFT = 1273var SQ_WAVE_GPR_ALLOC_ACCV_OFFSET_SIZE = 674#else75var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 876#endif77 78var SQ_WAVE_TRAPSTS_SAVECTX_MASK = 0x40079var SQ_WAVE_TRAPSTS_EXCP_MASK = 0x1FF80var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT = 1081var SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK = 0x8082var SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT = 783var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK = 0x10084var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT = 885var SQ_WAVE_TRAPSTS_HOST_TRAP_MASK = 0x40000086var SQ_WAVE_TRAPSTS_WAVE_BEGIN_MASK = 0x80000087var SQ_WAVE_TRAPSTS_WAVE_END_MASK = 0x100000088var SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK = 0x200000089var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK = 0x3FF90var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT = 0x091var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE = 1092var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK = 0xFFFFF80093var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT = 1194var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE = 2195var SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK = 0x80096var SQ_WAVE_TRAPSTS_EXCP_HI_MASK = 0x700097var SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK = 0x1000000098 99var SQ_WAVE_MODE_EXCP_EN_SHIFT = 12100var SQ_WAVE_MODE_EXCP_EN_ADDR_WATCH_SHIFT = 19101 102var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT = 15 //FIXME103var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK = 0x1F8000104 105var SQ_WAVE_MODE_DEBUG_EN_MASK = 0x800106 107var TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT = 26 // bits [31:26] unused by SPI debug data108var TTMP_SAVE_RCNT_FIRST_REPLAY_MASK = 0xFC000000109var TTMP_DEBUG_TRAP_ENABLED_SHIFT = 23110var TTMP_DEBUG_TRAP_ENABLED_MASK = 0x800000111 112/* Save */113var S_SAVE_BUF_RSRC_WORD1_STRIDE = 0x00040000 //stride is 4 bytes114var S_SAVE_BUF_RSRC_WORD3_MISC = 0x00807FAC //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE115var S_SAVE_PC_HI_TRAP_ID_MASK = 0x00FF0000116var S_SAVE_PC_HI_HT_MASK = 0x01000000117var S_SAVE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG118var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26119 120var s_save_spi_init_lo = exec_lo121var s_save_spi_init_hi = exec_hi122 123var s_save_pc_lo = ttmp0 //{TTMP1, TTMP0} = {3'h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}124var s_save_pc_hi = ttmp1125var s_save_exec_lo = ttmp2126var s_save_exec_hi = ttmp3127var s_save_tmp = ttmp14128var s_save_trapsts = ttmp15 //not really used until the end of the SAVE routine129var s_save_xnack_mask_lo = ttmp6130var s_save_xnack_mask_hi = ttmp7131var s_save_buf_rsrc0 = ttmp8132var s_save_buf_rsrc1 = ttmp9133var s_save_buf_rsrc2 = ttmp10134var s_save_buf_rsrc3 = ttmp11135var s_save_status = ttmp12136var s_save_mem_offset = ttmp4137var s_save_alloc_size = s_save_trapsts //conflict138var s_save_m0 = ttmp5139var s_save_ttmps_lo = s_save_tmp //no conflict140var s_save_ttmps_hi = s_save_trapsts //no conflict141#if ASIC_FAMILY >= CHIP_GC_9_4_3142var s_save_ib_sts = ttmp13143#else144var s_save_ib_sts = ttmp11145#endif146 147/* Restore */148var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE149var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC150 151var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 //bit[26]: FirstWaveInTG152var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26153 154var s_restore_spi_init_lo = exec_lo155var s_restore_spi_init_hi = exec_hi156 157var s_restore_mem_offset = ttmp12158var s_restore_tmp2 = ttmp13159var s_restore_alloc_size = ttmp3160var s_restore_tmp = ttmp2161var s_restore_mem_offset_save = s_restore_tmp //no conflict162var s_restore_accvgpr_offset_save = ttmp7163 164var s_restore_m0 = s_restore_alloc_size //no conflict165 166var s_restore_mode = s_restore_accvgpr_offset_save167 168var s_restore_pc_lo = ttmp0169var s_restore_pc_hi = ttmp1170var s_restore_exec_lo = ttmp4171var s_restore_exec_hi = ttmp5172var s_restore_status = ttmp14173var s_restore_trapsts = ttmp15174var s_restore_xnack_mask_lo = xnack_mask_lo175var s_restore_xnack_mask_hi = xnack_mask_hi176var s_restore_buf_rsrc0 = ttmp8177var s_restore_buf_rsrc1 = ttmp9178var s_restore_buf_rsrc2 = ttmp10179var s_restore_buf_rsrc3 = ttmp11180var s_restore_ttmps_lo = s_restore_tmp //no conflict181var s_restore_ttmps_hi = s_restore_alloc_size //no conflict182 183/**************************************************************************/184/* trap handler entry points */185/**************************************************************************/186/* Shader Main*/187 188shader main189 asic(DEFAULT)190 type(CS)191 192 193 s_branch L_SKIP_RESTORE //NOT restore. might be a regular trap or save194 195L_JUMP_TO_RESTORE:196 s_branch L_RESTORE //restore197 198L_SKIP_RESTORE:199 200 s_getreg_b32 s_save_status, hwreg(HW_REG_STATUS) //save STATUS since we will change SCC201 202 // Clear SPI_PRIO: do not save with elevated priority.203 // Clear ECC_ERR: prevents SQC store and triggers FATAL_HALT if setreg'd.204 s_andn2_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK|SQ_WAVE_STATUS_ECC_ERR_MASK205 206 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)207 208 s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK209 s_cbranch_scc0 L_NOT_HALTED210 211L_HALTED:212 // Host trap may occur while wave is halted.213 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK214 s_cbranch_scc1 L_FETCH_2ND_TRAP215 216L_CHECK_SAVE:217 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK //check whether this is for save218 s_cbranch_scc1 L_SAVE //this is the operation for save219 220 // Wave is halted but neither host trap nor SAVECTX is raised.221 // Caused by instruction fetch memory violation.222 // Spin wait until context saved to prevent interrupt storm.223 s_sleep 0x10224 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)225 s_branch L_CHECK_SAVE226 227L_NOT_HALTED:228 // Let second-level handle non-SAVECTX exception or trap.229 // Any concurrent SAVECTX will be handled upon re-entry once halted.230 231 // Check non-maskable exceptions. memory_violation, illegal_instruction232 // and debugger (host trap, wave start/end, trap after instruction)233 // exceptions always cause the wave to enter the trap handler.234 s_and_b32 ttmp2, s_save_trapsts, \235 SQ_WAVE_TRAPSTS_MEM_VIOL_MASK | \236 SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK | \237 SQ_WAVE_TRAPSTS_HOST_TRAP_MASK | \238 SQ_WAVE_TRAPSTS_WAVE_BEGIN_MASK | \239 SQ_WAVE_TRAPSTS_WAVE_END_MASK | \240 SQ_WAVE_TRAPSTS_TRAP_AFTER_INST_MASK241 s_cbranch_scc1 L_FETCH_2ND_TRAP242 243 // Check for maskable exceptions in trapsts.excp and trapsts.excp_hi.244 // Maskable exceptions only cause the wave to enter the trap handler if245 // their respective bit in mode.excp_en is set.246 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCP_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK247 s_cbranch_scc0 L_CHECK_TRAP_ID248 249 s_and_b32 ttmp3, s_save_trapsts, SQ_WAVE_TRAPSTS_ADDR_WATCH_MASK|SQ_WAVE_TRAPSTS_EXCP_HI_MASK250 s_cbranch_scc0 L_NOT_ADDR_WATCH251 s_bitset1_b32 ttmp2, SQ_WAVE_TRAPSTS_ADDR_WATCH_SHIFT // Check all addr_watch[123] exceptions against excp_en.addr_watch252 253L_NOT_ADDR_WATCH:254 s_getreg_b32 ttmp3, hwreg(HW_REG_MODE)255 s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT256 s_and_b32 ttmp2, ttmp2, ttmp3257 s_cbranch_scc1 L_FETCH_2ND_TRAP258 259L_CHECK_TRAP_ID:260 // Check trap_id != 0261 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK262 s_cbranch_scc1 L_FETCH_2ND_TRAP263 264if SINGLE_STEP_MISSED_WORKAROUND265 // Prioritize single step exception over context save.266 // Second-level trap will halt wave and RFE, re-entering for SAVECTX.267 s_getreg_b32 ttmp2, hwreg(HW_REG_MODE)268 s_and_b32 ttmp2, ttmp2, SQ_WAVE_MODE_DEBUG_EN_MASK269 s_cbranch_scc1 L_FETCH_2ND_TRAP270end271 272 s_and_b32 ttmp2, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK273 s_cbranch_scc1 L_SAVE274 275L_FETCH_2ND_TRAP:276 // Preserve and clear scalar XNACK state before issuing scalar reads.277 save_and_clear_ib_sts(ttmp14)278 279 // Read second-level TBA/TMA from first-level TMA and jump if available.280 // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)281 // ttmp12 holds SQ_WAVE_STATUS282 s_getreg_b32 ttmp14, hwreg(HW_REG_SQ_SHADER_TMA_LO)283 s_getreg_b32 ttmp15, hwreg(HW_REG_SQ_SHADER_TMA_HI)284 s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8285 286 s_bitcmp1_b32 ttmp15, 0xF287 s_cbranch_scc0 L_NO_SIGN_EXTEND_TMA288 s_or_b32 ttmp15, ttmp15, 0xFFFF0000289L_NO_SIGN_EXTEND_TMA:290 291 s_load_dword ttmp2, [ttmp14, ttmp15], 0x10 glc:1 // debug trap enabled flag292 s_waitcnt lgkmcnt(0)293 s_lshl_b32 ttmp2, ttmp2, TTMP_DEBUG_TRAP_ENABLED_SHIFT294 s_andn2_b32 s_save_ib_sts, s_save_ib_sts, TTMP_DEBUG_TRAP_ENABLED_MASK295 s_or_b32 s_save_ib_sts, s_save_ib_sts, ttmp2296 297 s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 glc:1 // second-level TBA298 s_waitcnt lgkmcnt(0)299 s_load_dwordx2 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 glc:1 // second-level TMA300 s_waitcnt lgkmcnt(0)301 302 s_and_b64 [ttmp2, ttmp3], [ttmp2, ttmp3], [ttmp2, ttmp3]303 s_cbranch_scc0 L_NO_NEXT_TRAP // second-level trap handler not been set304 s_setpc_b64 [ttmp2, ttmp3] // jump to second-level trap handler305 306L_NO_NEXT_TRAP:307 // If not caused by trap then halt wave to prevent re-entry.308 s_and_b32 ttmp2, s_save_pc_hi, (S_SAVE_PC_HI_TRAP_ID_MASK|S_SAVE_PC_HI_HT_MASK)309 s_cbranch_scc1 L_TRAP_CASE310 s_or_b32 s_save_status, s_save_status, SQ_WAVE_STATUS_HALT_MASK311 312 // If the PC points to S_ENDPGM then context save will fail if STATUS.HALT is set.313 // Rewind the PC to prevent this from occurring.314 s_sub_u32 ttmp0, ttmp0, 0x8315 s_subb_u32 ttmp1, ttmp1, 0x0316 317 s_branch L_EXIT_TRAP318 319L_TRAP_CASE:320 // Host trap will not cause trap re-entry.321 s_and_b32 ttmp2, s_save_pc_hi, S_SAVE_PC_HI_HT_MASK322 s_cbranch_scc1 L_EXIT_TRAP323 324 // Advance past trap instruction to prevent re-entry.325 s_add_u32 ttmp0, ttmp0, 0x4326 s_addc_u32 ttmp1, ttmp1, 0x0327 328L_EXIT_TRAP:329 s_and_b32 ttmp1, ttmp1, 0xFFFF330 331 restore_ib_sts(ttmp14)332 333 // Restore SQ_WAVE_STATUS.334 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32335 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32336 set_status_without_spi_prio(s_save_status, ttmp2)337 338 s_rfe_b64 [ttmp0, ttmp1]339 340 // ********* End handling of non-CWSR traps *******************341 342/**************************************************************************/343/* save routine */344/**************************************************************************/345 346L_SAVE:347 s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32]348 349 s_mov_b32 s_save_tmp, 0 //clear saveCtx bit350 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp //clear saveCtx bit351 352 save_and_clear_ib_sts(s_save_tmp)353 354 /* inform SPI the readiness and wait for SPI's go signal */355 s_mov_b32 s_save_exec_lo, exec_lo //save EXEC and use EXEC for the go signal from SPI356 s_mov_b32 s_save_exec_hi, exec_hi357 s_mov_b64 exec, 0x0 //clear EXEC to get ready to receive358 359 s_sendmsg sendmsg(MSG_SAVEWAVE) //send SPI a message and wait for SPI's write to EXEC360 361 // Set SPI_PRIO=2 to avoid starving instruction fetch in the waves we're waiting for.362 s_or_b32 s_save_tmp, s_save_status, (2 << SQ_WAVE_STATUS_SPI_PRIO_SHIFT)363 s_setreg_b32 hwreg(HW_REG_STATUS), s_save_tmp364 365 L_SLEEP:366 s_sleep 0x2 // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0367 368 s_cbranch_execz L_SLEEP369 370 // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic371 // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40372 get_vgpr_size_bytes(s_save_ttmps_lo)373 get_sgpr_size_bytes(s_save_ttmps_hi)374 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi375 s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo376 s_addc_u32 s_save_ttmps_hi, s_save_spi_init_hi, 0x0377 s_and_b32 s_save_ttmps_hi, s_save_ttmps_hi, 0xFFFF378 s_store_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_save_ttmps_lo, s_save_ttmps_hi], 0x50 glc:1379 ack_sqc_store_workaround()380 s_store_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_save_ttmps_lo, s_save_ttmps_hi], 0x60 glc:1381 ack_sqc_store_workaround()382 s_store_dword ttmp13, [s_save_ttmps_lo, s_save_ttmps_hi], 0x74 glc:1383 ack_sqc_store_workaround()384 385 /* setup Resource Contants */386 s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo387 s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi388 s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE389 s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited390 s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC391 392 //FIXME right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi (might need to save them before using them?)393 s_mov_b32 s_save_m0, m0 //save M0394 395 /* global mem offset */396 s_mov_b32 s_save_mem_offset, 0x0 //mem offset initial value = 0397 398 399 400 401 /* save HW registers */402 //////////////////////////////403 404 L_SAVE_HWREG:405 // HWREG SR memory offset : size(VGPR)+size(SGPR)406 get_vgpr_size_bytes(s_save_mem_offset)407 get_sgpr_size_bytes(s_save_tmp)408 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp409 410 411 s_mov_b32 s_save_buf_rsrc2, 0x4 //NUM_RECORDS in bytes412 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes413 414 415 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset) //M0416 write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset) //PC417 write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)418 write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset) //EXEC419 write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)420 write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset) //STATUS421 422 //s_save_trapsts conflicts with s_save_alloc_size423 s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS)424 write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset) //TRAPSTS425 426 write_hwreg_to_mem(xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_LO427 write_hwreg_to_mem(xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset) //XNACK_MASK_HI428 429 //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2430 s_getreg_b32 s_save_m0, hwreg(HW_REG_MODE) //MODE431 write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)432 433 434 435 /* the first wave in the threadgroup */436 s_and_b32 s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK // extract fisrt wave bit437 s_mov_b32 s_save_exec_hi, 0x0438 s_or_b32 s_save_exec_hi, s_save_tmp, s_save_exec_hi // save first wave bit to s_save_exec_hi.bits[26]439 440 441 /* save SGPRs */442 // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...443 //////////////////////////////444 445 // SGPR SR memory offset : size(VGPR)446 get_vgpr_size_bytes(s_save_mem_offset)447 // TODO, change RSRC word to rearrange memory layout for SGPRS448 449 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size450 s_add_u32 s_save_alloc_size, s_save_alloc_size, 1451 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)452 453 s_lshl_b32 s_save_buf_rsrc2, s_save_alloc_size, 2 //NUM_RECORDS in bytes454 455 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes456 457 458 // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0459 //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0460 s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0461 s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset462 s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0463 464 s_mov_b32 m0, 0x0 //SGPR initial index value =0465 s_nop 0x0 //Manually inserted wait states466 L_SAVE_SGPR_LOOP:467 // SGPR is allocated in 16 SGPR granularity468 s_movrels_b64 s0, s0 //s0 = s[0+m0], s1 = s[1+m0]469 s_movrels_b64 s2, s2 //s2 = s[2+m0], s3 = s[3+m0]470 s_movrels_b64 s4, s4 //s4 = s[4+m0], s5 = s[5+m0]471 s_movrels_b64 s6, s6 //s6 = s[6+m0], s7 = s[7+m0]472 s_movrels_b64 s8, s8 //s8 = s[8+m0], s9 = s[9+m0]473 s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0]474 s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0]475 s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0]476 477 write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4478 s_add_u32 m0, m0, 16 //next sgpr index479 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0480 s_cbranch_scc1 L_SAVE_SGPR_LOOP //SGPR save is complete?481 // restore s_save_buf_rsrc0,1482 //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo483 s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo484 485 486 487 488 /* save first 4 VGPR, then LDS save could use */489 // each wave will alloc 4 vgprs at least...490 /////////////////////////////////////////////////////////////////////////////////////491 492 s_mov_b32 s_save_mem_offset, 0493 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on494 s_mov_b32 exec_hi, 0xFFFFFFFF495 s_mov_b32 xnack_mask_lo, 0x0496 s_mov_b32 xnack_mask_hi, 0x0497 498 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes499 500 501 // VGPR Allocated in 4-GPR granularity502 503if SAVE_AFTER_XNACK_ERROR504 check_if_tcp_store_ok()505 s_cbranch_scc1 L_SAVE_FIRST_VGPRS_WITH_TCP506 507 write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)508 s_branch L_SAVE_LDS509 510L_SAVE_FIRST_VGPRS_WITH_TCP:511end512 513 write_4vgprs_to_mem(s_save_buf_rsrc0, s_save_mem_offset)514 515 /* save LDS */516 //////////////////////////////517 518 L_SAVE_LDS:519 520 // Change EXEC to all threads...521 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on522 s_mov_b32 exec_hi, 0xFFFFFFFF523 524 s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size525 s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //lds_size is zero?526 s_cbranch_scc0 L_SAVE_LDS_DONE //no lds used? jump to L_SAVE_DONE527 528 s_barrier //LDS is used? wait for other waves in the same TG529 s_and_b32 s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK //exec is still used here530 s_cbranch_scc0 L_SAVE_LDS_DONE531 532 // first wave do LDS save;533 534 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 6 //LDS size in dwords = lds_size * 64dw535 s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 2 //LDS size in bytes536 s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes537 538 // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)539 //540 get_vgpr_size_bytes(s_save_mem_offset)541 get_sgpr_size_bytes(s_save_tmp)542 s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp543 s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()544 545 546 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes547 548 s_mov_b32 m0, 0x0 //lds_offset initial value = 0549 550 551 v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0552 v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2 // tid553 554if SAVE_AFTER_XNACK_ERROR555 check_if_tcp_store_ok()556 s_cbranch_scc1 L_SAVE_LDS_WITH_TCP557 558 v_lshlrev_b32 v2, 2, v3559L_SAVE_LDS_LOOP_SQC:560 ds_read2_b32 v[0:1], v2 offset0:0 offset1:0x40561 s_waitcnt lgkmcnt(0)562 563 write_vgprs_to_mem_with_sqc(v0, 2, s_save_buf_rsrc0, s_save_mem_offset)564 565 v_add_u32 v2, 0x200, v2566 v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size567 s_cbranch_vccnz L_SAVE_LDS_LOOP_SQC568 569 s_branch L_SAVE_LDS_DONE570 571L_SAVE_LDS_WITH_TCP:572end573 574 v_mul_i32_i24 v2, v3, 8 // tid*8575 v_mov_b32 v3, 256*2576 s_mov_b32 m0, 0x10000577 s_mov_b32 s0, s_save_buf_rsrc3578 s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF // disable add_tid579 s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000 //DFMT580 581L_SAVE_LDS_LOOP_VECTOR:582 ds_read_b64 v[0:1], v2 //x =LDS[a], byte address583 s_waitcnt lgkmcnt(0)584 buffer_store_dwordx2 v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1 glc:1 slc:1585// s_waitcnt vmcnt(0)586// v_add_u32 v2, vcc[0:1], v2, v3587 v_add_u32 v2, v2, v3588 v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size589 s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR590 591 // restore rsrc3592 s_mov_b32 s_save_buf_rsrc3, s0593 594L_SAVE_LDS_DONE:595 596 597 /* save VGPRs - set the Rest VGPRs */598 //////////////////////////////////////////////////////////////////////////////////////599 L_SAVE_VGPR:600 // VGPR SR memory offset: 0601 // TODO rearrange the RSRC words to use swizzle for VGPR save...602 603 s_mov_b32 s_save_mem_offset, (0+256*4) // for the rest VGPRs604 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on605 s_mov_b32 exec_hi, 0xFFFFFFFF606 607 get_num_arch_vgprs(s_save_alloc_size)608 s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes609 610 611 // VGPR store using dw burst612 s_mov_b32 m0, 0x4 //VGPR initial index value =0613 s_cmp_lt_u32 m0, s_save_alloc_size614 s_cbranch_scc0 L_SAVE_VGPR_END615 616 617 s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1618 s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later619 620if SAVE_AFTER_XNACK_ERROR621 check_if_tcp_store_ok()622 s_cbranch_scc1 L_SAVE_VGPR_LOOP623 624L_SAVE_VGPR_LOOP_SQC:625 write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)626 627 s_add_u32 m0, m0, 4628 s_cmp_lt_u32 m0, s_save_alloc_size629 s_cbranch_scc1 L_SAVE_VGPR_LOOP_SQC630 631 s_set_gpr_idx_off632 s_branch L_SAVE_VGPR_END633end634 635 L_SAVE_VGPR_LOOP:636 v_mov_b32 v0, v0 //v0 = v[0+m0]637 v_mov_b32 v1, v1 //v0 = v[0+m0]638 v_mov_b32 v2, v2 //v0 = v[0+m0]639 v_mov_b32 v3, v3 //v0 = v[0+m0]640 641 write_4vgprs_to_mem(s_save_buf_rsrc0, s_save_mem_offset)642 643 s_add_u32 m0, m0, 4 //next vgpr index644 s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes645 s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0646 s_cbranch_scc1 L_SAVE_VGPR_LOOP //VGPR save is complete?647 s_set_gpr_idx_off648 649L_SAVE_VGPR_END:650 651#if ASIC_FAMILY >= CHIP_ARCTURUS652 // Save ACC VGPRs653 654#if ASIC_FAMILY >= CHIP_ALDEBARAN655 // ACC VGPR count may differ from ARCH VGPR count.656 get_num_acc_vgprs(s_save_alloc_size, s_save_tmp)657 s_and_b32 s_save_alloc_size, s_save_alloc_size, s_save_alloc_size658 s_cbranch_scc0 L_SAVE_ACCVGPR_END659 s_add_u32 s_save_alloc_size, s_save_alloc_size, 0x1000 //add 0x1000 since we compare m0 against it later660#endif661 662 s_mov_b32 m0, 0x0 //VGPR initial index value =0663 s_set_gpr_idx_on m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1664 665if SAVE_AFTER_XNACK_ERROR666 check_if_tcp_store_ok()667 s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP668 669L_SAVE_ACCVGPR_LOOP_SQC:670 for var vgpr = 0; vgpr < 4; ++ vgpr671 v_accvgpr_read v[vgpr], acc[vgpr] // v[N] = acc[N+m0]672 end673 674 write_vgprs_to_mem_with_sqc(v0, 4, s_save_buf_rsrc0, s_save_mem_offset)675 676 s_add_u32 m0, m0, 4677 s_cmp_lt_u32 m0, s_save_alloc_size678 s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP_SQC679 680 s_set_gpr_idx_off681 s_branch L_SAVE_ACCVGPR_END682end683 684L_SAVE_ACCVGPR_LOOP:685 for var vgpr = 0; vgpr < 4; ++ vgpr686 v_accvgpr_read v[vgpr], acc[vgpr] // v[N] = acc[N+m0]687 end688 689 write_4vgprs_to_mem(s_save_buf_rsrc0, s_save_mem_offset)690 691 s_add_u32 m0, m0, 4692 s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4693 s_cmp_lt_u32 m0, s_save_alloc_size694 s_cbranch_scc1 L_SAVE_ACCVGPR_LOOP695 s_set_gpr_idx_off696 697L_SAVE_ACCVGPR_END:698#endif699 700 s_branch L_END_PGM701 702 703 704/**************************************************************************/705/* restore routine */706/**************************************************************************/707 708L_RESTORE:709 /* Setup Resource Contants */710 s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo711 s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi712 s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE713 s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes)714 s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC715 716 /* global mem offset */717// s_mov_b32 s_restore_mem_offset, 0x0 //mem offset initial value = 0718 719 /* the first wave in the threadgroup */720 s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK721 s_cbranch_scc0 L_RESTORE_VGPR722 723 /* restore LDS */724 //////////////////////////////725 L_RESTORE_LDS:726 727 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead728 s_mov_b32 exec_hi, 0xFFFFFFFF729 730 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) //lds_size731 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero?732 s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR733 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 6 //LDS size in dwords = lds_size * 64dw734 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 2 //LDS size in bytes735 s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes736 737 // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)738 //739 get_vgpr_size_bytes(s_restore_mem_offset)740 get_sgpr_size_bytes(s_restore_tmp)741 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp742 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes() //FIXME, Check if offset overflow???743 744 745 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes746 s_mov_b32 m0, 0x0 //lds_offset initial value = 0747 748 L_RESTORE_LDS_LOOP:749 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 // first 64DW750 buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256 // second 64DW751 s_add_u32 m0, m0, 256*2 // 128 DW752 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*2 //mem offset increased by 128DW753 s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0754 s_cbranch_scc1 L_RESTORE_LDS_LOOP //LDS restore is complete?755 756 757 /* restore VGPRs */758 //////////////////////////////759 L_RESTORE_VGPR:760 s_mov_b32 exec_lo, 0xFFFFFFFF //need every thread from now on //be consistent with SAVE although can be moved ahead761 s_mov_b32 exec_hi, 0xFFFFFFFF762 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes763 764 // Save ARCH VGPRs 4-N, then all ACC VGPRs, then ARCH VGPRs 0-3.765 get_num_arch_vgprs(s_restore_alloc_size)766 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 0x8000 //add 0x8000 since we compare m0 against it later767 768 // ARCH VGPRs at offset: 0769 s_mov_b32 s_restore_mem_offset, 0x0770 s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last771 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4772 s_mov_b32 m0, 4 //VGPR initial index value = 1773 s_set_gpr_idx_on m0, 0x8 //M0[7:0] = M0[7:0] and M0[15:12] = 0x8774 775 L_RESTORE_VGPR_LOOP:776 read_4vgprs_from_mem(s_restore_buf_rsrc0, s_restore_mem_offset)777 v_mov_b32 v0, v0 //v[0+m0] = v0778 v_mov_b32 v1, v1779 v_mov_b32 v2, v2780 v_mov_b32 v3, v3781 s_add_u32 m0, m0, 4 //next vgpr index782 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes783 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0784 s_cbranch_scc1 L_RESTORE_VGPR_LOOP //VGPR restore (except v0) is complete?785 786#if ASIC_FAMILY >= CHIP_ALDEBARAN787 // ACC VGPR count may differ from ARCH VGPR count.788 get_num_acc_vgprs(s_restore_alloc_size, s_restore_tmp2)789 s_and_b32 s_restore_alloc_size, s_restore_alloc_size, s_restore_alloc_size790 s_cbranch_scc0 L_RESTORE_ACCVGPR_END791 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 0x8000 //add 0x8000 since we compare m0 against it later792#endif793 794#if ASIC_FAMILY >= CHIP_ARCTURUS795 // ACC VGPRs at offset: size(ARCH VGPRs)796 s_mov_b32 m0, 0797 s_set_gpr_idx_on m0, 0x8 //M0[7:0] = M0[7:0] and M0[15:12] = 0x8798 799 L_RESTORE_ACCVGPR_LOOP:800 read_4vgprs_from_mem(s_restore_buf_rsrc0, s_restore_mem_offset)801 802 for var vgpr = 0; vgpr < 4; ++ vgpr803 v_accvgpr_write acc[vgpr], v[vgpr]804 end805 806 s_add_u32 m0, m0, 4 //next vgpr index807 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 //every buffer_load_dword does 256 bytes808 s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0809 s_cbranch_scc1 L_RESTORE_ACCVGPR_LOOP //VGPR restore (except v0) is complete?810 L_RESTORE_ACCVGPR_END:811#endif812 813 s_set_gpr_idx_off814 815 // Restore VGPRs 0-3 last, no longer needed.816 read_4vgprs_from_mem(s_restore_buf_rsrc0, s_restore_mem_offset_save)817 818 /* restore SGPRs */819 //////////////////////////////820 821 // SGPR SR memory offset : size(VGPR)822 get_vgpr_size_bytes(s_restore_mem_offset)823 get_sgpr_size_bytes(s_restore_tmp)824 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp825 s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4 // restore SGPR from S[n] to S[0], by 16 sgprs group826 // TODO, change RSRC word to rearrange memory layout for SGPRS827 828 s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size829 s_add_u32 s_restore_alloc_size, s_restore_alloc_size, 1830 s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 4 //Number of SGPRs = (sgpr_size + 1) * 16 (non-zero value)831 832 s_lshl_b32 s_restore_buf_rsrc2, s_restore_alloc_size, 2 //NUM_RECORDS in bytes833 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes834 835 s_mov_b32 m0, s_restore_alloc_size836 837 L_RESTORE_SGPR_LOOP:838 read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) //PV: further performance improvement can be made839 s_waitcnt lgkmcnt(0) //ensure data ready840 841 s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0]842 s_nop 0 // hazard SALU M0=> S_MOVREL843 844 s_movreld_b64 s0, s0 //s[0+m0] = s0845 s_movreld_b64 s2, s2846 s_movreld_b64 s4, s4847 s_movreld_b64 s6, s6848 s_movreld_b64 s8, s8849 s_movreld_b64 s10, s10850 s_movreld_b64 s12, s12851 s_movreld_b64 s14, s14852 853 s_cmp_eq_u32 m0, 0 //scc = (m0 < s_restore_alloc_size) ? 1 : 0854 s_cbranch_scc0 L_RESTORE_SGPR_LOOP //SGPR restore (except s0) is complete?855 856 /* restore HW registers */857 //////////////////////////////858 L_RESTORE_HWREG:859 860 861 // HWREG SR memory offset : size(VGPR)+size(SGPR)862 get_vgpr_size_bytes(s_restore_mem_offset)863 get_sgpr_size_bytes(s_restore_tmp)864 s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp865 866 867 s_mov_b32 s_restore_buf_rsrc2, 0x4 //NUM_RECORDS in bytes868 s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes869 870 read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset) //M0871 read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //PC872 read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)873 read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //EXEC874 read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)875 read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset) //STATUS876 read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset) //TRAPSTS877 read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_LO878 read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset) //XNACK_MASK_HI879 read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset) //MODE880 881 s_waitcnt lgkmcnt(0) //from now on, it is safe to restore STATUS and IB_STS882 883 s_mov_b32 m0, s_restore_m0884 s_mov_b32 exec_lo, s_restore_exec_lo885 s_mov_b32 exec_hi, s_restore_exec_hi886 887 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts888 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0889 s_and_b32 s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts890 s_lshr_b32 s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT891 s_setreg_b32 hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0892 //s_setreg_b32 hwreg(HW_REG_TRAPSTS), s_restore_trapsts //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore893 s_setreg_b32 hwreg(HW_REG_MODE), s_restore_mode894 895 // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic896 // ttmp SR memory offset : size(VGPR)+size(SGPR)+0x40897 get_vgpr_size_bytes(s_restore_ttmps_lo)898 get_sgpr_size_bytes(s_restore_ttmps_hi)899 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi900 s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0901 s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0902 s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF903 s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 glc:1904 s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 glc:1905 s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 glc:1906 s_waitcnt lgkmcnt(0)907 908 restore_ib_sts(s_restore_tmp)909 910 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS911 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32912 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32913 set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu914 915 s_barrier //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time916 917 s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution918 919 920/**************************************************************************/921/* the END */922/**************************************************************************/923L_END_PGM:924 s_endpgm_saved925 926end927 928 929/**************************************************************************/930/* the helper functions */931/**************************************************************************/932 933//Only for save hwreg to mem934function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)935 s_mov_b32 exec_lo, m0 //assuming exec_lo is not needed anymore from this point on936 s_mov_b32 m0, s_mem_offset937 s_buffer_store_dword s, s_rsrc, m0 glc:1938 ack_sqc_store_workaround()939 s_add_u32 s_mem_offset, s_mem_offset, 4940 s_mov_b32 m0, exec_lo941end942 943 944// HWREG are saved before SGPRs, so all HWREG could be use.945function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)946 947 s_buffer_store_dwordx4 s[0], s_rsrc, 0 glc:1948 ack_sqc_store_workaround()949 s_buffer_store_dwordx4 s[4], s_rsrc, 16 glc:1950 ack_sqc_store_workaround()951 s_buffer_store_dwordx4 s[8], s_rsrc, 32 glc:1952 ack_sqc_store_workaround()953 s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1954 ack_sqc_store_workaround()955 s_add_u32 s_rsrc[0], s_rsrc[0], 4*16956 s_addc_u32 s_rsrc[1], s_rsrc[1], 0x0 // +scc957end958 959 960function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)961 s_buffer_load_dword s, s_rsrc, s_mem_offset glc:1962 s_add_u32 s_mem_offset, s_mem_offset, 4963end964 965function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)966 s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset glc:1967 s_sub_u32 s_mem_offset, s_mem_offset, 4*16968end969 970function check_if_tcp_store_ok971 // If STATUS.ALLOW_REPLAY=0 and TRAPSTS.XNACK_ERROR=1 then TCP stores will fail.972 s_and_b32 s_save_tmp, s_save_status, SQ_WAVE_STATUS_ALLOW_REPLAY_MASK973 s_cbranch_scc1 L_TCP_STORE_CHECK_DONE974 975 s_getreg_b32 s_save_tmp, hwreg(HW_REG_TRAPSTS)976 s_andn2_b32 s_save_tmp, SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK, s_save_tmp977 978L_TCP_STORE_CHECK_DONE:979end980 981function write_4vgprs_to_mem(s_rsrc, s_mem_offset)982 buffer_store_dword v0, v0, s_rsrc, s_mem_offset slc:1 glc:1983 buffer_store_dword v1, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256984 buffer_store_dword v2, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256*2985 buffer_store_dword v3, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256*3986end987 988function read_4vgprs_from_mem(s_rsrc, s_mem_offset)989 buffer_load_dword v0, v0, s_rsrc, s_mem_offset slc:1 glc:1990 buffer_load_dword v1, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256991 buffer_load_dword v2, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256*2992 buffer_load_dword v3, v0, s_rsrc, s_mem_offset slc:1 glc:1 offset:256*3993 s_waitcnt vmcnt(0)994end995 996function write_vgpr_to_mem_with_sqc(v, s_rsrc, s_mem_offset)997 s_mov_b32 s4, 0998 999L_WRITE_VGPR_LANE_LOOP:1000 for var lane = 0; lane < 4; ++ lane1001 v_readlane_b32 s[lane], v, s41002 s_add_u32 s4, s4, 11003 end1004 1005 s_buffer_store_dwordx4 s[0:3], s_rsrc, s_mem_offset glc:11006 ack_sqc_store_workaround()1007 1008 s_add_u32 s_mem_offset, s_mem_offset, 0x101009 s_cmp_eq_u32 s4, 0x401010 s_cbranch_scc0 L_WRITE_VGPR_LANE_LOOP1011end1012 1013function write_vgprs_to_mem_with_sqc(v, n_vgprs, s_rsrc, s_mem_offset)1014 for var vgpr = 0; vgpr < n_vgprs; ++ vgpr1015 write_vgpr_to_mem_with_sqc(v[vgpr], s_rsrc, s_mem_offset)1016 end1017end1018 1019function get_lds_size_bytes(s_lds_size_byte)1020 // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW1021 s_getreg_b32 s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE) // lds_size1022 s_lshl_b32 s_lds_size_byte, s_lds_size_byte, 8 //LDS size in dwords = lds_size * 64 *4Bytes // granularity 64DW1023end1024 1025function get_vgpr_size_bytes(s_vgpr_size_byte)1026 s_getreg_b32 s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE) //vpgr_size1027 s_add_u32 s_vgpr_size_byte, s_vgpr_size_byte, 11028 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4 (non-zero value) //FIXME for GFX, zero is possible1029 1030#if ASIC_FAMILY >= CHIP_ARCTURUS1031 s_lshl_b32 s_vgpr_size_byte, s_vgpr_size_byte, 1 // Double size for ACC VGPRs1032#endif1033end1034 1035function get_sgpr_size_bytes(s_sgpr_size_byte)1036 s_getreg_b32 s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE) //spgr_size1037 s_add_u32 s_sgpr_size_byte, s_sgpr_size_byte, 11038 s_lshl_b32 s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4 (non-zero value)1039end1040 1041function get_hwreg_size_bytes1042 return 128 //HWREG size 128 bytes1043end1044 1045function get_num_arch_vgprs(s_num_arch_vgprs)1046#if ASIC_FAMILY >= CHIP_ALDEBARAN1047 // VGPR count includes ACC VGPRs, use ACC VGPR offset for ARCH VGPR count.1048 s_getreg_b32 s_num_arch_vgprs, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_ACCV_OFFSET_SHIFT,SQ_WAVE_GPR_ALLOC_ACCV_OFFSET_SIZE)1049#else1050 s_getreg_b32 s_num_arch_vgprs, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)1051#endif1052 1053 // Number of VGPRs = (vgpr_size + 1) * 41054 s_add_u32 s_num_arch_vgprs, s_num_arch_vgprs, 11055 s_lshl_b32 s_num_arch_vgprs, s_num_arch_vgprs, 21056end1057 1058#if ASIC_FAMILY >= CHIP_ALDEBARAN1059function get_num_acc_vgprs(s_num_acc_vgprs, s_tmp)1060 // VGPR count = (GPR_ALLOC.VGPR_SIZE + 1) * 81061 s_getreg_b32 s_num_acc_vgprs, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)1062 s_add_u32 s_num_acc_vgprs, s_num_acc_vgprs, 11063 s_lshl_b32 s_num_acc_vgprs, s_num_acc_vgprs, 31064 1065 // ACC VGPR count = VGPR count - ARCH VGPR count.1066 get_num_arch_vgprs(s_tmp)1067 s_sub_u32 s_num_acc_vgprs, s_num_acc_vgprs, s_tmp1068end1069#endif1070 1071function ack_sqc_store_workaround1072 if ACK_SQC_STORE1073 s_waitcnt lgkmcnt(0)1074 end1075end1076 1077function set_status_without_spi_prio(status, tmp)1078 // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.1079 s_lshr_b32 tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT1080 s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp1081 s_nop 0x2 // avoid S_SETREG => S_SETREG hazard1082 s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status1083end1084 1085function save_and_clear_ib_sts(tmp)1086 // Save IB_STS.FIRST_REPLAY[15] and IB_STS.RCNT[20:16] into unused space s_save_ib_sts[31:26].1087 s_getreg_b32 tmp, hwreg(HW_REG_IB_STS)1088 s_and_b32 tmp, tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK1089 s_lshl_b32 tmp, tmp, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)1090 s_andn2_b32 s_save_ib_sts, s_save_ib_sts, TTMP_SAVE_RCNT_FIRST_REPLAY_MASK1091 s_or_b32 s_save_ib_sts, s_save_ib_sts, tmp1092 s_setreg_imm32_b32 hwreg(HW_REG_IB_STS), 0x01093end1094 1095function restore_ib_sts(tmp)1096 s_lshr_b32 tmp, s_save_ib_sts, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)1097 s_and_b32 tmp, tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK1098 s_setreg_b32 hwreg(HW_REG_IB_STS), tmp1099end1100