354 lines · plain
1# RUN: llc -mtriple=amdgcn -mcpu=carrizo -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,XNACK %s2# RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-xnack -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN %s3 4# RUN: llc -mtriple=amdgcn -mcpu=fiji -mattr=-xnack -passes post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN %s5 6---7# Trivial clause at beginning of program8name: trivial_smem_clause_load_smrd4_x19 10body: |11 bb.0:12 ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x113 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 014 ; GCN-NEXT: S_ENDPGM 015 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 016 S_ENDPGM 017...18---19# Trivial clause at beginning of program20name: trivial_smem_clause_load_smrd4_x221 22body: |23 bb.0:24 ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x225 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 026 ; GCN-NEXT: $sgpr1 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 027 ; GCN-NEXT: S_ENDPGM 028 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 029 $sgpr1 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 030 S_ENDPGM 031...32---33# Trivial clause at beginning of program34name: trivial_smem_clause_load_smrd4_x335 36body: |37 bb.0:38 ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x339 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 040 ; GCN-NEXT: $sgpr1 = S_LOAD_DWORD_IMM $sgpr6_sgpr7, 0, 041 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 042 ; GCN-NEXT: S_ENDPGM 043 $sgpr0 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 044 $sgpr1 = S_LOAD_DWORD_IMM $sgpr6_sgpr7, 0, 045 $sgpr2 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 046 S_ENDPGM 047...48---49# Trivial clause at beginning of program50name: trivial_smem_clause_load_smrd4_x451 52body: |53 bb.0:54 ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x455 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 056 ; GCN-NEXT: $sgpr1 = S_LOAD_DWORD_IMM $sgpr8_sgpr9, 0, 057 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 058 ; GCN-NEXT: $sgpr3 = S_LOAD_DWORD_IMM $sgpr16_sgpr17, 0, 059 ; GCN-NEXT: S_ENDPGM 060 $sgpr0 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 061 $sgpr1 = S_LOAD_DWORD_IMM $sgpr8_sgpr9, 0, 062 $sgpr2 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 063 $sgpr3 = S_LOAD_DWORD_IMM $sgpr16_sgpr17, 0, 064 S_ENDPGM 065...66---67# Reuse of same input pointer is OK68name: trivial_smem_clause_load_smrd4_x2_sameptr69body: |70 bb.0:71 ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x2_sameptr72 ; GCN: $sgpr12 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 073 ; GCN-NEXT: $sgpr13 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 074 ; GCN-NEXT: S_ENDPGM 075 $sgpr12 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 076 $sgpr13 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 077 S_ENDPGM 078...79---80# 32-bit load partially clobbers its own ptr reg81name: smrd_load4_overwrite_ptr_lo82 83body: |84 bb.0:85 ; GCN-LABEL: name: smrd_load4_overwrite_ptr_lo86 ; GCN: $sgpr10 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 087 ; GCN-NEXT: S_ENDPGM 088 $sgpr10 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 089 S_ENDPGM 090...91---92# 32-bit load partially clobbers its own ptr reg93name: smrd_load4_overwrite_ptr_hi94 95body: |96 bb.0:97 ; GCN-LABEL: name: smrd_load4_overwrite_ptr_hi98 ; GCN: $sgpr11 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 099 ; GCN-NEXT: S_ENDPGM 0100 $sgpr11 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0101 S_ENDPGM 0102...103---104# 64-bit load clobbers its own ptr reg105name: smrd_load8_overwrite_ptr106 107body: |108 bb.0:109 ; GCN-LABEL: name: smrd_load8_overwrite_ptr110 ; GCN: $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0111 ; GCN-NEXT: S_ENDPGM 0112 $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0113 S_ENDPGM 0114...115---116# vmcnt has 4 bits, so maximum 16 outstanding loads. The waitcnt117# breaks the clause.118 119name: break_smem_clause_at_max_smem_clause_size_smrd_load4120 121body: |122 bb.0:123 ; GCN-LABEL: name: break_smem_clause_at_max_smem_clause_size_smrd_load4124 ; GCN: $sgpr13 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0125 ; GCN-NEXT: $sgpr14 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0126 ; GCN-NEXT: $sgpr15 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0127 ; GCN-NEXT: $sgpr16 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0128 ; GCN-NEXT: $sgpr17 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0129 ; GCN-NEXT: $sgpr18 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0130 ; GCN-NEXT: $sgpr19 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0131 ; GCN-NEXT: $sgpr20 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0132 ; GCN-NEXT: $sgpr21 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0133 ; GCN-NEXT: $sgpr22 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0134 ; GCN-NEXT: $sgpr23 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0135 ; GCN-NEXT: $sgpr24 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0136 ; GCN-NEXT: $sgpr25 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0137 ; GCN-NEXT: $sgpr26 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0138 ; GCN-NEXT: $sgpr27 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0139 ; GCN-NEXT: $sgpr28 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0140 ; GCN-NEXT: $sgpr0 = S_LOAD_DWORD_IMM $sgpr30_sgpr31, 0, 0141 ; GCN-NEXT: $sgpr0 = S_MOV_B32 $sgpr0, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28142 ; GCN-NEXT: S_ENDPGM 0143 $sgpr13 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0144 $sgpr14 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0145 $sgpr15 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0146 $sgpr16 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0147 148 $sgpr17 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0149 $sgpr18 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0150 $sgpr19 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0151 $sgpr20 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0152 153 $sgpr21 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0154 $sgpr22 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0155 $sgpr23 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0156 $sgpr24 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0157 158 $sgpr25 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0159 $sgpr26 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0160 $sgpr27 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0161 $sgpr28 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0162 163 $sgpr0 = S_LOAD_DWORD_IMM $sgpr30_sgpr31, 0, 0164 $sgpr0 = S_MOV_B32 $sgpr0, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $sgpr16, implicit $sgpr17, implicit $sgpr18, implicit $sgpr19, implicit $sgpr20, implicit $sgpr21, implicit $sgpr22, implicit $sgpr23, implicit $sgpr24, implicit $sgpr25, implicit $sgpr26, implicit $sgpr27, implicit $sgpr28165 S_ENDPGM 0166...167---168 169name: break_smem_clause_simple_load_smrd4_lo_ptr170 171body: |172 bb.0:173 ; GCN-LABEL: name: break_smem_clause_simple_load_smrd4_lo_ptr174 ; GCN: $sgpr10 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0175 ; XNACK-NEXT: S_NOP 0176 ; GCN-NEXT: $sgpr12 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0177 ; GCN-NEXT: S_ENDPGM 0178 $sgpr10 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0179 $sgpr12 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0180 S_ENDPGM 0181...182---183 184name: break_smem_clause_simple_load_smrd4_hi_ptr185 186body: |187 bb.0:188 ; GCN-LABEL: name: break_smem_clause_simple_load_smrd4_hi_ptr189 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0190 ; GCN-NEXT: $sgpr3 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0191 ; GCN-NEXT: S_ENDPGM 0192 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0193 $sgpr3 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0194 S_ENDPGM 0195...196---197 198name: break_smem_clause_simple_load_smrd8_ptr199 200body: |201 bb.0:202 ; GCN-LABEL: name: break_smem_clause_simple_load_smrd8_ptr203 ; GCN: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0204 ; XNACK-NEXT: S_NOP 0205 ; GCN-NEXT: $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr12_sgpr13, 0, 0206 ; GCN-NEXT: S_ENDPGM 0207 $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0208 $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr12_sgpr13, 0, 0209 S_ENDPGM 0210...211---212 213name: break_smem_clause_simple_load_smrd16_ptr214 215body: |216 bb.0:217 ; GCN-LABEL: name: break_smem_clause_simple_load_smrd16_ptr218 ; GCN: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0219 ; GCN-NEXT: $sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX4_IMM $sgpr6_sgpr7, 0, 0220 ; GCN-NEXT: S_ENDPGM 0221 $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0222 $sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX4_IMM $sgpr6_sgpr7, 0, 0223 S_ENDPGM 0224...225---226 227name: break_smem_clause_block_boundary_load_smrd8_ptr228 229body: |230 ; GCN-LABEL: name: break_smem_clause_block_boundary_load_smrd8_ptr231 ; GCN: bb.0:232 ; GCN: successors: %bb.1(0x80000000)233 ; GCN: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0234 ; GCN: bb.1:235 ; XNACK-NEXT: S_NOP 0236 ; GCN-NEXT: $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr12_sgpr13, 0, 0237 ; GCN-NEXT: S_ENDPGM 0238 bb.0:239 $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0240 241 bb.1:242 $sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM $sgpr12_sgpr13, 0, 0243 S_ENDPGM 0244...245---246# The load clobbers the pointer of the store, so it needs to break.247 248name: break_smem_clause_store_load_into_ptr_smrd4249 250body: |251 bb.0:252 ; GCN-LABEL: name: break_smem_clause_store_load_into_ptr_smrd4253 ; GCN: S_STORE_DWORD_IMM $sgpr16, $sgpr10_sgpr11, 0, 0254 ; GCN-NEXT: $sgpr12 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 0255 ; GCN-NEXT: S_ENDPGM 0256 S_STORE_DWORD_IMM $sgpr16, $sgpr10_sgpr11, 0, 0257 $sgpr12 = S_LOAD_DWORD_IMM $sgpr14_sgpr15, 0, 0258 S_ENDPGM 0259...260---261# The load clobbers the data of the store, so it needs to break.262# FIXME: Would it be better to s_nop and wait later?263 264name: break_smem_clause_store_load_into_data_smrd4265 266body: |267 bb.0:268 ; GCN-LABEL: name: break_smem_clause_store_load_into_data_smrd4269 ; GCN: S_STORE_DWORD_IMM $sgpr8, $sgpr10_sgpr11, 0, 0270 ; GCN-NEXT: $sgpr8 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0271 ; GCN-NEXT: S_ENDPGM 0272 S_STORE_DWORD_IMM $sgpr8, $sgpr10_sgpr11, 0, 0273 $sgpr8 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0274 S_ENDPGM 0275...276---277# Regular VALU instruction breaks clause, no nop needed278name: valu_inst_breaks_smem_clause279 280body: |281 bb.0:282 ; GCN-LABEL: name: valu_inst_breaks_smem_clause283 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0284 ; GCN-NEXT: $vgpr8 = V_MOV_B32_e32 0, implicit $exec285 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0286 ; GCN-NEXT: S_ENDPGM 0287 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0288 $vgpr8 = V_MOV_B32_e32 0, implicit $exec289 $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0290 S_ENDPGM 0291...292---293# Regular SALU instruction breaks clause, no nop needed294name: salu_inst_breaks_smem_clause295 296body: |297 bb.0:298 ; GCN-LABEL: name: salu_inst_breaks_smem_clause299 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0300 ; GCN-NEXT: $sgpr8 = S_MOV_B32 0301 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0302 ; GCN-NEXT: S_ENDPGM 0303 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0304 $sgpr8 = S_MOV_B32 0305 $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0306 S_ENDPGM 0307...308---309name: ds_inst_breaks_smem_clause310 311body: |312 bb.0:313 ; GCN-LABEL: name: ds_inst_breaks_smem_clause314 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0315 ; GCN-NEXT: $vgpr8 = DS_READ_B32 $vgpr9, 0, 0, implicit $m0, implicit $exec316 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0317 ; GCN-NEXT: S_ENDPGM 0318 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0319 $vgpr8 = DS_READ_B32 $vgpr9, 0, 0, implicit $m0, implicit $exec320 $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0321 S_ENDPGM 0322...323---324 325name: flat_inst_breaks_smem_clause326 327body: |328 bb.0:329 ; GCN-LABEL: name: flat_inst_breaks_smem_clause330 ; GCN: $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0331 ; GCN-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr332 ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0333 ; GCN-NEXT: S_ENDPGM 0334 $sgpr0 = S_LOAD_DWORD_IMM $sgpr10_sgpr11, 0, 0335 $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr336 $sgpr2 = S_LOAD_DWORD_IMM $sgpr12_sgpr13, 0, 0337 S_ENDPGM 0338...339---340# FIXME: Should this be handled?341name: implicit_use_breaks_smem_clause342 343body: |344 bb.0:345 ; GCN-LABEL: name: implicit_use_breaks_smem_clause346 ; GCN: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0, implicit $sgpr12_sgpr13347 ; XNACK-NEXT: S_NOP 0348 ; GCN-NEXT: $sgpr12_sgpr13 = S_LOAD_DWORDX2_IMM $sgpr6_sgpr7, 0, 0349 ; GCN-NEXT: S_ENDPGM 0350 $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0, implicit $sgpr12_sgpr13351 $sgpr12_sgpr13 = S_LOAD_DWORDX2_IMM $sgpr6_sgpr7, 0, 0352 S_ENDPGM 0353...354