221 lines · plain
1# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s2 3# Check that SILoadStoreOptimizer honors memory dependencies between moved4# instructions.5#6# The following IR snippet would usually be optimized by the peephole optimizer.7# However, an equivalent situation can occur with buffer instructions as well.8 9# CHECK-LABEL: name: mem_dependency10# CHECK: DS_READ2_B32 %0, 0, 1,11# CHECK: DS_WRITE_B32 %0, killed %1, 64,12# CHECK: DS_READ2_B32 %0, 16, 17,13# CHECK: DS_WRITE_B32 killed %0, %5, 014 15--- |16 define amdgpu_kernel void @mem_dependency(i32 addrspace(3)* %ptr.0) nounwind {17 %ptr.4 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 118 %ptr.64 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 1619 %1 = load i32, i32 addrspace(3)* %ptr.020 store i32 %1, i32 addrspace(3)* %ptr.6421 %2 = load i32, i32 addrspace(3)* %ptr.6422 %3 = load i32, i32 addrspace(3)* %ptr.423 %4 = add i32 %2, %324 store i32 %4, i32 addrspace(3)* %ptr.025 ret void26 }27 28 @lds0 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 429 @lds1 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 430 @lds2 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 431 @lds3 = external dso_local unnamed_addr addrspace(3) global [256 x i32], align 432 33 define void @asm_defines_address() #0 {34 bb:35 %tmp1 = load i32, i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0), align 436 %0 = and i32 %tmp1, 25537 %tmp3 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 poison), align 438 %tmp6 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 poison), align 439 %tmp7 = tail call i32 asm "v_or_b32 $0, 0, $1", "=v,v"(i32 %tmp6) #140 %tmp10 = lshr i32 %tmp7, 1641 %tmp11 = and i32 %tmp10, 25542 %tmp12 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp1143 %tmp13 = load i32, i32 addrspace(3)* %tmp12, align 444 %tmp14 = xor i32 %tmp3, %tmp1345 %tmp15 = lshr i32 %tmp14, 846 %tmp16 = and i32 %tmp15, 1671168047 %tmp19 = lshr i32 %tmp16, 1648 %tmp20 = and i32 %tmp19, 25549 %tmp21 = getelementptr inbounds [256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 %tmp2050 %tmp22 = load i32, i32 addrspace(3)* %tmp21, align 451 %tmp24 = load i32, i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 poison), align 452 %tmp25 = xor i32 %tmp22, %tmp2453 %tmp26 = and i32 %tmp25, -1677721654 %tmp28 = or i32 %0, %tmp2655 store volatile i32 %tmp28, i32 addrspace(1)* undef56 ret void57 }58 59 attributes #0 = { convergent nounwind }60 attributes #1 = { convergent nounwind readnone }61 62 define amdgpu_kernel void @move_waw_hazards() #0 {63 ret void64 }65 66 attributes #0 = { convergent nounwind }67 68 define amdgpu_kernel void @merge_mmos(i32 addrspace(1)* %ptr_addr1) { ret void }69 define amdgpu_kernel void @reorder_offsets(i32 addrspace(1)* %reorder_addr1) { ret void }70 71...72---73name: mem_dependency74alignment: 175exposesReturnsTwice: false76legalized: false77regBankSelected: false78selected: false79tracksRegLiveness: true80liveins:81 - { reg: '$vgpr0', virtual-reg: '%1' }82frameInfo:83 isFrameAddressTaken: false84 isReturnAddressTaken: false85 hasStackMap: false86 hasPatchPoint: false87 stackSize: 088 offsetAdjustment: 089 maxAlignment: 090 adjustsStack: false91 hasCalls: false92 maxCallFrameSize: 093 hasOpaqueSPAdjustment: false94 hasVAStart: false95 hasMustTailInVarArgFunc: false96body: |97 bb.0:98 liveins: $vgpr099 100 %1:vgpr_32 = COPY $vgpr0101 $m0 = S_MOV_B32 -1102 %2:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load (s32) from %ir.ptr.0)103 DS_WRITE_B32 %1, killed %2, 64, 0, implicit $m0, implicit $exec :: (store (s32) into %ir.ptr.64)104 105 ; Make this load unmergeable, to tempt SILoadStoreOptimizer into merging the106 ; other two loads.107 %6:vreg_64 = DS_READ2_B32 %1, 16, 17, 0, implicit $m0, implicit $exec :: (load (s64) from %ir.ptr.64, align 4)108 %3:vgpr_32 = COPY %6.sub0109 %4:vgpr_32 = DS_READ_B32 %1, 4, 0, implicit $m0, implicit $exec :: (load (s32) from %ir.ptr.4)110 %5:vgpr_32 = V_ADD_CO_U32_e32 killed %3, killed %4, implicit-def $vcc, implicit $exec111 DS_WRITE_B32 killed %1, %5, 0, 0, implicit killed $m0, implicit $exec :: (store (s32) into %ir.ptr.0)112 S_ENDPGM 0113 114...115---116# Make sure the asm def isn't moved after the point where it's used for117# the address.118# CHECK-LABEL: name: asm_defines_address119# CHECK: DS_READ2ST64_B32120# CHECK: DS_READ2ST64_B32121# CHECK: INLINEASM122# CHECK: DS_READ_B32123# CHECK: DS_READ_B32124name: asm_defines_address125tracksRegLiveness: true126registers:127 - { id: 0, class: vgpr_32, preferred-register: '' }128body: |129 bb.0:130 %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec131 %2:vgpr_32 = DS_READ_B32 %1, 3072, 0, implicit $m0, implicit $exec :: (dereferenceable load (s32) from `i32 addrspace(3)* getelementptr inbounds ([256 x i32], [256 x i32] addrspace(3)* @lds0, i32 0, i32 0)`, addrspace 3)132 %3:vgpr_32 = DS_READ_B32 %1, 2048, 0, implicit $m0, implicit $exec :: (load (s32) from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds1, i32 0, i32 poison)`, addrspace 3)133 %4:vgpr_32 = DS_READ_B32 %1, 1024, 0, implicit $m0, implicit $exec :: (load (s32) from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds3, i32 0, i32 poison)`, addrspace 3)134 INLINEASM &"v_or_b32 $0, 0, $1", 32, 327690, def %0, 327689, %4135 %5:vgpr_32 = DS_READ_B32 %0, 2048, 0, implicit $m0, implicit $exec :: (load (s32) from %ir.tmp12, addrspace 3)136 %6:vgpr_32 = DS_READ_B32 %5, 2048, 0, implicit $m0, implicit $exec :: (load (s32) from %ir.tmp21, addrspace 3)137 %7:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit $m0, implicit $exec :: (load (s32) from `i32 addrspace(3)* getelementptr ([256 x i32], [256 x i32] addrspace(3)* @lds2, i32 0, i32 poison)`, addrspace 3)138 S_SETPC_B64_return undef $sgpr30_sgpr31, implicit %6, implicit %7139 140...141---142# Make sure Write-after-Write hazards are correctly detected and the143# instructions moved accordingly.144# operations.145# CHECK-LABEL: name: move_waw_hazards146# CHECK: S_AND_B64147# CHECK: S_CMP_EQ_U32148name: move_waw_hazards149tracksRegLiveness: true150body: |151 bb.0:152 liveins: $sgpr0_sgpr1153 154 %3:sgpr_64 = COPY $sgpr0_sgpr1155 %6:sreg_32_xm0_xexec = S_MOV_B32 0156 %7:sreg_32_xm0 = S_MOV_B32 0157 %8:sreg_64_xexec = REG_SEQUENCE killed %6, %subreg.sub0, %7, %subreg.sub1158 %9:sgpr_128 = S_LOAD_DWORDX4_IMM killed %8, 0, 0 :: (invariant load (s128), addrspace 6)159 %31:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %9, 0, 0 :: (dereferenceable invariant load (s32))160 %10:sreg_32_xm0_xexec = COPY %31.sub0161 %11:sreg_32_xm0_xexec = COPY killed %31.sub1162 %12:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 2, 0 :: (dereferenceable invariant load (s32))163 %13:sreg_64 = V_CMP_NE_U32_e64 killed %11, 0, implicit $exec164 %15:sreg_64 = V_CMP_NE_U32_e64 killed %12, 0, implicit $exec165 %17:sreg_64_xexec = S_AND_B64 killed %13, killed %15, implicit-def dead $scc166 S_CMP_EQ_U32 killed %10, 0, implicit-def $scc167 %18:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %9, 3, 0 :: (dereferenceable invariant load (s32))168 S_ENDPGM 0169...170---171# CHECK-LABEL: merge_mmos172# CHECK: S_BUFFER_LOAD_DWORDX2_IMM %0, 0, 0 :: (dereferenceable invariant load (s64), align 4)173# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 4)174# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s64), align 4)175# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 64, 0, 0, implicit $exec :: (dereferenceable load (s64) from %ir.ptr_addr1 + 64, align 4176# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 64, 0, 0, implicit $exec :: (dereferenceable store (s64) into %ir.ptr_addr1 + 64, align 4177name: merge_mmos178tracksRegLiveness: true179body: |180 bb.0:181 liveins: $sgpr0_sgpr1_sgpr2_sgpr3182 183 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3184 %1:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 0, 0 :: (dereferenceable invariant load (s32))185 %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 1, 0 :: (dereferenceable invariant load (s32))186 %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32))187 %4:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 4, 0, 0, implicit $exec :: (dereferenceable load (s32))188 BUFFER_STORE_DWORD_OFFSET_exact %3, %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32))189 BUFFER_STORE_DWORD_OFFSET_exact %4, %0, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32))190 %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 64, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.ptr_addr1 + 64)191 %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 68, 0, 0, implicit $exec :: (dereferenceable load (s32) from %ir.ptr_addr1 + 68)192 BUFFER_STORE_DWORD_OFFSET_exact %5, %0, 0, 64, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.ptr_addr1 + 64)193 BUFFER_STORE_DWORD_OFFSET_exact %6, %0, 0, 68, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.ptr_addr1 + 68)194 195 S_ENDPGM 0196 197...198---199# CHECK-LABEL: reorder_offsets200# CHECK-DAG: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 16, 0, 0, implicit $exec :: (dereferenceable store (s64) into %ir.reorder_addr1 + 16, align 4, addrspace 1)201# CHECK-DAG: BUFFER_STORE_DWORDX4_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.reorder_addr1, align 4, addrspace 1)202 203name: reorder_offsets204tracksRegLiveness: true205body: |206 bb.0:207 liveins: $sgpr0_sgpr1_sgpr2_sgpr3208 209 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3210 %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec211 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 4, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1 + 4)212 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 8, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1 + 8)213 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 12, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1 + 12)214 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 16, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1 + 16)215 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 20, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1 + 20)216 BUFFER_STORE_DWORD_OFFSET_exact %1, %0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32) into %ir.reorder_addr1)217 S_ENDPGM 0218 219 220...221