271 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefix=GFX9 %s3 4; Make sure that AMDGPUCodeGenPrepare introduces mul24 intrinsics5; after SLSR, as the intrinsics would interfere. It's unclear if these6; should be introduced before LSR or not. It seems to help in some7; cases, and hurt others.8 9define void @lsr_order_mul24_0(i32 %arg, i32 %arg2, i32 %arg6, i32 %arg13, i32 %arg16) #0 {10; GFX9-LABEL: lsr_order_mul24_0:11; GFX9: ; %bb.0: ; %bb12; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13; GFX9-NEXT: global_load_dword v5, v[0:1], off14; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v215; GFX9-NEXT: v_sub_u32_e32 v4, v4, v116; GFX9-NEXT: s_mov_b64 s[4:5], 017; GFX9-NEXT: s_waitcnt vmcnt(0)18; GFX9-NEXT: ds_write_b32 v0, v519; GFX9-NEXT: .LBB0_1: ; %bb2320; GFX9-NEXT: ; =>This Inner Loop Header: Depth=121; GFX9-NEXT: v_mul_u32_u24_e32 v5, v0, v222; GFX9-NEXT: v_add_u32_e32 v0, v0, v123; GFX9-NEXT: v_sub_u32_e32 v5, v4, v524; GFX9-NEXT: v_add_u32_e32 v5, v5, v025; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v5, v326; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]27; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]28; GFX9-NEXT: s_cbranch_execnz .LBB0_129; GFX9-NEXT: ; %bb.2: ; %.loopexit30; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]31; GFX9-NEXT: s_waitcnt lgkmcnt(0)32; GFX9-NEXT: s_setpc_b64 s[30:31]33bb:34 %tmp22 = and i32 %arg6, 1677721535 br label %bb2336 37.loopexit: ; preds = %bb2338 ret void39 40bb23: ; preds = %bb23, %bb41 %tmp24 = phi i32 [ %arg, %bb ], [ %tmp47, %bb23 ]42 %tmp28 = and i32 %tmp24, 1677721543 %tmp29 = mul i32 %tmp28, %tmp2244 %tmp30 = sub i32 %tmp24, %tmp2945 %tmp31 = add i32 %tmp30, %arg1646 %tmp37 = icmp ult i32 %tmp31, %arg1347 %tmp44 = load float, ptr addrspace(1) poison, align 448 store float %tmp44, ptr addrspace(3) poison, align 449 %tmp47 = add i32 %tmp24, %arg250 br i1 %tmp37, label %bb23, label %.loopexit51}52 53define void @lsr_order_mul24_1(i32 %arg, i32 %arg1, i32 %arg2, ptr addrspace(3) nocapture %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, ptr addrspace(1) nocapture readonly %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15, i32 %arg16, i1 zeroext %arg17, i1 zeroext %arg18) #0 {54; GFX9-LABEL: lsr_order_mul24_1:55; GFX9: ; %bb.0: ; %bb56; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)57; GFX9-NEXT: v_and_b32_e32 v5, 1, v1858; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v559; GFX9-NEXT: v_cmp_lt_u32_e64 s[4:5], v0, v160; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]61; GFX9-NEXT: s_cbranch_execz .LBB1_362; GFX9-NEXT: ; %bb.1: ; %bb1963; GFX9-NEXT: v_cvt_f32_u32_e32 v7, v664; GFX9-NEXT: v_add_u32_e32 v4, v4, v065; GFX9-NEXT: v_and_b32_e32 v5, 0xffffff, v666; GFX9-NEXT: v_lshl_add_u32 v6, v4, 2, v367; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v768; GFX9-NEXT: v_lshlrev_b32_e32 v8, 2, v269; GFX9-NEXT: v_add_u32_e32 v9, v17, v1270; GFX9-NEXT: s_mov_b64 s[10:11], 071; GFX9-NEXT: v_mov_b32_e32 v4, 072; GFX9-NEXT: ; implicit-def: $vgpr373; GFX9-NEXT: .LBB1_2: ; %bb2374; GFX9-NEXT: ; =>This Inner Loop Header: Depth=175; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v076; GFX9-NEXT: v_add_u32_e32 v12, v17, v077; GFX9-NEXT: v_madak_f32 v3, v3, v7, 0x3727c5ac78; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v379; GFX9-NEXT: v_mul_u32_u24_e32 v18, v3, v580; GFX9-NEXT: v_add_u32_e32 v19, v3, v1681; GFX9-NEXT: v_add_u32_e32 v3, v9, v082; GFX9-NEXT: v_sub_u32_e32 v3, v3, v1883; GFX9-NEXT: v_sub_u32_e32 v12, v12, v1884; GFX9-NEXT: v_cmp_lt_u32_e64 s[4:5], v19, v1385; GFX9-NEXT: v_mad_u64_u32 v[18:19], s[6:7], v19, v15, v[3:4]86; GFX9-NEXT: v_cmp_lt_u32_e64 s[6:7], v12, v1487; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]88; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], vcc89; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, v18, s[4:5]90; GFX9-NEXT: v_lshlrev_b64 v[18:19], 2, v[3:4]91; GFX9-NEXT: v_add_u32_e32 v0, v0, v292; GFX9-NEXT: v_add_co_u32_e64 v18, s[6:7], v10, v1893; GFX9-NEXT: v_addc_co_u32_e64 v19, s[6:7], v11, v19, s[6:7]94; GFX9-NEXT: global_load_dword v3, v[18:19], off95; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v0, v196; GFX9-NEXT: s_or_b64 s[10:11], s[6:7], s[10:11]97; GFX9-NEXT: s_waitcnt vmcnt(0)98; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]99; GFX9-NEXT: ds_write_b32 v6, v3100; GFX9-NEXT: v_add_u32_e32 v6, v6, v8101; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]102; GFX9-NEXT: s_cbranch_execnz .LBB1_2103; GFX9-NEXT: .LBB1_3: ; %Flow2104; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]105; GFX9-NEXT: s_waitcnt lgkmcnt(0)106; GFX9-NEXT: s_setpc_b64 s[30:31]107bb:108 %tmp = icmp ult i32 %arg, %arg1109 br i1 %tmp, label %bb19, label %.loopexit110 111bb19: ; preds = %bb112 %tmp20 = uitofp i32 %arg6 to float113 %tmp21 = fdiv float 1.000000e+00, %tmp20, !fpmath !0114 %tmp22 = and i32 %arg6, 16777215115 br label %bb23116 117.loopexit: ; preds = %bb23, %bb118 ret void119 120bb23: ; preds = %bb19, %bb23121 %tmp24 = phi i32 [ %arg, %bb19 ], [ %tmp47, %bb23 ]122 %tmp25 = uitofp i32 %tmp24 to float123 %tmp26 = tail call float @llvm.fmuladd.f32(float %tmp25, float %tmp21, float 0x3EE4F8B580000000) #2124 %tmp27 = fptoui float %tmp26 to i32125 %tmp28 = and i32 %tmp27, 16777215126 %tmp29 = mul i32 %tmp28, %tmp22127 %tmp30 = sub i32 %tmp24, %tmp29128 %tmp31 = add i32 %tmp30, %arg16129 %tmp32 = add i32 %tmp27, %arg15130 %tmp33 = mul i32 %tmp32, %arg14131 %tmp34 = add i32 %tmp33, %arg11132 %tmp35 = add i32 %tmp34, %tmp31133 %tmp36 = add i32 %tmp24, %arg4134 %tmp37 = icmp ult i32 %tmp31, %arg13135 %tmp38 = icmp ult i32 %tmp32, %arg12136 %tmp39 = and i1 %tmp38, %tmp37137 %tmp40 = and i1 %tmp39, %arg17138 %tmp41 = zext i32 %tmp35 to i64139 %tmp42 = select i1 %tmp40, i64 %tmp41, i64 0140 %tmp43 = getelementptr inbounds float, ptr addrspace(1) %arg10, i64 %tmp42141 %tmp44 = load float, ptr addrspace(1) %tmp43, align 4142 %tmp45 = select i1 %tmp40, float %tmp44, float 0.000000e+00143 %tmp46 = getelementptr inbounds float, ptr addrspace(3) %arg3, i32 %tmp36144 store float %tmp45, ptr addrspace(3) %tmp46, align 4145 %tmp47 = add i32 %tmp24, %arg2146 %tmp48 = icmp ult i32 %tmp47, %arg1147 br i1 %tmp48, label %bb23, label %.loopexit148}149 150define void @slsr1_0(i32 %b.arg, i32 %s.arg) #0 {151; GFX9-LABEL: slsr1_0:152; GFX9: ; %bb.0:153; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)154; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v1155; GFX9-NEXT: v_mul_u32_u24_e32 v3, v0, v1156; GFX9-NEXT: global_store_dword v[0:1], v3, off157; GFX9-NEXT: s_waitcnt vmcnt(0)158; GFX9-NEXT: v_mad_u32_u24 v0, v0, v1, v2159; GFX9-NEXT: global_store_dword v[0:1], v0, off160; GFX9-NEXT: s_waitcnt vmcnt(0)161; GFX9-NEXT: v_add_u32_e32 v0, v0, v2162; GFX9-NEXT: global_store_dword v[0:1], v0, off163; GFX9-NEXT: s_waitcnt vmcnt(0)164; GFX9-NEXT: s_setpc_b64 s[30:31]165 %b = and i32 %b.arg, 16777215166 %s = and i32 %s.arg, 16777215167 168; CHECK-LABEL: @slsr1(169 ; foo(b * s);170 %mul0 = mul i32 %b, %s171; CHECK: mul i32172; CHECK-NOT: mul i32173 store volatile i32 %mul0, ptr addrspace(1) poison174 175 ; foo((b + 1) * s);176 %b1 = add i32 %b, 1177 %mul1 = mul i32 %b1, %s178 store volatile i32 %mul1, ptr addrspace(1) poison179 180 ; foo((b + 2) * s);181 %b2 = add i32 %b, 2182 %mul2 = mul i32 %b2, %s183 store volatile i32 %mul2, ptr addrspace(1) poison184 ret void185}186 187define void @slsr1_1(i32 %b.arg, i32 %s.arg) #0 {188; GFX9-LABEL: slsr1_1:189; GFX9: ; %bb.0:190; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)191; GFX9-NEXT: s_mov_b32 s4, s33192; GFX9-NEXT: s_mov_b32 s33, s32193; GFX9-NEXT: s_or_saveexec_b64 s[6:7], -1194; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s33 offset:12 ; 4-byte Folded Spill195; GFX9-NEXT: s_mov_b64 exec, s[6:7]196; GFX9-NEXT: v_writelane_b32 v43, s4, 5197; GFX9-NEXT: v_writelane_b32 v43, s30, 0198; GFX9-NEXT: v_writelane_b32 v43, s31, 1199; GFX9-NEXT: s_addk_i32 s32, 0x800200; GFX9-NEXT: v_writelane_b32 v43, s34, 2201; GFX9-NEXT: v_writelane_b32 v43, s36, 3202; GFX9-NEXT: s_getpc_b64 s[4:5]203; GFX9-NEXT: s_add_u32 s4, s4, foo@gotpcrel32@lo+4204; GFX9-NEXT: s_addc_u32 s5, s5, foo@gotpcrel32@hi+12205; GFX9-NEXT: v_writelane_b32 v43, s37, 4206; GFX9-NEXT: s_load_dwordx2 s[36:37], s[4:5], 0x0207; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill208; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill209; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s33 ; 4-byte Folded Spill210; GFX9-NEXT: v_mov_b32_e32 v40, v1211; GFX9-NEXT: v_mov_b32_e32 v41, v0212; GFX9-NEXT: v_mul_u32_u24_e32 v0, v41, v40213; GFX9-NEXT: s_mov_b32 s34, s15214; GFX9-NEXT: v_and_b32_e32 v42, 0xffffff, v40215; GFX9-NEXT: s_waitcnt lgkmcnt(0)216; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]217; GFX9-NEXT: v_mad_u32_u24 v40, v41, v40, v42218; GFX9-NEXT: s_mov_b32 s15, s34219; GFX9-NEXT: v_mov_b32_e32 v0, v40220; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]221; GFX9-NEXT: v_add_u32_e32 v0, v40, v42222; GFX9-NEXT: s_mov_b32 s15, s34223; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]224; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s33 ; 4-byte Folded Reload225; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload226; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload227; GFX9-NEXT: v_readlane_b32 s37, v43, 4228; GFX9-NEXT: v_readlane_b32 s36, v43, 3229; GFX9-NEXT: v_readlane_b32 s34, v43, 2230; GFX9-NEXT: v_readlane_b32 s31, v43, 1231; GFX9-NEXT: v_readlane_b32 s30, v43, 0232; GFX9-NEXT: s_mov_b32 s32, s33233; GFX9-NEXT: v_readlane_b32 s4, v43, 5234; GFX9-NEXT: s_or_saveexec_b64 s[6:7], -1235; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s33 offset:12 ; 4-byte Folded Reload236; GFX9-NEXT: s_mov_b64 exec, s[6:7]237; GFX9-NEXT: s_mov_b32 s33, s4238; GFX9-NEXT: s_waitcnt vmcnt(0)239; GFX9-NEXT: s_setpc_b64 s[30:31]240 %b = and i32 %b.arg, 16777215241 %s = and i32 %s.arg, 16777215242 243; CHECK-LABEL: @slsr1(244 ; foo(b * s);245 %mul0 = mul i32 %b, %s246; CHECK: mul i32247; CHECK-NOT: mul i32248 call void @foo(i32 %mul0)249 250 ; foo((b + 1) * s);251 %b1 = add i32 %b, 1252 %mul1 = mul i32 %b1, %s253 call void @foo(i32 %mul1)254 255 ; foo((b + 2) * s);256 %b2 = add i32 %b, 2257 %mul2 = mul i32 %b2, %s258 call void @foo(i32 %mul2)259 260 ret void261}262 263declare void @foo(i32) #2264declare float @llvm.fmuladd.f32(float, float, float) #1265 266attributes #0 = { nounwind willreturn "denormal-fp-math-f32"="preserve-sign,preserve-sign" }267attributes #1 = { nounwind readnone speculatable }268attributes #2 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-cluster-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-cluster-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-cluster-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }269 270!0 = !{float 2.500000e+00}271