404 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -enable-misched -asm-verbose -disable-block-placement -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=SI %s3 4declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone5 6; Make sure the i1 values created by the cfg structurizer pass are7; moved using VALU instructions8 9 10; waitcnt should be inserted after exec modification11; v_mov should be after exec modification12define amdgpu_kernel void @test_if(i32 %b, ptr addrspace(1) %src, ptr addrspace(1) %dst) #1 {13; SI-LABEL: test_if:14; SI: ; %bb.0: ; %entry15; SI-NEXT: s_load_dword s8, s[4:5], 0x916; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd17; SI-NEXT: v_cmp_lt_i32_e32 vcc, 1, v018; SI-NEXT: s_mov_b64 s[10:11], 019; SI-NEXT: s_mov_b64 s[2:3], 020; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc21; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]22; SI-NEXT: s_cbranch_execz .LBB0_323; SI-NEXT: ; %bb.1: ; %LeafBlock324; SI-NEXT: v_cmp_eq_u32_e32 vcc, 2, v025; SI-NEXT: s_mov_b64 s[2:3], -126; SI-NEXT: s_and_saveexec_b64 s[6:7], vcc27; SI-NEXT: s_cbranch_execnz .LBB0_928; SI-NEXT: .LBB0_2: ; %Flow729; SI-NEXT: s_or_b64 exec, exec, s[6:7]30; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec31; SI-NEXT: .LBB0_3: ; %Flow632; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]33; SI-NEXT: s_cbranch_execz .LBB0_534; SI-NEXT: ; %bb.4: ; %LeafBlock35; SI-NEXT: s_mov_b64 s[10:11], exec36; SI-NEXT: v_cmp_ne_u32_e32 vcc, 1, v037; SI-NEXT: s_andn2_b64 s[2:3], s[2:3], exec38; SI-NEXT: s_and_b64 s[6:7], vcc, exec39; SI-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]40; SI-NEXT: .LBB0_5: ; %Flow841; SI-NEXT: s_or_b64 exec, exec, s[4:5]42; SI-NEXT: s_and_saveexec_b64 s[4:5], s[2:3]43; SI-NEXT: s_xor_b64 s[2:3], exec, s[4:5]44; SI-NEXT: s_cbranch_execnz .LBB0_1045; SI-NEXT: .LBB0_6: ; %Flow946; SI-NEXT: s_or_b64 exec, exec, s[2:3]47; SI-NEXT: s_and_saveexec_b64 s[2:3], s[10:11]48; SI-NEXT: s_cbranch_execz .LBB0_849; SI-NEXT: ; %bb.7: ; %case150; SI-NEXT: s_waitcnt lgkmcnt(0)51; SI-NEXT: s_ashr_i32 s9, s8, 3152; SI-NEXT: s_mov_b32 s3, 0xf00053; SI-NEXT: s_mov_b32 s2, 054; SI-NEXT: s_lshl_b64 s[4:5], s[8:9], 255; SI-NEXT: v_mov_b32_e32 v2, 1356; SI-NEXT: s_waitcnt expcnt(0)57; SI-NEXT: v_mov_b32_e32 v0, s458; SI-NEXT: v_mov_b32_e32 v1, s559; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr6460; SI-NEXT: .LBB0_8: ; %end61; SI-NEXT: s_endpgm62; SI-NEXT: .LBB0_9: ; %case263; SI-NEXT: s_waitcnt lgkmcnt(0)64; SI-NEXT: s_ashr_i32 s9, s8, 3165; SI-NEXT: s_mov_b32 s3, 0xf00066; SI-NEXT: s_mov_b32 s2, 067; SI-NEXT: s_lshl_b64 s[12:13], s[8:9], 268; SI-NEXT: v_mov_b32_e32 v3, 1769; SI-NEXT: v_mov_b32_e32 v1, s1270; SI-NEXT: v_mov_b32_e32 v2, s1371; SI-NEXT: buffer_store_dword v3, v[1:2], s[0:3], 0 addr6472; SI-NEXT: s_xor_b64 s[2:3], exec, -173; SI-NEXT: s_branch .LBB0_274; SI-NEXT: .LBB0_10: ; %default75; SI-NEXT: v_cmp_ne_u32_e32 vcc, 2, v076; SI-NEXT: s_waitcnt lgkmcnt(0)77; SI-NEXT: s_ashr_i32 s9, s8, 3178; SI-NEXT: s_lshl_b64 s[4:5], s[8:9], 279; SI-NEXT: s_add_u32 s4, s0, s480; SI-NEXT: s_addc_u32 s5, s1, s581; SI-NEXT: s_and_saveexec_b64 s[6:7], vcc82; SI-NEXT: s_xor_b64 s[12:13], exec, s[6:7]83; SI-NEXT: s_cbranch_execnz .LBB0_1484; SI-NEXT: .LBB0_11: ; %Flow85; SI-NEXT: s_andn2_saveexec_b64 s[12:13], s[12:13]86; SI-NEXT: s_cbranch_execz .LBB0_1387; SI-NEXT: ; %bb.12: ; %if88; SI-NEXT: s_mov_b32 s7, 0xf00089; SI-NEXT: s_mov_b32 s6, -190; SI-NEXT: s_waitcnt expcnt(0)91; SI-NEXT: v_mov_b32_e32 v0, 1992; SI-NEXT: buffer_store_dword v0, off, s[4:7], 093; SI-NEXT: .LBB0_13: ; %Flow594; SI-NEXT: s_or_b64 exec, exec, s[12:13]95; SI-NEXT: s_andn2_b64 s[10:11], s[10:11], exec96; SI-NEXT: s_branch .LBB0_697; SI-NEXT: .LBB0_14: ; %else98; SI-NEXT: s_mov_b32 s7, 0xf00099; SI-NEXT: s_mov_b32 s6, -1100; SI-NEXT: v_mov_b32_e32 v0, 21101; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0102; SI-NEXT: s_branch .LBB0_11103entry:104 %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone105 switch i32 %tid, label %default [106 i32 1, label %case1107 i32 2, label %case2108 ]109 110case1:111 %arrayidx1 = getelementptr i32, ptr addrspace(1) %dst, i32 %b112 store i32 13, ptr addrspace(1) %arrayidx1, align 4113 br label %end114 115case2:116 %arrayidx5 = getelementptr i32, ptr addrspace(1) %dst, i32 %b117 store i32 17, ptr addrspace(1) %arrayidx5, align 4118 br label %end119 120default:121 %cmp8 = icmp eq i32 %tid, 2122 %arrayidx10 = getelementptr i32, ptr addrspace(1) %dst, i32 %b123 br i1 %cmp8, label %if, label %else124 125if:126 store i32 19, ptr addrspace(1) %arrayidx10, align 4127 br label %end128 129else:130 store i32 21, ptr addrspace(1) %arrayidx10, align 4131 br label %end132 133end:134 ret void135}136 137define amdgpu_kernel void @simple_test_v_if(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {138; SI-LABEL: simple_test_v_if:139; SI: ; %bb.0:140; SI-NEXT: s_mov_b32 s2, 0141; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0142; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc143; SI-NEXT: s_cbranch_execz .LBB1_2144; SI-NEXT: ; %bb.1: ; %then145; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9146; SI-NEXT: s_mov_b32 s3, 0xf000147; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0148; SI-NEXT: v_mov_b32_e32 v1, 0149; SI-NEXT: v_mov_b32_e32 v2, 0x3e7150; SI-NEXT: s_waitcnt lgkmcnt(0)151; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64152; SI-NEXT: .LBB1_2: ; %exit153; SI-NEXT: s_endpgm154 %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone155 %is.0 = icmp ne i32 %tid, 0156 br i1 %is.0, label %then, label %exit157 158then:159 %gep = getelementptr i32, ptr addrspace(1) %dst, i32 %tid160 store i32 999, ptr addrspace(1) %gep161 br label %exit162 163exit:164 ret void165}166 167; FIXME: It would be better to endpgm in the then block.168define amdgpu_kernel void @simple_test_v_if_ret_else_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {169; SI-LABEL: simple_test_v_if_ret_else_ret:170; SI: ; %bb.0:171; SI-NEXT: s_mov_b32 s2, 0172; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0173; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc174; SI-NEXT: s_cbranch_execz .LBB2_2175; SI-NEXT: ; %bb.1: ; %then176; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9177; SI-NEXT: s_mov_b32 s3, 0xf000178; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0179; SI-NEXT: v_mov_b32_e32 v1, 0180; SI-NEXT: v_mov_b32_e32 v2, 0x3e7181; SI-NEXT: s_waitcnt lgkmcnt(0)182; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64183; SI-NEXT: .LBB2_2: ; %UnifiedReturnBlock184; SI-NEXT: s_endpgm185 %tid = call i32 @llvm.amdgcn.workitem.id.x()186 %is.0 = icmp ne i32 %tid, 0187 br i1 %is.0, label %then, label %exit188 189then:190 %gep = getelementptr i32, ptr addrspace(1) %dst, i32 %tid191 store i32 999, ptr addrspace(1) %gep192 ret void193 194exit:195 ret void196}197 198; Final block has more than a ret to execute. This was miscompiled199; before function exit blocks were unified since the endpgm would200; terminate the then wavefront before reaching the store.201define amdgpu_kernel void @simple_test_v_if_ret_else_code_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {202; SI-LABEL: simple_test_v_if_ret_else_code_ret:203; SI: ; %bb.0:204; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0205; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc206; SI-NEXT: s_xor_b64 s[0:1], exec, s[0:1]207; SI-NEXT: s_cbranch_execnz .LBB3_4208; SI-NEXT: .LBB3_1: ; %Flow209; SI-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]210; SI-NEXT: s_cbranch_execz .LBB3_3211; SI-NEXT: ; %bb.2: ; %then212; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9213; SI-NEXT: s_mov_b32 s3, 0xf000214; SI-NEXT: s_mov_b32 s2, 0215; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0216; SI-NEXT: v_mov_b32_e32 v1, 0217; SI-NEXT: v_mov_b32_e32 v2, 0x3e7218; SI-NEXT: s_waitcnt lgkmcnt(0)219; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64220; SI-NEXT: .LBB3_3: ; %UnifiedReturnBlock221; SI-NEXT: s_endpgm222; SI-NEXT: .LBB3_4: ; %exit223; SI-NEXT: v_mov_b32_e32 v0, 7224; SI-NEXT: s_mov_b32 m0, -1225; SI-NEXT: ds_write_b32 v0, v0226; SI-NEXT: ; implicit-def: $vgpr0227; SI-NEXT: s_branch .LBB3_1228 %tid = call i32 @llvm.amdgcn.workitem.id.x()229 %is.0 = icmp ne i32 %tid, 0230 br i1 %is.0, label %then, label %exit231 232then:233 %gep = getelementptr i32, ptr addrspace(1) %dst, i32 %tid234 store i32 999, ptr addrspace(1) %gep235 ret void236 237exit:238 store volatile i32 7, ptr addrspace(3) poison239 ret void240}241 242define amdgpu_kernel void @simple_test_v_loop(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {243; SI-LABEL: simple_test_v_loop:244; SI: ; %bb.0: ; %entry245; SI-NEXT: s_mov_b32 s2, 0246; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0247; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc248; SI-NEXT: s_cbranch_execz .LBB4_3249; SI-NEXT: ; %bb.1: ; %loop.preheader250; SI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9251; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0252; SI-NEXT: s_mov_b64 s[0:1], 0253; SI-NEXT: s_mov_b32 s3, 0xf000254; SI-NEXT: s_waitcnt lgkmcnt(0)255; SI-NEXT: v_mov_b32_e32 v1, s9256; SI-NEXT: v_add_i32_e32 v0, vcc, s8, v0257; SI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc258; SI-NEXT: s_mov_b32 s6, -1259; SI-NEXT: s_mov_b32 s4, s10260; SI-NEXT: s_mov_b32 s5, s11261; SI-NEXT: s_mov_b32 s7, s3262; SI-NEXT: .LBB4_2: ; %loop263; SI-NEXT: ; =>This Inner Loop Header: Depth=1264; SI-NEXT: s_waitcnt expcnt(0)265; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0266; SI-NEXT: s_waitcnt vmcnt(0)267; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64268; SI-NEXT: s_add_u32 s0, s0, 4269; SI-NEXT: s_addc_u32 s1, s1, 0270; SI-NEXT: s_cmpk_lg_i32 s0, 0x100271; SI-NEXT: s_cbranch_scc1 .LBB4_2272; SI-NEXT: .LBB4_3: ; %exit273; SI-NEXT: s_endpgm274entry:275 %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone276 %is.0 = icmp ne i32 %tid, 0277 %limit = add i32 %tid, 64278 br i1 %is.0, label %loop, label %exit279 280loop:281 %i = phi i32 [%tid, %entry], [%i.inc, %loop]282 %gep.src = getelementptr i32, ptr addrspace(1) %src, i32 %i283 %gep.dst = getelementptr i32, ptr addrspace(1) %dst, i32 %i284 %load = load i32, ptr addrspace(1) %src285 store i32 %load, ptr addrspace(1) %gep.dst286 %i.inc = add nsw i32 %i, 1287 %cmp = icmp eq i32 %limit, %i.inc288 br i1 %cmp, label %exit, label %loop289 290exit:291 ret void292}293 294; Load loop limit from buffer295; Branch to exit if uniformly not taken296; Initialize inner condition to false297; Clear exec bits for workitems that load -1s298define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %arg, ptr addrspace(1) noalias nocapture readonly %arg1, ptr addrspace(1) noalias nocapture readonly %arg2, ptr addrspace(1) noalias nocapture readonly %arg3) #1 {299; SI-LABEL: multi_vcond_loop:300; SI: ; %bb.0: ; %bb301; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xf302; SI-NEXT: v_mov_b32_e32 v7, 0303; SI-NEXT: s_mov_b32 s11, 0xf000304; SI-NEXT: s_mov_b32 s10, 0305; SI-NEXT: v_lshlrev_b32_e32 v6, 2, v0306; SI-NEXT: s_waitcnt lgkmcnt(0)307; SI-NEXT: buffer_load_dword v0, v[6:7], s[8:11], 0 addr64308; SI-NEXT: s_waitcnt vmcnt(0)309; SI-NEXT: v_cmp_lt_i32_e32 vcc, 0, v0310; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc311; SI-NEXT: s_cbranch_execz .LBB5_5312; SI-NEXT: ; %bb.1: ; %bb10.preheader313; SI-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x9314; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd315; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0316; SI-NEXT: s_mov_b64 s[2:3], 0317; SI-NEXT: s_mov_b32 s8, s10318; SI-NEXT: s_mov_b32 s9, s10319; SI-NEXT: ; implicit-def: $sgpr4_sgpr5320; SI-NEXT: s_waitcnt lgkmcnt(0)321; SI-NEXT: v_mov_b32_e32 v3, s13322; SI-NEXT: v_add_i32_e32 v2, vcc, s12, v6323; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc324; SI-NEXT: v_mov_b32_e32 v5, s1325; SI-NEXT: v_add_i32_e32 v4, vcc, s0, v6326; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc327; SI-NEXT: v_mov_b32_e32 v7, s15328; SI-NEXT: v_add_i32_e32 v6, vcc, s14, v6329; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc330; SI-NEXT: s_mov_b64 s[6:7], 0331; SI-NEXT: .LBB5_2: ; %bb10332; SI-NEXT: ; =>This Inner Loop Header: Depth=1333; SI-NEXT: s_waitcnt expcnt(0)334; SI-NEXT: buffer_load_dword v8, v[6:7], s[8:11], 0 addr64335; SI-NEXT: buffer_load_dword v9, v[4:5], s[8:11], 0 addr64336; SI-NEXT: s_waitcnt vmcnt(1)337; SI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v8338; SI-NEXT: s_waitcnt vmcnt(0)339; SI-NEXT: v_cmp_ne_u32_e64 s[0:1], -1, v9340; SI-NEXT: s_and_b64 s[12:13], vcc, s[0:1]341; SI-NEXT: s_or_b64 s[4:5], s[4:5], exec342; SI-NEXT: s_and_saveexec_b64 s[0:1], s[12:13]343; SI-NEXT: s_cbranch_execz .LBB5_4344; SI-NEXT: ; %bb.3: ; %bb20345; SI-NEXT: ; in Loop: Header=BB5_2 Depth=1346; SI-NEXT: v_add_i32_e32 v8, vcc, v9, v8347; SI-NEXT: s_add_u32 s6, s6, 1348; SI-NEXT: v_add_i32_e32 v4, vcc, 4, v4349; SI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc350; SI-NEXT: v_add_i32_e32 v6, vcc, 4, v6351; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc352; SI-NEXT: buffer_store_dword v8, v[2:3], s[8:11], 0 addr64353; SI-NEXT: s_addc_u32 s7, s7, 0354; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v2355; SI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc356; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]357; SI-NEXT: s_andn2_b64 s[4:5], s[4:5], exec358; SI-NEXT: s_and_b64 s[12:13], vcc, exec359; SI-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]360; SI-NEXT: .LBB5_4: ; %Flow361; SI-NEXT: ; in Loop: Header=BB5_2 Depth=1362; SI-NEXT: s_or_b64 exec, exec, s[0:1]363; SI-NEXT: s_and_b64 s[0:1], exec, s[4:5]364; SI-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]365; SI-NEXT: s_andn2_b64 exec, exec, s[2:3]366; SI-NEXT: s_cbranch_execnz .LBB5_2367; SI-NEXT: .LBB5_5: ; %bb26368; SI-NEXT: s_endpgm369bb:370 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0371 %tmp4 = sext i32 %tmp to i64372 %tmp5 = getelementptr inbounds i32, ptr addrspace(1) %arg3, i64 %tmp4373 %tmp6 = load i32, ptr addrspace(1) %tmp5, align 4374 %tmp7 = icmp sgt i32 %tmp6, 0375 %tmp8 = sext i32 %tmp6 to i64376 br i1 %tmp7, label %bb10, label %bb26377 378bb10: ; preds = %bb, %bb20379 %tmp11 = phi i64 [ %tmp23, %bb20 ], [ 0, %bb ]380 %tmp12 = add nsw i64 %tmp11, %tmp4381 %tmp13 = getelementptr inbounds i32, ptr addrspace(1) %arg1, i64 %tmp12382 %tmp14 = load i32, ptr addrspace(1) %tmp13, align 4383 %tmp15 = getelementptr inbounds i32, ptr addrspace(1) %arg2, i64 %tmp12384 %tmp16 = load i32, ptr addrspace(1) %tmp15, align 4385 %tmp17 = icmp ne i32 %tmp14, -1386 %tmp18 = icmp ne i32 %tmp16, -1387 %tmp19 = and i1 %tmp17, %tmp18388 br i1 %tmp19, label %bb20, label %bb26389 390bb20: ; preds = %bb10391 %tmp21 = add nsw i32 %tmp16, %tmp14392 %tmp22 = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 %tmp12393 store i32 %tmp21, ptr addrspace(1) %tmp22, align 4394 %tmp23 = add nuw nsw i64 %tmp11, 1395 %tmp24 = icmp slt i64 %tmp23, %tmp8396 br i1 %tmp24, label %bb10, label %bb26397 398bb26: ; preds = %bb10, %bb20, %bb399 ret void400}401 402attributes #0 = { nounwind readnone }403attributes #1 = { nounwind }404