3708 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9-W64 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize32 < %s | FileCheck -check-prefixes=GFX10-W32 %s4 5; Check that WQM isn't triggered by image load/store intrinsics.6define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, i32 %c) {7; GFX9-W64-LABEL: test1:8; GFX9-W64: ; %bb.0: ; %main_body9; GFX9-W64-NEXT: v_mov_b32_e32 v4, v010; GFX9-W64-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf unorm11; GFX9-W64-NEXT: s_waitcnt vmcnt(0)12; GFX9-W64-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf unorm13; GFX9-W64-NEXT: s_waitcnt vmcnt(0)14; GFX9-W64-NEXT: ; return to shader part epilog15;16; GFX10-W32-LABEL: test1:17; GFX10-W32: ; %bb.0: ; %main_body18; GFX10-W32-NEXT: v_mov_b32_e32 v4, v019; GFX10-W32-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm20; GFX10-W32-NEXT: s_waitcnt vmcnt(0)21; GFX10-W32-NEXT: image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm22; GFX10-W32-NEXT: ; return to shader part epilog23main_body:24 %tex = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0)25 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %tex, i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0)26 ret <4 x float> %tex27}28 29; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible30define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {31; GFX9-W64-LABEL: test2:32; GFX9-W64: ; %bb.0: ; %main_body33; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec34; GFX9-W64-NEXT: s_wqm_b64 exec, exec35; GFX9-W64-NEXT: s_mov_b32 m0, s336; GFX9-W64-NEXT: s_nop 037; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x38; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y39; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x40; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y41; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]42; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf43; GFX9-W64-NEXT: s_waitcnt vmcnt(0)44; GFX9-W64-NEXT: ; return to shader part epilog45;46; GFX10-W32-LABEL: test2:47; GFX10-W32: ; %bb.0: ; %main_body48; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo49; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo50; GFX10-W32-NEXT: s_mov_b32 m0, s351; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x52; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y53; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x54; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y55; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s056; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D57; GFX10-W32-NEXT: s_waitcnt vmcnt(0)58; GFX10-W32-NEXT: ; return to shader part epilog59main_body:60 %inst23 = extractelement <2 x float> %pos, i32 061 %inst24 = extractelement <2 x float> %pos, i32 162 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)63 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)64 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)65 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)66 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #067 ret <4 x float> %tex68}69 70; ... but disabled for stores (and, in this simple case, not re-enabled) ...71define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) {72; GFX9-W64-LABEL: test3:73; GFX9-W64: ; %bb.0: ; %main_body74; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec75; GFX9-W64-NEXT: s_wqm_b64 exec, exec76; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]77; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf78; GFX9-W64-NEXT: s_waitcnt vmcnt(0)79; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen80; GFX9-W64-NEXT: s_waitcnt vmcnt(0)81; GFX9-W64-NEXT: ; return to shader part epilog82;83; GFX10-W32-LABEL: test3:84; GFX10-W32: ; %bb.0: ; %main_body85; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo86; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo87; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s1288; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D89; GFX10-W32-NEXT: s_waitcnt vmcnt(0)90; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen91; GFX10-W32-NEXT: ; return to shader part epilog92main_body:93 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #094 %tex.1 = bitcast <4 x float> %tex to <4 x i32>95 %tex.2 = extractelement <4 x i32> %tex.1, i32 096 97 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %tex, <4 x i32> poison, i32 %tex.2, i32 0, i32 0, i32 0)98 99 ret <4 x float> %tex100}101 102define amdgpu_ps <4 x float> @test3_ptr_buf(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) {103; GFX9-W64-LABEL: test3_ptr_buf:104; GFX9-W64: ; %bb.0: ; %main_body105; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec106; GFX9-W64-NEXT: s_wqm_b64 exec, exec107; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]108; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf109; GFX9-W64-NEXT: s_waitcnt vmcnt(0)110; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen111; GFX9-W64-NEXT: s_waitcnt vmcnt(0)112; GFX9-W64-NEXT: ; return to shader part epilog113;114; GFX10-W32-LABEL: test3_ptr_buf:115; GFX10-W32: ; %bb.0: ; %main_body116; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo117; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo118; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12119; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D120; GFX10-W32-NEXT: s_waitcnt vmcnt(0)121; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v0, s[0:3], 0 idxen122; GFX10-W32-NEXT: ; return to shader part epilog123main_body:124 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0125 %tex.1 = bitcast <4 x float> %tex to <4 x i32>126 %tex.2 = extractelement <4 x i32> %tex.1, i32 0127 128 call void @llvm.amdgcn.struct.ptr.buffer.store.v4f32(<4 x float> %tex, ptr addrspace(8) poison, i32 %tex.2, i32 0, i32 0, i32 0)129 130 ret <4 x float> %tex131}132 133; ... and disabled for export.134define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {135; GFX9-W64-LABEL: test3x:136; GFX9-W64: ; %bb.0: ; %main_body137; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec138; GFX9-W64-NEXT: s_wqm_b64 exec, exec139; GFX9-W64-NEXT: s_mov_b32 m0, s3140; GFX9-W64-NEXT: s_nop 0141; GFX9-W64-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x142; GFX9-W64-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y143; GFX9-W64-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x144; GFX9-W64-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y145; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]146; GFX9-W64-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf147; GFX9-W64-NEXT: s_waitcnt vmcnt(0)148; GFX9-W64-NEXT: exp mrt0 v0, v1, v2, v3 done vm149; GFX9-W64-NEXT: s_endpgm150;151; GFX10-W32-LABEL: test3x:152; GFX10-W32: ; %bb.0: ; %main_body153; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo154; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo155; GFX10-W32-NEXT: s_mov_b32 m0, s3156; GFX10-W32-NEXT: v_interp_p1_f32_e32 v2, v0, attr0.x157; GFX10-W32-NEXT: v_interp_p1_f32_e32 v3, v0, attr0.y158; GFX10-W32-NEXT: v_interp_p2_f32_e32 v2, v1, attr0.x159; GFX10-W32-NEXT: v_interp_p2_f32_e32 v3, v1, attr0.y160; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s0161; GFX10-W32-NEXT: image_sample v[0:3], v[2:3], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D162; GFX10-W32-NEXT: s_waitcnt vmcnt(0)163; GFX10-W32-NEXT: exp mrt0 v0, v1, v2, v3 done vm164; GFX10-W32-NEXT: s_endpgm165main_body:166 %inst23 = extractelement <2 x float> %pos, i32 0167 %inst24 = extractelement <2 x float> %pos, i32 1168 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)169 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)170 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)171 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)172 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0173 %tex.0 = extractelement <4 x float> %tex, i32 0174 %tex.1 = extractelement <4 x float> %tex, i32 1175 %tex.2 = extractelement <4 x float> %tex, i32 2176 %tex.3 = extractelement <4 x float> %tex, i32 3177 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true)178 ret void179}180 181; Check that WQM is re-enabled when required.182define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, ptr addrspace(1) inreg %ptr, i32 %c, i32 %d, float %data) {183; GFX9-W64-LABEL: test4:184; GFX9-W64: ; %bb.0: ; %main_body185; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec186; GFX9-W64-NEXT: s_wqm_b64 exec, exec187; GFX9-W64-NEXT: v_mul_lo_u32 v4, v0, v1188; GFX9-W64-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1189; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]190; GFX9-W64-NEXT: s_waitcnt vmcnt(0)191; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf192; GFX9-W64-NEXT: s_waitcnt vmcnt(0)193; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen194; GFX9-W64-NEXT: s_waitcnt vmcnt(0)195; GFX9-W64-NEXT: ; return to shader part epilog196;197; GFX10-W32-LABEL: test4:198; GFX10-W32: ; %bb.0: ; %main_body199; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo200; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo201; GFX10-W32-NEXT: v_mul_lo_u32 v4, v0, v1202; GFX10-W32-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D203; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12204; GFX10-W32-NEXT: s_waitcnt vmcnt(0)205; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D206; GFX10-W32-NEXT: s_waitcnt vmcnt(0)207; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen208; GFX10-W32-NEXT: ; return to shader part epilog209main_body:210 %c.1 = mul i32 %c, %d211 212 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> poison, <4 x i32> poison, i32 %c.1, i32 0, i32 0, i32 0)213 %c.1.bc = bitcast i32 %c.1 to float214 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0215 %tex0 = extractelement <4 x float> %tex, i32 0216 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0217 ret <4 x float> %dtex218}219 220define amdgpu_ps <4 x float> @test4_ptr_buf(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, ptr addrspace(1) inreg %ptr, i32 %c, i32 %d, float %data) {221; GFX9-W64-LABEL: test4_ptr_buf:222; GFX9-W64: ; %bb.0: ; %main_body223; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec224; GFX9-W64-NEXT: s_wqm_b64 exec, exec225; GFX9-W64-NEXT: v_mul_lo_u32 v4, v0, v1226; GFX9-W64-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1227; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]228; GFX9-W64-NEXT: s_waitcnt vmcnt(0)229; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf230; GFX9-W64-NEXT: s_waitcnt vmcnt(0)231; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen232; GFX9-W64-NEXT: s_waitcnt vmcnt(0)233; GFX9-W64-NEXT: ; return to shader part epilog234;235; GFX10-W32-LABEL: test4_ptr_buf:236; GFX10-W32: ; %bb.0: ; %main_body237; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo238; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo239; GFX10-W32-NEXT: v_mul_lo_u32 v4, v0, v1240; GFX10-W32-NEXT: image_sample v0, v4, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D241; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12242; GFX10-W32-NEXT: s_waitcnt vmcnt(0)243; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D244; GFX10-W32-NEXT: s_waitcnt vmcnt(0)245; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen246; GFX10-W32-NEXT: ; return to shader part epilog247main_body:248 %c.1 = mul i32 %c, %d249 250 call void @llvm.amdgcn.struct.ptr.buffer.store.v4f32(<4 x float> poison, ptr addrspace(8) poison, i32 %c.1, i32 0, i32 0, i32 0)251 %c.1.bc = bitcast i32 %c.1 to float252 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0253 %tex0 = extractelement <4 x float> %tex, i32 0254 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0255 ret <4 x float> %dtex256}257 258; Check that WQM is triggered by the wqm intrinsic.259; WQM was inserting an unecessary v_mov to self after the v_add. Make sure this260; does not happen - the v_add should write the return reg directly.261define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) {262; GFX9-W64-LABEL: test5:263; GFX9-W64: ; %bb.0: ; %main_body264; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec265; GFX9-W64-NEXT: s_wqm_b64 exec, exec266; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0267; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1268; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen269; GFX9-W64-NEXT: s_nop 0270; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen271; GFX9-W64-NEXT: s_waitcnt vmcnt(0)272; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1273; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec274; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]275; GFX9-W64-NEXT: ; return to shader part epilog276;277; GFX10-W32-LABEL: test5:278; GFX10-W32: ; %bb.0: ; %main_body279; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo280; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo281; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0282; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1283; GFX10-W32-NEXT: s_clause 0x1284; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen285; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen286; GFX10-W32-NEXT: s_waitcnt vmcnt(0)287; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1288; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec289; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2290; GFX10-W32-NEXT: ; return to shader part epilog291main_body:292 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> poison, i32 %idx0, i32 0, i32 0, i32 0)293 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> poison, i32 %idx1, i32 0, i32 0, i32 0)294 %out = fadd float %src0, %src1295 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)296 ret float %out.0297}298 299define amdgpu_ps float @test5_ptr_buf(i32 inreg %idx0, i32 inreg %idx1) {300; GFX9-W64-LABEL: test5_ptr_buf:301; GFX9-W64: ; %bb.0: ; %main_body302; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec303; GFX9-W64-NEXT: s_wqm_b64 exec, exec304; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0305; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1306; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen307; GFX9-W64-NEXT: s_nop 0308; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen309; GFX9-W64-NEXT: s_waitcnt vmcnt(0)310; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1311; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec312; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]313; GFX9-W64-NEXT: ; return to shader part epilog314;315; GFX10-W32-LABEL: test5_ptr_buf:316; GFX10-W32: ; %bb.0: ; %main_body317; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo318; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo319; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0320; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1321; GFX10-W32-NEXT: s_clause 0x1322; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen323; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen324; GFX10-W32-NEXT: s_waitcnt vmcnt(0)325; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1326; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec327; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2328; GFX10-W32-NEXT: ; return to shader part epilog329main_body:330 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)331 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)332 %out = fadd float %src0, %src1333 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)334 ret float %out.0335}336 337; Check that the wqm intrinsic works correctly for integers.338define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) {339; GFX9-W64-LABEL: test6:340; GFX9-W64: ; %bb.0: ; %main_body341; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec342; GFX9-W64-NEXT: s_wqm_b64 exec, exec343; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0344; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1345; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen346; GFX9-W64-NEXT: s_nop 0347; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen348; GFX9-W64-NEXT: s_waitcnt vmcnt(0)349; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1350; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec351; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]352; GFX9-W64-NEXT: ; return to shader part epilog353;354; GFX10-W32-LABEL: test6:355; GFX10-W32: ; %bb.0: ; %main_body356; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo357; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo358; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0359; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1360; GFX10-W32-NEXT: s_clause 0x1361; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen362; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen363; GFX10-W32-NEXT: s_waitcnt vmcnt(0)364; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1365; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec366; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2367; GFX10-W32-NEXT: ; return to shader part epilog368main_body:369 %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> poison, i32 %idx0, i32 0, i32 0, i32 0)370 %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> poison, i32 %idx1, i32 0, i32 0, i32 0)371 %out = fadd float %src0, %src1372 %out.0 = bitcast float %out to i32373 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0)374 %out.2 = bitcast i32 %out.1 to float375 ret float %out.2376}377 378define amdgpu_ps float @test6_ptr_buf(i32 inreg %idx0, i32 inreg %idx1) {379; GFX9-W64-LABEL: test6_ptr_buf:380; GFX9-W64: ; %bb.0: ; %main_body381; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec382; GFX9-W64-NEXT: s_wqm_b64 exec, exec383; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0384; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1385; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen386; GFX9-W64-NEXT: s_nop 0387; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen388; GFX9-W64-NEXT: s_waitcnt vmcnt(0)389; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v1390; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec391; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]392; GFX9-W64-NEXT: ; return to shader part epilog393;394; GFX10-W32-LABEL: test6_ptr_buf:395; GFX10-W32: ; %bb.0: ; %main_body396; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo397; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo398; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0399; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1400; GFX10-W32-NEXT: s_clause 0x1401; GFX10-W32-NEXT: buffer_load_dword v0, v0, s[0:3], 0 idxen402; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen403; GFX10-W32-NEXT: s_waitcnt vmcnt(0)404; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v1405; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec406; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2407; GFX10-W32-NEXT: ; return to shader part epilog408main_body:409 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)410 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)411 %out = fadd float %src0, %src1412 %out.0 = bitcast float %out to i32413 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0)414 %out.2 = bitcast i32 %out.1 to float415 ret float %out.2416}417 418; NOTE: llvm.amdgcn.wwm is deprecated, use llvm.amdgcn.strict.wwm instead.419 420; Check that WWM is triggered by the wwm intrinsic.421define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) {422; GFX9-W64-LABEL: test_wwm1:423; GFX9-W64: ; %bb.0: ; %main_body424; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1425; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0426; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1427; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen428; GFX9-W64-NEXT: s_nop 0429; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen430; GFX9-W64-NEXT: s_waitcnt vmcnt(0)431; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2432; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]433; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1434; GFX9-W64-NEXT: ; return to shader part epilog435;436; GFX10-W32-LABEL: test_wwm1:437; GFX10-W32: ; %bb.0: ; %main_body438; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1439; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0440; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1441; GFX10-W32-NEXT: s_clause 0x1442; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen443; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen444; GFX10-W32-NEXT: s_waitcnt vmcnt(0)445; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2446; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2447; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1448; GFX10-W32-NEXT: ; return to shader part epilog449main_body:450 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)451 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)452 %out = fadd float %src0, %src1453 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)454 ret float %out.0455}456 457; Same as above, but with an integer type.458define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) {459; GFX9-W64-LABEL: test_wwm2:460; GFX9-W64: ; %bb.0: ; %main_body461; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1462; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0463; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1464; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen465; GFX9-W64-NEXT: s_nop 0466; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen467; GFX9-W64-NEXT: s_waitcnt vmcnt(0)468; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2469; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]470; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1471; GFX9-W64-NEXT: ; return to shader part epilog472;473; GFX10-W32-LABEL: test_wwm2:474; GFX10-W32: ; %bb.0: ; %main_body475; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1476; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0477; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1478; GFX10-W32-NEXT: s_clause 0x1479; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen480; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen481; GFX10-W32-NEXT: s_waitcnt vmcnt(0)482; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2483; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2484; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1485; GFX10-W32-NEXT: ; return to shader part epilog486main_body:487 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)488 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)489 %src0.0 = bitcast float %src0 to i32490 %src1.0 = bitcast float %src1 to i32491 %out = add i32 %src0.0, %src1.0492 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)493 %out.1 = bitcast i32 %out.0 to float494 ret float %out.1495}496 497; Check that we don't leave WWM on for computations that don't require WWM,498; since that will lead clobbering things that aren't supposed to be clobbered499; in cases like this.500; We enforce this by checking that v_add gets emitted in the same block as501; WWM computations.502define amdgpu_ps float @test_wwm3(i32 inreg %idx) {503; GFX9-W64-LABEL: test_wwm3:504; GFX9-W64: ; %bb.0: ; %main_body505; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0506; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0507; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0508; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0509; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc510; GFX9-W64-NEXT: s_cbranch_execz .LBB13_2511; GFX9-W64-NEXT: ; %bb.1: ; %if512; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1513; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0514; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen515; GFX9-W64-NEXT: s_waitcnt vmcnt(0)516; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1517; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]518; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2519; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0520; GFX9-W64-NEXT: .LBB13_2: ; %endif521; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]522; GFX9-W64-NEXT: ; return to shader part epilog523;524; GFX10-W32-LABEL: test_wwm3:525; GFX10-W32: ; %bb.0: ; %main_body526; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0527; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0528; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0529; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0530; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo531; GFX10-W32-NEXT: s_cbranch_execz .LBB13_2532; GFX10-W32-NEXT: ; %bb.1: ; %if533; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1534; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0535; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen536; GFX10-W32-NEXT: s_waitcnt vmcnt(0)537; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1538; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2539; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2540; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0541; GFX10-W32-NEXT: .LBB13_2: ; %endif542; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1543; GFX10-W32-NEXT: ; return to shader part epilog544main_body:545 ; use mbcnt to make sure the branch is divergent546 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)547 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)548 %cc = icmp uge i32 %hi, 16549 br i1 %cc, label %endif, label %if550 551if:552 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)553 %out = fadd float %src, %src554 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)555 %out.1 = fadd float %src, %out.0556 br label %endif557 558endif:559 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]560 ret float %out.2561}562 563; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM564; write could clobber disabled channels in the non-WWM one.565; We enforce this by checking that v_mov gets emitted in the same block as566; WWM computations.567define amdgpu_ps float @test_wwm4(i32 inreg %idx) {568; GFX9-W64-LABEL: test_wwm4:569; GFX9-W64: ; %bb.0: ; %main_body570; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0571; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0572; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0573; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0574; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc575; GFX9-W64-NEXT: s_cbranch_execz .LBB14_2576; GFX9-W64-NEXT: ; %bb.1: ; %if577; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1578; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0579; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen580; GFX9-W64-NEXT: s_waitcnt vmcnt(0)581; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1582; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]583; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1584; GFX9-W64-NEXT: .LBB14_2: ; %endif585; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]586; GFX9-W64-NEXT: ; return to shader part epilog587;588; GFX10-W32-LABEL: test_wwm4:589; GFX10-W32: ; %bb.0: ; %main_body590; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0591; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0592; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0593; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0594; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo595; GFX10-W32-NEXT: s_cbranch_execz .LBB14_2596; GFX10-W32-NEXT: ; %bb.1: ; %if597; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -1598; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0599; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen600; GFX10-W32-NEXT: s_waitcnt vmcnt(0)601; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1602; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2603; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1604; GFX10-W32-NEXT: .LBB14_2: ; %endif605; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1606; GFX10-W32-NEXT: ; return to shader part epilog607main_body:608 ; use mbcnt to make sure the branch is divergent609 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)610 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)611 %cc = icmp uge i32 %hi, 16612 br i1 %cc, label %endif, label %if613 614if:615 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)616 %out = fadd float %src, %src617 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)618 br label %endif619 620endif:621 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]622 ret float %out.1623}624 625; Make sure the transition from Exact to WWM then WQM works properly.626define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) {627; GFX9-W64-LABEL: test_wwm5:628; GFX9-W64: ; %bb.0: ; %main_body629; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec630; GFX9-W64-NEXT: v_mov_b32_e32 v0, s0631; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen632; GFX9-W64-NEXT: s_waitcnt vmcnt(0)633; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen634; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -1635; GFX9-W64-NEXT: v_mov_b32_e32 v1, s1636; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen637; GFX9-W64-NEXT: s_waitcnt vmcnt(0)638; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v1639; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]640; GFX9-W64-NEXT: s_wqm_b64 exec, exec641; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1642; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v0643; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec644; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]645; GFX9-W64-NEXT: ; return to shader part epilog646;647; GFX10-W32-LABEL: test_wwm5:648; GFX10-W32: ; %bb.0: ; %main_body649; GFX10-W32-NEXT: v_mov_b32_e32 v0, s0650; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo651; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen652; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1653; GFX10-W32-NEXT: v_mov_b32_e32 v1, s1654; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0655; GFX10-W32-NEXT: s_waitcnt vmcnt(0)656; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen657; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1658; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen659; GFX10-W32-NEXT: s_waitcnt vmcnt(0)660; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v1661; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0662; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo663; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1664; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v0665; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec666; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s2667; GFX10-W32-NEXT: ; return to shader part epilog668main_body:669 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)670 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %src0, ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)671 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)672 %temp = fadd float %src1, %src1673 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp)674 %out = fadd float %temp.0, %temp.0675 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)676 ret float %out.0677}678 679; Check that WWM is turned on correctly across basic block boundaries.680; if..then..endif version681;SI-CHECK: buffer_load_dword682;VI-CHECK: flat_load_dword683;SI-CHECK: buffer_load_dword684;VI-CHECK: flat_load_dword685define amdgpu_ps float @test_wwm6_then() {686; GFX9-W64-LABEL: test_wwm6_then:687; GFX9-W64: ; %bb.0: ; %main_body688; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1689; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc690; GFX9-W64-NEXT: s_waitcnt vmcnt(0)691; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]692; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0693; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0694; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0695; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0696; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc697; GFX9-W64-NEXT: s_cbranch_execz .LBB16_2698; GFX9-W64-NEXT: ; %bb.1: ; %if699; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1700; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc701; GFX9-W64-NEXT: s_waitcnt vmcnt(0)702; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2703; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]704; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1705; GFX9-W64-NEXT: .LBB16_2: ; %endif706; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]707; GFX9-W64-NEXT: ; return to shader part epilog708;709; GFX10-W32-LABEL: test_wwm6_then:710; GFX10-W32: ; %bb.0: ; %main_body711; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1712; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc713; GFX10-W32-NEXT: s_waitcnt vmcnt(0)714; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0715; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0716; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0717; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0718; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0719; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo720; GFX10-W32-NEXT: s_cbranch_execz .LBB16_2721; GFX10-W32-NEXT: ; %bb.1: ; %if722; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1723; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc724; GFX10-W32-NEXT: s_waitcnt vmcnt(0)725; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2726; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1727; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1728; GFX10-W32-NEXT: .LBB16_2: ; %endif729; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0730; GFX10-W32-NEXT: ; return to shader part epilog731main_body:732 %src0 = load volatile float, ptr addrspace(1) poison733 ; use mbcnt to make sure the branch is divergent734 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)735 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)736 %cc = icmp uge i32 %hi, 16737 br i1 %cc, label %endif, label %if738 739if:740 %src1 = load volatile float, ptr addrspace(1) poison741 %out = fadd float %src0, %src1742 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)743 br label %endif744 745endif:746 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]747 ret float %out.1748}749 750; Check that WWM is turned on correctly across basic block boundaries.751; loop version752;SI-CHECK: buffer_load_dword753;VI-CHECK: flat_load_dword754;SI-CHECK: buffer_load_dword755;VI-CHECK: flat_load_dword756define amdgpu_ps float @test_wwm6_loop() {757; GFX9-W64-LABEL: test_wwm6_loop:758; GFX9-W64: ; %bb.0: ; %main_body759; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1760; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc761; GFX9-W64-NEXT: s_waitcnt vmcnt(0)762; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]763; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0764; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0765; GFX9-W64-NEXT: s_mov_b64 s[0:1], 0766; GFX9-W64-NEXT: .LBB17_1: ; %loop767; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=1768; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1769; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc770; GFX9-W64-NEXT: s_waitcnt vmcnt(0)771; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]772; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v3773; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3774; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -1775; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v2776; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]777; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1]778; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2779; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1]780; GFX9-W64-NEXT: s_cbranch_execnz .LBB17_1781; GFX9-W64-NEXT: ; %bb.2: ; %endloop782; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]783; GFX9-W64-NEXT: ; return to shader part epilog784;785; GFX10-W32-LABEL: test_wwm6_loop:786; GFX10-W32: ; %bb.0: ; %main_body787; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1788; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc789; GFX10-W32-NEXT: s_waitcnt vmcnt(0)790; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0791; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0792; GFX10-W32-NEXT: s_mov_b32 s0, 0793; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v0794; GFX10-W32-NEXT: .LBB17_1: ; %loop795; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=1796; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1797; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc798; GFX10-W32-NEXT: s_waitcnt vmcnt(0)799; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1800; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v3801; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -1802; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v2803; GFX10-W32-NEXT: s_mov_b32 exec_lo, s1804; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3805; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2806; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s0807; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s0808; GFX10-W32-NEXT: s_cbranch_execnz .LBB17_1809; GFX10-W32-NEXT: ; %bb.2: ; %endloop810; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s0811; GFX10-W32-NEXT: ; return to shader part epilog812main_body:813 %src0 = load volatile float, ptr addrspace(1) poison814 ; use mbcnt to make sure the branch is divergent815 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)816 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)817 br label %loop818 819loop:820 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ]821 %src1 = load volatile float, ptr addrspace(1) poison822 %out = fadd float %src0, %src1823 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)824 %counter.1 = sub i32 %counter, 1825 %cc = icmp ne i32 %counter.1, 0826 br i1 %cc, label %loop, label %endloop827 828endloop:829 ret float %out.0830}831 832; Check that @llvm.amdgcn.set.inactive disables WWM.833define amdgpu_ps void @test_wwm_set_inactive1(i32 inreg %idx) {834; GFX9-W64-LABEL: test_wwm_set_inactive1:835; GFX9-W64: ; %bb.0: ; %main_body836; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0837; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen838; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -1839; GFX9-W64-NEXT: s_waitcnt vmcnt(0)840; GFX9-W64-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[0:1]841; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v0842; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]843; GFX9-W64-NEXT: v_mov_b32_e32 v2, v0844; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen845; GFX9-W64-NEXT: s_endpgm846;847; GFX10-W32-LABEL: test_wwm_set_inactive1:848; GFX10-W32: ; %bb.0: ; %main_body849; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0850; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen851; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -1852; GFX10-W32-NEXT: s_waitcnt vmcnt(0)853; GFX10-W32-NEXT: v_cndmask_b32_e64 v0, 0, v2, s0854; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v0855; GFX10-W32-NEXT: s_mov_b32 exec_lo, s0856; GFX10-W32-NEXT: v_mov_b32_e32 v2, v0857; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen858; GFX10-W32-NEXT: s_endpgm859main_body:860 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)861 %src.0 = bitcast float %src to i32862 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0)863 %out = add i32 %src.1, %src.1864 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)865 %out.1 = bitcast i32 %out.0 to float866 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %out.1, ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)867 ret void868}869 870; Check that Strict WQM is triggered by the strict_wqm intrinsic.871define amdgpu_ps float @test_strict_wqm1(i32 inreg %idx0, i32 inreg %idx1) {872; GFX9-W64-LABEL: test_strict_wqm1:873; GFX9-W64: ; %bb.0: ; %main_body874; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec875; GFX9-W64-NEXT: s_wqm_b64 exec, exec876; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0877; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1878; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen879; GFX9-W64-NEXT: s_nop 0880; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen881; GFX9-W64-NEXT: s_waitcnt vmcnt(0)882; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v2883; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]884; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1885; GFX9-W64-NEXT: ; return to shader part epilog886;887; GFX10-W32-LABEL: test_strict_wqm1:888; GFX10-W32: ; %bb.0: ; %main_body889; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo890; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo891; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0892; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1893; GFX10-W32-NEXT: s_clause 0x1894; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen895; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen896; GFX10-W32-NEXT: s_waitcnt vmcnt(0)897; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v2898; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2899; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1900; GFX10-W32-NEXT: ; return to shader part epilog901main_body:902 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)903 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)904 %out = fadd float %src0, %src1905 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)906 ret float %out.0907}908 909; Same as above, but with an integer type.910define amdgpu_ps float @test_strict_wqm2(i32 inreg %idx0, i32 inreg %idx1) {911; GFX9-W64-LABEL: test_strict_wqm2:912; GFX9-W64: ; %bb.0: ; %main_body913; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec914; GFX9-W64-NEXT: s_wqm_b64 exec, exec915; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0916; GFX9-W64-NEXT: v_mov_b32_e32 v2, s1917; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen918; GFX9-W64-NEXT: s_nop 0919; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen920; GFX9-W64-NEXT: s_waitcnt vmcnt(0)921; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v2922; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]923; GFX9-W64-NEXT: v_mov_b32_e32 v0, v1924; GFX9-W64-NEXT: ; return to shader part epilog925;926; GFX10-W32-LABEL: test_strict_wqm2:927; GFX10-W32: ; %bb.0: ; %main_body928; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo929; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo930; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0931; GFX10-W32-NEXT: v_mov_b32_e32 v2, s1932; GFX10-W32-NEXT: s_clause 0x1933; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen934; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen935; GFX10-W32-NEXT: s_waitcnt vmcnt(0)936; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v2937; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2938; GFX10-W32-NEXT: v_mov_b32_e32 v0, v1939; GFX10-W32-NEXT: ; return to shader part epilog940main_body:941 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)942 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)943 %src0.0 = bitcast float %src0 to i32944 %src1.0 = bitcast float %src1 to i32945 %out = add i32 %src0.0, %src1.0946 %out.0 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %out)947 %out.1 = bitcast i32 %out.0 to float948 ret float %out.1949}950 951; Check that we don't leave Strict WQM on for computations that don't require it,952; since that will lead clobbering things that aren't supposed to be clobbered953; in cases like this.954; We enforce this by checking that v_add gets emitted in the same block as955; WWM computations.956define amdgpu_ps float @test_strict_wqm3(i32 inreg %idx) {957; GFX9-W64-LABEL: test_strict_wqm3:958; GFX9-W64: ; %bb.0: ; %main_body959; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0960; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0961; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0962; GFX9-W64-NEXT: v_mov_b32_e32 v0, 0963; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc964; GFX9-W64-NEXT: s_cbranch_execz .LBB21_2965; GFX9-W64-NEXT: ; %bb.1: ; %if966; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec967; GFX9-W64-NEXT: s_wqm_b64 exec, exec968; GFX9-W64-NEXT: v_mov_b32_e32 v1, s0969; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen970; GFX9-W64-NEXT: s_waitcnt vmcnt(0)971; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v1972; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]973; GFX9-W64-NEXT: v_mov_b32_e32 v0, v2974; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v0975; GFX9-W64-NEXT: .LBB21_2: ; %endif976; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]977; GFX9-W64-NEXT: ; return to shader part epilog978;979; GFX10-W32-LABEL: test_strict_wqm3:980; GFX10-W32: ; %bb.0: ; %main_body981; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0982; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v0983; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v0984; GFX10-W32-NEXT: v_mov_b32_e32 v0, 0985; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo986; GFX10-W32-NEXT: s_cbranch_execz .LBB21_2987; GFX10-W32-NEXT: ; %bb.1: ; %if988; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo989; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo990; GFX10-W32-NEXT: v_mov_b32_e32 v1, s0991; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen992; GFX10-W32-NEXT: s_waitcnt vmcnt(0)993; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v1994; GFX10-W32-NEXT: s_mov_b32 exec_lo, s2995; GFX10-W32-NEXT: v_mov_b32_e32 v0, v2996; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v0997; GFX10-W32-NEXT: .LBB21_2: ; %endif998; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s1999; GFX10-W32-NEXT: ; return to shader part epilog1000main_body:1001 ; use mbcnt to make sure the branch is divergent1002 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)1003 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)1004 %cc = icmp uge i32 %hi, 161005 br i1 %cc, label %endif, label %if1006 1007if:1008 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)1009 %out = fadd float %src, %src1010 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)1011 %out.1 = fadd float %src, %out.01012 br label %endif1013 1014endif:1015 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]1016 ret float %out.21017}1018 1019; Check that Strict WQM writes aren't coalesced with non-strict writes, since1020; the Strict WQM write could clobber disabled channels in the non-strict one.1021; We enforce this by checking that v_mov gets emitted in the same block as1022; WWM computations.1023define amdgpu_ps float @test_strict_wqm4(i32 inreg %idx) {1024; GFX9-W64-LABEL: test_strict_wqm4:1025; GFX9-W64: ; %bb.0: ; %main_body1026; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01027; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v01028; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v01029; GFX9-W64-NEXT: v_mov_b32_e32 v0, 01030; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc1031; GFX9-W64-NEXT: s_cbranch_execz .LBB22_21032; GFX9-W64-NEXT: ; %bb.1: ; %if1033; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec1034; GFX9-W64-NEXT: s_wqm_b64 exec, exec1035; GFX9-W64-NEXT: v_mov_b32_e32 v1, s01036; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen1037; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1038; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v11039; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]1040; GFX9-W64-NEXT: v_mov_b32_e32 v0, v11041; GFX9-W64-NEXT: .LBB22_2: ; %endif1042; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]1043; GFX9-W64-NEXT: ; return to shader part epilog1044;1045; GFX10-W32-LABEL: test_strict_wqm4:1046; GFX10-W32: ; %bb.0: ; %main_body1047; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01048; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v01049; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v01050; GFX10-W32-NEXT: v_mov_b32_e32 v0, 01051; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo1052; GFX10-W32-NEXT: s_cbranch_execz .LBB22_21053; GFX10-W32-NEXT: ; %bb.1: ; %if1054; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo1055; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1056; GFX10-W32-NEXT: v_mov_b32_e32 v1, s01057; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen1058; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1059; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v11060; GFX10-W32-NEXT: s_mov_b32 exec_lo, s21061; GFX10-W32-NEXT: v_mov_b32_e32 v0, v11062; GFX10-W32-NEXT: .LBB22_2: ; %endif1063; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s11064; GFX10-W32-NEXT: ; return to shader part epilog1065main_body:1066 ; use mbcnt to make sure the branch is divergent1067 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)1068 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)1069 %cc = icmp uge i32 %hi, 161070 br i1 %cc, label %endif, label %if1071 1072if:1073 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)1074 %out = fadd float %src, %src1075 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)1076 br label %endif1077 1078endif:1079 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]1080 ret float %out.11081}1082 1083; Make sure the transition from Exact to Strict WQM then WQM works properly.1084define amdgpu_ps float @test_strict_wqm5(i32 inreg %idx0, i32 inreg %idx1) {1085; GFX9-W64-LABEL: test_strict_wqm5:1086; GFX9-W64: ; %bb.0: ; %main_body1087; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec1088; GFX9-W64-NEXT: v_mov_b32_e32 v0, s01089; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen1090; GFX9-W64-NEXT: s_mov_b64 s[4:5], exec1091; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1092; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1093; GFX9-W64-NEXT: s_wqm_b64 exec, exec1094; GFX9-W64-NEXT: v_mov_b32_e32 v1, s11095; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen1096; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1097; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v11098; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]1099; GFX9-W64-NEXT: s_wqm_b64 exec, exec1100; GFX9-W64-NEXT: v_mov_b32_e32 v0, v11101; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v01102; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec1103; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]1104; GFX9-W64-NEXT: ; return to shader part epilog1105;1106; GFX10-W32-LABEL: test_strict_wqm5:1107; GFX10-W32: ; %bb.0: ; %main_body1108; GFX10-W32-NEXT: v_mov_b32_e32 v0, s01109; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo1110; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen1111; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1112; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1113; GFX10-W32-NEXT: v_mov_b32_e32 v1, s11114; GFX10-W32-NEXT: s_mov_b32 exec_lo, s01115; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1116; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1117; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1118; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1119; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen1120; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1121; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v11122; GFX10-W32-NEXT: s_mov_b32 exec_lo, s01123; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1124; GFX10-W32-NEXT: v_mov_b32_e32 v0, v11125; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v01126; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec1127; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s21128; GFX10-W32-NEXT: ; return to shader part epilog1129main_body:1130 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)1131 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %src0, ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)1132 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)1133 %temp = fadd float %src1, %src11134 %temp.0 = call float @llvm.amdgcn.strict.wqm.f32(float %temp)1135 %out = fadd float %temp.0, %temp.01136 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)1137 ret float %out.01138}1139 1140; Check that Strict WQM is turned on correctly across basic block boundaries.1141; if..then..endif version1142;SI-CHECK: buffer_load_dword1143;VI-CHECK: flat_load_dword1144;SI-CHECK: buffer_load_dword1145;VI-CHECK: flat_load_dword1146define amdgpu_ps float @test_strict_wqm6_then() {1147; GFX9-W64-LABEL: test_strict_wqm6_then:1148; GFX9-W64: ; %bb.0: ; %main_body1149; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec1150; GFX9-W64-NEXT: s_wqm_b64 exec, exec1151; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc1152; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1153; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]1154; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01155; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v01156; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v01157; GFX9-W64-NEXT: v_mov_b32_e32 v0, 01158; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1159; GFX9-W64-NEXT: s_cbranch_execz .LBB24_21160; GFX9-W64-NEXT: ; %bb.1: ; %if1161; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec1162; GFX9-W64-NEXT: s_wqm_b64 exec, exec1163; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc1164; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1165; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v21166; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]1167; GFX9-W64-NEXT: v_mov_b32_e32 v0, v11168; GFX9-W64-NEXT: .LBB24_2: ; %endif1169; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]1170; GFX9-W64-NEXT: ; return to shader part epilog1171;1172; GFX10-W32-LABEL: test_strict_wqm6_then:1173; GFX10-W32: ; %bb.0: ; %main_body1174; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1175; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1176; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc1177; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1178; GFX10-W32-NEXT: s_mov_b32 exec_lo, s01179; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01180; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v01181; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v01182; GFX10-W32-NEXT: v_mov_b32_e32 v0, 01183; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo1184; GFX10-W32-NEXT: s_cbranch_execz .LBB24_21185; GFX10-W32-NEXT: ; %bb.1: ; %if1186; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo1187; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1188; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc1189; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1190; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v21191; GFX10-W32-NEXT: s_mov_b32 exec_lo, s11192; GFX10-W32-NEXT: v_mov_b32_e32 v0, v11193; GFX10-W32-NEXT: .LBB24_2: ; %endif1194; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s01195; GFX10-W32-NEXT: ; return to shader part epilog1196main_body:1197 %src0 = load volatile float, ptr addrspace(1) poison1198 ; use mbcnt to make sure the branch is divergent1199 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)1200 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)1201 %cc = icmp uge i32 %hi, 161202 br i1 %cc, label %endif, label %if1203 1204if:1205 %src1 = load volatile float, ptr addrspace(1) poison1206 %out = fadd float %src0, %src11207 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)1208 br label %endif1209 1210endif:1211 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]1212 ret float %out.11213}1214 1215; Check that Strict WQM is turned on correctly across basic block boundaries.1216; loop version1217;SI-CHECK: buffer_load_dword1218;VI-CHECK: flat_load_dword1219;SI-CHECK: buffer_load_dword1220;VI-CHECK: flat_load_dword1221define amdgpu_ps float @test_strict_wqm6_loop() {1222; GFX9-W64-LABEL: test_strict_wqm6_loop:1223; GFX9-W64: ; %bb.0: ; %main_body1224; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec1225; GFX9-W64-NEXT: s_wqm_b64 exec, exec1226; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc1227; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1228; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]1229; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01230; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v01231; GFX9-W64-NEXT: s_mov_b64 s[0:1], 01232; GFX9-W64-NEXT: .LBB25_1: ; %loop1233; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=11234; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec1235; GFX9-W64-NEXT: s_wqm_b64 exec, exec1236; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc1237; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1238; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]1239; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v31240; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31241; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec1242; GFX9-W64-NEXT: s_wqm_b64 exec, exec1243; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v21244; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]1245; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1246; GFX9-W64-NEXT: v_mov_b32_e32 v0, v21247; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1]1248; GFX9-W64-NEXT: s_cbranch_execnz .LBB25_11249; GFX9-W64-NEXT: ; %bb.2: ; %endloop1250; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]1251; GFX9-W64-NEXT: ; return to shader part epilog1252;1253; GFX10-W32-LABEL: test_strict_wqm6_loop:1254; GFX10-W32: ; %bb.0: ; %main_body1255; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1256; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1257; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc1258; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1259; GFX10-W32-NEXT: s_mov_b32 exec_lo, s01260; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 01261; GFX10-W32-NEXT: s_mov_b32 s0, 01262; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v01263; GFX10-W32-NEXT: .LBB25_1: ; %loop1264; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=11265; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo1266; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1267; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc1268; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1269; GFX10-W32-NEXT: s_mov_b32 exec_lo, s11270; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v31271; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo1272; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1273; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v21274; GFX10-W32-NEXT: s_mov_b32 exec_lo, s11275; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v31276; GFX10-W32-NEXT: v_mov_b32_e32 v0, v21277; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s01278; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s01279; GFX10-W32-NEXT: s_cbranch_execnz .LBB25_11280; GFX10-W32-NEXT: ; %bb.2: ; %endloop1281; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s01282; GFX10-W32-NEXT: ; return to shader part epilog1283main_body:1284 %src0 = load volatile float, ptr addrspace(1) poison1285 ; use mbcnt to make sure the branch is divergent1286 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)1287 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)1288 br label %loop1289 1290loop:1291 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ]1292 %src1 = load volatile float, ptr addrspace(1) poison1293 %out = fadd float %src0, %src11294 %out.0 = call float @llvm.amdgcn.strict.wqm.f32(float %out)1295 %counter.1 = sub i32 %counter, 11296 %cc = icmp ne i32 %counter.1, 01297 br i1 %cc, label %loop, label %endloop1298 1299endloop:1300 ret float %out.01301}1302 1303; Check that enabling WQM anywhere enables WQM for the set.inactive source.1304define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) {1305; GFX9-W64-LABEL: test_set_inactive2:1306; GFX9-W64: ; %bb.0: ; %main_body1307; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec1308; GFX9-W64-NEXT: s_wqm_b64 exec, exec1309; GFX9-W64-NEXT: v_mov_b32_e32 v0, s11310; GFX9-W64-NEXT: v_mov_b32_e32 v2, s01311; GFX9-W64-NEXT: buffer_load_dword v1, v0, s[0:3], 0 idxen1312; GFX9-W64-NEXT: s_nop 01313; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen1314; GFX9-W64-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $exec1315; GFX9-W64-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $exec1316; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]1317; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1318; GFX9-W64-NEXT: v_add_u32_e32 v1, v2, v11319; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen1320; GFX9-W64-NEXT: s_endpgm1321;1322; GFX10-W32-LABEL: test_set_inactive2:1323; GFX10-W32: ; %bb.0: ; %main_body1324; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo1325; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1326; GFX10-W32-NEXT: v_mov_b32_e32 v0, s11327; GFX10-W32-NEXT: v_mov_b32_e32 v1, s01328; GFX10-W32-NEXT: s_clause 0x11329; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen1330; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen1331; GFX10-W32-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $exec1332; GFX10-W32-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $exec1333; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s21334; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1335; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v21336; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen1337; GFX10-W32-NEXT: s_endpgm1338main_body:1339 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)1340 %src1.0 = bitcast float %src1 to i321341 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 poison)1342 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)1343 %src0.0 = bitcast float %src0 to i321344 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0)1345 %out = add i32 %src0.1, %src1.11346 %out.0 = bitcast i32 %out to float1347 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %out.0, ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)1348 ret void1349}1350 1351; Check a case of one branch of an if-else requiring WQM, the other requiring1352; exact.1353; Note: In this particular case, the save-and-restore could be avoided if the1354; analysis understood that the two branches of the if-else are mutually1355; exclusive.1356define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {1357; GFX9-W64-LABEL: test_control_flow_0:1358; GFX9-W64: ; %bb.0: ; %main_body1359; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1360; GFX9-W64-NEXT: s_wqm_b64 exec, exec1361; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11362; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc1363; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15]1364; GFX9-W64-NEXT: s_cbranch_execz .LBB27_21365; GFX9-W64-NEXT: ; %bb.1: ; %ELSE1366; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13]1367; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1368; GFX9-W64-NEXT: ; implicit-def: $vgpr01369; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17]1370; GFX9-W64-NEXT: .LBB27_2: ; %Flow1371; GFX9-W64-NEXT: s_andn2_saveexec_b64 s[14:15], s[14:15]1372; GFX9-W64-NEXT: s_cbranch_execz .LBB27_41373; GFX9-W64-NEXT: ; %bb.3: ; %IF1374; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x11375; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1376; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x11377; GFX9-W64-NEXT: .LBB27_4: ; %END1378; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]1379; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1380; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1381; GFX9-W64-NEXT: v_mov_b32_e32 v0, v21382; GFX9-W64-NEXT: ; return to shader part epilog1383;1384; GFX10-W32-LABEL: test_control_flow_0:1385; GFX10-W32: ; %bb.0: ; %main_body1386; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1387; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1388; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo1389; GFX10-W32-NEXT: v_cmpx_ne_u32_e32 0, v11390; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s131391; GFX10-W32-NEXT: s_cbranch_execz .LBB27_21392; GFX10-W32-NEXT: ; %bb.1: ; %ELSE1393; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s121394; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1395; GFX10-W32-NEXT: ; implicit-def: $vgpr01396; GFX10-W32-NEXT: s_mov_b32 exec_lo, s141397; GFX10-W32-NEXT: .LBB27_2: ; %Flow1398; GFX10-W32-NEXT: s_andn2_saveexec_b32 s13, s131399; GFX10-W32-NEXT: s_cbranch_execz .LBB27_41400; GFX10-W32-NEXT: ; %bb.3: ; %IF1401; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1402; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1403; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1404; GFX10-W32-NEXT: .LBB27_4: ; %END1405; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s131406; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121407; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1408; GFX10-W32-NEXT: v_mov_b32_e32 v0, v21409; GFX10-W32-NEXT: ; return to shader part epilog1410main_body:1411 %cmp = icmp eq i32 %z, 01412 br i1 %cmp, label %IF, label %ELSE1413 1414IF:1415 %c.bc = bitcast i32 %c to float1416 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01417 %tex0 = extractelement <4 x float> %tex, i32 01418 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01419 %data.if = extractelement <4 x float> %dtex, i32 01420 br label %END1421 1422ELSE:1423 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 %c, i32 0, i32 0, i32 0)1424 br label %END1425 1426END:1427 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]1428 ret float %r1429}1430 1431; Reverse branch order compared to the previous test.1432define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {1433; GFX9-W64-LABEL: test_control_flow_1:1434; GFX9-W64: ; %bb.0: ; %main_body1435; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1436; GFX9-W64-NEXT: s_wqm_b64 exec, exec1437; GFX9-W64-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11438; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc1439; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15]1440; GFX9-W64-NEXT: s_cbranch_execz .LBB28_21441; GFX9-W64-NEXT: ; %bb.1: ; %IF1442; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x11443; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1444; GFX9-W64-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x11445; GFX9-W64-NEXT: ; implicit-def: $vgpr01446; GFX9-W64-NEXT: .LBB28_2: ; %Flow1447; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], s[14:15]1448; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1449; GFX9-W64-NEXT: s_and_b64 s[0:1], exec, s[0:1]1450; GFX9-W64-NEXT: s_xor_b64 exec, exec, s[0:1]1451; GFX9-W64-NEXT: s_cbranch_execz .LBB28_41452; GFX9-W64-NEXT: ; %bb.3: ; %ELSE1453; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1454; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1455; GFX9-W64-NEXT: .LBB28_4: ; %END1456; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]1457; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1458; GFX9-W64-NEXT: v_mov_b32_e32 v0, v21459; GFX9-W64-NEXT: ; return to shader part epilog1460;1461; GFX10-W32-LABEL: test_control_flow_1:1462; GFX10-W32: ; %bb.0: ; %main_body1463; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1464; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1465; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo1466; GFX10-W32-NEXT: v_cmpx_ne_u32_e32 0, v11467; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s131468; GFX10-W32-NEXT: s_cbranch_execz .LBB28_21469; GFX10-W32-NEXT: ; %bb.1: ; %IF1470; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1471; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1472; GFX10-W32-NEXT: image_sample v2, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1473; GFX10-W32-NEXT: ; implicit-def: $vgpr01474; GFX10-W32-NEXT: .LBB28_2: ; %Flow1475; GFX10-W32-NEXT: s_or_saveexec_b32 s0, s131476; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121477; GFX10-W32-NEXT: s_and_b32 s0, exec_lo, s01478; GFX10-W32-NEXT: s_xor_b32 exec_lo, exec_lo, s01479; GFX10-W32-NEXT: s_cbranch_execz .LBB28_41480; GFX10-W32-NEXT: ; %bb.3: ; %ELSE1481; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1482; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1483; GFX10-W32-NEXT: .LBB28_4: ; %END1484; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s01485; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1486; GFX10-W32-NEXT: v_mov_b32_e32 v0, v21487; GFX10-W32-NEXT: ; return to shader part epilog1488main_body:1489 %cmp = icmp eq i32 %z, 01490 br i1 %cmp, label %ELSE, label %IF1491 1492IF:1493 %c.bc = bitcast i32 %c to float1494 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01495 %tex0 = extractelement <4 x float> %tex, i32 01496 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01497 %data.if = extractelement <4 x float> %dtex, i32 01498 br label %END1499 1500ELSE:1501 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 %c, i32 0, i32 0, i32 0)1502 br label %END1503 1504END:1505 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]1506 ret float %r1507}1508 1509; Check that branch conditions are properly marked as needing WQM...1510define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) {1511; GFX9-W64-LABEL: test_control_flow_2:1512; GFX9-W64: ; %bb.0: ; %main_body1513; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1514; GFX9-W64-NEXT: s_wqm_b64 exec, exec1515; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1516; GFX9-W64-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen1517; GFX9-W64-NEXT: s_wqm_b64 exec, exec1518; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen1519; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1520; GFX9-W64-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen1521; GFX9-W64-NEXT: s_wqm_b64 exec, exec1522; GFX9-W64-NEXT: s_waitcnt vmcnt(1)1523; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v01524; GFX9-W64-NEXT: ; implicit-def: $vgpr01525; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc1526; GFX9-W64-NEXT: s_xor_b64 s[14:15], exec, s[14:15]1527; GFX9-W64-NEXT: ; %bb.1: ; %ELSE1528; GFX9-W64-NEXT: v_lshlrev_b32_e32 v0, 2, v51529; GFX9-W64-NEXT: ; implicit-def: $vgpr51530; GFX9-W64-NEXT: ; %bb.2: ; %Flow1531; GFX9-W64-NEXT: s_andn2_saveexec_b64 s[14:15], s[14:15]1532; GFX9-W64-NEXT: ; %bb.3: ; %IF1533; GFX9-W64-NEXT: v_lshl_add_u32 v0, v5, 1, v51534; GFX9-W64-NEXT: ; %bb.4: ; %END1535; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]1536; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1537; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf1538; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1539; GFX9-W64-NEXT: ; return to shader part epilog1540;1541; GFX10-W32-LABEL: test_control_flow_2:1542; GFX10-W32: ; %bb.0: ; %main_body1543; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1544; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1545; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121546; GFX10-W32-NEXT: buffer_store_dword v3, v0, s[0:3], 0 idxen1547; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1548; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[0:3], 0 idxen1549; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1550; GFX10-W32-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0, v01551; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121552; GFX10-W32-NEXT: buffer_store_dword v4, v2, s[0:3], 0 idxen1553; GFX10-W32-NEXT: ; implicit-def: $vgpr01554; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1555; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo1556; GFX10-W32-NEXT: s_xor_b32 s13, exec_lo, s131557; GFX10-W32-NEXT: ; %bb.1: ; %ELSE1558; GFX10-W32-NEXT: v_lshlrev_b32_e32 v0, 2, v51559; GFX10-W32-NEXT: ; implicit-def: $vgpr51560; GFX10-W32-NEXT: ; %bb.2: ; %Flow1561; GFX10-W32-NEXT: s_andn2_saveexec_b32 s13, s131562; GFX10-W32-NEXT: ; %bb.3: ; %IF1563; GFX10-W32-NEXT: v_lshl_add_u32 v0, v5, 1, v51564; GFX10-W32-NEXT: ; %bb.4: ; %END1565; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s131566; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121567; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D1568; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1569; GFX10-W32-NEXT: ; return to shader part epilog1570main_body:1571 %idx.1 = extractelement <3 x i32> %idx, i32 01572 %data.1 = extractelement <2 x float> %data, i32 01573 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data.1, ptr addrspace(8) poison, i32 %idx.1, i32 0, i32 0, i32 0)1574 1575 ; The load that determines the branch (and should therefore be WQM) is1576 ; surrounded by stores that require disabled WQM.1577 %idx.2 = extractelement <3 x i32> %idx, i32 11578 %z = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx.2, i32 0, i32 0, i32 0)1579 1580 %idx.3 = extractelement <3 x i32> %idx, i32 21581 %data.3 = extractelement <2 x float> %data, i32 11582 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data.3, ptr addrspace(8) poison, i32 %idx.3, i32 0, i32 0, i32 0)1583 1584 %cc = fcmp ogt float %z, 0.01585 br i1 %cc, label %IF, label %ELSE1586 1587IF:1588 %coord.IF = mul i32 %coord, 31589 br label %END1590 1591ELSE:1592 %coord.ELSE = mul i32 %coord, 41593 br label %END1594 1595END:1596 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ]1597 %coord.END.bc = bitcast i32 %coord.END to float1598 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01599 ret <4 x float> %tex1600}1601 1602; ... but only if they really do need it.1603define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) {1604; GFX9-W64-LABEL: test_control_flow_3:1605; GFX9-W64: ; %bb.0: ; %main_body1606; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1607; GFX9-W64-NEXT: s_wqm_b64 exec, exec1608; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x11609; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1610; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1611; GFX9-W64-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x11612; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1613; GFX9-W64-NEXT: v_cmp_nlt_f32_e32 vcc, 0, v11614; GFX9-W64-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen1615; GFX9-W64-NEXT: ; implicit-def: $vgpr01616; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc1617; GFX9-W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1618; GFX9-W64-NEXT: s_cbranch_execnz .LBB30_31619; GFX9-W64-NEXT: ; %bb.1: ; %Flow1620; GFX9-W64-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1621; GFX9-W64-NEXT: s_cbranch_execnz .LBB30_41622; GFX9-W64-NEXT: .LBB30_2: ; %END1623; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]1624; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1625; GFX9-W64-NEXT: s_branch .LBB30_51626; GFX9-W64-NEXT: .LBB30_3: ; %ELSE1627; GFX9-W64-NEXT: v_mul_f32_e32 v0, 4.0, v11628; GFX9-W64-NEXT: ; implicit-def: $vgpr11629; GFX9-W64-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1630; GFX9-W64-NEXT: s_cbranch_execz .LBB30_21631; GFX9-W64-NEXT: .LBB30_4: ; %IF1632; GFX9-W64-NEXT: v_mul_f32_e32 v0, 0x40400000, v11633; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]1634; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1635; GFX9-W64-NEXT: s_branch .LBB30_51636; GFX9-W64-NEXT: .LBB30_5:1637;1638; GFX10-W32-LABEL: test_control_flow_3:1639; GFX10-W32: ; %bb.0: ; %main_body1640; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1641; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1642; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1643; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121644; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1645; GFX10-W32-NEXT: image_sample v1, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1646; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1647; GFX10-W32-NEXT: buffer_store_dword v1, v0, s[0:3], 0 idxen1648; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1649; GFX10-W32-NEXT: ; implicit-def: $vgpr01650; GFX10-W32-NEXT: v_cmpx_nlt_f32_e32 0, v11651; GFX10-W32-NEXT: s_xor_b32 s0, exec_lo, s01652; GFX10-W32-NEXT: s_cbranch_execnz .LBB30_31653; GFX10-W32-NEXT: ; %bb.1: ; %Flow1654; GFX10-W32-NEXT: s_andn2_saveexec_b32 s0, s01655; GFX10-W32-NEXT: s_cbranch_execnz .LBB30_41656; GFX10-W32-NEXT: .LBB30_2: ; %END1657; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s01658; GFX10-W32-NEXT: s_branch .LBB30_51659; GFX10-W32-NEXT: .LBB30_3: ; %ELSE1660; GFX10-W32-NEXT: v_mul_f32_e32 v0, 4.0, v11661; GFX10-W32-NEXT: ; implicit-def: $vgpr11662; GFX10-W32-NEXT: s_andn2_saveexec_b32 s0, s01663; GFX10-W32-NEXT: s_cbranch_execz .LBB30_21664; GFX10-W32-NEXT: .LBB30_4: ; %IF1665; GFX10-W32-NEXT: v_mul_f32_e32 v0, 0x40400000, v11666; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s01667; GFX10-W32-NEXT: s_branch .LBB30_51668; GFX10-W32-NEXT: .LBB30_5:1669main_body:1670 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01671 %tex0 = extractelement <4 x float> %tex, i32 01672 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01673 %dtex.1 = extractelement <4 x float> %dtex, i32 01674 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %dtex.1, ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)1675 1676 %cc = fcmp ogt float %dtex.1, 0.01677 br i1 %cc, label %IF, label %ELSE1678 1679IF:1680 %tex.IF = fmul float %dtex.1, 3.01681 br label %END1682 1683ELSE:1684 %tex.ELSE = fmul float %dtex.1, 4.01685 br label %END1686 1687END:1688 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ]1689 ret float %tex.END1690}1691 1692; Another test that failed at some point because of terminator handling.1693define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) {1694; GFX9-W64-LABEL: test_control_flow_4:1695; GFX9-W64: ; %bb.0: ; %main_body1696; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1697; GFX9-W64-NEXT: s_wqm_b64 exec, exec1698; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11699; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc1700; GFX9-W64-NEXT: s_cbranch_execz .LBB31_21701; GFX9-W64-NEXT: ; %bb.1: ; %IF1702; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13]1703; GFX9-W64-NEXT: buffer_load_dword v1, off, s[0:3], 01704; GFX9-W64-NEXT: v_mov_b32_e32 v2, 11705; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1706; GFX9-W64-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen1707; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17]1708; GFX9-W64-NEXT: .LBB31_2: ; %END1709; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]1710; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x11711; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1712; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1713; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf1714; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1715; GFX9-W64-NEXT: ; return to shader part epilog1716;1717; GFX10-W32-LABEL: test_control_flow_4:1718; GFX10-W32: ; %bb.0: ; %main_body1719; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1720; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1721; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo1722; GFX10-W32-NEXT: v_cmpx_eq_u32_e32 0, v11723; GFX10-W32-NEXT: s_cbranch_execz .LBB31_21724; GFX10-W32-NEXT: ; %bb.1: ; %IF1725; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s121726; GFX10-W32-NEXT: buffer_load_dword v1, off, s[0:3], 01727; GFX10-W32-NEXT: v_mov_b32_e32 v2, 11728; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1729; GFX10-W32-NEXT: buffer_store_dword v1, v2, s[0:3], 0 idxen1730; GFX10-W32-NEXT: s_mov_b32 exec_lo, s141731; GFX10-W32-NEXT: .LBB31_2: ; %END1732; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s131733; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1734; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121735; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1736; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D1737; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1738; GFX10-W32-NEXT: ; return to shader part epilog1739main_body:1740 %cond = icmp eq i32 %y, 01741 br i1 %cond, label %IF, label %END1742 1743IF:1744 %data = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 0, i32 0, i32 0)1745 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 1, i32 0, i32 0, i32 0)1746 br label %END1747 1748END:1749 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01750 %tex0 = extractelement <4 x float> %tex, i32 01751 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01752 ret <4 x float> %dtex1753}1754 1755; Kill is performed in WQM mode so that uniform kill behaves correctly ...1756define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, ptr addrspace(1) inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) {1757; GFX9-W64-LABEL: test_kill_0:1758; GFX9-W64: ; %bb.0: ; %main_body1759; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1760; GFX9-W64-NEXT: s_wqm_b64 exec, exec1761; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1762; GFX9-W64-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf1763; GFX9-W64-NEXT: s_nop 01764; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1765; GFX9-W64-NEXT: s_wqm_b64 exec, exec1766; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v61767; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc1768; GFX9-W64-NEXT: s_cbranch_scc0 .LBB32_21769; GFX9-W64-NEXT: ; %bb.1: ; %main_body1770; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc1771; GFX9-W64-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x11772; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1773; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1774; GFX9-W64-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf1775; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1776; GFX9-W64-NEXT: v_add_f32_e32 v0, v7, v111777; GFX9-W64-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen1778; GFX9-W64-NEXT: v_add_f32_e32 v1, v8, v121779; GFX9-W64-NEXT: v_add_f32_e32 v2, v9, v131780; GFX9-W64-NEXT: v_add_f32_e32 v3, v10, v141781; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1782; GFX9-W64-NEXT: s_branch .LBB32_31783; GFX9-W64-NEXT: .LBB32_2:1784; GFX9-W64-NEXT: s_mov_b64 exec, 01785; GFX9-W64-NEXT: exp null off, off, off, off done vm1786; GFX9-W64-NEXT: s_endpgm1787; GFX9-W64-NEXT: .LBB32_3:1788;1789; GFX10-W32-LABEL: test_kill_0:1790; GFX10-W32: ; %bb.0: ; %main_body1791; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1792; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1793; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121794; GFX10-W32-NEXT: image_sample v[7:10], v4, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D1795; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen1796; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1797; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v61798; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo1799; GFX10-W32-NEXT: s_cbranch_scc0 .LBB32_21800; GFX10-W32-NEXT: ; %bb.1: ; %main_body1801; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo1802; GFX10-W32-NEXT: image_sample v0, v5, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1803; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121804; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1805; GFX10-W32-NEXT: image_sample v[11:14], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D1806; GFX10-W32-NEXT: buffer_store_dword v3, v1, s[0:3], 0 idxen1807; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1808; GFX10-W32-NEXT: v_add_f32_e32 v4, v8, v121809; GFX10-W32-NEXT: v_add_f32_e32 v5, v10, v141810; GFX10-W32-NEXT: v_add_f32_e32 v0, v7, v111811; GFX10-W32-NEXT: v_add_f32_e32 v2, v9, v131812; GFX10-W32-NEXT: v_mov_b32_e32 v1, v41813; GFX10-W32-NEXT: v_mov_b32_e32 v3, v51814; GFX10-W32-NEXT: s_branch .LBB32_31815; GFX10-W32-NEXT: .LBB32_2:1816; GFX10-W32-NEXT: s_mov_b32 exec_lo, 01817; GFX10-W32-NEXT: exp null off, off, off, off done vm1818; GFX10-W32-NEXT: s_endpgm1819; GFX10-W32-NEXT: .LBB32_3:1820main_body:1821 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01822 %idx.0 = extractelement <2 x i32> %idx, i32 01823 %data.0 = extractelement <2 x float> %data, i32 01824 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data.0, ptr addrspace(8) poison, i32 %idx.0, i32 0, i32 0, i32 0)1825 1826 %z.cmp = fcmp olt float %z, 0.01827 call void @llvm.amdgcn.kill(i1 %z.cmp)1828 1829 %idx.1 = extractelement <2 x i32> %idx, i32 11830 %data.1 = extractelement <2 x float> %data, i32 11831 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data.1, ptr addrspace(8) poison, i32 %idx.1, i32 0, i32 0, i32 0)1832 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01833 %tex2.0 = extractelement <4 x float> %tex2, i32 01834 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex2.0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01835 %out = fadd <4 x float> %tex, %dtex1836 1837 ret <4 x float> %out1838}1839 1840; ... but only if WQM is necessary.1841define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) {1842; GFX9-W64-LABEL: test_kill_1:1843; GFX9-W64: ; %bb.0: ; %main_body1844; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec1845; GFX9-W64-NEXT: s_wqm_b64 exec, exec1846; GFX9-W64-NEXT: v_mov_b32_e32 v5, v01847; GFX9-W64-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x11848; GFX9-W64-NEXT: v_mov_b32_e32 v4, v21849; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]1850; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1851; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf1852; GFX9-W64-NEXT: v_cmp_ngt_f32_e32 vcc, 0, v41853; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], vcc1854; GFX9-W64-NEXT: buffer_store_dword v5, off, s[0:3], 01855; GFX9-W64-NEXT: s_cbranch_scc0 .LBB33_21856; GFX9-W64-NEXT: ; %bb.1: ; %main_body1857; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc1858; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1859; GFX9-W64-NEXT: s_branch .LBB33_31860; GFX9-W64-NEXT: .LBB33_2:1861; GFX9-W64-NEXT: s_mov_b64 exec, 01862; GFX9-W64-NEXT: exp null off, off, off, off done vm1863; GFX9-W64-NEXT: s_endpgm1864; GFX9-W64-NEXT: .LBB33_3:1865;1866; GFX10-W32-LABEL: test_kill_1:1867; GFX10-W32: ; %bb.0: ; %main_body1868; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo1869; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1870; GFX10-W32-NEXT: v_mov_b32_e32 v5, v01871; GFX10-W32-NEXT: image_sample v0, v1, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D1872; GFX10-W32-NEXT: v_mov_b32_e32 v4, v21873; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s121874; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1875; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D1876; GFX10-W32-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0, v41877; GFX10-W32-NEXT: buffer_store_dword v5, off, s[0:3], 01878; GFX10-W32-NEXT: s_andn2_b32 s12, s12, vcc_lo1879; GFX10-W32-NEXT: s_cbranch_scc0 .LBB33_21880; GFX10-W32-NEXT: ; %bb.1: ; %main_body1881; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo1882; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1883; GFX10-W32-NEXT: s_branch .LBB33_31884; GFX10-W32-NEXT: .LBB33_2:1885; GFX10-W32-NEXT: s_mov_b32 exec_lo, 01886; GFX10-W32-NEXT: exp null off, off, off, off done vm1887; GFX10-W32-NEXT: s_endpgm1888; GFX10-W32-NEXT: .LBB33_3:1889main_body:1890 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01891 %tex0 = extractelement <4 x float> %tex, i32 01892 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #01893 1894 call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 0, i32 0, i32 0)1895 1896 %z.cmp = fcmp olt float %z, 0.01897 call void @llvm.amdgcn.kill(i1 %z.cmp)1898 1899 ret <4 x float> %dtex1900}1901 1902; Check prolog shaders.1903define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 {1904; GFX9-W64-LABEL: test_prolog_1:1905; GFX9-W64: ; %bb.0: ; %main_body1906; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec1907; GFX9-W64-NEXT: s_wqm_b64 exec, exec1908; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v11909; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]1910; GFX9-W64-NEXT: ; return to shader part epilog1911;1912; GFX10-W32-LABEL: test_prolog_1:1913; GFX10-W32: ; %bb.0: ; %main_body1914; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1915; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1916; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v11917; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s01918; GFX10-W32-NEXT: ; return to shader part epilog1919main_body:1920 %s = fadd float %a, %b1921 ret float %s1922}1923 1924define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind {1925; GFX9-W64-LABEL: test_loop_vcc:1926; GFX9-W64: ; %bb.0: ; %entry1927; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec1928; GFX9-W64-NEXT: s_wqm_b64 exec, exec1929; GFX9-W64-NEXT: v_mov_b32_e32 v7, v31930; GFX9-W64-NEXT: v_mov_b32_e32 v6, v21931; GFX9-W64-NEXT: v_mov_b32_e32 v5, v11932; GFX9-W64-NEXT: v_mov_b32_e32 v4, v01933; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]1934; GFX9-W64-NEXT: image_store v[4:7], v0, s[0:7] dmask:0xf unorm1935; GFX9-W64-NEXT: s_wqm_b64 exec, exec1936; GFX9-W64-NEXT: v_mov_b32_e32 v8, 01937; GFX9-W64-NEXT: s_mov_b32 s4, 0x40e000001938; GFX9-W64-NEXT: s_branch .LBB35_21939; GFX9-W64-NEXT: .LBB35_1: ; %body1940; GFX9-W64-NEXT: ; in Loop: Header=BB35_2 Depth=11941; GFX9-W64-NEXT: image_sample v[4:7], v0, s[0:7], s[0:3] dmask:0xf1942; GFX9-W64-NEXT: v_add_f32_e32 v8, 2.0, v81943; GFX9-W64-NEXT: s_cbranch_execz .LBB35_41944; GFX9-W64-NEXT: .LBB35_2: ; %loop1945; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=11946; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1947; GFX9-W64-NEXT: v_mov_b32_e32 v0, v41948; GFX9-W64-NEXT: v_cmp_lt_f32_e32 vcc, s4, v81949; GFX9-W64-NEXT: v_mov_b32_e32 v1, v51950; GFX9-W64-NEXT: v_mov_b32_e32 v2, v61951; GFX9-W64-NEXT: v_mov_b32_e32 v3, v71952; GFX9-W64-NEXT: s_cbranch_vccz .LBB35_11953; GFX9-W64-NEXT: ; %bb.3:1954; GFX9-W64-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr71955; GFX9-W64-NEXT: ; implicit-def: $vgpr81956; GFX9-W64-NEXT: .LBB35_4: ; %break1957; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]1958; GFX9-W64-NEXT: s_waitcnt vmcnt(0)1959; GFX9-W64-NEXT: ; return to shader part epilog1960;1961; GFX10-W32-LABEL: test_loop_vcc:1962; GFX10-W32: ; %bb.0: ; %entry1963; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo1964; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1965; GFX10-W32-NEXT: v_mov_b32_e32 v8, 01966; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s01967; GFX10-W32-NEXT: image_store v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm1968; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo1969; GFX10-W32-NEXT: s_branch .LBB35_21970; GFX10-W32-NEXT: .p2align 61971; GFX10-W32-NEXT: .LBB35_1: ; %body1972; GFX10-W32-NEXT: ; in Loop: Header=BB35_2 Depth=11973; GFX10-W32-NEXT: image_sample v[0:3], v4, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D1974; GFX10-W32-NEXT: v_add_f32_e32 v8, 2.0, v81975; GFX10-W32-NEXT: s_cbranch_execz .LBB35_41976; GFX10-W32-NEXT: .LBB35_2: ; %loop1977; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=11978; GFX10-W32-NEXT: v_cmp_lt_f32_e32 vcc_lo, 0x40e00000, v81979; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1980; GFX10-W32-NEXT: v_mov_b32_e32 v7, v31981; GFX10-W32-NEXT: v_mov_b32_e32 v6, v21982; GFX10-W32-NEXT: v_mov_b32_e32 v5, v11983; GFX10-W32-NEXT: v_mov_b32_e32 v4, v01984; GFX10-W32-NEXT: s_cbranch_vccz .LBB35_11985; GFX10-W32-NEXT: ; %bb.3:1986; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr31987; GFX10-W32-NEXT: ; implicit-def: $vgpr81988; GFX10-W32-NEXT: .LBB35_4: ; %break1989; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s01990; GFX10-W32-NEXT: s_waitcnt vmcnt(0)1991; GFX10-W32-NEXT: v_mov_b32_e32 v0, v41992; GFX10-W32-NEXT: v_mov_b32_e32 v1, v51993; GFX10-W32-NEXT: v_mov_b32_e32 v2, v61994; GFX10-W32-NEXT: v_mov_b32_e32 v3, v71995; GFX10-W32-NEXT: ; return to shader part epilog1996entry:1997 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %in, i32 15, i32 poison, <8 x i32> poison, i32 0, i32 0)1998 br label %loop1999 2000loop:2001 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ]2002 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ]2003 %cc = fcmp ogt float %ctr.iv, 7.02004 br i1 %cc, label %break, label %body2005 2006body:2007 %c.iv0 = extractelement <4 x float> %c.iv, i32 02008 %c.next = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.iv0, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02009 %ctr.next = fadd float %ctr.iv, 2.02010 br label %loop2011 2012break:2013 ret <4 x float> %c.iv2014}2015 2016; Only intrinsic stores need exact execution -- other stores do not have2017; externally visible effects and may require WQM for correctness.2018define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind {2019; GFX9-W64-LABEL: test_alloca:2020; GFX9-W64: ; %bb.0: ; %entry2021; GFX9-W64-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD02022; GFX9-W64-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD12023; GFX9-W64-NEXT: s_mov_b32 s10, -12024; GFX9-W64-NEXT: s_mov_b32 s11, 0xe000002025; GFX9-W64-NEXT: s_add_u32 s8, s8, s02026; GFX9-W64-NEXT: s_addc_u32 s9, s9, 02027; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec2028; GFX9-W64-NEXT: s_wqm_b64 exec, exec2029; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]2030; GFX9-W64-NEXT: buffer_store_dword v0, off, s[0:3], 02031; GFX9-W64-NEXT: s_wqm_b64 exec, exec2032; GFX9-W64-NEXT: s_mov_b32 s2, 02033; GFX9-W64-NEXT: buffer_store_dword v1, off, s[8:11], 02034; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2035; GFX9-W64-NEXT: v_lshl_add_u32 v1, v2, 2, s22036; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[8:11], 0 offen2037; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]2038; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2039; GFX9-W64-NEXT: image_sample v[1:4], v1, s[0:7], s[0:3] dmask:0xf2040; GFX9-W64-NEXT: v_mov_b32_e32 v5, 12041; GFX9-W64-NEXT: buffer_store_dword v0, v5, s[0:3], 0 idxen2042; GFX9-W64-NEXT: s_waitcnt vmcnt(1)2043; GFX9-W64-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 02044; GFX9-W64-NEXT: s_endpgm2045;2046; GFX10-W32-LABEL: test_alloca:2047; GFX10-W32: ; %bb.0: ; %entry2048; GFX10-W32-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD02049; GFX10-W32-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD12050; GFX10-W32-NEXT: s_mov_b32 s10, -12051; GFX10-W32-NEXT: s_mov_b32 s11, 0x31c160002052; GFX10-W32-NEXT: s_add_u32 s8, s8, s02053; GFX10-W32-NEXT: s_addc_u32 s9, s9, 02054; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo2055; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2056; GFX10-W32-NEXT: v_lshl_add_u32 v2, v2, 2, 02057; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s02058; GFX10-W32-NEXT: buffer_store_dword v0, off, s[0:3], 02059; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2060; GFX10-W32-NEXT: buffer_store_dword v1, off, s[8:11], 02061; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x02062; GFX10-W32-NEXT: buffer_load_dword v1, v2, s[8:11], 0 offen2063; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s02064; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2065; GFX10-W32-NEXT: image_sample v[1:4], v1, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D2066; GFX10-W32-NEXT: v_mov_b32_e32 v5, 12067; GFX10-W32-NEXT: buffer_store_dword v0, v5, s[0:3], 0 idxen2068; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2069; GFX10-W32-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 02070; GFX10-W32-NEXT: s_endpgm2071entry:2072 %array = alloca [32 x i32], align 4, addrspace(5)2073 2074 call void @llvm.amdgcn.raw.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 0, i32 0, i32 0)2075 2076 store volatile i32 %a, ptr addrspace(5) %array, align 42077 2078 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %data, ptr addrspace(8) poison, i32 1, i32 0, i32 0, i32 0)2079 2080 %c.gep = getelementptr [32 x i32], ptr addrspace(5) %array, i32 0, i32 %idx2081 %c = load i32, ptr addrspace(5) %c.gep, align 42082 %c.bc = bitcast i32 %c to float2083 %t = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02084 call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> %t, ptr addrspace(8) poison, i32 0, i32 0, i32 0)2085 2086 ret void2087}2088 2089; Must return to exact at the end of a non-void returning shader,2090; otherwise the EXEC mask exported by the epilog will be wrong. This is true2091; even if the shader has no kills, because a kill could have happened in a2092; previous shader fragment.2093define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind {2094; GFX9-W64-LABEL: test_nonvoid_return:2095; GFX9-W64: ; %bb.0:2096; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec2097; GFX9-W64-NEXT: s_wqm_b64 exec, exec2098; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x12099; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]2100; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2101; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf2102; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2103; GFX9-W64-NEXT: ; return to shader part epilog2104;2105; GFX10-W32-LABEL: test_nonvoid_return:2106; GFX10-W32: ; %bb.0:2107; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo2108; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2109; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D2110; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s02111; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2112; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D2113; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2114; GFX10-W32-NEXT: ; return to shader part epilog2115 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float poison, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02116 %tex0 = extractelement <4 x float> %tex, i32 02117 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02118 ret <4 x float> %dtex2119}2120 2121define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind {2122; GFX9-W64-LABEL: test_nonvoid_return_unreachable:2123; GFX9-W64: ; %bb.0: ; %entry2124; GFX9-W64-NEXT: s_wqm_b64 exec, exec2125; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x12126; GFX9-W64-NEXT: s_and_b64 exec, exec, exec2127; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2128; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf2129; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 12130; GFX9-W64-NEXT: s_cbranch_scc0 .LBB38_22131; GFX9-W64-NEXT: ; %bb.1: ; %else2132; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2133; GFX9-W64-NEXT: s_branch .LBB38_32134; GFX9-W64-NEXT: .LBB38_2: ; %if2135; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2136; GFX9-W64-NEXT: global_store_dwordx4 v[0:1], v[0:3], off2137; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2138; GFX9-W64-NEXT: .LBB38_3:2139;2140; GFX10-W32-LABEL: test_nonvoid_return_unreachable:2141; GFX10-W32: ; %bb.0: ; %entry2142; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2143; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_1D2144; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, exec_lo2145; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2146; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D2147; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 12148; GFX10-W32-NEXT: s_cbranch_scc0 .LBB38_22149; GFX10-W32-NEXT: ; %bb.1: ; %else2150; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2151; GFX10-W32-NEXT: s_branch .LBB38_32152; GFX10-W32-NEXT: .LBB38_2: ; %if2153; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2154; GFX10-W32-NEXT: global_store_dwordx4 v[0:1], v[0:3], off2155; GFX10-W32-NEXT: s_waitcnt_vscnt null, 0x02156; GFX10-W32-NEXT: .LBB38_3:2157entry:2158 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float poison, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02159 %tex0 = extractelement <4 x float> %tex, i32 02160 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02161 %cc = icmp sgt i32 %c, 02162 br i1 %cc, label %if, label %else2163 2164if:2165 store volatile <4 x float> %dtex, ptr addrspace(1) poison2166 unreachable2167 2168else:2169 ret <4 x float> %dtex2170}2171 2172; Test awareness that s_wqm_b64 clobbers SCC.2173define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 {2174; GFX9-W64-LABEL: test_scc:2175; GFX9-W64: ; %bb.0: ; %main_body2176; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec2177; GFX9-W64-NEXT: s_wqm_b64 exec, exec2178; GFX9-W64-NEXT: v_mov_b32_e32 v4, v02179; GFX9-W64-NEXT: s_cmp_lt_i32 s0, 12180; GFX9-W64-NEXT: s_cbranch_scc0 .LBB39_22181; GFX9-W64-NEXT: ; %bb.1: ; %else2182; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02183; GFX9-W64-NEXT: v_mov_b32_e32 v1, 12184; GFX9-W64-NEXT: image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf2185; GFX9-W64-NEXT: s_cbranch_execz .LBB39_32186; GFX9-W64-NEXT: s_branch .LBB39_42187; GFX9-W64-NEXT: .LBB39_2:2188; GFX9-W64-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr32189; GFX9-W64-NEXT: .LBB39_3: ; %if2190; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2191; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02192; GFX9-W64-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf2193; GFX9-W64-NEXT: .LBB39_4: ; %end2194; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]2195; GFX9-W64-NEXT: v_mov_b32_e32 v5, 1.02196; GFX9-W64-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen2197; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2198; GFX9-W64-NEXT: ; return to shader part epilog2199;2200; GFX10-W32-LABEL: test_scc:2201; GFX10-W32: ; %bb.0: ; %main_body2202; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo2203; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2204; GFX10-W32-NEXT: v_mov_b32_e32 v4, v02205; GFX10-W32-NEXT: s_cmp_lt_i32 s0, 12206; GFX10-W32-NEXT: s_cbranch_scc0 .LBB39_22207; GFX10-W32-NEXT: ; %bb.1: ; %else2208; GFX10-W32-NEXT: v_mov_b32_e32 v1, 12209; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02210; GFX10-W32-NEXT: image_sample v[0:3], v[0:1], s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_2D2211; GFX10-W32-NEXT: s_cbranch_execz .LBB39_32212; GFX10-W32-NEXT: s_branch .LBB39_42213; GFX10-W32-NEXT: .LBB39_2:2214; GFX10-W32-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr32215; GFX10-W32-NEXT: .LBB39_3: ; %if2216; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2217; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02218; GFX10-W32-NEXT: image_sample v[0:3], v0, s[0:7], s[0:3] dmask:0xf dim:SQ_RSRC_IMG_1D2219; GFX10-W32-NEXT: .LBB39_4: ; %end2220; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s12221; GFX10-W32-NEXT: v_mov_b32_e32 v5, 1.02222; GFX10-W32-NEXT: buffer_store_dword v5, v4, s[0:3], 0 idxen2223; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2224; GFX10-W32-NEXT: ; return to shader part epilog2225main_body:2226 %cc = icmp sgt i32 %sel, 02227 br i1 %cc, label %if, label %else2228 2229if:2230 %r.if = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float 0.0, <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02231 br label %end2232 2233else:2234 %r.else = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float 0.0, float bitcast (i32 1 to float), <8 x i32> poison, <4 x i32> poison, i1 false, i32 0, i32 0) #02235 br label %end2236 2237end:2238 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ]2239 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float 1.0, ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)2240 ret <4 x float> %r2241}2242 2243; Check a case of a block being entirely WQM except for a bit of WWM.2244; There was a bug where it forgot to enter and leave WWM.2245define amdgpu_ps float @test_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {2246; GFX9-W64-LABEL: test_wwm_within_wqm:2247; GFX9-W64: ; %bb.0: ; %main_body2248; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec2249; GFX9-W64-NEXT: s_wqm_b64 exec, exec2250; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12251; GFX9-W64-NEXT: v_mov_b32_e32 v1, 02252; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc2253; GFX9-W64-NEXT: s_cbranch_execz .LBB40_22254; GFX9-W64-NEXT: ; %bb.1: ; %IF2255; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12256; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2257; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12258; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2259; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v0, v02260; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -12261; GFX9-W64-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]2262; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2263; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2264; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)2265; GFX9-W64-NEXT: v_mov_b32_e32 v0, v22266; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v02267; GFX9-W64-NEXT: .LBB40_2: ; %ENDIF2268; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]2269; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]2270; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12271; GFX9-W64-NEXT: ; return to shader part epilog2272;2273; GFX10-W32-LABEL: test_wwm_within_wqm:2274; GFX10-W32: ; %bb.0: ; %main_body2275; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo2276; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2277; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v12278; GFX10-W32-NEXT: v_mov_b32_e32 v1, 02279; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo2280; GFX10-W32-NEXT: s_cbranch_execz .LBB40_22281; GFX10-W32-NEXT: ; %bb.1: ; %IF2282; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2283; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2284; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2285; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2286; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v0, v02287; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12288; GFX10-W32-NEXT: v_cndmask_b32_e64 v2, 0, v0, s02289; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2290; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02291; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)2292; GFX10-W32-NEXT: v_mov_b32_e32 v0, v22293; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v02294; GFX10-W32-NEXT: .LBB40_2: ; %ENDIF2295; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s132296; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s122297; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12298; GFX10-W32-NEXT: ; return to shader part epilog2299main_body:2300 %cmp = icmp eq i32 %z, 02301 br i1 %cmp, label %IF, label %ENDIF2302 2303IF:2304 %c.bc = bitcast i32 %c to float2305 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02306 %tex0 = extractelement <4 x float> %tex, i32 02307 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02308 %dataf = extractelement <4 x float> %dtex, i32 02309 %data1 = fptosi float %dataf to i322310 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0)2311 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079)2312 %data4 = call i32 @llvm.amdgcn.wwm.i32(i32 %data3)2313 %data4f = sitofp i32 %data4 to float2314 br label %ENDIF2315 2316ENDIF:2317 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ]2318 ret float %r2319}2320 2321; Check that WWM is triggered by the strict_wwm intrinsic.2322define amdgpu_ps float @test_strict_wwm1(i32 inreg %idx0, i32 inreg %idx1) {2323; GFX9-W64-LABEL: test_strict_wwm1:2324; GFX9-W64: ; %bb.0: ; %main_body2325; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -12326; GFX9-W64-NEXT: v_mov_b32_e32 v1, s02327; GFX9-W64-NEXT: v_mov_b32_e32 v2, s12328; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2329; GFX9-W64-NEXT: s_nop 02330; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen2331; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2332; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v22333; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]2334; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12335; GFX9-W64-NEXT: ; return to shader part epilog2336;2337; GFX10-W32-LABEL: test_strict_wwm1:2338; GFX10-W32: ; %bb.0: ; %main_body2339; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -12340; GFX10-W32-NEXT: v_mov_b32_e32 v1, s02341; GFX10-W32-NEXT: v_mov_b32_e32 v2, s12342; GFX10-W32-NEXT: s_clause 0x12343; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2344; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen2345; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2346; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v22347; GFX10-W32-NEXT: s_mov_b32 exec_lo, s22348; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12349; GFX10-W32-NEXT: ; return to shader part epilog2350main_body:2351 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)2352 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)2353 %out = fadd float %src0, %src12354 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out)2355 ret float %out.02356}2357 2358; Same as above, but with an integer type.2359define amdgpu_ps float @test_strict_wwm2(i32 inreg %idx0, i32 inreg %idx1) {2360; GFX9-W64-LABEL: test_strict_wwm2:2361; GFX9-W64: ; %bb.0: ; %main_body2362; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -12363; GFX9-W64-NEXT: v_mov_b32_e32 v1, s02364; GFX9-W64-NEXT: v_mov_b32_e32 v2, s12365; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2366; GFX9-W64-NEXT: s_nop 02367; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen2368; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2369; GFX9-W64-NEXT: v_add_u32_e32 v1, v1, v22370; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]2371; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12372; GFX9-W64-NEXT: ; return to shader part epilog2373;2374; GFX10-W32-LABEL: test_strict_wwm2:2375; GFX10-W32: ; %bb.0: ; %main_body2376; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -12377; GFX10-W32-NEXT: v_mov_b32_e32 v1, s02378; GFX10-W32-NEXT: v_mov_b32_e32 v2, s12379; GFX10-W32-NEXT: s_clause 0x12380; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2381; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[0:3], 0 idxen2382; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2383; GFX10-W32-NEXT: v_add_nc_u32_e32 v1, v1, v22384; GFX10-W32-NEXT: s_mov_b32 exec_lo, s22385; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12386; GFX10-W32-NEXT: ; return to shader part epilog2387main_body:2388 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)2389 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)2390 %src0.0 = bitcast float %src0 to i322391 %src1.0 = bitcast float %src1 to i322392 %out = add i32 %src0.0, %src1.02393 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out)2394 %out.1 = bitcast i32 %out.0 to float2395 ret float %out.12396}2397 2398; Check that we don't leave WWM on for computations that don't require WWM,2399; since that will lead clobbering things that aren't supposed to be clobbered2400; in cases like this.2401; We enforce this by checking that v_add gets emitted in the same block as2402; WWM computations.2403define amdgpu_ps float @test_strict_wwm3(i32 inreg %idx) {2404; GFX9-W64-LABEL: test_strict_wwm3:2405; GFX9-W64: ; %bb.0: ; %main_body2406; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02407; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02408; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v02409; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02410; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc2411; GFX9-W64-NEXT: s_cbranch_execz .LBB43_22412; GFX9-W64-NEXT: ; %bb.1: ; %if2413; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -12414; GFX9-W64-NEXT: v_mov_b32_e32 v1, s02415; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2416; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2417; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v12418; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]2419; GFX9-W64-NEXT: v_mov_b32_e32 v0, v22420; GFX9-W64-NEXT: v_add_f32_e32 v0, v1, v02421; GFX9-W64-NEXT: .LBB43_2: ; %endif2422; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]2423; GFX9-W64-NEXT: ; return to shader part epilog2424;2425; GFX10-W32-LABEL: test_strict_wwm3:2426; GFX10-W32: ; %bb.0: ; %main_body2427; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02428; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02429; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v02430; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02431; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo2432; GFX10-W32-NEXT: s_cbranch_execz .LBB43_22433; GFX10-W32-NEXT: ; %bb.1: ; %if2434; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -12435; GFX10-W32-NEXT: v_mov_b32_e32 v1, s02436; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2437; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2438; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v12439; GFX10-W32-NEXT: s_mov_b32 exec_lo, s22440; GFX10-W32-NEXT: v_mov_b32_e32 v0, v22441; GFX10-W32-NEXT: v_add_f32_e32 v0, v1, v02442; GFX10-W32-NEXT: .LBB43_2: ; %endif2443; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s12444; GFX10-W32-NEXT: ; return to shader part epilog2445main_body:2446 ; use mbcnt to make sure the branch is divergent2447 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)2448 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)2449 %cc = icmp uge i32 %hi, 162450 br i1 %cc, label %endif, label %if2451 2452if:2453 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)2454 %out = fadd float %src, %src2455 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out)2456 %out.1 = fadd float %src, %out.02457 br label %endif2458 2459endif:2460 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]2461 ret float %out.22462}2463 2464; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM2465; write could clobber disabled channels in the non-WWM one.2466; We enforce this by checking that v_mov gets emitted in the same block as2467; WWM computations.2468define amdgpu_ps float @test_strict_wwm4(i32 inreg %idx) {2469; GFX9-W64-LABEL: test_strict_wwm4:2470; GFX9-W64: ; %bb.0: ; %main_body2471; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02472; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02473; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v02474; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02475; GFX9-W64-NEXT: s_and_saveexec_b64 s[2:3], vcc2476; GFX9-W64-NEXT: s_cbranch_execz .LBB44_22477; GFX9-W64-NEXT: ; %bb.1: ; %if2478; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -12479; GFX9-W64-NEXT: v_mov_b32_e32 v1, s02480; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2481; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2482; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v12483; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]2484; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12485; GFX9-W64-NEXT: .LBB44_2: ; %endif2486; GFX9-W64-NEXT: s_or_b64 exec, exec, s[2:3]2487; GFX9-W64-NEXT: ; return to shader part epilog2488;2489; GFX10-W32-LABEL: test_strict_wwm4:2490; GFX10-W32: ; %bb.0: ; %main_body2491; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02492; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02493; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v02494; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02495; GFX10-W32-NEXT: s_and_saveexec_b32 s1, vcc_lo2496; GFX10-W32-NEXT: s_cbranch_execz .LBB44_22497; GFX10-W32-NEXT: ; %bb.1: ; %if2498; GFX10-W32-NEXT: s_or_saveexec_b32 s2, -12499; GFX10-W32-NEXT: v_mov_b32_e32 v1, s02500; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2501; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2502; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v12503; GFX10-W32-NEXT: s_mov_b32 exec_lo, s22504; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12505; GFX10-W32-NEXT: .LBB44_2: ; %endif2506; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s12507; GFX10-W32-NEXT: ; return to shader part epilog2508main_body:2509 ; use mbcnt to make sure the branch is divergent2510 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)2511 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)2512 %cc = icmp uge i32 %hi, 162513 br i1 %cc, label %endif, label %if2514 2515if:2516 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)2517 %out = fadd float %src, %src2518 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out)2519 br label %endif2520 2521endif:2522 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]2523 ret float %out.12524}2525 2526; Make sure the transition from Exact to WWM then WQM works properly.2527define amdgpu_ps float @test_strict_wwm5(i32 inreg %idx0, i32 inreg %idx1) {2528; GFX9-W64-LABEL: test_strict_wwm5:2529; GFX9-W64: ; %bb.0: ; %main_body2530; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec2531; GFX9-W64-NEXT: v_mov_b32_e32 v0, s02532; GFX9-W64-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen2533; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2534; GFX9-W64-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen2535; GFX9-W64-NEXT: s_or_saveexec_b64 s[4:5], -12536; GFX9-W64-NEXT: v_mov_b32_e32 v1, s12537; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2538; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2539; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v12540; GFX9-W64-NEXT: s_mov_b64 exec, s[4:5]2541; GFX9-W64-NEXT: s_wqm_b64 exec, exec2542; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12543; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v02544; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec2545; GFX9-W64-NEXT: s_and_b64 exec, exec, s[2:3]2546; GFX9-W64-NEXT: ; return to shader part epilog2547;2548; GFX10-W32-LABEL: test_strict_wwm5:2549; GFX10-W32: ; %bb.0: ; %main_body2550; GFX10-W32-NEXT: v_mov_b32_e32 v0, s02551; GFX10-W32-NEXT: s_mov_b32 s2, exec_lo2552; GFX10-W32-NEXT: buffer_load_dword v2, v0, s[0:3], 0 idxen2553; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12554; GFX10-W32-NEXT: v_mov_b32_e32 v1, s12555; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02556; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2557; GFX10-W32-NEXT: buffer_store_dword v2, v0, s[0:3], 0 idxen2558; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12559; GFX10-W32-NEXT: buffer_load_dword v1, v1, s[0:3], 0 idxen2560; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2561; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v12562; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02563; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2564; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12565; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v02566; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec2567; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s22568; GFX10-W32-NEXT: ; return to shader part epilog2569main_body:2570 %src0 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)2571 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %src0, ptr addrspace(8) poison, i32 %idx0, i32 0, i32 0, i32 0)2572 %src1 = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx1, i32 0, i32 0, i32 0)2573 %temp = fadd float %src1, %src12574 %temp.0 = call float @llvm.amdgcn.strict.wwm.f32(float %temp)2575 %out = fadd float %temp.0, %temp.02576 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)2577 ret float %out.02578}2579 2580; Check that WWM is turned on correctly across basic block boundaries.2581; if..then..endif version2582;SI-CHECK: buffer_load_dword2583;VI-CHECK: flat_load_dword2584;SI-CHECK: buffer_load_dword2585;VI-CHECK: flat_load_dword2586define amdgpu_ps float @test_strict_wwm6_then() {2587; GFX9-W64-LABEL: test_strict_wwm6_then:2588; GFX9-W64: ; %bb.0: ; %main_body2589; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -12590; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc2591; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2592; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2593; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02594; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02595; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v02596; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02597; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc2598; GFX9-W64-NEXT: s_cbranch_execz .LBB46_22599; GFX9-W64-NEXT: ; %bb.1: ; %if2600; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -12601; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc2602; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2603; GFX9-W64-NEXT: v_add_f32_e32 v1, v1, v22604; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]2605; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12606; GFX9-W64-NEXT: .LBB46_2: ; %endif2607; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]2608; GFX9-W64-NEXT: ; return to shader part epilog2609;2610; GFX10-W32-LABEL: test_strict_wwm6_then:2611; GFX10-W32: ; %bb.0: ; %main_body2612; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12613; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc2614; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2615; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02616; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02617; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v02618; GFX10-W32-NEXT: v_cmp_gt_u32_e32 vcc_lo, 16, v02619; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02620; GFX10-W32-NEXT: s_and_saveexec_b32 s0, vcc_lo2621; GFX10-W32-NEXT: s_cbranch_execz .LBB46_22622; GFX10-W32-NEXT: ; %bb.1: ; %if2623; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -12624; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc2625; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2626; GFX10-W32-NEXT: v_add_f32_e32 v1, v1, v22627; GFX10-W32-NEXT: s_mov_b32 exec_lo, s12628; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12629; GFX10-W32-NEXT: .LBB46_2: ; %endif2630; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s02631; GFX10-W32-NEXT: ; return to shader part epilog2632main_body:2633 %src0 = load volatile float, ptr addrspace(1) poison2634 ; use mbcnt to make sure the branch is divergent2635 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)2636 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)2637 %cc = icmp uge i32 %hi, 162638 br i1 %cc, label %endif, label %if2639 2640if:2641 %src1 = load volatile float, ptr addrspace(1) poison2642 %out = fadd float %src0, %src12643 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out)2644 br label %endif2645 2646endif:2647 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]2648 ret float %out.12649}2650 2651; Check that WWM is turned on correctly across basic block boundaries.2652; loop version2653define amdgpu_ps float @test_strict_wwm6_loop() {2654; GFX9-W64-LABEL: test_strict_wwm6_loop:2655; GFX9-W64: ; %bb.0: ; %main_body2656; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -12657; GFX9-W64-NEXT: global_load_dword v1, v[3:4], off glc2658; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2659; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2660; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02661; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v02662; GFX9-W64-NEXT: s_mov_b64 s[0:1], 02663; GFX9-W64-NEXT: .LBB47_1: ; %loop2664; GFX9-W64-NEXT: ; =>This Inner Loop Header: Depth=12665; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -12666; GFX9-W64-NEXT: global_load_dword v2, v[3:4], off glc2667; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2668; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]2669; GFX9-W64-NEXT: v_add_u32_e32 v3, -1, v32670; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v32671; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -12672; GFX9-W64-NEXT: v_add_f32_e32 v2, v1, v22673; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]2674; GFX9-W64-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2675; GFX9-W64-NEXT: v_mov_b32_e32 v0, v22676; GFX9-W64-NEXT: s_andn2_b64 exec, exec, s[0:1]2677; GFX9-W64-NEXT: s_cbranch_execnz .LBB47_12678; GFX9-W64-NEXT: ; %bb.2: ; %endloop2679; GFX9-W64-NEXT: s_or_b64 exec, exec, s[0:1]2680; GFX9-W64-NEXT: ; return to shader part epilog2681;2682; GFX10-W32-LABEL: test_strict_wwm6_loop:2683; GFX10-W32: ; %bb.0: ; %main_body2684; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12685; GFX10-W32-NEXT: global_load_dword v1, v[3:4], off glc dlc2686; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2687; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02688; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 02689; GFX10-W32-NEXT: s_mov_b32 s0, 02690; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v02691; GFX10-W32-NEXT: .LBB47_1: ; %loop2692; GFX10-W32-NEXT: ; =>This Inner Loop Header: Depth=12693; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -12694; GFX10-W32-NEXT: global_load_dword v2, v[3:4], off glc dlc2695; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2696; GFX10-W32-NEXT: s_mov_b32 exec_lo, s12697; GFX10-W32-NEXT: v_add_nc_u32_e32 v3, -1, v32698; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -12699; GFX10-W32-NEXT: v_add_f32_e32 v2, v1, v22700; GFX10-W32-NEXT: s_mov_b32 exec_lo, s12701; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v32702; GFX10-W32-NEXT: v_mov_b32_e32 v0, v22703; GFX10-W32-NEXT: s_or_b32 s0, vcc_lo, s02704; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, s02705; GFX10-W32-NEXT: s_cbranch_execnz .LBB47_12706; GFX10-W32-NEXT: ; %bb.2: ; %endloop2707; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s02708; GFX10-W32-NEXT: ; return to shader part epilog2709main_body:2710 %src0 = load volatile float, ptr addrspace(1) poison2711 ; use mbcnt to make sure the branch is divergent2712 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)2713 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)2714 br label %loop2715 2716loop:2717 %counter = phi i32 [ %hi, %main_body ], [ %counter.1, %loop ]2718 %src1 = load volatile float, ptr addrspace(1) poison2719 %out = fadd float %src0, %src12720 %out.0 = call float @llvm.amdgcn.strict.wwm.f32(float %out)2721 %counter.1 = sub i32 %counter, 12722 %cc = icmp ne i32 %counter.1, 02723 br i1 %cc, label %loop, label %endloop2724 2725endloop:2726 ret float %out.02727}2728 2729; Check that @llvm.amdgcn.set.inactive disables WWM.2730define amdgpu_ps void @test_strict_wwm_set_inactive1(i32 inreg %idx) {2731; GFX9-W64-LABEL: test_strict_wwm_set_inactive1:2732; GFX9-W64: ; %bb.0: ; %main_body2733; GFX9-W64-NEXT: v_mov_b32_e32 v1, s02734; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen2735; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -12736; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2737; GFX9-W64-NEXT: v_cndmask_b32_e64 v0, 0, v2, s[0:1]2738; GFX9-W64-NEXT: v_add_u32_e32 v0, v0, v02739; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2740; GFX9-W64-NEXT: v_mov_b32_e32 v2, v02741; GFX9-W64-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen2742; GFX9-W64-NEXT: s_endpgm2743;2744; GFX10-W32-LABEL: test_strict_wwm_set_inactive1:2745; GFX10-W32: ; %bb.0: ; %main_body2746; GFX10-W32-NEXT: v_mov_b32_e32 v1, s02747; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[0:3], 0 idxen2748; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12749; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2750; GFX10-W32-NEXT: v_cndmask_b32_e64 v0, 0, v2, s02751; GFX10-W32-NEXT: v_add_nc_u32_e32 v0, v0, v02752; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02753; GFX10-W32-NEXT: v_mov_b32_e32 v2, v02754; GFX10-W32-NEXT: buffer_store_dword v2, v1, s[0:3], 0 idxen2755; GFX10-W32-NEXT: s_endpgm2756main_body:2757 %src = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)2758 %src.0 = bitcast float %src to i322759 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0)2760 %out = add i32 %src.1, %src.12761 %out.0 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %out)2762 %out.1 = bitcast i32 %out.0 to float2763 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %out.1, ptr addrspace(8) poison, i32 %idx, i32 0, i32 0, i32 0)2764 ret void2765}2766 2767; Check a case of a block being entirely WQM except for a bit of WWM.2768; There was a bug where it forgot to enter and leave WWM.2769define amdgpu_ps float @test_strict_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {2770; GFX9-W64-LABEL: test_strict_wwm_within_wqm:2771; GFX9-W64: ; %bb.0: ; %main_body2772; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec2773; GFX9-W64-NEXT: s_wqm_b64 exec, exec2774; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12775; GFX9-W64-NEXT: v_mov_b32_e32 v1, 02776; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc2777; GFX9-W64-NEXT: s_cbranch_execz .LBB49_22778; GFX9-W64-NEXT: ; %bb.1: ; %IF2779; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12780; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2781; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12782; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2783; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v0, v02784; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -12785; GFX9-W64-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]2786; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2787; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2788; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)2789; GFX9-W64-NEXT: v_mov_b32_e32 v0, v22790; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v02791; GFX9-W64-NEXT: .LBB49_2: ; %ENDIF2792; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]2793; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]2794; GFX9-W64-NEXT: v_mov_b32_e32 v0, v12795; GFX9-W64-NEXT: ; return to shader part epilog2796;2797; GFX10-W32-LABEL: test_strict_wwm_within_wqm:2798; GFX10-W32: ; %bb.0: ; %main_body2799; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo2800; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2801; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v12802; GFX10-W32-NEXT: v_mov_b32_e32 v1, 02803; GFX10-W32-NEXT: s_and_saveexec_b32 s13, vcc_lo2804; GFX10-W32-NEXT: s_cbranch_execz .LBB49_22805; GFX10-W32-NEXT: ; %bb.1: ; %IF2806; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2807; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2808; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2809; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2810; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v0, v02811; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -12812; GFX10-W32-NEXT: v_cndmask_b32_e64 v2, 0, v0, s02813; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2814; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02815; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)2816; GFX10-W32-NEXT: v_mov_b32_e32 v0, v22817; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v02818; GFX10-W32-NEXT: .LBB49_2: ; %ENDIF2819; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s132820; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s122821; GFX10-W32-NEXT: v_mov_b32_e32 v0, v12822; GFX10-W32-NEXT: ; return to shader part epilog2823main_body:2824 %cmp = icmp eq i32 %z, 02825 br i1 %cmp, label %IF, label %ENDIF2826 2827IF:2828 %c.bc = bitcast i32 %c to float2829 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02830 %tex0 = extractelement <4 x float> %tex, i32 02831 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02832 %dataf = extractelement <4 x float> %dtex, i32 02833 %data1 = fptosi float %dataf to i322834 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0)2835 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079)2836 %data4 = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %data3)2837 %data4f = sitofp i32 %data4 to float2838 br label %ENDIF2839 2840ENDIF:2841 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ]2842 ret float %r2843}2844 2845; Check a case of a block being entirely WQM except for a bit of STRICT WQM.2846define amdgpu_ps float @test_strict_wqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {2847; GFX9-W64-LABEL: test_strict_wqm_within_wqm:2848; GFX9-W64: ; %bb.0: ; %main_body2849; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec2850; GFX9-W64-NEXT: s_wqm_b64 exec, exec2851; GFX9-W64-NEXT: s_mov_b64 s[14:15], exec2852; GFX9-W64-NEXT: s_wqm_b64 exec, exec2853; GFX9-W64-NEXT: v_mov_b32_e32 v2, v02854; GFX9-W64-NEXT: s_mov_b64 exec, s[14:15]2855; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12856; GFX9-W64-NEXT: v_mov_b32_e32 v0, 02857; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc2858; GFX9-W64-NEXT: s_cbranch_execz .LBB50_22859; GFX9-W64-NEXT: ; %bb.1: ; %IF2860; GFX9-W64-NEXT: s_mov_b64 s[16:17], exec2861; GFX9-W64-NEXT: s_wqm_b64 exec, exec2862; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x12863; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2864; GFX9-W64-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x12865; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2866; GFX9-W64-NEXT: v_cvt_i32_f32_e32 v2, v22867; GFX9-W64-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2868; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17]2869; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)2870; GFX9-W64-NEXT: v_mov_b32_e32 v0, v22871; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v0, v02872; GFX9-W64-NEXT: .LBB50_2: ; %ENDIF2873; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]2874; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]2875; GFX9-W64-NEXT: ; return to shader part epilog2876;2877; GFX10-W32-LABEL: test_strict_wqm_within_wqm:2878; GFX10-W32: ; %bb.0: ; %main_body2879; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo2880; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2881; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo2882; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2883; GFX10-W32-NEXT: v_mov_b32_e32 v2, v02884; GFX10-W32-NEXT: s_mov_b32 exec_lo, s132885; GFX10-W32-NEXT: v_mov_b32_e32 v0, 02886; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo2887; GFX10-W32-NEXT: v_cmpx_eq_u32_e32 0, v12888; GFX10-W32-NEXT: s_cbranch_execz .LBB50_22889; GFX10-W32-NEXT: ; %bb.1: ; %IF2890; GFX10-W32-NEXT: s_mov_b32 s14, exec_lo2891; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2892; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2893; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2894; GFX10-W32-NEXT: image_sample v2, v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2895; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2896; GFX10-W32-NEXT: v_cvt_i32_f32_e32 v2, v22897; GFX10-W32-NEXT: ds_swizzle_b32 v2, v2 offset:swizzle(SWAP,2)2898; GFX10-W32-NEXT: s_mov_b32 exec_lo, s142899; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)2900; GFX10-W32-NEXT: v_mov_b32_e32 v0, v22901; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v0, v02902; GFX10-W32-NEXT: .LBB50_2: ; %ENDIF2903; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s132904; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s122905; GFX10-W32-NEXT: ; return to shader part epilog2906main_body:2907 %cmp = icmp eq i32 %z, 02908 br i1 %cmp, label %IF, label %ENDIF2909 2910IF:2911 %c.bc = bitcast i32 %c to float2912 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02913 %tex0 = extractelement <4 x float> %tex, i32 02914 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #02915 %dataf = extractelement <4 x float> %dtex, i32 02916 %data1 = fptosi float %dataf to i322917 %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079)2918 %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2)2919 %data3f = sitofp i32 %data3 to float2920 br label %ENDIF2921 2922ENDIF:2923 %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ]2924 ret float %r2925}2926 2927; WQM -> StrictWQM transition must be preserved because kill breaks WQM mask2928define amdgpu_ps float @test_strict_wqm_within_wqm_with_kill(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data, i32 %wqm_data) {2929; GFX9-W64-LABEL: test_strict_wqm_within_wqm_with_kill:2930; GFX9-W64: ; %bb.0: ; %main_body2931; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec2932; GFX9-W64-NEXT: s_mov_b64 s[14:15], exec2933; GFX9-W64-NEXT: s_wqm_b64 exec, exec2934; GFX9-W64-NEXT: v_mov_b32_e32 v3, v22935; GFX9-W64-NEXT: s_mov_b64 exec, s[14:15]2936; GFX9-W64-NEXT: s_wqm_b64 exec, exec2937; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12938; GFX9-W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v12939; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2940; GFX9-W64-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x12941; GFX9-W64-NEXT: s_andn2_b64 s[0:1], exec, vcc2942; GFX9-W64-NEXT: s_andn2_b64 s[12:13], s[12:13], s[0:1]2943; GFX9-W64-NEXT: s_cbranch_scc0 .LBB51_22944; GFX9-W64-NEXT: ; %bb.1: ; %main_body2945; GFX9-W64-NEXT: s_and_b64 exec, exec, vcc2946; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec2947; GFX9-W64-NEXT: s_wqm_b64 exec, exec2948; GFX9-W64-NEXT: ds_swizzle_b32 v3, v3 offset:swizzle(SWAP,2)2949; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]2950; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)2951; GFX9-W64-NEXT: v_mov_b32_e32 v1, v32952; GFX9-W64-NEXT: v_cvt_f32_i32_e32 v1, v12953; GFX9-W64-NEXT: s_waitcnt vmcnt(0)2954; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v12955; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec2956; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]2957; GFX9-W64-NEXT: s_branch .LBB51_32958; GFX9-W64-NEXT: .LBB51_2:2959; GFX9-W64-NEXT: s_mov_b64 exec, 02960; GFX9-W64-NEXT: exp null off, off, off, off done vm2961; GFX9-W64-NEXT: s_endpgm2962; GFX9-W64-NEXT: .LBB51_3:2963;2964; GFX10-W32-LABEL: test_strict_wqm_within_wqm_with_kill:2965; GFX10-W32: ; %bb.0: ; %main_body2966; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo2967; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo2968; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2969; GFX10-W32-NEXT: v_mov_b32_e32 v3, v22970; GFX10-W32-NEXT: s_mov_b32 exec_lo, s132971; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2972; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2973; GFX10-W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v12974; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2975; GFX10-W32-NEXT: image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D2976; GFX10-W32-NEXT: s_andn2_b32 s0, exec_lo, vcc_lo2977; GFX10-W32-NEXT: s_andn2_b32 s12, s12, s02978; GFX10-W32-NEXT: s_cbranch_scc0 .LBB51_22979; GFX10-W32-NEXT: ; %bb.1: ; %main_body2980; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, vcc_lo2981; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo2982; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo2983; GFX10-W32-NEXT: ds_swizzle_b32 v3, v3 offset:swizzle(SWAP,2)2984; GFX10-W32-NEXT: s_mov_b32 exec_lo, s02985; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)2986; GFX10-W32-NEXT: v_mov_b32_e32 v1, v32987; GFX10-W32-NEXT: v_cvt_f32_i32_e32 v1, v12988; GFX10-W32-NEXT: s_waitcnt vmcnt(0)2989; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v12990; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec2991; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s122992; GFX10-W32-NEXT: s_branch .LBB51_32993; GFX10-W32-NEXT: .LBB51_2:2994; GFX10-W32-NEXT: s_mov_b32 exec_lo, 02995; GFX10-W32-NEXT: exp null off, off, off, off done vm2996; GFX10-W32-NEXT: s_endpgm2997; GFX10-W32-NEXT: .LBB51_3:2998main_body:2999 %c.bc = bitcast i32 %c to float3000 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03001 %tex0 = extractelement <4 x float> %tex, i32 03002 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03003 %cmp = icmp eq i32 %z, 03004 call void @llvm.amdgcn.kill(i1 %cmp)3005 %dataf = extractelement <4 x float> %dtex, i32 03006 %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %wqm_data, i32 2079)3007 %data3 = call i32 @llvm.amdgcn.strict.wqm.i32(i32 %data2)3008 %data3f = sitofp i32 %data3 to float3009 %result.f = fadd float %dataf, %data3f3010 %result.i = bitcast float %result.f to i323011 %result.wqm = call i32 @llvm.amdgcn.wqm.i32(i32 %result.i)3012 %result = bitcast i32 %result.wqm to float3013 ret float %result3014}3015 3016;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again.3017define amdgpu_ps float @test_strict_wqm_strict_wwm_wqm(i32 inreg %idx0, i32 inreg %idx1, ptr addrspace(8) inreg %res, ptr addrspace(8) inreg %res2, float %inp, <8 x i32> inreg %res3) {3018; GFX9-W64-LABEL: test_strict_wqm_strict_wwm_wqm:3019; GFX9-W64: ; %bb.0: ; %main_body3020; GFX9-W64-NEXT: s_mov_b64 s[28:29], exec3021; GFX9-W64-NEXT: s_mov_b32 s19, s173022; GFX9-W64-NEXT: s_mov_b64 s[30:31], exec3023; GFX9-W64-NEXT: s_wqm_b64 exec, exec3024; GFX9-W64-NEXT: s_mov_b32 s23, s53025; GFX9-W64-NEXT: s_mov_b32 s22, s43026; GFX9-W64-NEXT: s_mov_b32 s21, s33027; GFX9-W64-NEXT: s_mov_b32 s20, s23028; GFX9-W64-NEXT: s_mov_b32 s27, s93029; GFX9-W64-NEXT: s_mov_b32 s26, s83030; GFX9-W64-NEXT: s_mov_b32 s25, s73031; GFX9-W64-NEXT: s_mov_b32 s24, s63032; GFX9-W64-NEXT: s_mov_b32 s18, s163033; GFX9-W64-NEXT: s_mov_b32 s17, s153034; GFX9-W64-NEXT: s_mov_b32 s16, s143035; GFX9-W64-NEXT: s_mov_b32 s15, s133036; GFX9-W64-NEXT: s_mov_b32 s14, s123037; GFX9-W64-NEXT: s_mov_b32 s13, s113038; GFX9-W64-NEXT: s_mov_b32 s12, s103039; GFX9-W64-NEXT: v_mov_b32_e32 v1, s13040; GFX9-W64-NEXT: s_mov_b64 exec, s[30:31]3041; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen3042; GFX9-W64-NEXT: s_mov_b64 s[2:3], exec3043; GFX9-W64-NEXT: s_wqm_b64 exec, exec3044; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen3045; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]3046; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -13047; GFX9-W64-NEXT: v_mov_b32_e32 v3, s03048; GFX9-W64-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen3049; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]3050; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec3051; GFX9-W64-NEXT: s_wqm_b64 exec, exec3052; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3053; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v23054; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]3055; GFX9-W64-NEXT: s_wqm_b64 exec, exec3056; GFX9-W64-NEXT: v_mov_b32_e32 v0, v23057; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v03058; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3059; GFX9-W64-NEXT: v_mov_b32_e32 v4, v33060; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v43061; GFX9-W64-NEXT: s_and_b64 exec, exec, s[28:29]3062; GFX9-W64-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x13063; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3064; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen3065; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen3066; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3067; GFX9-W64-NEXT: ; return to shader part epilog3068;3069; GFX10-W32-LABEL: test_strict_wqm_strict_wwm_wqm:3070; GFX10-W32: ; %bb.0: ; %main_body3071; GFX10-W32-NEXT: s_mov_b32 s28, exec_lo3072; GFX10-W32-NEXT: s_mov_b32 s19, s173073; GFX10-W32-NEXT: s_mov_b32 s29, exec_lo3074; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3075; GFX10-W32-NEXT: s_mov_b32 s23, s53076; GFX10-W32-NEXT: s_mov_b32 s22, s43077; GFX10-W32-NEXT: s_mov_b32 s21, s33078; GFX10-W32-NEXT: s_mov_b32 s20, s23079; GFX10-W32-NEXT: s_mov_b32 s27, s93080; GFX10-W32-NEXT: s_mov_b32 s26, s83081; GFX10-W32-NEXT: s_mov_b32 s25, s73082; GFX10-W32-NEXT: s_mov_b32 s24, s63083; GFX10-W32-NEXT: s_mov_b32 s18, s163084; GFX10-W32-NEXT: s_mov_b32 s17, s153085; GFX10-W32-NEXT: s_mov_b32 s16, s143086; GFX10-W32-NEXT: s_mov_b32 s15, s133087; GFX10-W32-NEXT: s_mov_b32 s14, s123088; GFX10-W32-NEXT: s_mov_b32 s13, s113089; GFX10-W32-NEXT: s_mov_b32 s12, s103090; GFX10-W32-NEXT: v_mov_b32_e32 v1, s13091; GFX10-W32-NEXT: s_mov_b32 exec_lo, s293092; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen3093; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo3094; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3095; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[20:23], 0 idxen3096; GFX10-W32-NEXT: s_mov_b32 exec_lo, s13097; GFX10-W32-NEXT: s_or_saveexec_b32 s1, -13098; GFX10-W32-NEXT: v_mov_b32_e32 v3, s03099; GFX10-W32-NEXT: buffer_load_dword v3, v3, s[24:27], 0 idxen3100; GFX10-W32-NEXT: s_mov_b32 exec_lo, s13101; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo3102; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3103; GFX10-W32-NEXT: s_waitcnt vmcnt(1)3104; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v23105; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03106; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3107; GFX10-W32-NEXT: v_mov_b32_e32 v0, v23108; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3109; GFX10-W32-NEXT: v_mov_b32_e32 v4, v33110; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v03111; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v43112; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s283113; GFX10-W32-NEXT: image_sample v0, v0, s[12:19], s[20:23] dmask:0x1 dim:SQ_RSRC_IMG_1D3114; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3115; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[20:23], 0 idxen3116; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[20:23], 0 idxen3117; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3118; GFX10-W32-NEXT: ; return to shader part epilog3119main_body:3120 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %inp, ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3121 %reload = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3122 %temp = fadd float %reload, %reload3123 %temp2 = call float @llvm.amdgcn.strict.wqm.f32(float %temp)3124 %temp3 = fadd float %temp2, %temp23125 %reload_wwm = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res2, i32 %idx0, i32 0, i32 0, i32 0)3126 %temp4 = call float @llvm.amdgcn.strict.wwm.f32(float %reload_wwm)3127 %temp5 = fadd float %temp3, %temp43128 %res.int = ptrtoint ptr addrspace(8) %res to i1283129 %res.vec = bitcast i128 %res.int to <4 x i32>3130 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res.vec, i1 false, i32 0, i32 0)3131 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %tex, ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3132 %out = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3133 ret float %out3134}3135 3136define amdgpu_ps float @test_strict_wwm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, ptr addrspace(8) inreg %res, float %inp, <8 x i32> inreg %res2) {3137; GFX9-W64-LABEL: test_strict_wwm_strict_wqm_wqm:3138; GFX9-W64: ; %bb.0: ; %main_body3139; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec3140; GFX9-W64-NEXT: s_mov_b32 s15, s133141; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec3142; GFX9-W64-NEXT: s_wqm_b64 exec, exec3143; GFX9-W64-NEXT: s_mov_b32 s19, s53144; GFX9-W64-NEXT: s_mov_b32 s18, s43145; GFX9-W64-NEXT: s_mov_b32 s17, s33146; GFX9-W64-NEXT: s_mov_b32 s16, s23147; GFX9-W64-NEXT: s_mov_b32 s14, s123148; GFX9-W64-NEXT: s_mov_b32 s13, s113149; GFX9-W64-NEXT: s_mov_b32 s12, s103150; GFX9-W64-NEXT: s_mov_b32 s11, s93151; GFX9-W64-NEXT: s_mov_b32 s10, s83152; GFX9-W64-NEXT: s_mov_b32 s9, s73153; GFX9-W64-NEXT: s_mov_b32 s8, s63154; GFX9-W64-NEXT: v_mov_b32_e32 v1, s03155; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23]3156; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3157; GFX9-W64-NEXT: s_or_saveexec_b64 s[2:3], -13158; GFX9-W64-NEXT: v_mov_b32_e32 v2, s13159; GFX9-W64-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen3160; GFX9-W64-NEXT: s_mov_b64 exec, s[2:3]3161; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec3162; GFX9-W64-NEXT: s_wqm_b64 exec, exec3163; GFX9-W64-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen3164; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]3165; GFX9-W64-NEXT: s_or_saveexec_b64 s[0:1], -13166; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3167; GFX9-W64-NEXT: v_add_f32_e32 v2, v2, v23168; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]3169; GFX9-W64-NEXT: s_wqm_b64 exec, exec3170; GFX9-W64-NEXT: v_mov_b32_e32 v0, v23171; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v03172; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3173; GFX9-W64-NEXT: v_mov_b32_e32 v4, v33174; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v43175; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21]3176; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x13177; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3178; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3179; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen3180; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3181; GFX9-W64-NEXT: ; return to shader part epilog3182;3183; GFX10-W32-LABEL: test_strict_wwm_strict_wqm_wqm:3184; GFX10-W32: ; %bb.0: ; %main_body3185; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo3186; GFX10-W32-NEXT: s_mov_b32 s15, s133187; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo3188; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3189; GFX10-W32-NEXT: s_mov_b32 s19, s53190; GFX10-W32-NEXT: s_mov_b32 s18, s43191; GFX10-W32-NEXT: s_mov_b32 s17, s33192; GFX10-W32-NEXT: s_mov_b32 s16, s23193; GFX10-W32-NEXT: s_mov_b32 s14, s123194; GFX10-W32-NEXT: s_mov_b32 s13, s113195; GFX10-W32-NEXT: s_mov_b32 s12, s103196; GFX10-W32-NEXT: s_mov_b32 s11, s93197; GFX10-W32-NEXT: s_mov_b32 s10, s83198; GFX10-W32-NEXT: s_mov_b32 s9, s73199; GFX10-W32-NEXT: s_mov_b32 s8, s63200; GFX10-W32-NEXT: v_mov_b32_e32 v1, s03201; GFX10-W32-NEXT: s_mov_b32 exec_lo, s213202; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -13203; GFX10-W32-NEXT: v_mov_b32_e32 v2, s13204; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03205; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3206; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -13207; GFX10-W32-NEXT: buffer_load_dword v2, v2, s[16:19], 0 idxen3208; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03209; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo3210; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3211; GFX10-W32-NEXT: buffer_load_dword v3, v1, s[16:19], 0 idxen3212; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03213; GFX10-W32-NEXT: s_or_saveexec_b32 s0, -13214; GFX10-W32-NEXT: s_waitcnt vmcnt(1)3215; GFX10-W32-NEXT: v_add_f32_e32 v2, v2, v23216; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03217; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3218; GFX10-W32-NEXT: v_mov_b32_e32 v0, v23219; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3220; GFX10-W32-NEXT: v_mov_b32_e32 v4, v33221; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v03222; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v43223; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s203224; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D3225; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3226; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3227; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen3228; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3229; GFX10-W32-NEXT: ; return to shader part epilog3230main_body:3231 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %inp, ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3232 %reload = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3233 %temp = fadd float %reload, %reload3234 %temp2 = call float @llvm.amdgcn.strict.wwm.f32(float %temp)3235 %temp3 = fadd float %temp2, %temp23236 %reload_wwm = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3237 %temp4 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm)3238 %temp5 = fadd float %temp3, %temp43239 %res.int = ptrtoint ptr addrspace(8) %res to i1283240 %res.vec = bitcast i128 %res.int to <4 x i32>3241 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res.vec, i1 false, i32 0, i32 0)3242 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %tex, ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3243 %out = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3244 ret float %out3245}3246 3247;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again.3248define amdgpu_ps float @test_wqm_strict_wqm_wqm(i32 inreg %idx0, i32 inreg %idx1, ptr addrspace(8) inreg %res, float %inp, <8 x i32> inreg %res2) {3249; GFX9-W64-LABEL: test_wqm_strict_wqm_wqm:3250; GFX9-W64: ; %bb.0: ; %main_body3251; GFX9-W64-NEXT: s_mov_b64 s[20:21], exec3252; GFX9-W64-NEXT: s_mov_b32 s15, s133253; GFX9-W64-NEXT: s_mov_b64 s[22:23], exec3254; GFX9-W64-NEXT: s_wqm_b64 exec, exec3255; GFX9-W64-NEXT: s_mov_b32 s19, s53256; GFX9-W64-NEXT: s_mov_b32 s18, s43257; GFX9-W64-NEXT: s_mov_b32 s17, s33258; GFX9-W64-NEXT: s_mov_b32 s16, s23259; GFX9-W64-NEXT: s_mov_b32 s14, s123260; GFX9-W64-NEXT: s_mov_b32 s13, s113261; GFX9-W64-NEXT: s_mov_b32 s12, s103262; GFX9-W64-NEXT: s_mov_b32 s11, s93263; GFX9-W64-NEXT: s_mov_b32 s10, s83264; GFX9-W64-NEXT: s_mov_b32 s9, s73265; GFX9-W64-NEXT: s_mov_b32 s8, s63266; GFX9-W64-NEXT: v_mov_b32_e32 v1, s03267; GFX9-W64-NEXT: s_mov_b64 exec, s[22:23]3268; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3269; GFX9-W64-NEXT: s_wqm_b64 exec, exec3270; GFX9-W64-NEXT: v_mov_b32_e32 v0, s13271; GFX9-W64-NEXT: buffer_load_dword v0, v0, s[16:19], 0 idxen3272; GFX9-W64-NEXT: s_mov_b64 s[0:1], exec3273; GFX9-W64-NEXT: s_wqm_b64 exec, exec3274; GFX9-W64-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen3275; GFX9-W64-NEXT: s_mov_b64 exec, s[0:1]3276; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3277; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v03278; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x13279; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3280; GFX9-W64-NEXT: v_mov_b32_e32 v3, v23281; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3282; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v03283; GFX9-W64-NEXT: v_add_f32_e32 v0, v0, v33284; GFX9-W64-NEXT: s_and_b64 exec, exec, s[20:21]3285; GFX9-W64-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x13286; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3287; GFX9-W64-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3288; GFX9-W64-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen3289; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3290; GFX9-W64-NEXT: ; return to shader part epilog3291;3292; GFX10-W32-LABEL: test_wqm_strict_wqm_wqm:3293; GFX10-W32: ; %bb.0: ; %main_body3294; GFX10-W32-NEXT: s_mov_b32 s20, exec_lo3295; GFX10-W32-NEXT: s_mov_b32 s15, s133296; GFX10-W32-NEXT: s_mov_b32 s21, exec_lo3297; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3298; GFX10-W32-NEXT: s_mov_b32 s19, s53299; GFX10-W32-NEXT: s_mov_b32 s18, s43300; GFX10-W32-NEXT: s_mov_b32 s17, s33301; GFX10-W32-NEXT: s_mov_b32 s16, s23302; GFX10-W32-NEXT: s_mov_b32 s14, s123303; GFX10-W32-NEXT: s_mov_b32 s13, s113304; GFX10-W32-NEXT: s_mov_b32 s12, s103305; GFX10-W32-NEXT: s_mov_b32 s11, s93306; GFX10-W32-NEXT: s_mov_b32 s10, s83307; GFX10-W32-NEXT: s_mov_b32 s9, s73308; GFX10-W32-NEXT: s_mov_b32 s8, s63309; GFX10-W32-NEXT: v_mov_b32_e32 v1, s03310; GFX10-W32-NEXT: s_mov_b32 exec_lo, s213311; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3312; GFX10-W32-NEXT: v_mov_b32_e32 v3, s13313; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s203314; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3315; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3316; GFX10-W32-NEXT: buffer_load_dword v0, v3, s[16:19], 0 idxen3317; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo3318; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3319; GFX10-W32-NEXT: buffer_load_dword v2, v1, s[16:19], 0 idxen3320; GFX10-W32-NEXT: s_mov_b32 exec_lo, s03321; GFX10-W32-NEXT: s_waitcnt vmcnt(1)3322; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v03323; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3324; GFX10-W32-NEXT: v_mov_b32_e32 v3, v23325; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D3326; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3327; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v03328; GFX10-W32-NEXT: v_add_f32_e32 v0, v0, v33329; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s203330; GFX10-W32-NEXT: image_sample v0, v0, s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_1D3331; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3332; GFX10-W32-NEXT: buffer_store_dword v0, v1, s[16:19], 0 idxen3333; GFX10-W32-NEXT: buffer_load_dword v0, v1, s[16:19], 0 idxen3334; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3335; GFX10-W32-NEXT: ; return to shader part epilog3336main_body:3337 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %inp, ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3338 %reload = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx1, i32 0, i32 0, i32 0)3339 %temp = fadd float %reload, %reload3340 %res.int = ptrtoint ptr addrspace(8) %res to i1283341 %res.vec = bitcast i128 %res.int to <4 x i32>3342 %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res.vec, i1 false, i32 0, i32 0)3343 %temp2 = fadd float %tex, %tex3344 %reload_wwm = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3345 %temp3 = call float @llvm.amdgcn.strict.wqm.f32(float %reload_wwm)3346 %temp4 = fadd float %temp2, %temp33347 %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res.vec, i1 false, i32 0, i32 0)3348 call void @llvm.amdgcn.struct.ptr.buffer.store.f32(float %tex2, ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3349 %out = call float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8) %res, i32 %idx0, i32 0, i32 0, i32 0)3350 ret float %out3351}3352 3353; Check if the correct VCC register is selected. WQM pass incorrectly uses VCC for3354; vector comparisons in Wave32 mode.3355define amdgpu_ps void @test_for_deactivating_lanes_in_wave32(ptr addrspace(6) inreg %0) {3356; GFX9-W64-LABEL: test_for_deactivating_lanes_in_wave32:3357; GFX9-W64: ; %bb.0: ; %main_body3358; GFX9-W64-NEXT: s_mov_b32 s3, 0x31016fac3359; GFX9-W64-NEXT: s_mov_b32 s2, 323360; GFX9-W64-NEXT: s_mov_b32 s1, 0x80003361; GFX9-W64-NEXT: s_buffer_load_dword s0, s[0:3], 0x03362; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)3363; GFX9-W64-NEXT: v_cmp_le_f32_e64 vcc, s0, 03364; GFX9-W64-NEXT: s_andn2_b64 exec, exec, vcc3365; GFX9-W64-NEXT: s_cbranch_scc0 .LBB55_13366; GFX9-W64-NEXT: s_endpgm3367; GFX9-W64-NEXT: .LBB55_1:3368; GFX9-W64-NEXT: s_mov_b64 exec, 03369; GFX9-W64-NEXT: exp null off, off, off, off done vm3370; GFX9-W64-NEXT: s_endpgm3371;3372; GFX10-W32-LABEL: test_for_deactivating_lanes_in_wave32:3373; GFX10-W32: ; %bb.0: ; %main_body3374; GFX10-W32-NEXT: s_mov_b32 s3, 0x31016fac3375; GFX10-W32-NEXT: s_mov_b32 s2, 323376; GFX10-W32-NEXT: s_mov_b32 s1, 0x80003377; GFX10-W32-NEXT: s_buffer_load_dword s0, s[0:3], 0x03378; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)3379; GFX10-W32-NEXT: v_cmp_le_f32_e64 vcc_lo, s0, 03380; GFX10-W32-NEXT: s_andn2_b32 exec_lo, exec_lo, vcc_lo3381; GFX10-W32-NEXT: s_cbranch_scc0 .LBB55_13382; GFX10-W32-NEXT: s_endpgm3383; GFX10-W32-NEXT: .LBB55_1:3384; GFX10-W32-NEXT: s_mov_b32 exec_lo, 03385; GFX10-W32-NEXT: exp null off, off, off, off done vm3386; GFX10-W32-NEXT: s_endpgm3387main_body:3388 %1 = ptrtoint ptr addrspace(6) %0 to i323389 %2 = insertelement <4 x i32> <i32 poison, i32 32768, i32 32, i32 822177708>, i32 %1, i32 03390 %3 = call nsz arcp float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %2, i32 0, i32 0) #33391 %4 = fcmp nsz arcp ugt float %3, 0.000000e+003392 call void @llvm.amdgcn.kill(i1 %4) #13393 ret void3394}3395 3396; Test the interaction between wqm and llvm.amdgcn.init.exec.3397define amdgpu_gs void @wqm_init_exec() {3398; GFX9-W64-LABEL: wqm_init_exec:3399; GFX9-W64: ; %bb.0: ; %bb3400; GFX9-W64-NEXT: s_mov_b64 exec, -13401; GFX9-W64-NEXT: s_mov_b32 s0, 03402; GFX9-W64-NEXT: v_mov_b32_e32 v0, 03403; GFX9-W64-NEXT: s_mov_b32 s1, s03404; GFX9-W64-NEXT: s_mov_b32 s2, s03405; GFX9-W64-NEXT: s_mov_b32 s3, s03406; GFX9-W64-NEXT: v_mov_b32_e32 v1, v03407; GFX9-W64-NEXT: v_mov_b32_e32 v2, v03408; GFX9-W64-NEXT: v_mov_b32_e32 v3, v03409; GFX9-W64-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 03410; GFX9-W64-NEXT: s_wqm_b64 exec, exec3411; GFX9-W64-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $exec3412; GFX9-W64-NEXT: v_mov_b32_e32 v1, s03413; GFX9-W64-NEXT: ds_write_b32 v0, v13414; GFX9-W64-NEXT: s_endpgm3415;3416; GFX10-W32-LABEL: wqm_init_exec:3417; GFX10-W32: ; %bb.0: ; %bb3418; GFX10-W32-NEXT: s_mov_b32 exec_lo, -13419; GFX10-W32-NEXT: s_mov_b32 s1, exec_lo3420; GFX10-W32-NEXT: v_mov_b32_e32 v0, 03421; GFX10-W32-NEXT: s_mov_b32 s0, 03422; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3423; GFX10-W32-NEXT: s_mov_b32 s2, s03424; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s13425; GFX10-W32-NEXT: v_mov_b32_e32 v1, v03426; GFX10-W32-NEXT: v_mov_b32_e32 v2, v03427; GFX10-W32-NEXT: v_mov_b32_e32 v3, v03428; GFX10-W32-NEXT: v_mov_b32_e32 v4, s03429; GFX10-W32-NEXT: s_mov_b32 s1, s03430; GFX10-W32-NEXT: s_mov_b32 s3, s03431; GFX10-W32-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 03432; GFX10-W32-NEXT: ds_write_b32 v0, v43433; GFX10-W32-NEXT: s_endpgm3434bb:3435 call void @llvm.amdgcn.init.exec(i64 -1)3436 call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> zeroinitializer, <4 x i32> zeroinitializer, i32 0, i32 0, i32 0)3437 %i = call i32 @llvm.amdgcn.wqm.i32(i32 0)3438 store i32 %i, i32 addrspace(3)* null, align 43439 ret void3440}3441 3442; Test a case that failed machine verification.3443define amdgpu_gs void @wqm_init_exec_switch(i32 %arg) {3444; GFX9-W64-LABEL: wqm_init_exec_switch:3445; GFX9-W64: ; %bb.0:3446; GFX9-W64-NEXT: s_mov_b64 exec, 03447; GFX9-W64-NEXT: v_cmp_lt_i32_e32 vcc, 0, v03448; GFX9-W64-NEXT: s_and_saveexec_b64 s[0:1], vcc3449; GFX9-W64-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3450; GFX9-W64-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3451; GFX9-W64-NEXT: s_endpgm3452;3453; GFX10-W32-LABEL: wqm_init_exec_switch:3454; GFX10-W32: ; %bb.0:3455; GFX10-W32-NEXT: s_mov_b32 exec_lo, 03456; GFX10-W32-NEXT: s_mov_b32 s0, exec_lo3457; GFX10-W32-NEXT: v_cmpx_lt_i32_e32 0, v03458; GFX10-W32-NEXT: s_xor_b32 s0, exec_lo, s03459; GFX10-W32-NEXT: s_andn2_saveexec_b32 s0, s03460; GFX10-W32-NEXT: s_endpgm3461 call void @llvm.amdgcn.init.exec(i64 0)3462 switch i32 %arg, label %bb1 [3463 i32 0, label %bb33464 i32 1, label %bb23465 ]3466bb1:3467 ret void3468bb2:3469 ret void3470bb3:3471 ret void3472}3473 3474define amdgpu_gs void @wqm_init_exec_wwm() {3475; GFX9-W64-LABEL: wqm_init_exec_wwm:3476; GFX9-W64: ; %bb.0:3477; GFX9-W64-NEXT: s_mov_b64 exec, 03478; GFX9-W64-NEXT: s_mov_b32 s1, 03479; GFX9-W64-NEXT: s_mov_b32 s0, s13480; GFX9-W64-NEXT: s_cmp_eq_u64 s[0:1], 03481; GFX9-W64-NEXT: s_cselect_b64 s[0:1], -1, 03482; GFX9-W64-NEXT: v_mov_b32_e32 v0, 03483; GFX9-W64-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]3484; GFX9-W64-NEXT: exp mrt0 off, off, off, off3485; GFX9-W64-NEXT: s_endpgm3486;3487; GFX10-W32-LABEL: wqm_init_exec_wwm:3488; GFX10-W32: ; %bb.0:3489; GFX10-W32-NEXT: s_mov_b32 exec_lo, 03490; GFX10-W32-NEXT: s_mov_b32 s1, 03491; GFX10-W32-NEXT: v_mov_b32_e32 v0, 03492; GFX10-W32-NEXT: s_mov_b32 s0, s13493; GFX10-W32-NEXT: s_cmp_eq_u64 s[0:1], 03494; GFX10-W32-NEXT: s_cselect_b32 s0, -1, 03495; GFX10-W32-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s03496; GFX10-W32-NEXT: exp mrt0 off, off, off, off3497; GFX10-W32-NEXT: s_endpgm3498 call void @llvm.amdgcn.init.exec(i64 0)3499 %i = call i64 @llvm.amdgcn.ballot.i64(i1 true)3500 %i1 = call i32 @llvm.amdgcn.wwm.i32(i32 0)3501 %i2 = insertelement <2 x i32> zeroinitializer, i32 %i1, i64 03502 %i3 = bitcast <2 x i32> %i2 to i643503 %i4 = icmp ne i64 %i, 03504 %i5 = icmp ne i64 %i3, 03505 %i6 = xor i1 %i4, %i53506 %i7 = uitofp i1 %i6 to float3507 call void @llvm.amdgcn.exp.f32(i32 0, i32 0, float %i7, float 0.0, float 0.0, float 0.0, i1 false, i1 false)3508 ret void3509}3510 3511; Check that exact regions with execz affected instructions are as short as possible3512define amdgpu_ps float @short_exact_regions(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c, ptr addrspace(4) %p) {3513; GFX9-W64-LABEL: short_exact_regions:3514; GFX9-W64: ; %bb.0: ; %main_body3515; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec3516; GFX9-W64-NEXT: s_wqm_b64 exec, exec3517; GFX9-W64-NEXT: image_sample v[3:6], v0, s[0:7], s[8:11] dmask:0xf3518; GFX9-W64-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 03519; GFX9-W64-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v03520; GFX9-W64-NEXT: v_cmp_gt_u32_e32 vcc, 16, v03521; GFX9-W64-NEXT: s_and_saveexec_b64 s[14:15], vcc3522; GFX9-W64-NEXT: s_cbranch_execz .LBB59_23523; GFX9-W64-NEXT: ; %bb.1: ; %if3524; GFX9-W64-NEXT: global_load_dword v0, v[1:2], off3525; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3526; GFX9-W64-NEXT: v_readfirstlane_b32 s16, v03527; GFX9-W64-NEXT: s_buffer_load_dword s16, s[8:11], s16 offset:0x03528; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)3529; GFX9-W64-NEXT: v_mov_b32_e32 v0, s163530; GFX9-W64-NEXT: s_and_saveexec_b64 s[16:17], s[12:13]3531; GFX9-W64-NEXT: buffer_store_dwordx4 v[3:6], v0, s[0:3], 0 idxen3532; GFX9-W64-NEXT: s_mov_b64 exec, s[16:17]3533; GFX9-W64-NEXT: .LBB59_2: ; %endif3534; GFX9-W64-NEXT: s_or_b64 exec, exec, s[14:15]3535; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3536; GFX9-W64-NEXT: image_sample v0, v3, s[0:7], s[8:11] dmask:0x43537; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3538; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v03539; GFX9-W64-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec3540; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]3541; GFX9-W64-NEXT: ; return to shader part epilog3542;3543; GFX10-W32-LABEL: short_exact_regions:3544; GFX10-W32: ; %bb.0: ; %main_body3545; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo3546; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3547; GFX10-W32-NEXT: image_sample v[3:6], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D3548; GFX10-W32-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 03549; GFX10-W32-NEXT: s_mov_b32 s13, exec_lo3550; GFX10-W32-NEXT: v_mbcnt_hi_u32_b32 v0, -1, v03551; GFX10-W32-NEXT: v_cmpx_gt_u32_e32 16, v03552; GFX10-W32-NEXT: s_cbranch_execz .LBB59_23553; GFX10-W32-NEXT: ; %bb.1: ; %if3554; GFX10-W32-NEXT: global_load_dword v0, v[1:2], off3555; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3556; GFX10-W32-NEXT: v_readfirstlane_b32 s14, v03557; GFX10-W32-NEXT: s_buffer_load_dword s14, s[8:11], s14 offset:0x03558; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)3559; GFX10-W32-NEXT: v_mov_b32_e32 v0, s143560; GFX10-W32-NEXT: s_and_saveexec_b32 s14, s123561; GFX10-W32-NEXT: buffer_store_dwordx4 v[3:6], v0, s[0:3], 0 idxen3562; GFX10-W32-NEXT: s_mov_b32 exec_lo, s143563; GFX10-W32-NEXT: .LBB59_2: ; %endif3564; GFX10-W32-NEXT: s_or_b32 exec_lo, exec_lo, s133565; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3566; GFX10-W32-NEXT: image_sample v0, v3, s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_1D3567; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3568; GFX10-W32-NEXT: v_add_f32_e32 v0, v4, v03569; GFX10-W32-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec3570; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s123571; GFX10-W32-NEXT: ; return to shader part epilog3572main_body:3573 %tex1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03574 %idx0 = load <4 x i32>, ptr addrspace(4) %p, align 43575 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)3576 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)3577 %cc = icmp uge i32 %hi, 163578 br i1 %cc, label %endif, label %if3579 3580if:3581 %idx1 = extractelement <4 x i32> %idx0, i64 03582 %idx2 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %idx1)3583 %idx3 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %sampler, i32 %idx2, i32 0)3584 3585 call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %tex1, <4 x i32> poison, i32 %idx3, i32 0, i32 0, i32 0)3586 br label %endif3587 3588endif:3589 %d = extractelement <4 x float> %tex1, i64 03590 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %d, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03591 %r0 = extractelement <4 x float> %tex1, i64 13592 %r1 = extractelement <4 x float> %tex2, i64 23593 %r2 = fadd float %r0, %r13594 %out = call float @llvm.amdgcn.wqm.f32(float %r2)3595 3596 ret float %out3597}3598 3599; Check that exact regions shortening doesn't prevent early WQM exit3600define amdgpu_ps float @short_exact_regions_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c, ptr addrspace(4) %p) {3601; GFX9-W64-LABEL: short_exact_regions_2:3602; GFX9-W64: ; %bb.0: ; %main_body3603; GFX9-W64-NEXT: s_mov_b64 s[12:13], exec3604; GFX9-W64-NEXT: s_wqm_b64 exec, exec3605; GFX9-W64-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x33606; GFX9-W64-NEXT: s_and_b64 exec, exec, s[12:13]3607; GFX9-W64-NEXT: global_load_dword v0, v[1:2], off3608; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3609; GFX9-W64-NEXT: image_sample v5, v3, s[0:7], s[8:11] dmask:0x43610; GFX9-W64-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6 killed $sgpr73611; GFX9-W64-NEXT: ; kill: killed $vgpr33612; GFX9-W64-NEXT: ; kill: killed $vgpr1 killed $vgpr23613; GFX9-W64-NEXT: s_waitcnt vmcnt(1)3614; GFX9-W64-NEXT: v_readfirstlane_b32 s0, v03615; GFX9-W64-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x03616; GFX9-W64-NEXT: s_waitcnt vmcnt(0)3617; GFX9-W64-NEXT: v_add_f32_e32 v0, v4, v53618; GFX9-W64-NEXT: s_waitcnt lgkmcnt(0)3619; GFX9-W64-NEXT: v_add_f32_e32 v0, s0, v03620; GFX9-W64-NEXT: ; return to shader part epilog3621;3622; GFX10-W32-LABEL: short_exact_regions_2:3623; GFX10-W32: ; %bb.0: ; %main_body3624; GFX10-W32-NEXT: s_mov_b32 s12, exec_lo3625; GFX10-W32-NEXT: s_wqm_b32 exec_lo, exec_lo3626; GFX10-W32-NEXT: image_sample v[3:4], v0, s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D3627; GFX10-W32-NEXT: s_and_b32 exec_lo, exec_lo, s123628; GFX10-W32-NEXT: global_load_dword v0, v[1:2], off3629; GFX10-W32-NEXT: s_waitcnt vmcnt(1)3630; GFX10-W32-NEXT: image_sample v1, v3, s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_1D3631; GFX10-W32-NEXT: s_waitcnt vmcnt(1)3632; GFX10-W32-NEXT: v_readfirstlane_b32 s0, v03633; GFX10-W32-NEXT: s_waitcnt vmcnt(0)3634; GFX10-W32-NEXT: v_add_f32_e32 v0, v4, v13635; GFX10-W32-NEXT: s_buffer_load_dword s0, s[8:11], s0 offset:0x03636; GFX10-W32-NEXT: s_waitcnt lgkmcnt(0)3637; GFX10-W32-NEXT: v_add_f32_e32 v0, s0, v03638; GFX10-W32-NEXT: ; return to shader part epilog3639main_body:3640 %tex1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03641 %idx0 = load <4 x i32>, ptr addrspace(4) %p, align 43642 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)3643 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)3644 %idx1 = extractelement <4 x i32> %idx0, i64 03645 %d = extractelement <4 x float> %tex1, i64 03646 3647 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %d, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #03648 3649 %idx2 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %idx1)3650 %idx3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %sampler, i32 %idx2, i32 0)3651 3652 %r0 = extractelement <4 x float> %tex1, i64 13653 %r1 = extractelement <4 x float> %tex2, i64 23654 %r2 = fadd float %r0, %r13655 %out = fadd float %r2, %idx33656 3657 ret float %out3658}3659 3660declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #13661declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #13662 3663declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg) #23664declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #23665declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg) #23666declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #23667declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #33668declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #33669 3670declare void @llvm.amdgcn.struct.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg) #23671declare void @llvm.amdgcn.struct.ptr.buffer.store.v4f32(<4 x float>, ptr addrspace(8), i32, i32, i32, i32 immarg) #23672declare void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float>, ptr addrspace(8), i32, i32, i32 immarg) #23673declare void @llvm.amdgcn.raw.ptr.buffer.store.f32(float, ptr addrspace(8), i32, i32, i32 immarg) #23674declare float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32) #33675declare float @llvm.amdgcn.struct.ptr.buffer.load.f32(ptr addrspace(8), i32, i32, i32, i32) #33676 3677declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #33678declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #33679declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #33680declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #33681declare void @llvm.amdgcn.kill(i1) #13682declare float @llvm.amdgcn.wqm.f32(float) #33683declare i32 @llvm.amdgcn.wqm.i32(i32) #33684declare float @llvm.amdgcn.strict.wwm.f32(float) #33685declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #33686declare float @llvm.amdgcn.wwm.f32(float) #33687declare i32 @llvm.amdgcn.wwm.i32(i32) #33688declare float @llvm.amdgcn.strict.wqm.f32(float) #33689declare i32 @llvm.amdgcn.strict.wqm.i32(i32) #33690declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #43691declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #33692declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #33693declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #33694declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #13695declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #23696declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #23697declare i32 @llvm.amdgcn.ds.swizzle(i32, i32)3698declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32 immarg) #73699declare i32 @llvm.amdgcn.readfirstlane.i32(i32)3700 3701attributes #1 = { nounwind }3702attributes #2 = { nounwind readonly }3703attributes #3 = { nounwind readnone }3704attributes #4 = { nounwind readnone convergent }3705attributes #5 = { "amdgpu-ps-wqm-outputs" }3706attributes #6 = { nounwind "InitialPSInputAddr"="2" }3707attributes #7 = { nounwind readnone willreturn }3708