2928 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 < %s | FileCheck --check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9-NODL %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck --check-prefixes=GFX9-DL %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck --check-prefixes=GFX10-DL %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 < %s | FileCheck --check-prefixes=GFX10-DL %s8 9; add(mul(S0.x, S1.y),10; add (mul (S0.y, S1.y), S3)) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)11 12define amdgpu_kernel void @udot2(ptr addrspace(1) %src1,13; GFX7-LABEL: udot2:14; GFX7: ; %bb.0: ; %entry15; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x916; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd17; GFX7-NEXT: s_mov_b32 s7, 0xf00018; GFX7-NEXT: s_mov_b32 s10, 019; GFX7-NEXT: s_mov_b32 s11, s720; GFX7-NEXT: s_waitcnt lgkmcnt(0)21; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]22; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v023; GFX7-NEXT: v_mov_b32_e32 v1, 024; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr6425; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]26; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr6427; GFX7-NEXT: s_load_dword s0, s[4:5], 0x028; GFX7-NEXT: s_mov_b32 s6, -129; GFX7-NEXT: s_waitcnt vmcnt(1)30; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v231; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v232; GFX7-NEXT: s_waitcnt vmcnt(0)33; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v034; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v035; GFX7-NEXT: s_waitcnt lgkmcnt(0)36; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s037; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v138; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 039; GFX7-NEXT: s_endpgm40;41; GFX8-LABEL: udot2:42; GFX8: ; %bb.0: ; %entry43; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2444; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3445; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v046; GFX8-NEXT: s_waitcnt lgkmcnt(0)47; GFX8-NEXT: v_mov_b32_e32 v1, s148; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v249; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc50; GFX8-NEXT: flat_load_dword v3, v[0:1]51; GFX8-NEXT: v_mov_b32_e32 v1, s352; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v253; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc54; GFX8-NEXT: flat_load_dword v0, v[0:1]55; GFX8-NEXT: s_load_dword s0, s[4:5], 0x056; GFX8-NEXT: s_waitcnt vmcnt(1)57; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v358; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v359; GFX8-NEXT: s_waitcnt vmcnt(0)60; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v061; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v062; GFX8-NEXT: s_waitcnt lgkmcnt(0)63; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s064; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v065; GFX8-NEXT: v_mov_b32_e32 v0, s466; GFX8-NEXT: v_mov_b32_e32 v1, s567; GFX8-NEXT: flat_store_dword v[0:1], v268; GFX8-NEXT: s_endpgm69;70; GFX9-NODL-LABEL: udot2:71; GFX9-NODL: ; %bb.0: ; %entry72; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2473; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3474; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v075; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)76; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]77; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]78; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x079; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 080; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)81; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_082; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_183; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)84; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v385; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]86; GFX9-NODL-NEXT: s_endpgm87;88; GFX9-DL-LABEL: udot2:89; GFX9-DL: ; %bb.0: ; %entry90; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2491; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3492; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v093; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)94; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]95; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]96; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x097; GFX9-DL-NEXT: v_mov_b32_e32 v0, 098; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)99; GFX9-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s0100; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]101; GFX9-DL-NEXT: s_endpgm102;103; GFX10-DL-LABEL: udot2:104; GFX10-DL: ; %bb.0: ; %entry105; GFX10-DL-NEXT: s_clause 0x1106; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24107; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34108; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0109; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)110; GFX10-DL-NEXT: s_clause 0x1111; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]112; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]113; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)114; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0115; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0116; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)117; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s0118; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]119; GFX10-DL-NEXT: s_endpgm120 ptr addrspace(1) %src2,121 ptr addrspace(1) nocapture %dst) {122entry:123 %idx = call i32 @llvm.amdgcn.workitem.id.x()124 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx125 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1126 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx127 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2128 129 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0130 %conv = zext i16 %s1.elt1 to i32131 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0132 %conv2 = zext i16 %s2.elt1 to i32133 %mul1 = mul nuw i32 %conv2, %conv134 135 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1136 %conv3 = zext i16 %s1.elt2 to i32137 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1138 %conv4 = zext i16 %s2.elt2 to i32139 %mul2 = mul nuw i32 %conv4, %conv3140 141 %s3 = load i32, ptr addrspace(1) %dst, align 4142 %add = add i32 %mul2, %s3143 %add6 = add i32 %add, %mul1144 store i32 %add6, ptr addrspace(1) %dst, align 4145 ret void146}147 148; TODO: Support this pattern149; add(S3,150; add (mul (S0.y, S1.y), mul (S0.y, S1.y))) -> v_dot2_{I|U}32_{I|U}16(S1, S2, S3)151define amdgpu_kernel void @udot2_MulMul(ptr addrspace(1) %src1,152; GFX7-LABEL: udot2_MulMul:153; GFX7: ; %bb.0: ; %entry154; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9155; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd156; GFX7-NEXT: s_mov_b32 s7, 0xf000157; GFX7-NEXT: s_mov_b32 s10, 0158; GFX7-NEXT: s_mov_b32 s11, s7159; GFX7-NEXT: s_waitcnt lgkmcnt(0)160; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]161; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0162; GFX7-NEXT: v_mov_b32_e32 v1, 0163; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64164; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]165; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64166; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0167; GFX7-NEXT: s_mov_b32 s6, -1168; GFX7-NEXT: s_waitcnt vmcnt(1)169; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2170; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2171; GFX7-NEXT: s_waitcnt vmcnt(0)172; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0173; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0174; GFX7-NEXT: v_mul_u32_u24_e32 v0, v0, v2175; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v0176; GFX7-NEXT: s_waitcnt lgkmcnt(0)177; GFX7-NEXT: v_add_i32_e32 v0, vcc, s0, v0178; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0179; GFX7-NEXT: s_endpgm180;181; GFX8-LABEL: udot2_MulMul:182; GFX8: ; %bb.0: ; %entry183; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24184; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34185; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0186; GFX8-NEXT: s_waitcnt lgkmcnt(0)187; GFX8-NEXT: v_mov_b32_e32 v1, s1188; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2189; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc190; GFX8-NEXT: flat_load_dword v3, v[0:1]191; GFX8-NEXT: v_mov_b32_e32 v1, s3192; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2193; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc194; GFX8-NEXT: flat_load_dword v0, v[0:1]195; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0196; GFX8-NEXT: s_waitcnt vmcnt(1)197; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3198; GFX8-NEXT: s_waitcnt vmcnt(0)199; GFX8-NEXT: v_mul_u32_u24_sdwa v1, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0200; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0201; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v1202; GFX8-NEXT: s_waitcnt lgkmcnt(0)203; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v0204; GFX8-NEXT: v_mov_b32_e32 v0, s4205; GFX8-NEXT: v_mov_b32_e32 v1, s5206; GFX8-NEXT: flat_store_dword v[0:1], v2207; GFX8-NEXT: s_endpgm208;209; GFX9-NODL-LABEL: udot2_MulMul:210; GFX9-NODL: ; %bb.0: ; %entry211; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24212; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34213; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0214; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)215; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]216; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]217; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0218; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0219; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)220; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0221; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1222; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)223; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, s0224; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]225; GFX9-NODL-NEXT: s_endpgm226;227; GFX9-DL-LABEL: udot2_MulMul:228; GFX9-DL: ; %bb.0: ; %entry229; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24230; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34231; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0232; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)233; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]234; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]235; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0236; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0237; GFX9-DL-NEXT: s_waitcnt vmcnt(0)238; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0239; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1240; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)241; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, s0242; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]243; GFX9-DL-NEXT: s_endpgm244;245; GFX10-DL-LABEL: udot2_MulMul:246; GFX10-DL: ; %bb.0: ; %entry247; GFX10-DL-NEXT: s_clause 0x1248; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24249; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34250; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0251; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)252; GFX10-DL-NEXT: s_clause 0x1253; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]254; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]255; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)256; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0257; GFX10-DL-NEXT: s_waitcnt vmcnt(0)258; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0259; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1260; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0261; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)262; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, s0263; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]264; GFX10-DL-NEXT: s_endpgm265 ptr addrspace(1) %src2,266 ptr addrspace(1) nocapture %dst) {267entry:268 %idx = call i32 @llvm.amdgcn.workitem.id.x()269 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx270 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1271 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx272 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2273 274 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0275 %conv = zext i16 %s1.elt1 to i32276 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0277 %conv2 = zext i16 %s2.elt1 to i32278 %mul1 = mul nuw i32 %conv2, %conv279 280 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1281 %conv3 = zext i16 %s1.elt2 to i32282 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1283 %conv4 = zext i16 %s2.elt2 to i32284 %mul2 = mul nuw i32 %conv4, %conv3285 %s3 = load i32, ptr addrspace(1) %dst, align 4286 %add = add i32 %mul2, %mul1287 %add6 = add i32 %add, %s3288 store i32 %add6, ptr addrspace(1) %dst, align 4289 ret void290}291 292define amdgpu_kernel void @idot2(ptr addrspace(1) %src1,293; GFX7-LABEL: idot2:294; GFX7: ; %bb.0: ; %entry295; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9296; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd297; GFX7-NEXT: s_mov_b32 s7, 0xf000298; GFX7-NEXT: s_mov_b32 s10, 0299; GFX7-NEXT: s_mov_b32 s11, s7300; GFX7-NEXT: s_waitcnt lgkmcnt(0)301; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]302; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0303; GFX7-NEXT: v_mov_b32_e32 v1, 0304; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64305; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]306; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64307; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0308; GFX7-NEXT: s_mov_b32 s6, -1309; GFX7-NEXT: s_waitcnt vmcnt(1)310; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16311; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2312; GFX7-NEXT: s_waitcnt vmcnt(0)313; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 16314; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0315; GFX7-NEXT: s_waitcnt lgkmcnt(0)316; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s0317; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0318; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0319; GFX7-NEXT: s_endpgm320;321; GFX8-LABEL: idot2:322; GFX8: ; %bb.0: ; %entry323; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24324; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34325; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0326; GFX8-NEXT: s_waitcnt lgkmcnt(0)327; GFX8-NEXT: v_mov_b32_e32 v1, s1328; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2329; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc330; GFX8-NEXT: flat_load_dword v3, v[0:1]331; GFX8-NEXT: v_mov_b32_e32 v1, s3332; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2333; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc334; GFX8-NEXT: flat_load_dword v0, v[0:1]335; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0336; GFX8-NEXT: s_waitcnt vmcnt(1)337; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16338; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3339; GFX8-NEXT: s_waitcnt vmcnt(0)340; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16341; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0342; GFX8-NEXT: s_waitcnt lgkmcnt(0)343; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s0344; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0345; GFX8-NEXT: v_mov_b32_e32 v0, s4346; GFX8-NEXT: v_mov_b32_e32 v1, s5347; GFX8-NEXT: flat_store_dword v[0:1], v2348; GFX8-NEXT: s_endpgm349;350; GFX9-NODL-LABEL: idot2:351; GFX9-NODL: ; %bb.0: ; %entry352; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24353; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34354; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0355; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)356; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]357; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]358; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0359; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0360; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)361; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0362; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1363; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)364; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3365; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]366; GFX9-NODL-NEXT: s_endpgm367;368; GFX9-DL-LABEL: idot2:369; GFX9-DL: ; %bb.0: ; %entry370; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24371; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34372; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0373; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)374; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]375; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]376; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0377; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0378; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)379; GFX9-DL-NEXT: v_dot2_i32_i16 v1, v2, v1, s0380; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]381; GFX9-DL-NEXT: s_endpgm382;383; GFX10-DL-LABEL: idot2:384; GFX10-DL: ; %bb.0: ; %entry385; GFX10-DL-NEXT: s_clause 0x1386; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24387; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34388; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0389; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)390; GFX10-DL-NEXT: s_clause 0x1391; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]392; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]393; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)394; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0395; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0396; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)397; GFX10-DL-NEXT: v_dot2_i32_i16 v1, v2, v1, s0398; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]399; GFX10-DL-NEXT: s_endpgm400 ptr addrspace(1) %src2,401 ptr addrspace(1) nocapture %dst) {402entry:403 %idx = call i32 @llvm.amdgcn.workitem.id.x()404 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx405 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1406 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx407 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2408 409 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0410 %conv = sext i16 %s1.elt1 to i32411 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0412 %conv2 = sext i16 %s2.elt1 to i32413 %mul1 = mul nuw i32 %conv2, %conv414 415 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1416 %conv3 = sext i16 %s1.elt2 to i32417 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1418 %conv4 = sext i16 %s2.elt2 to i32419 %mul2 = mul nuw i32 %conv4, %conv3420 421 %s3 = load i32, ptr addrspace(1) %dst, align 4422 %add = add i32 %mul2, %s3423 %add6 = add i32 %add, %mul1424 store i32 %add6, ptr addrspace(1) %dst, align 4425 ret void426}427 428define amdgpu_kernel void @idot2_MixedTypedMul(ptr addrspace(1) %src1,429; GFX7-LABEL: idot2_MixedTypedMul:430; GFX7: ; %bb.0: ; %entry431; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9432; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd433; GFX7-NEXT: s_mov_b32 s7, 0xf000434; GFX7-NEXT: s_mov_b32 s10, 0435; GFX7-NEXT: s_mov_b32 s11, s7436; GFX7-NEXT: s_waitcnt lgkmcnt(0)437; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]438; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0439; GFX7-NEXT: v_mov_b32_e32 v1, 0440; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64441; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]442; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64443; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0444; GFX7-NEXT: s_mov_b32 s6, -1445; GFX7-NEXT: s_waitcnt vmcnt(1)446; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2447; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16448; GFX7-NEXT: s_waitcnt vmcnt(0)449; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0450; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16451; GFX7-NEXT: s_waitcnt lgkmcnt(0)452; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s0453; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v1454; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0455; GFX7-NEXT: s_endpgm456;457; GFX8-LABEL: idot2_MixedTypedMul:458; GFX8: ; %bb.0: ; %entry459; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24460; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34461; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0462; GFX8-NEXT: s_waitcnt lgkmcnt(0)463; GFX8-NEXT: v_mov_b32_e32 v1, s1464; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2465; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc466; GFX8-NEXT: flat_load_dword v3, v[0:1]467; GFX8-NEXT: v_mov_b32_e32 v1, s3468; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2469; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc470; GFX8-NEXT: flat_load_dword v0, v[0:1]471; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0472; GFX8-NEXT: s_waitcnt vmcnt(1)473; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16474; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3475; GFX8-NEXT: s_waitcnt vmcnt(0)476; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 16477; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0478; GFX8-NEXT: s_waitcnt lgkmcnt(0)479; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s0480; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0481; GFX8-NEXT: v_mov_b32_e32 v0, s4482; GFX8-NEXT: v_mov_b32_e32 v1, s5483; GFX8-NEXT: flat_store_dword v[0:1], v2484; GFX8-NEXT: s_endpgm485;486; GFX9-NODL-LABEL: idot2_MixedTypedMul:487; GFX9-NODL: ; %bb.0: ; %entry488; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24489; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34490; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0491; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)492; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]493; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]494; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0495; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0496; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)497; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0498; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1499; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)500; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3501; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]502; GFX9-NODL-NEXT: s_endpgm503;504; GFX9-DL-LABEL: idot2_MixedTypedMul:505; GFX9-DL: ; %bb.0: ; %entry506; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24507; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34508; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0509; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)510; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]511; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]512; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0513; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0514; GFX9-DL-NEXT: s_waitcnt vmcnt(0)515; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0516; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1517; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)518; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3519; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]520; GFX9-DL-NEXT: s_endpgm521;522; GFX10-DL-LABEL: idot2_MixedTypedMul:523; GFX10-DL: ; %bb.0: ; %entry524; GFX10-DL-NEXT: s_clause 0x1525; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24526; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34527; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0528; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)529; GFX10-DL-NEXT: s_clause 0x1530; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]531; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]532; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)533; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0534; GFX10-DL-NEXT: s_waitcnt vmcnt(0)535; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0536; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1537; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0538; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)539; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v0540; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]541; GFX10-DL-NEXT: s_endpgm542 ptr addrspace(1) %src2,543 ptr addrspace(1) nocapture %dst) {544entry:545 %idx = call i32 @llvm.amdgcn.workitem.id.x()546 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx547 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1548 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx549 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2550 551 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0552 %conv = sext i16 %s1.elt1 to i32553 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0554 %conv2 = sext i16 %s2.elt1 to i32555 %mul1 = mul nuw i32 %conv2, %conv556 557 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1558 %conv3 = zext i16 %s1.elt2 to i32559 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1560 %conv4 = zext i16 %s2.elt2 to i32561 %mul2 = mul nuw i32 %conv4, %conv3562 563 %s3 = load i32, ptr addrspace(1) %dst, align 4564 %add = add i32 %mul2, %s3565 %add6 = add i32 %add, %mul1566 store i32 %add6, ptr addrspace(1) %dst, align 4567 ret void568}569 570define amdgpu_kernel void @udot2_alt_AddOperands(ptr addrspace(1) %src1,571; GFX7-LABEL: udot2_alt_AddOperands:572; GFX7: ; %bb.0: ; %entry573; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9574; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd575; GFX7-NEXT: s_mov_b32 s7, 0xf000576; GFX7-NEXT: s_mov_b32 s10, 0577; GFX7-NEXT: s_mov_b32 s11, s7578; GFX7-NEXT: s_waitcnt lgkmcnt(0)579; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]580; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0581; GFX7-NEXT: v_mov_b32_e32 v1, 0582; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64583; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]584; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64585; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0586; GFX7-NEXT: s_mov_b32 s6, -1587; GFX7-NEXT: s_waitcnt vmcnt(1)588; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2589; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2590; GFX7-NEXT: s_waitcnt vmcnt(0)591; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0592; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0593; GFX7-NEXT: s_waitcnt lgkmcnt(0)594; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s0595; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1596; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0597; GFX7-NEXT: s_endpgm598;599; GFX8-LABEL: udot2_alt_AddOperands:600; GFX8: ; %bb.0: ; %entry601; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24602; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34603; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0604; GFX8-NEXT: s_waitcnt lgkmcnt(0)605; GFX8-NEXT: v_mov_b32_e32 v1, s1606; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2607; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc608; GFX8-NEXT: flat_load_dword v3, v[0:1]609; GFX8-NEXT: v_mov_b32_e32 v1, s3610; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2611; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc612; GFX8-NEXT: flat_load_dword v0, v[0:1]613; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0614; GFX8-NEXT: s_waitcnt vmcnt(1)615; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v3616; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3617; GFX8-NEXT: s_waitcnt vmcnt(0)618; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0619; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0620; GFX8-NEXT: s_waitcnt lgkmcnt(0)621; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s0622; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v0623; GFX8-NEXT: v_mov_b32_e32 v0, s4624; GFX8-NEXT: v_mov_b32_e32 v1, s5625; GFX8-NEXT: flat_store_dword v[0:1], v2626; GFX8-NEXT: s_endpgm627;628; GFX9-NODL-LABEL: udot2_alt_AddOperands:629; GFX9-NODL: ; %bb.0: ; %entry630; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24631; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34632; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0633; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)634; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]635; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]636; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0637; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0638; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)639; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xffff, v1640; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)641; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xffff, v2642; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v1643; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v2644; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)645; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s0646; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v4, v3, v1647; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]648; GFX9-NODL-NEXT: s_endpgm649;650; GFX9-DL-LABEL: udot2_alt_AddOperands:651; GFX9-DL: ; %bb.0: ; %entry652; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24653; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34654; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0655; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)656; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]657; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]658; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0659; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0660; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)661; GFX9-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s0662; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]663; GFX9-DL-NEXT: s_endpgm664;665; GFX10-DL-LABEL: udot2_alt_AddOperands:666; GFX10-DL: ; %bb.0: ; %entry667; GFX10-DL-NEXT: s_clause 0x1668; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24669; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34670; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0671; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)672; GFX10-DL-NEXT: s_clause 0x1673; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]674; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]675; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)676; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0677; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0678; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)679; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s0680; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]681; GFX10-DL-NEXT: s_endpgm682 ptr addrspace(1) %src2,683 ptr addrspace(1) nocapture %dst) {684entry:685 %idx = call i32 @llvm.amdgcn.workitem.id.x()686 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx687 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1688 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx689 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2690 691 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0692 %conv = zext i16 %s1.elt1 to i32693 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0694 %conv2 = zext i16 %s2.elt1 to i32695 %mul1 = mul nuw i32 %conv2, %conv696 697 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1698 %conv3 = zext i16 %s1.elt2 to i32699 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1700 %conv4 = zext i16 %s2.elt2 to i32701 %mul2 = mul nuw i32 %conv4, %conv3702 703 %s3 = load i32, ptr addrspace(1) %dst, align 4704 %add = add i32 %s3, %mul2705 %add6 = add i32 %mul1, %add706 store i32 %add6, ptr addrspace(1) %dst, align 4707 ret void708}709 710define amdgpu_kernel void @idot2_MixedExt(ptr addrspace(1) %src1,711; GFX7-LABEL: idot2_MixedExt:712; GFX7: ; %bb.0: ; %entry713; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9714; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd715; GFX7-NEXT: s_mov_b32 s7, 0xf000716; GFX7-NEXT: s_mov_b32 s10, 0717; GFX7-NEXT: s_mov_b32 s11, s7718; GFX7-NEXT: s_waitcnt lgkmcnt(0)719; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]720; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0721; GFX7-NEXT: v_mov_b32_e32 v1, 0722; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64723; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]724; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64725; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0726; GFX7-NEXT: s_mov_b32 s6, -1727; GFX7-NEXT: s_waitcnt vmcnt(1)728; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 16729; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v2730; GFX7-NEXT: s_waitcnt vmcnt(0)731; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v0732; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v0733; GFX7-NEXT: s_waitcnt lgkmcnt(0)734; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s0735; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v0736; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0737; GFX7-NEXT: s_endpgm738;739; GFX8-LABEL: idot2_MixedExt:740; GFX8: ; %bb.0: ; %entry741; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24742; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34743; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0744; GFX8-NEXT: s_waitcnt lgkmcnt(0)745; GFX8-NEXT: v_mov_b32_e32 v1, s1746; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2747; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc748; GFX8-NEXT: flat_load_dword v3, v[0:1]749; GFX8-NEXT: v_mov_b32_e32 v1, s3750; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2751; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc752; GFX8-NEXT: flat_load_dword v0, v[0:1]753; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0754; GFX8-NEXT: s_waitcnt vmcnt(1)755; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 16756; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v3757; GFX8-NEXT: s_waitcnt vmcnt(0)758; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0759; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v0760; GFX8-NEXT: s_waitcnt lgkmcnt(0)761; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s0762; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v0763; GFX8-NEXT: v_mov_b32_e32 v0, s4764; GFX8-NEXT: v_mov_b32_e32 v1, s5765; GFX8-NEXT: flat_store_dword v[0:1], v2766; GFX8-NEXT: s_endpgm767;768; GFX9-NODL-LABEL: idot2_MixedExt:769; GFX9-NODL: ; %bb.0: ; %entry770; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24771; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34772; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0773; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)774; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]775; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]776; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0777; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0778; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)779; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0780; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1781; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)782; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v3783; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]784; GFX9-NODL-NEXT: s_endpgm785;786; GFX9-DL-LABEL: idot2_MixedExt:787; GFX9-DL: ; %bb.0: ; %entry788; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24789; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34790; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0791; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)792; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]793; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]794; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0795; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0796; GFX9-DL-NEXT: s_waitcnt vmcnt(0)797; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0798; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1799; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)800; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v3801; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]802; GFX9-DL-NEXT: s_endpgm803;804; GFX10-DL-LABEL: idot2_MixedExt:805; GFX10-DL: ; %bb.0: ; %entry806; GFX10-DL-NEXT: s_clause 0x1807; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24808; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34809; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0810; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)811; GFX10-DL-NEXT: s_clause 0x1812; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]813; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]814; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)815; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0816; GFX10-DL-NEXT: s_waitcnt vmcnt(0)817; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v0, v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0818; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1819; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0820; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)821; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v0822; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]823; GFX10-DL-NEXT: s_endpgm824 ptr addrspace(1) %src2,825 ptr addrspace(1) nocapture %dst) {826entry:827 %idx = call i32 @llvm.amdgcn.workitem.id.x()828 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx829 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1830 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx831 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2832 833 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0834 %conv = sext i16 %s1.elt1 to i32835 %s2.elt1 = extractelement <2 x i16> %vec2, i64 0836 %conv2 = zext i16 %s2.elt1 to i32837 %mul1 = mul nuw i32 %conv2, %conv838 839 %s1.elt2 = extractelement <2 x i16> %vec1, i64 1840 %conv3 = sext i16 %s1.elt2 to i32841 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1842 %conv4 = sext i16 %s2.elt2 to i32843 %mul2 = mul nuw i32 %conv4, %conv3844 845 %s3 = load i32, ptr addrspace(1) %dst, align 4846 %add = add i32 %mul2, %s3847 %add6 = add i32 %add, %mul1848 store i32 %add6, ptr addrspace(1) %dst, align 4849 ret void850}851 852define amdgpu_kernel void @notudot2_SameVec(ptr addrspace(1) %src1,853; GFX7-LABEL: notudot2_SameVec:854; GFX7: ; %bb.0: ; %entry855; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9856; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd857; GFX7-NEXT: s_mov_b32 s7, 0xf000858; GFX7-NEXT: s_mov_b32 s10, 0859; GFX7-NEXT: s_mov_b32 s11, s7860; GFX7-NEXT: s_waitcnt lgkmcnt(0)861; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]862; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0863; GFX7-NEXT: v_mov_b32_e32 v1, 0864; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64865; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]866; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64867; GFX7-NEXT: s_load_dword s0, s[4:5], 0x0868; GFX7-NEXT: s_mov_b32 s6, -1869; GFX7-NEXT: s_waitcnt vmcnt(1)870; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2871; GFX7-NEXT: s_waitcnt vmcnt(0)872; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0873; GFX7-NEXT: s_waitcnt lgkmcnt(0)874; GFX7-NEXT: v_mad_u32_u24 v0, v0, v0, s0875; GFX7-NEXT: v_mad_u32_u24 v0, v1, v1, v0876; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0877; GFX7-NEXT: s_endpgm878;879; GFX8-LABEL: notudot2_SameVec:880; GFX8: ; %bb.0: ; %entry881; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24882; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34883; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0884; GFX8-NEXT: s_waitcnt lgkmcnt(0)885; GFX8-NEXT: v_mov_b32_e32 v1, s1886; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2887; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc888; GFX8-NEXT: flat_load_dword v3, v[0:1]889; GFX8-NEXT: v_mov_b32_e32 v1, s3890; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2891; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc892; GFX8-NEXT: flat_load_dword v0, v[0:1]893; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0894; GFX8-NEXT: s_waitcnt vmcnt(1)895; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v3896; GFX8-NEXT: s_waitcnt vmcnt(0)897; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0898; GFX8-NEXT: s_waitcnt lgkmcnt(0)899; GFX8-NEXT: v_mad_u32_u24 v0, v0, v0, s0900; GFX8-NEXT: v_mad_u32_u24 v2, v1, v1, v0901; GFX8-NEXT: v_mov_b32_e32 v0, s4902; GFX8-NEXT: v_mov_b32_e32 v1, s5903; GFX8-NEXT: flat_store_dword v[0:1], v2904; GFX8-NEXT: s_endpgm905;906; GFX9-NODL-LABEL: notudot2_SameVec:907; GFX9-NODL: ; %bb.0: ; %entry908; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24909; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34910; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0911; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)912; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]913; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]914; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0915; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0916; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)917; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0918; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)919; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1920; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)921; GFX9-NODL-NEXT: v_add3_u32 v1, v2, s0, v1922; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]923; GFX9-NODL-NEXT: s_endpgm924;925; GFX9-DL-LABEL: notudot2_SameVec:926; GFX9-DL: ; %bb.0: ; %entry927; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24928; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34929; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0930; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)931; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]932; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]933; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0934; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0935; GFX9-DL-NEXT: s_waitcnt vmcnt(1)936; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0937; GFX9-DL-NEXT: s_waitcnt vmcnt(0)938; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1939; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)940; GFX9-DL-NEXT: v_add3_u32 v1, v2, s0, v1941; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]942; GFX9-DL-NEXT: s_endpgm943;944; GFX10-DL-LABEL: notudot2_SameVec:945; GFX10-DL: ; %bb.0: ; %entry946; GFX10-DL-NEXT: s_clause 0x1947; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24948; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34949; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0950; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)951; GFX10-DL-NEXT: s_clause 0x1952; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]953; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]954; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)955; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0956; GFX10-DL-NEXT: s_waitcnt vmcnt(1)957; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0958; GFX10-DL-NEXT: s_waitcnt vmcnt(0)959; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1960; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0961; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)962; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v0963; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]964; GFX10-DL-NEXT: s_endpgm965 ptr addrspace(1) %src2,966 ptr addrspace(1) nocapture %dst) {967entry:968 %idx = call i32 @llvm.amdgcn.workitem.id.x()969 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx970 %vec1 = load <2 x i16>, ptr addrspace(1) %gep1971 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx972 %vec2 = load <2 x i16>, ptr addrspace(1) %gep2973 974 %s1.elt1 = extractelement <2 x i16> %vec1, i64 0975 %conv = zext i16 %s1.elt1 to i32976 %s2.elt1 = extractelement <2 x i16> %vec1, i64 0977 %conv2 = zext i16 %s2.elt1 to i32978 %mul1 = mul i32 %conv2, %conv979 980 %s1.elt2 = extractelement <2 x i16> %vec2, i64 1981 %conv3 = zext i16 %s1.elt2 to i32982 %s2.elt2 = extractelement <2 x i16> %vec2, i64 1983 %conv4 = zext i16 %s2.elt2 to i32984 %mul2 = mul i32 %conv4, %conv3985 986 %s3 = load i32, ptr addrspace(1) %dst, align 4987 %add = add i32 %mul2, %s3988 %add6 = add i32 %add, %mul1989 store i32 %add6, ptr addrspace(1) %dst, align 4990 ret void991}992 993define amdgpu_kernel void @udot2_v4i16(ptr addrspace(1) %src1,994; GFX7-LABEL: udot2_v4i16:995; GFX7: ; %bb.0: ; %entry996; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9997; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd998; GFX7-NEXT: s_mov_b32 s7, 0xf000999; GFX7-NEXT: s_mov_b32 s10, 01000; GFX7-NEXT: s_mov_b32 s11, s71001; GFX7-NEXT: s_waitcnt lgkmcnt(0)1002; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1003; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v01004; GFX7-NEXT: v_mov_b32_e32 v1, 01005; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]1006; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]1007; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641008; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr641009; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01010; GFX7-NEXT: s_mov_b32 s6, -11011; GFX7-NEXT: s_waitcnt vmcnt(1)1012; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v21013; GFX7-NEXT: s_waitcnt vmcnt(0)1014; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v01015; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v21016; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v01017; GFX7-NEXT: s_waitcnt lgkmcnt(0)1018; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s01019; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v01020; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01021; GFX7-NEXT: s_endpgm1022;1023; GFX8-LABEL: udot2_v4i16:1024; GFX8: ; %bb.0: ; %entry1025; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241026; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341027; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v01028; GFX8-NEXT: s_waitcnt lgkmcnt(0)1029; GFX8-NEXT: v_mov_b32_e32 v1, s11030; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21031; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1032; GFX8-NEXT: v_mov_b32_e32 v3, s31033; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v21034; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1035; GFX8-NEXT: flat_load_dword v0, v[0:1]1036; GFX8-NEXT: flat_load_dword v1, v[2:3]1037; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01038; GFX8-NEXT: s_waitcnt vmcnt(1)1039; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v01040; GFX8-NEXT: s_waitcnt vmcnt(0)1041; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v11042; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v01043; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v11044; GFX8-NEXT: s_waitcnt lgkmcnt(0)1045; GFX8-NEXT: v_mad_u32_u24 v0, v1, v0, s01046; GFX8-NEXT: v_mad_u32_u24 v2, v3, v2, v01047; GFX8-NEXT: v_mov_b32_e32 v0, s41048; GFX8-NEXT: v_mov_b32_e32 v1, s51049; GFX8-NEXT: flat_store_dword v[0:1], v21050; GFX8-NEXT: s_endpgm1051;1052; GFX9-NODL-LABEL: udot2_v4i16:1053; GFX9-NODL: ; %bb.0: ; %entry1054; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241055; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341056; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v01057; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1058; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1059; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1060; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01061; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01062; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1063; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01064; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_11065; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1066; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v31067; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1068; GFX9-NODL-NEXT: s_endpgm1069;1070; GFX9-DL-LABEL: udot2_v4i16:1071; GFX9-DL: ; %bb.0: ; %entry1072; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241073; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341074; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v01075; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1076; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1077; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1078; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01079; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01080; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1081; GFX9-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s01082; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1083; GFX9-DL-NEXT: s_endpgm1084;1085; GFX10-DL-LABEL: udot2_v4i16:1086; GFX10-DL: ; %bb.0: ; %entry1087; GFX10-DL-NEXT: s_clause 0x11088; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241089; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341090; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v01091; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1092; GFX10-DL-NEXT: s_clause 0x11093; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1094; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1095; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1096; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01097; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01098; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1099; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s01100; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]1101; GFX10-DL-NEXT: s_endpgm1102 ptr addrspace(1) %src2,1103 ptr addrspace(1) nocapture %dst) {1104entry:1105 %idx = call i32 @llvm.amdgcn.workitem.id.x()1106 %gep1 = getelementptr <4 x i16>, ptr addrspace(1) %src1, i32 %idx1107 %vec1 = load <4 x i16>, ptr addrspace(1) %gep11108 %gep2 = getelementptr <4 x i16>, ptr addrspace(1) %src2, i32 %idx1109 %vec2 = load <4 x i16>, ptr addrspace(1) %gep21110 1111 %s1.elt1 = extractelement <4 x i16> %vec1, i64 01112 %conv = zext i16 %s1.elt1 to i321113 %s2.elt1 = extractelement <4 x i16> %vec2, i64 01114 %conv2 = zext i16 %s2.elt1 to i321115 %mul1 = mul i32 %conv2, %conv1116 1117 %s1.elt2 = extractelement <4 x i16> %vec1, i64 11118 %conv3 = zext i16 %s1.elt2 to i321119 %s2.elt2 = extractelement <4 x i16> %vec2, i64 11120 %conv4 = zext i16 %s2.elt2 to i321121 %mul2 = mul i32 %conv4, %conv31122 1123 %s3 = load i32, ptr addrspace(1) %dst, align 41124 %add = add i32 %mul2, %s31125 %add6 = add i32 %add, %mul11126 store i32 %add6, ptr addrspace(1) %dst, align 41127 ret void1128}1129 1130define amdgpu_kernel void @udot2_v4i16_Hi(ptr addrspace(1) %src1,1131; GFX7-LABEL: udot2_v4i16_Hi:1132; GFX7: ; %bb.0: ; %entry1133; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91134; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1135; GFX7-NEXT: s_mov_b32 s7, 0xf0001136; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v01137; GFX7-NEXT: v_mov_b32_e32 v1, 01138; GFX7-NEXT: s_mov_b32 s10, 01139; GFX7-NEXT: s_mov_b32 s11, s71140; GFX7-NEXT: s_waitcnt lgkmcnt(0)1141; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1142; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:41143; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1144; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:41145; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01146; GFX7-NEXT: s_mov_b32 s6, -11147; GFX7-NEXT: s_waitcnt vmcnt(1)1148; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v21149; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v21150; GFX7-NEXT: s_waitcnt vmcnt(0)1151; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v01152; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v01153; GFX7-NEXT: s_waitcnt lgkmcnt(0)1154; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, s01155; GFX7-NEXT: v_mad_u32_u24 v0, v3, v1, v01156; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01157; GFX7-NEXT: s_endpgm1158;1159; GFX8-LABEL: udot2_v4i16_Hi:1160; GFX8: ; %bb.0: ; %entry1161; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241162; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341163; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v01164; GFX8-NEXT: s_waitcnt lgkmcnt(0)1165; GFX8-NEXT: v_mov_b32_e32 v1, s11166; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v01167; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1168; GFX8-NEXT: v_mov_b32_e32 v3, s31169; GFX8-NEXT: v_add_u32_e32 v4, vcc, s2, v01170; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1171; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v21172; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1173; GFX8-NEXT: flat_load_dword v2, v[0:1]1174; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v41175; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc1176; GFX8-NEXT: flat_load_dword v0, v[0:1]1177; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01178; GFX8-NEXT: s_waitcnt vmcnt(1)1179; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v21180; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v21181; GFX8-NEXT: s_waitcnt vmcnt(0)1182; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v01183; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v01184; GFX8-NEXT: s_waitcnt lgkmcnt(0)1185; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, s01186; GFX8-NEXT: v_mad_u32_u24 v2, v3, v1, v01187; GFX8-NEXT: v_mov_b32_e32 v0, s41188; GFX8-NEXT: v_mov_b32_e32 v1, s51189; GFX8-NEXT: flat_store_dword v[0:1], v21190; GFX8-NEXT: s_endpgm1191;1192; GFX9-NODL-LABEL: udot2_v4i16_Hi:1193; GFX9-NODL: ; %bb.0: ; %entry1194; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241195; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341196; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v01197; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1198; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1] offset:41199; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3] offset:41200; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01201; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01202; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1203; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01204; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_11205; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1206; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v31207; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1208; GFX9-NODL-NEXT: s_endpgm1209;1210; GFX9-DL-LABEL: udot2_v4i16_Hi:1211; GFX9-DL: ; %bb.0: ; %entry1212; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241213; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341214; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v01215; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1216; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1] offset:41217; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3] offset:41218; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01219; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01220; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1221; GFX9-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s01222; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1223; GFX9-DL-NEXT: s_endpgm1224;1225; GFX10-DL-LABEL: udot2_v4i16_Hi:1226; GFX10-DL: ; %bb.0: ; %entry1227; GFX10-DL-NEXT: s_clause 0x11228; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241229; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341230; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v01231; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1232; GFX10-DL-NEXT: s_clause 0x11233; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1] offset:41234; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3] offset:41235; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1236; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01237; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01238; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1239; GFX10-DL-NEXT: v_dot2_u32_u16 v1, v2, v1, s01240; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]1241; GFX10-DL-NEXT: s_endpgm1242 ptr addrspace(1) %src2,1243 ptr addrspace(1) nocapture %dst) {1244entry:1245 %idx = call i32 @llvm.amdgcn.workitem.id.x()1246 %gep1 = getelementptr <4 x i16>, ptr addrspace(1) %src1, i32 %idx1247 %vec1 = load <4 x i16>, ptr addrspace(1) %gep11248 %gep2 = getelementptr <4 x i16>, ptr addrspace(1) %src2, i32 %idx1249 %vec2 = load <4 x i16>, ptr addrspace(1) %gep21250 1251 %s1.elt1 = extractelement <4 x i16> %vec1, i64 21252 %conv = zext i16 %s1.elt1 to i321253 %s2.elt1 = extractelement <4 x i16> %vec2, i64 21254 %conv2 = zext i16 %s2.elt1 to i321255 %mul1 = mul i32 %conv2, %conv1256 1257 %s1.elt2 = extractelement <4 x i16> %vec1, i64 31258 %conv3 = zext i16 %s1.elt2 to i321259 %s2.elt2 = extractelement <4 x i16> %vec2, i64 31260 %conv4 = zext i16 %s2.elt2 to i321261 %mul2 = mul i32 %conv4, %conv31262 1263 %s3 = load i32, ptr addrspace(1) %dst, align 41264 %add = add i32 %mul2, %s31265 %add6 = add i32 %add, %mul11266 store i32 %add6, ptr addrspace(1) %dst, align 41267 ret void1268}1269 1270define amdgpu_kernel void @notudot2_v4i16_Even(ptr addrspace(1) %src1,1271; GFX7-LABEL: notudot2_v4i16_Even:1272; GFX7: ; %bb.0: ; %entry1273; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91274; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1275; GFX7-NEXT: s_mov_b32 s7, 0xf0001276; GFX7-NEXT: s_mov_b32 s10, 01277; GFX7-NEXT: s_mov_b32 s11, s71278; GFX7-NEXT: s_waitcnt lgkmcnt(0)1279; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1280; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v01281; GFX7-NEXT: v_mov_b32_e32 v1, 01282; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]1283; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]1284; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr641285; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr641286; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01287; GFX7-NEXT: s_mov_b32 s6, -11288; GFX7-NEXT: s_waitcnt vmcnt(1)1289; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v31290; GFX7-NEXT: s_waitcnt vmcnt(0)1291; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v11292; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v21293; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v01294; GFX7-NEXT: s_waitcnt lgkmcnt(0)1295; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s01296; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v11297; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01298; GFX7-NEXT: s_endpgm1299;1300; GFX8-LABEL: notudot2_v4i16_Even:1301; GFX8: ; %bb.0: ; %entry1302; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241303; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341304; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v01305; GFX8-NEXT: s_waitcnt lgkmcnt(0)1306; GFX8-NEXT: v_mov_b32_e32 v1, s11307; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21308; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1309; GFX8-NEXT: v_mov_b32_e32 v3, s31310; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v21311; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1312; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]1313; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]1314; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01315; GFX8-NEXT: s_waitcnt vmcnt(1)1316; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v11317; GFX8-NEXT: s_waitcnt vmcnt(0)1318; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v31319; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v01320; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v21321; GFX8-NEXT: s_waitcnt lgkmcnt(0)1322; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s01323; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v11324; GFX8-NEXT: v_mov_b32_e32 v0, s41325; GFX8-NEXT: v_mov_b32_e32 v1, s51326; GFX8-NEXT: flat_store_dword v[0:1], v21327; GFX8-NEXT: s_endpgm1328;1329; GFX9-NODL-LABEL: notudot2_v4i16_Even:1330; GFX9-NODL: ; %bb.0: ; %entry1331; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241332; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341333; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v01334; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1335; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1336; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1337; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01338; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 01339; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1340; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01341; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01342; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1343; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v01344; GFX9-NODL-NEXT: global_store_dword v4, v0, s[6:7]1345; GFX9-NODL-NEXT: s_endpgm1346;1347; GFX9-DL-LABEL: notudot2_v4i16_Even:1348; GFX9-DL: ; %bb.0: ; %entry1349; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241350; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341351; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v01352; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1353; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1354; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1355; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01356; GFX9-DL-NEXT: v_mov_b32_e32 v4, 01357; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1358; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01359; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01360; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1361; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v01362; GFX9-DL-NEXT: global_store_dword v4, v0, s[6:7]1363; GFX9-DL-NEXT: s_endpgm1364;1365; GFX10-DL-LABEL: notudot2_v4i16_Even:1366; GFX10-DL: ; %bb.0: ; %entry1367; GFX10-DL-NEXT: s_clause 0x11368; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241369; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341370; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v01371; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1372; GFX10-DL-NEXT: s_clause 0x11373; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1374; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1375; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1376; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01377; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1378; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01379; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01380; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01381; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1382; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v01383; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1384; GFX10-DL-NEXT: s_endpgm1385 ptr addrspace(1) %src2,1386 ptr addrspace(1) nocapture %dst) {1387entry:1388 %idx = call i32 @llvm.amdgcn.workitem.id.x()1389 %gep1 = getelementptr <4 x i16>, ptr addrspace(1) %src1, i32 %idx1390 %vec1 = load <4 x i16>, ptr addrspace(1) %gep11391 %gep2 = getelementptr <4 x i16>, ptr addrspace(1) %src2, i32 %idx1392 %vec2 = load <4 x i16>, ptr addrspace(1) %gep21393 1394 %s1.elt1 = extractelement <4 x i16> %vec1, i64 01395 %conv = zext i16 %s1.elt1 to i321396 %s2.elt1 = extractelement <4 x i16> %vec2, i64 01397 %conv2 = zext i16 %s2.elt1 to i321398 %mul1 = mul i32 %conv2, %conv1399 1400 %s1.elt2 = extractelement <4 x i16> %vec1, i64 21401 %conv3 = zext i16 %s1.elt2 to i321402 %s2.elt2 = extractelement <4 x i16> %vec2, i64 21403 %conv4 = zext i16 %s2.elt2 to i321404 %mul2 = mul i32 %conv4, %conv31405 1406 %s3 = load i32, ptr addrspace(1) %dst, align 41407 %add = add i32 %mul2, %s31408 %add6 = add i32 %add, %mul11409 store i32 %add6, ptr addrspace(1) %dst, align 41410 ret void1411}1412 1413define amdgpu_kernel void @notudot2_v4i16_Middle(ptr addrspace(1) %src1,1414; GFX7-LABEL: notudot2_v4i16_Middle:1415; GFX7: ; %bb.0: ; %entry1416; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91417; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1418; GFX7-NEXT: s_mov_b32 s7, 0xf0001419; GFX7-NEXT: s_mov_b32 s10, 01420; GFX7-NEXT: s_mov_b32 s11, s71421; GFX7-NEXT: s_waitcnt lgkmcnt(0)1422; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1423; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v01424; GFX7-NEXT: v_mov_b32_e32 v1, 01425; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]1426; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]1427; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr641428; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr641429; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01430; GFX7-NEXT: s_mov_b32 s6, -11431; GFX7-NEXT: s_waitcnt vmcnt(1)1432; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v31433; GFX7-NEXT: s_waitcnt vmcnt(0)1434; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v11435; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v21436; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v01437; GFX7-NEXT: s_waitcnt lgkmcnt(0)1438; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s01439; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v11440; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01441; GFX7-NEXT: s_endpgm1442;1443; GFX8-LABEL: notudot2_v4i16_Middle:1444; GFX8: ; %bb.0: ; %entry1445; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241446; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341447; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v01448; GFX8-NEXT: s_waitcnt lgkmcnt(0)1449; GFX8-NEXT: v_mov_b32_e32 v1, s11450; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21451; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1452; GFX8-NEXT: v_mov_b32_e32 v3, s31453; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v21454; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc1455; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]1456; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]1457; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01458; GFX8-NEXT: s_waitcnt vmcnt(1)1459; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v11460; GFX8-NEXT: s_waitcnt vmcnt(0)1461; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v31462; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v01463; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v21464; GFX8-NEXT: s_waitcnt lgkmcnt(0)1465; GFX8-NEXT: v_mad_u32_u24 v1, v3, v1, s01466; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v11467; GFX8-NEXT: v_mov_b32_e32 v0, s41468; GFX8-NEXT: v_mov_b32_e32 v1, s51469; GFX8-NEXT: flat_store_dword v[0:1], v21470; GFX8-NEXT: s_endpgm1471;1472; GFX9-NODL-LABEL: notudot2_v4i16_Middle:1473; GFX9-NODL: ; %bb.0: ; %entry1474; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241475; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341476; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 3, v01477; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1478; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1479; GFX9-NODL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1480; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01481; GFX9-NODL-NEXT: v_mov_b32_e32 v4, 01482; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1483; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_11484; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01485; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1486; GFX9-NODL-NEXT: v_add3_u32 v0, v1, s0, v01487; GFX9-NODL-NEXT: global_store_dword v4, v0, s[6:7]1488; GFX9-NODL-NEXT: s_endpgm1489;1490; GFX9-DL-LABEL: notudot2_v4i16_Middle:1491; GFX9-DL: ; %bb.0: ; %entry1492; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241493; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341494; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v01495; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1496; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1497; GFX9-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1498; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01499; GFX9-DL-NEXT: v_mov_b32_e32 v4, 01500; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1501; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_11502; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01503; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1504; GFX9-DL-NEXT: v_add3_u32 v0, v1, s0, v01505; GFX9-DL-NEXT: global_store_dword v4, v0, s[6:7]1506; GFX9-DL-NEXT: s_endpgm1507;1508; GFX10-DL-LABEL: notudot2_v4i16_Middle:1509; GFX10-DL: ; %bb.0: ; %entry1510; GFX10-DL-NEXT: s_clause 0x11511; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241512; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341513; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v01514; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1515; GFX10-DL-NEXT: s_clause 0x11516; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]1517; GFX10-DL-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]1518; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1519; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01520; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1521; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_11522; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01523; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01524; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1525; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v01526; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1527; GFX10-DL-NEXT: s_endpgm1528 ptr addrspace(1) %src2,1529 ptr addrspace(1) nocapture %dst) {1530entry:1531 %idx = call i32 @llvm.amdgcn.workitem.id.x()1532 %gep1 = getelementptr <4 x i16>, ptr addrspace(1) %src1, i32 %idx1533 %vec1 = load <4 x i16>, ptr addrspace(1) %gep11534 %gep2 = getelementptr <4 x i16>, ptr addrspace(1) %src2, i32 %idx1535 %vec2 = load <4 x i16>, ptr addrspace(1) %gep21536 1537 %s1.elt1 = extractelement <4 x i16> %vec1, i64 11538 %conv = zext i16 %s1.elt1 to i321539 %s2.elt1 = extractelement <4 x i16> %vec2, i64 11540 %conv2 = zext i16 %s2.elt1 to i321541 %mul1 = mul i32 %conv2, %conv1542 1543 %s1.elt2 = extractelement <4 x i16> %vec1, i64 21544 %conv3 = zext i16 %s1.elt2 to i321545 %s2.elt2 = extractelement <4 x i16> %vec2, i64 21546 %conv4 = zext i16 %s2.elt2 to i321547 %mul2 = mul i32 %conv4, %conv31548 1549 %s3 = load i32, ptr addrspace(1) %dst, align 41550 %add = add i32 %mul2, %s31551 %add6 = add i32 %add, %mul11552 store i32 %add6, ptr addrspace(1) %dst, align 41553 ret void1554}1555 1556define amdgpu_kernel void @notudot2_DiffIndex(ptr addrspace(1) %src1,1557; GFX7-LABEL: notudot2_DiffIndex:1558; GFX7: ; %bb.0: ; %entry1559; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91560; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1561; GFX7-NEXT: s_mov_b32 s7, 0xf0001562; GFX7-NEXT: s_mov_b32 s10, 01563; GFX7-NEXT: s_mov_b32 s11, s71564; GFX7-NEXT: s_waitcnt lgkmcnt(0)1565; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1566; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01567; GFX7-NEXT: v_mov_b32_e32 v1, 01568; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641569; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1570; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641571; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01572; GFX7-NEXT: s_mov_b32 s6, -11573; GFX7-NEXT: s_waitcnt vmcnt(1)1574; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v21575; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v21576; GFX7-NEXT: s_waitcnt vmcnt(0)1577; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v01578; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v01579; GFX7-NEXT: s_waitcnt lgkmcnt(0)1580; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, s01581; GFX7-NEXT: v_mad_u32_u24 v0, v3, v2, v01582; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01583; GFX7-NEXT: s_endpgm1584;1585; GFX8-LABEL: notudot2_DiffIndex:1586; GFX8: ; %bb.0: ; %entry1587; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241588; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341589; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01590; GFX8-NEXT: s_waitcnt lgkmcnt(0)1591; GFX8-NEXT: v_mov_b32_e32 v1, s11592; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21593; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1594; GFX8-NEXT: flat_load_dword v3, v[0:1]1595; GFX8-NEXT: v_mov_b32_e32 v1, s31596; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21597; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1598; GFX8-NEXT: flat_load_dword v0, v[0:1]1599; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01600; GFX8-NEXT: s_waitcnt vmcnt(1)1601; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v31602; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v31603; GFX8-NEXT: s_waitcnt vmcnt(0)1604; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v01605; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v01606; GFX8-NEXT: s_waitcnt lgkmcnt(0)1607; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s01608; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v01609; GFX8-NEXT: v_mov_b32_e32 v0, s41610; GFX8-NEXT: v_mov_b32_e32 v1, s51611; GFX8-NEXT: flat_store_dword v[0:1], v21612; GFX8-NEXT: s_endpgm1613;1614; GFX9-NODL-LABEL: notudot2_DiffIndex:1615; GFX9-NODL: ; %bb.0: ; %entry1616; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241617; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341618; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01619; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1620; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1621; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1622; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01623; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01624; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1625; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_01626; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_11627; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1628; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v31629; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1630; GFX9-NODL-NEXT: s_endpgm1631;1632; GFX9-DL-LABEL: notudot2_DiffIndex:1633; GFX9-DL: ; %bb.0: ; %entry1634; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241635; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341636; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01637; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1638; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1639; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1640; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01641; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01642; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1643; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_01644; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_11645; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1646; GFX9-DL-NEXT: v_add3_u32 v1, v1, s0, v31647; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1648; GFX9-DL-NEXT: s_endpgm1649;1650; GFX10-DL-LABEL: notudot2_DiffIndex:1651; GFX10-DL: ; %bb.0: ; %entry1652; GFX10-DL-NEXT: s_clause 0x11653; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241654; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341655; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01656; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1657; GFX10-DL-NEXT: s_clause 0x11658; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1659; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1660; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1661; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01662; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1663; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_01664; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_11665; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01666; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1667; GFX10-DL-NEXT: v_add3_u32 v0, v1, s0, v01668; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1669; GFX10-DL-NEXT: s_endpgm1670 ptr addrspace(1) %src2,1671 ptr addrspace(1) nocapture %dst) {1672entry:1673 %idx = call i32 @llvm.amdgcn.workitem.id.x()1674 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx1675 %vec1 = load <2 x i16>, ptr addrspace(1) %gep11676 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx1677 %vec2 = load <2 x i16>, ptr addrspace(1) %gep21678 1679 %s1.elt1 = extractelement <2 x i16> %vec1, i64 01680 %conv = zext i16 %s1.elt1 to i321681 %s2.elt1 = extractelement <2 x i16> %vec2, i64 11682 %conv2 = zext i16 %s2.elt1 to i321683 %mul1 = mul i32 %conv2, %conv1684 1685 %s1.elt2 = extractelement <2 x i16> %vec1, i64 11686 %conv3 = zext i16 %s1.elt2 to i321687 %s2.elt2 = extractelement <2 x i16> %vec2, i64 01688 %conv4 = zext i16 %s2.elt2 to i321689 %mul2 = mul i32 %conv4, %conv31690 1691 %s3 = load i32, ptr addrspace(1) %dst, align 41692 %add = add i32 %mul2, %s31693 %add6 = add i32 %add, %mul11694 store i32 %add6, ptr addrspace(1) %dst, align 41695 ret void1696}1697 1698define amdgpu_kernel void @udot2_MultipleUses_add1(ptr addrspace(1) %src1,1699; GFX7-LABEL: udot2_MultipleUses_add1:1700; GFX7: ; %bb.0: ; %entry1701; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91702; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1703; GFX7-NEXT: s_mov_b32 s7, 0xf0001704; GFX7-NEXT: s_mov_b32 s10, 01705; GFX7-NEXT: s_mov_b32 s11, s71706; GFX7-NEXT: s_waitcnt lgkmcnt(0)1707; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1708; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01709; GFX7-NEXT: v_mov_b32_e32 v1, 01710; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641711; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1712; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641713; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01714; GFX7-NEXT: s_mov_b32 s6, -11715; GFX7-NEXT: s_waitcnt vmcnt(1)1716; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v21717; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v21718; GFX7-NEXT: s_waitcnt vmcnt(0)1719; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v01720; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v01721; GFX7-NEXT: s_waitcnt lgkmcnt(0)1722; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, s01723; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v11724; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v11725; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01726; GFX7-NEXT: s_endpgm1727;1728; GFX8-LABEL: udot2_MultipleUses_add1:1729; GFX8: ; %bb.0: ; %entry1730; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241731; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341732; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01733; GFX8-NEXT: s_waitcnt lgkmcnt(0)1734; GFX8-NEXT: v_mov_b32_e32 v1, s11735; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21736; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1737; GFX8-NEXT: flat_load_dword v3, v[0:1]1738; GFX8-NEXT: v_mov_b32_e32 v1, s31739; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21740; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1741; GFX8-NEXT: flat_load_dword v0, v[0:1]1742; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01743; GFX8-NEXT: s_waitcnt vmcnt(1)1744; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v31745; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v31746; GFX8-NEXT: s_waitcnt vmcnt(0)1747; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v01748; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v01749; GFX8-NEXT: s_waitcnt lgkmcnt(0)1750; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, s01751; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, v01752; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v01753; GFX8-NEXT: v_mov_b32_e32 v0, s41754; GFX8-NEXT: v_mov_b32_e32 v1, s51755; GFX8-NEXT: flat_store_dword v[0:1], v21756; GFX8-NEXT: s_endpgm1757;1758; GFX9-NODL-LABEL: udot2_MultipleUses_add1:1759; GFX9-NODL: ; %bb.0: ; %entry1760; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241761; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341762; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01763; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1764; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1765; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1766; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01767; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01768; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1769; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01770; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v11771; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v21772; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1773; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s01774; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v11775; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1776; GFX9-NODL-NEXT: s_endpgm1777;1778; GFX9-DL-LABEL: udot2_MultipleUses_add1:1779; GFX9-DL: ; %bb.0: ; %entry1780; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241781; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341782; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01783; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1784; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1785; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1786; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01787; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01788; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1789; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01790; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v11791; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v21792; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1793; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s01794; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v11795; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1796; GFX9-DL-NEXT: s_endpgm1797;1798; GFX10-DL-LABEL: udot2_MultipleUses_add1:1799; GFX10-DL: ; %bb.0: ; %entry1800; GFX10-DL-NEXT: s_clause 0x11801; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241802; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341803; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01804; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1805; GFX10-DL-NEXT: s_clause 0x11806; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1807; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1808; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1809; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01810; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1811; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v11812; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1813; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v21814; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01815; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01816; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1817; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s01818; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v01819; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1820; GFX10-DL-NEXT: s_endpgm1821 ptr addrspace(1) %src2,1822 ptr addrspace(1) nocapture %dst) {1823entry:1824 %idx = call i32 @llvm.amdgcn.workitem.id.x()1825 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx1826 %vec1 = load <2 x i16>, ptr addrspace(1) %gep11827 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx1828 %vec2 = load <2 x i16>, ptr addrspace(1) %gep21829 1830 %s1.elt1 = extractelement <2 x i16> %vec1, i64 01831 %conv = zext i16 %s1.elt1 to i321832 %s2.elt1 = extractelement <2 x i16> %vec2, i64 01833 %conv2 = zext i16 %s2.elt1 to i321834 %mul1 = mul i32 %conv2, %conv1835 1836 %s1.elt2 = extractelement <2 x i16> %vec1, i64 11837 %conv3 = zext i16 %s1.elt2 to i321838 %s2.elt2 = extractelement <2 x i16> %vec2, i64 11839 %conv4 = zext i16 %s2.elt2 to i321840 %mul2 = mul i32 %conv4, %conv31841 1842 %s3 = load i32, ptr addrspace(1) %dst, align 41843 %add1 = add i32 %mul2, %s31844 %add2 = add i32 %add1, %mul11845 1846 %res = add i32 %add2, %add11847 store i32 %res, ptr addrspace(1) %dst, align 41848 ret void1849}1850 1851define amdgpu_kernel void @idot2_MultipleUses_add1(ptr addrspace(1) %src1,1852; GFX7-LABEL: idot2_MultipleUses_add1:1853; GFX7: ; %bb.0: ; %entry1854; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91855; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1856; GFX7-NEXT: s_mov_b32 s7, 0xf0001857; GFX7-NEXT: s_mov_b32 s10, 01858; GFX7-NEXT: s_mov_b32 s11, s71859; GFX7-NEXT: s_waitcnt lgkmcnt(0)1860; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1861; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01862; GFX7-NEXT: v_mov_b32_e32 v1, 01863; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641864; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1865; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641866; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01867; GFX7-NEXT: s_mov_b32 s6, -11868; GFX7-NEXT: s_waitcnt vmcnt(1)1869; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 161870; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v21871; GFX7-NEXT: s_waitcnt vmcnt(0)1872; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 161873; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v01874; GFX7-NEXT: s_waitcnt lgkmcnt(0)1875; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s01876; GFX7-NEXT: v_mad_i32_i24 v1, v3, v1, v01877; GFX7-NEXT: v_add_i32_e32 v0, vcc, v1, v01878; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01879; GFX7-NEXT: s_endpgm1880;1881; GFX8-LABEL: idot2_MultipleUses_add1:1882; GFX8: ; %bb.0: ; %entry1883; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241884; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341885; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01886; GFX8-NEXT: s_waitcnt lgkmcnt(0)1887; GFX8-NEXT: v_mov_b32_e32 v1, s11888; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21889; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1890; GFX8-NEXT: flat_load_dword v3, v[0:1]1891; GFX8-NEXT: v_mov_b32_e32 v1, s31892; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21893; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1894; GFX8-NEXT: flat_load_dword v0, v[0:1]1895; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01896; GFX8-NEXT: s_waitcnt vmcnt(1)1897; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 161898; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v31899; GFX8-NEXT: s_waitcnt vmcnt(0)1900; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 161901; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v01902; GFX8-NEXT: s_waitcnt lgkmcnt(0)1903; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s01904; GFX8-NEXT: v_mad_i32_i24 v1, v2, v1, v01905; GFX8-NEXT: v_add_u32_e32 v2, vcc, v1, v01906; GFX8-NEXT: v_mov_b32_e32 v0, s41907; GFX8-NEXT: v_mov_b32_e32 v1, s51908; GFX8-NEXT: flat_store_dword v[0:1], v21909; GFX8-NEXT: s_endpgm1910;1911; GFX9-NODL-LABEL: idot2_MultipleUses_add1:1912; GFX9-NODL: ; %bb.0: ; %entry1913; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241914; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341915; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01916; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1917; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1918; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1919; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01920; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01921; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1922; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01923; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v11924; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v21925; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1926; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s01927; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v11928; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1929; GFX9-NODL-NEXT: s_endpgm1930;1931; GFX9-DL-LABEL: idot2_MultipleUses_add1:1932; GFX9-DL: ; %bb.0: ; %entry1933; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241934; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341935; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01936; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1937; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1938; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1939; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01940; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01941; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1942; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01943; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v11944; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v21945; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1946; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s01947; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v11948; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1949; GFX9-DL-NEXT: s_endpgm1950;1951; GFX10-DL-LABEL: idot2_MultipleUses_add1:1952; GFX10-DL: ; %bb.0: ; %entry1953; GFX10-DL-NEXT: s_clause 0x11954; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241955; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341956; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01957; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1958; GFX10-DL-NEXT: s_clause 0x11959; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1960; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1961; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1962; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01963; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1964; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v11965; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1966; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v21967; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_01968; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01969; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1970; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s01971; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v01972; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1973; GFX10-DL-NEXT: s_endpgm1974 ptr addrspace(1) %src2,1975 ptr addrspace(1) nocapture %dst) {1976entry:1977 %idx = call i32 @llvm.amdgcn.workitem.id.x()1978 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx1979 %vec1 = load <2 x i16>, ptr addrspace(1) %gep11980 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx1981 %vec2 = load <2 x i16>, ptr addrspace(1) %gep21982 1983 %s1.elt1 = extractelement <2 x i16> %vec1, i64 01984 %conv = sext i16 %s1.elt1 to i321985 %s2.elt1 = extractelement <2 x i16> %vec2, i64 01986 %conv2 = sext i16 %s2.elt1 to i321987 %mul1 = mul i32 %conv2, %conv1988 1989 %s1.elt2 = extractelement <2 x i16> %vec1, i64 11990 %conv3 = sext i16 %s1.elt2 to i321991 %s2.elt2 = extractelement <2 x i16> %vec2, i64 11992 %conv4 = sext i16 %s2.elt2 to i321993 %mul2 = mul i32 %conv4, %conv31994 1995 %s3 = load i32, ptr addrspace(1) %dst, align 41996 %add1 = add i32 %mul2, %s31997 %add2 = add i32 %add1, %mul11998 1999 %res = add i32 %add2, %add12000 store i32 %res, ptr addrspace(1) %dst, align 42001 ret void2002}2003 2004define amdgpu_kernel void @udot2_MultipleUses_mul1(ptr addrspace(1) %src1,2005; GFX7-LABEL: udot2_MultipleUses_mul1:2006; GFX7: ; %bb.0: ; %entry2007; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92008; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2009; GFX7-NEXT: s_mov_b32 s7, 0xf0002010; GFX7-NEXT: s_mov_b32 s10, 02011; GFX7-NEXT: s_mov_b32 s11, s72012; GFX7-NEXT: s_waitcnt lgkmcnt(0)2013; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2014; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02015; GFX7-NEXT: v_mov_b32_e32 v1, 02016; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642017; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2018; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642019; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02020; GFX7-NEXT: s_mov_b32 s6, -12021; GFX7-NEXT: s_waitcnt vmcnt(1)2022; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v22023; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v22024; GFX7-NEXT: s_waitcnt vmcnt(0)2025; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v02026; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v02027; GFX7-NEXT: s_waitcnt lgkmcnt(0)2028; GFX7-NEXT: v_mad_u32_u24 v4, v0, v2, s02029; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v42030; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v12031; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02032; GFX7-NEXT: s_endpgm2033;2034; GFX8-LABEL: udot2_MultipleUses_mul1:2035; GFX8: ; %bb.0: ; %entry2036; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242037; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342038; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02039; GFX8-NEXT: s_waitcnt lgkmcnt(0)2040; GFX8-NEXT: v_mov_b32_e32 v1, s12041; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22042; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2043; GFX8-NEXT: flat_load_dword v3, v[0:1]2044; GFX8-NEXT: v_mov_b32_e32 v1, s32045; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22046; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2047; GFX8-NEXT: flat_load_dword v0, v[0:1]2048; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02049; GFX8-NEXT: s_waitcnt vmcnt(1)2050; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v32051; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v32052; GFX8-NEXT: s_waitcnt vmcnt(0)2053; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v02054; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v02055; GFX8-NEXT: s_waitcnt lgkmcnt(0)2056; GFX8-NEXT: v_mad_u32_u24 v4, v2, v1, s02057; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v42058; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v02059; GFX8-NEXT: v_mov_b32_e32 v0, s42060; GFX8-NEXT: v_mov_b32_e32 v1, s52061; GFX8-NEXT: flat_store_dword v[0:1], v22062; GFX8-NEXT: s_endpgm2063;2064; GFX9-NODL-LABEL: udot2_MultipleUses_mul1:2065; GFX9-NODL: ; %bb.0: ; %entry2066; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242067; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342068; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02069; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2070; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2071; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2072; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02073; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02074; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2075; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xffff, v12076; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2077; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xffff, v22078; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12079; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v4, v32080; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2081; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v3, s02082; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v22083; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2084; GFX9-NODL-NEXT: s_endpgm2085;2086; GFX9-DL-LABEL: udot2_MultipleUses_mul1:2087; GFX9-DL: ; %bb.0: ; %entry2088; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242089; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342090; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02091; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2092; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2093; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2094; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02095; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02096; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2097; GFX9-DL-NEXT: v_and_b32_e32 v3, 0xffff, v12098; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2099; GFX9-DL-NEXT: v_and_b32_e32 v4, 0xffff, v22100; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12101; GFX9-DL-NEXT: v_mul_u32_u24_e32 v2, v4, v32102; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2103; GFX9-DL-NEXT: v_mad_u32_u24 v3, v4, v3, s02104; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v22105; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2106; GFX9-DL-NEXT: s_endpgm2107;2108; GFX10-DL-LABEL: udot2_MultipleUses_mul1:2109; GFX10-DL: ; %bb.0: ; %entry2110; GFX10-DL-NEXT: s_clause 0x12111; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242112; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342113; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02114; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2115; GFX10-DL-NEXT: s_clause 0x12116; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2117; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2118; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2119; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02120; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2121; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xffff, v12122; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2123; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v22124; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12125; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v02126; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2127; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s02128; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02129; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v22130; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]2131; GFX10-DL-NEXT: s_endpgm2132 ptr addrspace(1) %src2,2133 ptr addrspace(1) nocapture %dst) {2134entry:2135 %idx = call i32 @llvm.amdgcn.workitem.id.x()2136 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx2137 %vec1 = load <2 x i16>, ptr addrspace(1) %gep12138 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx2139 %vec2 = load <2 x i16>, ptr addrspace(1) %gep22140 2141 %s1.elt1 = extractelement <2 x i16> %vec1, i64 02142 %conv = zext i16 %s1.elt1 to i322143 %s2.elt1 = extractelement <2 x i16> %vec2, i64 02144 %conv2 = zext i16 %s2.elt1 to i322145 %mul1 = mul i32 %conv2, %conv2146 2147 %s1.elt2 = extractelement <2 x i16> %vec1, i64 12148 %conv3 = zext i16 %s1.elt2 to i322149 %s2.elt2 = extractelement <2 x i16> %vec2, i64 12150 %conv4 = zext i16 %s2.elt2 to i322151 %mul2 = mul i32 %conv4, %conv32152 2153 %s3 = load i32, ptr addrspace(1) %dst, align 42154 %add0 = add i32 %mul1, %s32155 2156 %add1 = add i32 %mul2, %add02157 %add2 = add i32 %add1, %mul12158 2159 store i32 %add2, ptr addrspace(1) %dst, align 42160 ret void2161}2162 2163define amdgpu_kernel void @idot2_MultipleUses_mul1(ptr addrspace(1) %src1,2164; GFX7-LABEL: idot2_MultipleUses_mul1:2165; GFX7: ; %bb.0: ; %entry2166; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92167; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2168; GFX7-NEXT: s_mov_b32 s7, 0xf0002169; GFX7-NEXT: s_mov_b32 s10, 02170; GFX7-NEXT: s_mov_b32 s11, s72171; GFX7-NEXT: s_waitcnt lgkmcnt(0)2172; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2173; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02174; GFX7-NEXT: v_mov_b32_e32 v1, 02175; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642176; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2177; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642178; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02179; GFX7-NEXT: s_mov_b32 s6, -12180; GFX7-NEXT: s_waitcnt vmcnt(1)2181; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 162182; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v22183; GFX7-NEXT: s_waitcnt vmcnt(0)2184; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 162185; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v02186; GFX7-NEXT: s_waitcnt lgkmcnt(0)2187; GFX7-NEXT: v_mad_i32_i24 v4, v3, v1, s02188; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v42189; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v02190; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02191; GFX7-NEXT: s_endpgm2192;2193; GFX8-LABEL: idot2_MultipleUses_mul1:2194; GFX8: ; %bb.0: ; %entry2195; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242196; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342197; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02198; GFX8-NEXT: s_waitcnt lgkmcnt(0)2199; GFX8-NEXT: v_mov_b32_e32 v1, s12200; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22201; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2202; GFX8-NEXT: flat_load_dword v3, v[0:1]2203; GFX8-NEXT: v_mov_b32_e32 v1, s32204; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22205; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2206; GFX8-NEXT: flat_load_dword v0, v[0:1]2207; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02208; GFX8-NEXT: s_waitcnt vmcnt(1)2209; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 162210; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v32211; GFX8-NEXT: s_waitcnt vmcnt(0)2212; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 162213; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v02214; GFX8-NEXT: s_waitcnt lgkmcnt(0)2215; GFX8-NEXT: v_mad_i32_i24 v4, v2, v1, s02216; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v42217; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v02218; GFX8-NEXT: v_mov_b32_e32 v0, s42219; GFX8-NEXT: v_mov_b32_e32 v1, s52220; GFX8-NEXT: flat_store_dword v[0:1], v22221; GFX8-NEXT: s_endpgm2222;2223; GFX9-NODL-LABEL: idot2_MultipleUses_mul1:2224; GFX9-NODL: ; %bb.0: ; %entry2225; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242226; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342227; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02228; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2229; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2230; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2231; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02232; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02233; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2234; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 162235; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2236; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 162237; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12238; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v4, v32239; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2240; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v4, v3, s02241; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v22242; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2243; GFX9-NODL-NEXT: s_endpgm2244;2245; GFX9-DL-LABEL: idot2_MultipleUses_mul1:2246; GFX9-DL: ; %bb.0: ; %entry2247; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242248; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342249; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02250; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2251; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2252; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2253; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02254; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02255; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2256; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 162257; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2258; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 162259; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12260; GFX9-DL-NEXT: v_mul_i32_i24_e32 v2, v4, v32261; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2262; GFX9-DL-NEXT: v_mad_i32_i24 v3, v4, v3, s02263; GFX9-DL-NEXT: v_add3_u32 v1, v1, v3, v22264; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2265; GFX9-DL-NEXT: s_endpgm2266;2267; GFX10-DL-LABEL: idot2_MultipleUses_mul1:2268; GFX10-DL: ; %bb.0: ; %entry2269; GFX10-DL-NEXT: s_clause 0x12270; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242271; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342272; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02273; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2274; GFX10-DL-NEXT: s_clause 0x12275; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2276; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2277; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2278; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02279; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2280; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 162281; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2282; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 162283; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_12284; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v02285; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2286; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s02287; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02288; GFX10-DL-NEXT: v_add3_u32 v0, v1, v0, v22289; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]2290; GFX10-DL-NEXT: s_endpgm2291 ptr addrspace(1) %src2,2292 ptr addrspace(1) nocapture %dst) {2293entry:2294 %idx = call i32 @llvm.amdgcn.workitem.id.x()2295 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx2296 %vec1 = load <2 x i16>, ptr addrspace(1) %gep12297 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx2298 %vec2 = load <2 x i16>, ptr addrspace(1) %gep22299 2300 %s1.elt1 = extractelement <2 x i16> %vec1, i64 02301 %conv = sext i16 %s1.elt1 to i322302 %s2.elt1 = extractelement <2 x i16> %vec2, i64 02303 %conv2 = sext i16 %s2.elt1 to i322304 %mul1 = mul i32 %conv2, %conv2305 2306 %s1.elt2 = extractelement <2 x i16> %vec1, i64 12307 %conv3 = sext i16 %s1.elt2 to i322308 %s2.elt2 = extractelement <2 x i16> %vec2, i64 12309 %conv4 = sext i16 %s2.elt2 to i322310 %mul2 = mul i32 %conv4, %conv32311 2312 %s3 = load i32, ptr addrspace(1) %dst, align 42313 %add0 = add i32 %mul1, %s32314 2315 %add1 = add i32 %mul2, %add02316 %add2 = add i32 %add1, %mul12317 2318 store i32 %add2, ptr addrspace(1) %dst, align 42319 ret void2320}2321 2322define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,2323; GFX7-LABEL: udot2_MultipleUses_mul2:2324; GFX7: ; %bb.0: ; %entry2325; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92326; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2327; GFX7-NEXT: s_mov_b32 s7, 0xf0002328; GFX7-NEXT: s_mov_b32 s10, 02329; GFX7-NEXT: s_mov_b32 s11, s72330; GFX7-NEXT: s_waitcnt lgkmcnt(0)2331; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2332; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02333; GFX7-NEXT: v_mov_b32_e32 v1, 02334; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642335; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2336; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642337; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02338; GFX7-NEXT: s_mov_b32 s6, -12339; GFX7-NEXT: s_waitcnt vmcnt(1)2340; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v22341; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v22342; GFX7-NEXT: s_waitcnt vmcnt(0)2343; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v02344; GFX7-NEXT: s_waitcnt lgkmcnt(0)2345; GFX7-NEXT: v_mad_u32_u24 v4, v3, v1, s02346; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v02347; GFX7-NEXT: v_mad_u32_u24 v1, v3, v1, v42348; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v12349; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02350; GFX7-NEXT: s_endpgm2351;2352; GFX8-LABEL: udot2_MultipleUses_mul2:2353; GFX8: ; %bb.0: ; %entry2354; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242355; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342356; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02357; GFX8-NEXT: s_waitcnt lgkmcnt(0)2358; GFX8-NEXT: v_mov_b32_e32 v1, s12359; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22360; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2361; GFX8-NEXT: flat_load_dword v3, v[0:1]2362; GFX8-NEXT: v_mov_b32_e32 v1, s32363; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22364; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2365; GFX8-NEXT: flat_load_dword v0, v[0:1]2366; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02367; GFX8-NEXT: s_waitcnt vmcnt(1)2368; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v32369; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v32370; GFX8-NEXT: s_waitcnt vmcnt(0)2371; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v02372; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v02373; GFX8-NEXT: s_waitcnt lgkmcnt(0)2374; GFX8-NEXT: v_mad_u32_u24 v4, v0, v3, s02375; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v42376; GFX8-NEXT: v_mad_u32_u24 v2, v2, v1, v02377; GFX8-NEXT: v_mov_b32_e32 v0, s42378; GFX8-NEXT: v_mov_b32_e32 v1, s52379; GFX8-NEXT: flat_store_dword v[0:1], v22380; GFX8-NEXT: s_endpgm2381;2382; GFX9-NODL-LABEL: udot2_MultipleUses_mul2:2383; GFX9-NODL: ; %bb.0: ; %entry2384; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242385; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342386; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02387; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2388; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2389; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2390; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02391; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02392; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2393; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02394; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 16, v12395; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 16, v22396; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v4, v2, v12397; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2398; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v2, v1, s02399; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v32400; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2401; GFX9-NODL-NEXT: s_endpgm2402;2403; GFX9-DL-LABEL: udot2_MultipleUses_mul2:2404; GFX9-DL: ; %bb.0: ; %entry2405; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242406; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342407; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02408; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2409; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2410; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2411; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02412; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02413; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2414; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02415; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v12416; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v22417; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v2, v12418; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2419; GFX9-DL-NEXT: v_mad_u32_u24 v1, v2, v1, s02420; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v32421; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2422; GFX9-DL-NEXT: s_endpgm2423;2424; GFX10-DL-LABEL: udot2_MultipleUses_mul2:2425; GFX10-DL: ; %bb.0: ; %entry2426; GFX10-DL-NEXT: s_clause 0x12427; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242428; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342429; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02430; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2431; GFX10-DL-NEXT: s_clause 0x12432; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2433; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2434; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2435; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02436; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2437; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v12438; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2439; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v22440; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02441; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v02442; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2443; GFX10-DL-NEXT: v_mad_u32_u24 v0, v3, v0, s02444; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02445; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v12446; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]2447; GFX10-DL-NEXT: s_endpgm2448 ptr addrspace(1) %src2,2449 ptr addrspace(1) nocapture %dst) {2450entry:2451 %idx = call i32 @llvm.amdgcn.workitem.id.x()2452 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx2453 %vec1 = load <2 x i16>, ptr addrspace(1) %gep12454 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx2455 %vec2 = load <2 x i16>, ptr addrspace(1) %gep22456 2457 %s1.elt1 = extractelement <2 x i16> %vec1, i64 02458 %conv = zext i16 %s1.elt1 to i322459 %s2.elt1 = extractelement <2 x i16> %vec2, i64 02460 %conv2 = zext i16 %s2.elt1 to i322461 %mul1 = mul i32 %conv2, %conv2462 2463 %s1.elt2 = extractelement <2 x i16> %vec1, i64 12464 %conv3 = zext i16 %s1.elt2 to i322465 %s2.elt2 = extractelement <2 x i16> %vec2, i64 12466 %conv4 = zext i16 %s2.elt2 to i322467 %mul2 = mul i32 %conv4, %conv32468 2469 %s3 = load i32, ptr addrspace(1) %dst, align 42470 %add0 = add i32 %mul2, %s32471 2472 %add1 = add i32 %mul2, %add02473 %add2 = add i32 %add1, %mul12474 2475 store i32 %add2, ptr addrspace(1) %dst, align 42476 ret void2477}2478 2479define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,2480; GFX7-LABEL: idot2_MultipleUses_mul2:2481; GFX7: ; %bb.0: ; %entry2482; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92483; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2484; GFX7-NEXT: s_mov_b32 s7, 0xf0002485; GFX7-NEXT: s_mov_b32 s10, 02486; GFX7-NEXT: s_mov_b32 s11, s72487; GFX7-NEXT: s_waitcnt lgkmcnt(0)2488; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2489; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02490; GFX7-NEXT: v_mov_b32_e32 v1, 02491; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642492; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2493; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642494; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02495; GFX7-NEXT: s_mov_b32 s6, -12496; GFX7-NEXT: s_waitcnt vmcnt(1)2497; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 162498; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v22499; GFX7-NEXT: s_waitcnt vmcnt(0)2500; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 162501; GFX7-NEXT: v_ashrrev_i32_e32 v0, 16, v02502; GFX7-NEXT: s_waitcnt lgkmcnt(0)2503; GFX7-NEXT: v_mad_i32_i24 v4, v0, v2, s02504; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, v42505; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v02506; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02507; GFX7-NEXT: s_endpgm2508;2509; GFX8-LABEL: idot2_MultipleUses_mul2:2510; GFX8: ; %bb.0: ; %entry2511; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242512; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342513; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02514; GFX8-NEXT: s_waitcnt lgkmcnt(0)2515; GFX8-NEXT: v_mov_b32_e32 v1, s12516; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22517; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2518; GFX8-NEXT: flat_load_dword v3, v[0:1]2519; GFX8-NEXT: v_mov_b32_e32 v1, s32520; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22521; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2522; GFX8-NEXT: flat_load_dword v0, v[0:1]2523; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02524; GFX8-NEXT: s_waitcnt vmcnt(1)2525; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 162526; GFX8-NEXT: v_ashrrev_i32_e32 v3, 16, v32527; GFX8-NEXT: s_waitcnt vmcnt(0)2528; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 162529; GFX8-NEXT: v_ashrrev_i32_e32 v0, 16, v02530; GFX8-NEXT: s_waitcnt lgkmcnt(0)2531; GFX8-NEXT: v_mad_i32_i24 v4, v0, v3, s02532; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, v42533; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v02534; GFX8-NEXT: v_mov_b32_e32 v0, s42535; GFX8-NEXT: v_mov_b32_e32 v1, s52536; GFX8-NEXT: flat_store_dword v[0:1], v22537; GFX8-NEXT: s_endpgm2538;2539; GFX9-NODL-LABEL: idot2_MultipleUses_mul2:2540; GFX9-NODL: ; %bb.0: ; %entry2541; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242542; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342543; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02544; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2545; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2546; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2547; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02548; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02549; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2550; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02551; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v1, 16, v12552; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v2, 16, v22553; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v4, v2, v12554; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2555; GFX9-NODL-NEXT: v_mad_i32_i24 v1, v2, v1, s02556; GFX9-NODL-NEXT: v_add3_u32 v1, v4, v1, v32557; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2558; GFX9-NODL-NEXT: s_endpgm2559;2560; GFX9-DL-LABEL: idot2_MultipleUses_mul2:2561; GFX9-DL: ; %bb.0: ; %entry2562; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242563; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342564; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02565; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2566; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2567; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2568; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02569; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02570; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2571; GFX9-DL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02572; GFX9-DL-NEXT: v_ashrrev_i32_e32 v1, 16, v12573; GFX9-DL-NEXT: v_ashrrev_i32_e32 v2, 16, v22574; GFX9-DL-NEXT: v_mul_i32_i24_e32 v4, v2, v12575; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2576; GFX9-DL-NEXT: v_mad_i32_i24 v1, v2, v1, s02577; GFX9-DL-NEXT: v_add3_u32 v1, v4, v1, v32578; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2579; GFX9-DL-NEXT: s_endpgm2580;2581; GFX10-DL-LABEL: idot2_MultipleUses_mul2:2582; GFX10-DL: ; %bb.0: ; %entry2583; GFX10-DL-NEXT: s_clause 0x12584; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242585; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342586; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02587; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2588; GFX10-DL-NEXT: s_clause 0x12589; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2590; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2591; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2592; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02593; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2594; GFX10-DL-NEXT: v_ashrrev_i32_e32 v0, 16, v12595; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2596; GFX10-DL-NEXT: v_ashrrev_i32_e32 v3, 16, v22597; GFX10-DL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_02598; GFX10-DL-NEXT: v_mul_i32_i24_e32 v2, v3, v02599; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2600; GFX10-DL-NEXT: v_mad_i32_i24 v0, v3, v0, s02601; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02602; GFX10-DL-NEXT: v_add3_u32 v0, v2, v0, v12603; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]2604; GFX10-DL-NEXT: s_endpgm2605 ptr addrspace(1) %src2,2606 ptr addrspace(1) nocapture %dst) {2607entry:2608 %idx = call i32 @llvm.amdgcn.workitem.id.x()2609 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx2610 %vec1 = load <2 x i16>, ptr addrspace(1) %gep12611 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx2612 %vec2 = load <2 x i16>, ptr addrspace(1) %gep22613 2614 %s1.elt1 = extractelement <2 x i16> %vec1, i64 02615 %conv = sext i16 %s1.elt1 to i322616 %s2.elt1 = extractelement <2 x i16> %vec2, i64 02617 %conv2 = sext i16 %s2.elt1 to i322618 %mul1 = mul i32 %conv2, %conv2619 2620 %s1.elt2 = extractelement <2 x i16> %vec1, i64 12621 %conv3 = sext i16 %s1.elt2 to i322622 %s2.elt2 = extractelement <2 x i16> %vec2, i64 12623 %conv4 = sext i16 %s2.elt2 to i322624 %mul2 = mul i32 %conv4, %conv32625 2626 %s3 = load i32, ptr addrspace(1) %dst, align 42627 %add0 = add i32 %mul2, %s32628 2629 %add1 = add i32 %mul2, %add02630 %add2 = add i32 %add1, %mul12631 2632 store i32 %add2, ptr addrspace(1) %dst, align 42633 ret void2634}2635 2636define amdgpu_kernel void @udot2_acc16(ptr addrspace(1) %src1,2637; GFX7-LABEL: udot2_acc16:2638; GFX7: ; %bb.0: ; %entry2639; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92640; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2641; GFX7-NEXT: s_mov_b32 s7, 0xf0002642; GFX7-NEXT: s_mov_b32 s10, 02643; GFX7-NEXT: s_mov_b32 s11, s72644; GFX7-NEXT: s_waitcnt lgkmcnt(0)2645; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2646; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02647; GFX7-NEXT: v_mov_b32_e32 v1, 02648; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642649; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2650; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642651; GFX7-NEXT: s_mov_b32 s6, -12652; GFX7-NEXT: buffer_load_ushort v1, off, s[4:7], 02653; GFX7-NEXT: s_waitcnt vmcnt(2)2654; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v22655; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v22656; GFX7-NEXT: s_waitcnt vmcnt(1)2657; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v02658; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v02659; GFX7-NEXT: s_waitcnt vmcnt(0)2660; GFX7-NEXT: v_mad_u32_u24 v1, v3, v4, v12661; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12662; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 02663; GFX7-NEXT: s_endpgm2664;2665; GFX8-LABEL: udot2_acc16:2666; GFX8: ; %bb.0: ; %entry2667; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242668; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342669; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02670; GFX8-NEXT: s_waitcnt lgkmcnt(0)2671; GFX8-NEXT: v_mov_b32_e32 v1, s12672; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22673; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2674; GFX8-NEXT: flat_load_dword v3, v[0:1]2675; GFX8-NEXT: v_mov_b32_e32 v1, s32676; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22677; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2678; GFX8-NEXT: flat_load_dword v2, v[0:1]2679; GFX8-NEXT: v_mov_b32_e32 v0, s42680; GFX8-NEXT: v_mov_b32_e32 v1, s52681; GFX8-NEXT: flat_load_ushort v4, v[0:1]2682; GFX8-NEXT: s_waitcnt vmcnt(2)2683; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v32684; GFX8-NEXT: s_waitcnt vmcnt(1)2685; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v22686; GFX8-NEXT: s_waitcnt vmcnt(0)2687; GFX8-NEXT: v_mad_u16 v4, v5, v6, v42688; GFX8-NEXT: v_mad_u16 v2, v3, v2, v42689; GFX8-NEXT: flat_store_short v[0:1], v22690; GFX8-NEXT: s_endpgm2691;2692; GFX9-NODL-LABEL: udot2_acc16:2693; GFX9-NODL: ; %bb.0: ; %entry2694; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242695; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342696; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02697; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 02698; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2699; GFX9-NODL-NEXT: global_load_dword v2, v0, s[0:1]2700; GFX9-NODL-NEXT: global_load_dword v3, v0, s[2:3]2701; GFX9-NODL-NEXT: global_load_ushort v4, v1, s[6:7]2702; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2703; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 16, v22704; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2705; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v32706; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2707; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v42708; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v2, v3, v02709; GFX9-NODL-NEXT: global_store_short v1, v0, s[6:7]2710; GFX9-NODL-NEXT: s_endpgm2711;2712; GFX9-DL-LABEL: udot2_acc16:2713; GFX9-DL: ; %bb.0: ; %entry2714; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242715; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342716; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02717; GFX9-DL-NEXT: v_mov_b32_e32 v1, 02718; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2719; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]2720; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]2721; GFX9-DL-NEXT: global_load_ushort v4, v1, s[6:7]2722; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2723; GFX9-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v22724; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2725; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v32726; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2727; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v0, v5, v42728; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v2, v3, v02729; GFX9-DL-NEXT: global_store_short v1, v0, s[6:7]2730; GFX9-DL-NEXT: s_endpgm2731;2732; GFX10-DL-LABEL: udot2_acc16:2733; GFX10-DL: ; %bb.0: ; %entry2734; GFX10-DL-NEXT: s_clause 0x12735; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242736; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342737; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02738; GFX10-DL-NEXT: v_mov_b32_e32 v1, 02739; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2740; GFX10-DL-NEXT: s_clause 0x12741; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]2742; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]2743; GFX10-DL-NEXT: global_load_ushort v4, v1, s[6:7]2744; GFX10-DL-NEXT: s_waitcnt vmcnt(2)2745; GFX10-DL-NEXT: v_lshrrev_b32_e32 v0, 16, v22746; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2747; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v32748; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2749; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v42750; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v02751; GFX10-DL-NEXT: global_store_short v1, v0, s[6:7]2752; GFX10-DL-NEXT: s_endpgm2753 ptr addrspace(1) %src2,2754 ptr addrspace(1) nocapture %dst) {2755entry:2756 %idx = call i32 @llvm.amdgcn.workitem.id.x()2757 %gep1 = getelementptr <2 x i16>, ptr addrspace(1) %src1, i32 %idx2758 %v1 = load <2 x i16>, ptr addrspace(1) %gep12759 %gep2 = getelementptr <2 x i16>, ptr addrspace(1) %src2, i32 %idx2760 %v2 = load <2 x i16>, ptr addrspace(1) %gep22761 2762 %v1e1 = extractelement <2 x i16> %v1, i64 02763 %v2e1 = extractelement <2 x i16> %v2, i64 02764 %mul1 = mul i16 %v1e1, %v2e12765 2766 %v1e2 = extractelement <2 x i16> %v1, i64 12767 %v2e2 = extractelement <2 x i16> %v2, i64 12768 %mul2 = mul i16 %v1e2, %v2e22769 2770 %s2 = load i16, ptr addrspace(1) %dst, align 22771 %add1 = add i16 %mul2, %s22772 %add2 = add i16 %add1, %mul12773 store i16 %add2, ptr addrspace(1) %dst, align 22774 ret void2775}2776 2777define amdgpu_kernel void @notsdot2_sext8(ptr addrspace(1) %src1,2778; GFX7-LABEL: notsdot2_sext8:2779; GFX7: ; %bb.0: ; %entry2780; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92781; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2782; GFX7-NEXT: s_mov_b32 s7, 0xf0002783; GFX7-NEXT: s_mov_b32 s10, 02784; GFX7-NEXT: s_mov_b32 s11, s72785; GFX7-NEXT: s_waitcnt lgkmcnt(0)2786; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2787; GFX7-NEXT: v_lshlrev_b32_e32 v0, 1, v02788; GFX7-NEXT: v_mov_b32_e32 v1, 02789; GFX7-NEXT: buffer_load_ushort v2, v[0:1], s[8:11], 0 addr642790; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2791; GFX7-NEXT: buffer_load_ushort v0, v[0:1], s[8:11], 0 addr642792; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02793; GFX7-NEXT: s_mov_b32 s6, -12794; GFX7-NEXT: s_waitcnt vmcnt(1)2795; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 82796; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 82797; GFX7-NEXT: s_waitcnt vmcnt(0)2798; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 82799; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 82800; GFX7-NEXT: s_waitcnt lgkmcnt(0)2801; GFX7-NEXT: v_mad_i32_i24 v0, v0, v2, s02802; GFX7-NEXT: v_mad_i32_i24 v0, v3, v1, v02803; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02804; GFX7-NEXT: s_endpgm2805;2806; GFX8-LABEL: notsdot2_sext8:2807; GFX8: ; %bb.0: ; %entry2808; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242809; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342810; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v02811; GFX8-NEXT: s_waitcnt lgkmcnt(0)2812; GFX8-NEXT: v_mov_b32_e32 v1, s12813; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22814; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2815; GFX8-NEXT: flat_load_ushort v3, v[0:1]2816; GFX8-NEXT: v_mov_b32_e32 v1, s32817; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22818; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2819; GFX8-NEXT: flat_load_ushort v0, v[0:1]2820; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02821; GFX8-NEXT: s_waitcnt vmcnt(1)2822; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 82823; GFX8-NEXT: v_lshrrev_b16_e32 v3, 8, v32824; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 82825; GFX8-NEXT: s_waitcnt vmcnt(0)2826; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 82827; GFX8-NEXT: v_lshrrev_b16_e32 v0, 8, v02828; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 82829; GFX8-NEXT: s_waitcnt lgkmcnt(0)2830; GFX8-NEXT: v_mad_i32_i24 v0, v0, v3, s02831; GFX8-NEXT: v_mad_i32_i24 v2, v2, v1, v02832; GFX8-NEXT: v_mov_b32_e32 v0, s42833; GFX8-NEXT: v_mov_b32_e32 v1, s52834; GFX8-NEXT: flat_store_dword v[0:1], v22835; GFX8-NEXT: s_endpgm2836;2837; GFX9-NODL-LABEL: notsdot2_sext8:2838; GFX9-NODL: ; %bb.0: ; %entry2839; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242840; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342841; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 1, v02842; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2843; GFX9-NODL-NEXT: global_load_ushort v1, v0, s[0:1]2844; GFX9-NODL-NEXT: global_load_ushort v2, v0, s[2:3]2845; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02846; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02847; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2848; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_02849; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v1, 8, v12850; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v2, 8, v22851; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v2), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_02852; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2853; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s0, v32854; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2855; GFX9-NODL-NEXT: s_endpgm2856;2857; GFX9-DL-LABEL: notsdot2_sext8:2858; GFX9-DL: ; %bb.0: ; %entry2859; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242860; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342861; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v02862; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2863; GFX9-DL-NEXT: global_load_ushort v1, v0, s[0:1]2864; GFX9-DL-NEXT: global_load_ushort v2, v0, s[2:3]2865; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02866; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c00012867; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02868; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2869; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s12870; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2871; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s12872; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2873; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s02874; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2875; GFX9-DL-NEXT: s_endpgm2876;2877; GFX10-DL-LABEL: notsdot2_sext8:2878; GFX10-DL: ; %bb.0: ; %entry2879; GFX10-DL-NEXT: s_clause 0x12880; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242881; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342882; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 1, v02883; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02884; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2885; GFX10-DL-NEXT: s_clause 0x12886; GFX10-DL-NEXT: global_load_ushort v1, v0, s[0:1]2887; GFX10-DL-NEXT: global_load_ushort v2, v0, s[2:3]2888; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2889; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02890; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2891; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0c00012892; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2893; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0c00012894; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2895; GFX10-DL-NEXT: v_mov_b32_e32 v2, s02896; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v02897; GFX10-DL-NEXT: global_store_dword v3, v2, s[6:7]2898; GFX10-DL-NEXT: s_endpgm2899 ptr addrspace(1) %src2,2900 ptr addrspace(1) nocapture %dst) {2901entry:2902 %idx = call i32 @llvm.amdgcn.workitem.id.x()2903 %gep1 = getelementptr <2 x i8>, ptr addrspace(1) %src1, i32 %idx2904 %vec1 = load <2 x i8>, ptr addrspace(1) %gep12905 %gep2 = getelementptr <2 x i8>, ptr addrspace(1) %src2, i32 %idx2906 %vec2 = load <2 x i8>, ptr addrspace(1) %gep22907 2908 %s1.elt1 = extractelement <2 x i8> %vec1, i64 02909 %conv = sext i8 %s1.elt1 to i322910 %s2.elt1 = extractelement <2 x i8> %vec2, i64 02911 %conv2 = sext i8 %s2.elt1 to i322912 %mul1 = mul nuw i32 %conv2, %conv2913 2914 %s1.elt2 = extractelement <2 x i8> %vec1, i64 12915 %conv3 = sext i8 %s1.elt2 to i322916 %s2.elt2 = extractelement <2 x i8> %vec2, i64 12917 %conv4 = sext i8 %s2.elt2 to i322918 %mul2 = mul nuw i32 %conv4, %conv32919 2920 %s3 = load i32, ptr addrspace(1) %dst, align 42921 %add = add i32 %mul2, %s32922 %add6 = add i32 %add, %mul12923 store i32 %add6, ptr addrspace(1) %dst, align 42924 ret void2925}2926 2927declare i32 @llvm.amdgcn.workitem.id.x()2928