6319 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9-NODL %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck -check-prefixes=GFX9-DL %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck -check-prefixes=GFX10-DL %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 < %s | FileCheck -check-prefixes=GFX10-DL %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-DL,GFX11-DL-TRUE16 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-DL,GFX11-DL-FAKE16 %s10 11define amdgpu_kernel void @udot4_acc32(ptr addrspace(1) %src1,12; GFX7-LABEL: udot4_acc32:13; GFX7: ; %bb.0: ; %entry14; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x915; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd16; GFX7-NEXT: s_mov_b32 s3, 0xf00017; GFX7-NEXT: s_mov_b32 s6, 018; GFX7-NEXT: s_mov_b32 s7, s319; GFX7-NEXT: s_waitcnt lgkmcnt(0)20; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]21; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v022; GFX7-NEXT: v_mov_b32_e32 v1, 023; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr6424; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]25; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr6426; GFX7-NEXT: s_load_dword s4, s[0:1], 0x027; GFX7-NEXT: s_mov_b32 s2, -128; GFX7-NEXT: s_waitcnt vmcnt(1)29; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v230; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 831; GFX7-NEXT: s_waitcnt vmcnt(0)32; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v033; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 834; GFX7-NEXT: s_waitcnt lgkmcnt(0)35; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, s436; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 837; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 838; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v139; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v240; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v041; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v142; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v143; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 044; GFX7-NEXT: s_endpgm45;46; GFX8-LABEL: udot4_acc32:47; GFX8: ; %bb.0: ; %entry48; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2449; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3450; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v051; GFX8-NEXT: s_waitcnt lgkmcnt(0)52; GFX8-NEXT: v_mov_b32_e32 v1, s153; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v254; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc55; GFX8-NEXT: flat_load_dword v3, v[0:1]56; GFX8-NEXT: v_mov_b32_e32 v1, s357; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v258; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc59; GFX8-NEXT: flat_load_dword v0, v[0:1]60; GFX8-NEXT: s_load_dword s0, s[4:5], 0x061; GFX8-NEXT: s_waitcnt vmcnt(1)62; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v363; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 864; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 865; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v366; GFX8-NEXT: s_waitcnt vmcnt(0)67; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v068; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 869; GFX8-NEXT: s_waitcnt lgkmcnt(0)70; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s071; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 872; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v173; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v074; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v175; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v176; GFX8-NEXT: v_mov_b32_e32 v0, s477; GFX8-NEXT: v_mov_b32_e32 v1, s578; GFX8-NEXT: flat_store_dword v[0:1], v279; GFX8-NEXT: s_endpgm80;81; GFX9-NODL-LABEL: udot4_acc32:82; GFX9-NODL: ; %bb.0: ; %entry83; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2484; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3485; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v086; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)87; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]88; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]89; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x090; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 091; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)92; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_093; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_194; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_295; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_396; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)97; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v498; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v199; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]100; GFX9-NODL-NEXT: s_endpgm101;102; GFX9-DL-LABEL: udot4_acc32:103; GFX9-DL: ; %bb.0: ; %entry104; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24105; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34106; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0107; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)108; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]109; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]110; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0111; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0112; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)113; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s0114; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]115; GFX9-DL-NEXT: s_endpgm116;117; GFX10-DL-LABEL: udot4_acc32:118; GFX10-DL: ; %bb.0: ; %entry119; GFX10-DL-NEXT: s_clause 0x1120; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24121; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34122; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0123; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)124; GFX10-DL-NEXT: s_clause 0x1125; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]126; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]127; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)128; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0129; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0130; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)131; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s0132; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]133; GFX10-DL-NEXT: s_endpgm134;135; GFX11-DL-LABEL: udot4_acc32:136; GFX11-DL: ; %bb.0: ; %entry137; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24138; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v0139; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34140; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0141; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)142; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0143; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)144; GFX11-DL-NEXT: s_clause 0x1145; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]146; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]147; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x0148; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)149; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s0150; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]151; GFX11-DL-NEXT: s_endpgm152 ptr addrspace(1) %src2,153 ptr addrspace(1) nocapture %dst) {154entry:155 %idx = call i32 @llvm.amdgcn.workitem.id.x()156 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx157 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1158 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx159 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2160 161 %v1e0 = extractelement <4 x i8> %vec1, i64 0162 %cv1e0 = zext i8 %v1e0 to i32163 %v2e0 = extractelement <4 x i8> %vec2, i64 0164 %cv2e0 = zext i8 %v2e0 to i32165 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0166 167 %v1e1 = extractelement <4 x i8> %vec1, i64 1168 %cv1e1 = zext i8 %v1e1 to i32169 %v2e1 = extractelement <4 x i8> %vec2, i64 1170 %cv2e1 = zext i8 %v2e1 to i32171 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1172 173 %v1e2 = extractelement <4 x i8> %vec1, i64 2174 %cv1e2 = zext i8 %v1e2 to i32175 %v2e2 = extractelement <4 x i8> %vec2, i64 2176 %cv2e2 = zext i8 %v2e2 to i32177 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2178 179 %v1e3 = extractelement <4 x i8> %vec1, i64 3180 %cv1e3 = zext i8 %v1e3 to i32181 %v2e3 = extractelement <4 x i8> %vec2, i64 3182 %cv2e3 = zext i8 %v2e3 to i32183 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3184 185 %acc = load i32, ptr addrspace(1) %dst, align 4186 %mad1 = add i32 %mul1, %acc187 %mad2 = add i32 %mad1, %mul2188 %mad3 = add i32 %mad2, %mul3189 %mad4 = add i32 %mad3, %mul4190 191 store i32 %mad4, ptr addrspace(1) %dst, align 4192 ret void193}194 195define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,196; GFX7-LABEL: udot4_acc16:197; GFX7: ; %bb.0: ; %entry198; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9199; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd200; GFX7-NEXT: s_mov_b32 s7, 0xf000201; GFX7-NEXT: s_mov_b32 s10, 0202; GFX7-NEXT: s_mov_b32 s11, s7203; GFX7-NEXT: s_waitcnt lgkmcnt(0)204; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]205; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0206; GFX7-NEXT: v_mov_b32_e32 v1, 0207; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64208; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]209; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64210; GFX7-NEXT: s_mov_b32 s6, -1211; GFX7-NEXT: buffer_load_ushort v1, off, s[4:7], 0212; GFX7-NEXT: s_waitcnt vmcnt(2)213; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2214; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8215; GFX7-NEXT: s_waitcnt vmcnt(1)216; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0217; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8218; GFX7-NEXT: s_waitcnt vmcnt(0)219; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1220; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8221; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8222; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1223; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2224; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0225; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1226; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1227; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0228; GFX7-NEXT: s_endpgm229;230; GFX8-LABEL: udot4_acc16:231; GFX8: ; %bb.0: ; %entry232; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24233; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34234; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0235; GFX8-NEXT: v_mov_b32_e32 v5, 0xff236; GFX8-NEXT: s_waitcnt lgkmcnt(0)237; GFX8-NEXT: v_mov_b32_e32 v1, s1238; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2239; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc240; GFX8-NEXT: flat_load_dword v3, v[0:1]241; GFX8-NEXT: v_mov_b32_e32 v1, s3242; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2243; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc244; GFX8-NEXT: flat_load_dword v2, v[0:1]245; GFX8-NEXT: v_mov_b32_e32 v0, s4246; GFX8-NEXT: v_mov_b32_e32 v1, s5247; GFX8-NEXT: flat_load_ushort v4, v[0:1]248; GFX8-NEXT: s_waitcnt vmcnt(2)249; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v3250; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3251; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8252; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD253; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3254; GFX8-NEXT: s_waitcnt vmcnt(1)255; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2256; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2257; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9258; GFX8-NEXT: s_waitcnt vmcnt(0)259; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4260; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD261; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4262; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2263; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4264; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4265; GFX8-NEXT: flat_store_short v[0:1], v2266; GFX8-NEXT: s_endpgm267;268; GFX9-NODL-LABEL: udot4_acc16:269; GFX9-NODL: ; %bb.0: ; %entry270; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24271; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34272; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0273; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)274; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]275; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]276; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0277; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]278; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff279; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)280; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v1281; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)282; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2283; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1284; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2285; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6286; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7287; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)288; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3289; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD290; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD291; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v6, v7, v3292; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1293; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2294; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v9, v3295; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3296; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]297; GFX9-NODL-NEXT: s_endpgm298;299; GFX9-DL-LABEL: udot4_acc16:300; GFX9-DL: ; %bb.0: ; %entry301; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24302; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34303; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0304; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0305; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)306; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]307; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]308; GFX9-DL-NEXT: global_load_ushort v4, v1, s[6:7]309; GFX9-DL-NEXT: s_waitcnt vmcnt(0)310; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4311; GFX9-DL-NEXT: global_store_short v1, v0, s[6:7]312; GFX9-DL-NEXT: s_endpgm313;314; GFX10-DL-LABEL: udot4_acc16:315; GFX10-DL: ; %bb.0: ; %entry316; GFX10-DL-NEXT: s_clause 0x1317; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24318; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34319; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0320; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0321; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)322; GFX10-DL-NEXT: s_clause 0x1323; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]324; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]325; GFX10-DL-NEXT: global_load_ushort v4, v1, s[6:7]326; GFX10-DL-NEXT: s_waitcnt vmcnt(0)327; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4328; GFX10-DL-NEXT: global_store_short v1, v0, s[6:7]329; GFX10-DL-NEXT: s_endpgm330;331; GFX11-DL-LABEL: udot4_acc16:332; GFX11-DL: ; %bb.0: ; %entry333; GFX11-DL-NEXT: s_clause 0x1334; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24335; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34336; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0337; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)338; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0339; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)340; GFX11-DL-NEXT: s_clause 0x1341; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]342; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]343; GFX11-DL-NEXT: global_load_u16 v3, v1, s[4:5]344; GFX11-DL-NEXT: s_waitcnt vmcnt(0)345; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, v3346; GFX11-DL-NEXT: global_store_b16 v1, v0, s[4:5]347; GFX11-DL-NEXT: s_endpgm348 ptr addrspace(1) %src2,349 ptr addrspace(1) nocapture %dst) {350entry:351 %idx = call i32 @llvm.amdgcn.workitem.id.x()352 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx353 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1354 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx355 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2356 357 %v1e0 = extractelement <4 x i8> %vec1, i64 0358 %cv1e0 = zext i8 %v1e0 to i16359 %v2e0 = extractelement <4 x i8> %vec2, i64 0360 %cv2e0 = zext i8 %v2e0 to i16361 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0362 363 %v1e1 = extractelement <4 x i8> %vec1, i64 1364 %cv1e1 = zext i8 %v1e1 to i16365 %v2e1 = extractelement <4 x i8> %vec2, i64 1366 %cv2e1 = zext i8 %v2e1 to i16367 %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1368 369 %v1e2 = extractelement <4 x i8> %vec1, i64 2370 %cv1e2 = zext i8 %v1e2 to i16371 %v2e2 = extractelement <4 x i8> %vec2, i64 2372 %cv2e2 = zext i8 %v2e2 to i16373 %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2374 375 %v1e3 = extractelement <4 x i8> %vec1, i64 3376 %cv1e3 = zext i8 %v1e3 to i16377 %v2e3 = extractelement <4 x i8> %vec2, i64 3378 %cv2e3 = zext i8 %v2e3 to i16379 %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3380 381 %acc = load i16, ptr addrspace(1) %dst, align 2382 %mad1 = add i16 %mul1, %acc383 %mad2 = add i16 %mad1, %mul2384 %mad3 = add i16 %mad2, %mul3385 %mad4 = add i16 %mad3, %mul4386 387 store i16 %mad4, ptr addrspace(1) %dst, align 2388 ret void389}390 391define amdgpu_kernel void @udot4_acc8(ptr addrspace(1) %src1,392; GFX7-LABEL: udot4_acc8:393; GFX7: ; %bb.0: ; %entry394; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9395; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd396; GFX7-NEXT: s_mov_b32 s7, 0xf000397; GFX7-NEXT: s_mov_b32 s10, 0398; GFX7-NEXT: s_mov_b32 s11, s7399; GFX7-NEXT: s_waitcnt lgkmcnt(0)400; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]401; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0402; GFX7-NEXT: v_mov_b32_e32 v1, 0403; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64404; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]405; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64406; GFX7-NEXT: s_mov_b32 s6, -1407; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0408; GFX7-NEXT: s_waitcnt vmcnt(2)409; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2410; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8411; GFX7-NEXT: s_waitcnt vmcnt(1)412; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0413; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8414; GFX7-NEXT: s_waitcnt vmcnt(0)415; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1416; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8417; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8418; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1419; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2420; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0421; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1422; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1423; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0424; GFX7-NEXT: s_endpgm425;426; GFX8-LABEL: udot4_acc8:427; GFX8: ; %bb.0: ; %entry428; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24429; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34430; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0431; GFX8-NEXT: s_waitcnt lgkmcnt(0)432; GFX8-NEXT: v_mov_b32_e32 v1, s1433; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2434; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc435; GFX8-NEXT: flat_load_dword v3, v[0:1]436; GFX8-NEXT: v_mov_b32_e32 v1, s3437; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2438; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc439; GFX8-NEXT: flat_load_dword v2, v[0:1]440; GFX8-NEXT: v_mov_b32_e32 v0, s4441; GFX8-NEXT: v_mov_b32_e32 v1, s5442; GFX8-NEXT: flat_load_ubyte v4, v[0:1]443; GFX8-NEXT: s_waitcnt vmcnt(2)444; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3445; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3446; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3447; GFX8-NEXT: s_waitcnt vmcnt(1)448; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2449; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2450; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2451; GFX8-NEXT: s_waitcnt vmcnt(0)452; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4453; GFX8-NEXT: v_mad_u16 v2, v7, v8, v2454; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2455; GFX8-NEXT: v_mad_u16 v2, v9, v10, v2456; GFX8-NEXT: flat_store_byte v[0:1], v2457; GFX8-NEXT: s_endpgm458;459; GFX9-NODL-LABEL: udot4_acc8:460; GFX9-NODL: ; %bb.0: ; %entry461; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24462; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34463; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0464; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)465; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]466; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]467; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0468; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[6:7]469; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)470; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1471; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1472; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)473; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2474; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1475; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)476; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3477; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2478; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v6, v7, v1479; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2480; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1481; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v8, v9, v1482; GFX9-NODL-NEXT: global_store_byte v0, v1, s[6:7]483; GFX9-NODL-NEXT: s_endpgm484;485; GFX9-DL-LABEL: udot4_acc8:486; GFX9-DL: ; %bb.0: ; %entry487; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24488; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34489; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0490; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0491; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)492; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]493; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]494; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[6:7]495; GFX9-DL-NEXT: s_waitcnt vmcnt(0)496; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4497; GFX9-DL-NEXT: global_store_byte v1, v0, s[6:7]498; GFX9-DL-NEXT: s_endpgm499;500; GFX10-DL-LABEL: udot4_acc8:501; GFX10-DL: ; %bb.0: ; %entry502; GFX10-DL-NEXT: s_clause 0x1503; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24504; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34505; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0506; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0507; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)508; GFX10-DL-NEXT: s_clause 0x1509; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]510; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]511; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]512; GFX10-DL-NEXT: s_waitcnt vmcnt(0)513; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4514; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]515; GFX10-DL-NEXT: s_endpgm516;517; GFX11-DL-LABEL: udot4_acc8:518; GFX11-DL: ; %bb.0: ; %entry519; GFX11-DL-NEXT: s_clause 0x1520; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24521; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34522; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0523; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)524; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0525; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)526; GFX11-DL-NEXT: s_clause 0x1527; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]528; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]529; GFX11-DL-NEXT: global_load_u8 v3, v1, s[4:5]530; GFX11-DL-NEXT: s_waitcnt vmcnt(0)531; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, v3532; GFX11-DL-NEXT: global_store_b8 v1, v0, s[4:5]533; GFX11-DL-NEXT: s_endpgm534 ptr addrspace(1) %src2,535 ptr addrspace(1) nocapture %dst) {536entry:537 %idx = call i32 @llvm.amdgcn.workitem.id.x()538 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx539 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1540 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx541 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2542 543 %v1e0 = extractelement <4 x i8> %vec1, i64 0544 %v2e0 = extractelement <4 x i8> %vec2, i64 0545 %mul1 = mul nuw nsw i8 %v1e0, %v2e0546 547 %v1e1 = extractelement <4 x i8> %vec1, i64 1548 %v2e1 = extractelement <4 x i8> %vec2, i64 1549 %mul2 = mul nuw nsw i8 %v1e1, %v2e1550 551 %v1e2 = extractelement <4 x i8> %vec1, i64 2552 %v2e2 = extractelement <4 x i8> %vec2, i64 2553 %mul3 = mul nuw nsw i8 %v1e2, %v2e2554 555 %v1e3 = extractelement <4 x i8> %vec1, i64 3556 %v2e3 = extractelement <4 x i8> %vec2, i64 3557 %mul4 = mul nuw nsw i8 %v1e3, %v2e3558 559 %acc = load i8, ptr addrspace(1) %dst, align 2560 %mad1 = add i8 %mul1, %acc561 %mad2 = add i8 %mad1, %mul2562 %mad3 = add i8 %mad2, %mul3563 %mad4 = add i8 %mad3, %mul4564 565 store i8 %mad4, ptr addrspace(1) %dst, align 2566 ret void567}568 569define amdgpu_kernel void @udot2_8(ptr addrspace(1) %src1,570; GFX7-LABEL: udot2_8:571; GFX7: ; %bb.0: ; %entry572; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9573; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd574; GFX7-NEXT: s_mov_b32 s7, 0xf000575; GFX7-NEXT: s_mov_b32 s10, 0576; GFX7-NEXT: s_mov_b32 s11, s7577; GFX7-NEXT: s_waitcnt lgkmcnt(0)578; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]579; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0580; GFX7-NEXT: v_mov_b32_e32 v1, 0581; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64582; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]583; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64584; GFX7-NEXT: s_mov_b32 s6, -1585; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0586; GFX7-NEXT: s_waitcnt vmcnt(2)587; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2588; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8589; GFX7-NEXT: s_waitcnt vmcnt(1)590; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0591; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8592; GFX7-NEXT: s_waitcnt vmcnt(0)593; GFX7-NEXT: v_mad_u32_u24 v1, v3, v4, v1594; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1595; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0596; GFX7-NEXT: s_endpgm597;598; GFX8-LABEL: udot2_8:599; GFX8: ; %bb.0: ; %entry600; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24601; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34602; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0603; GFX8-NEXT: s_waitcnt lgkmcnt(0)604; GFX8-NEXT: v_mov_b32_e32 v1, s1605; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2606; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc607; GFX8-NEXT: flat_load_dword v3, v[0:1]608; GFX8-NEXT: v_mov_b32_e32 v1, s3609; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2610; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc611; GFX8-NEXT: flat_load_dword v2, v[0:1]612; GFX8-NEXT: v_mov_b32_e32 v0, s4613; GFX8-NEXT: v_mov_b32_e32 v1, s5614; GFX8-NEXT: flat_load_ubyte v4, v[0:1]615; GFX8-NEXT: s_waitcnt vmcnt(2)616; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v3617; GFX8-NEXT: s_waitcnt vmcnt(1)618; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v2619; GFX8-NEXT: s_waitcnt vmcnt(0)620; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4621; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2622; GFX8-NEXT: flat_store_byte v[0:1], v2623; GFX8-NEXT: s_endpgm624;625; GFX9-NODL-LABEL: udot2_8:626; GFX9-NODL: ; %bb.0: ; %entry627; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24628; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34629; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0630; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0631; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)632; GFX9-NODL-NEXT: global_load_dword v2, v0, s[0:1]633; GFX9-NODL-NEXT: global_load_dword v3, v0, s[2:3]634; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[6:7]635; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)636; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 8, v2637; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)638; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v3639; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)640; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4641; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2642; GFX9-NODL-NEXT: global_store_byte v1, v0, s[6:7]643; GFX9-NODL-NEXT: s_endpgm644;645; GFX9-DL-LABEL: udot2_8:646; GFX9-DL: ; %bb.0: ; %entry647; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24648; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34649; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0650; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)651; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]652; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]653; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0654; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]655; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0100656; GFX9-DL-NEXT: s_waitcnt vmcnt(2)657; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s0658; GFX9-DL-NEXT: s_waitcnt vmcnt(1)659; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s0660; GFX9-DL-NEXT: s_waitcnt vmcnt(0)661; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3662; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]663; GFX9-DL-NEXT: s_endpgm664;665; GFX10-DL-LABEL: udot2_8:666; GFX10-DL: ; %bb.0: ; %entry667; GFX10-DL-NEXT: s_clause 0x1668; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24669; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34670; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0671; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)672; GFX10-DL-NEXT: s_clause 0x1673; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]674; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]675; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0676; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]677; GFX10-DL-NEXT: s_waitcnt vmcnt(2)678; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100679; GFX10-DL-NEXT: s_waitcnt vmcnt(1)680; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0100681; GFX10-DL-NEXT: s_waitcnt vmcnt(0)682; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3683; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]684; GFX10-DL-NEXT: s_endpgm685;686; GFX11-DL-LABEL: udot2_8:687; GFX11-DL: ; %bb.0: ; %entry688; GFX11-DL-NEXT: s_clause 0x1689; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24690; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34691; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v0692; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0693; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)694; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0695; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)696; GFX11-DL-NEXT: s_clause 0x1697; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]698; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]699; GFX11-DL-NEXT: global_load_u8 v3, v2, s[4:5]700; GFX11-DL-NEXT: s_waitcnt vmcnt(2)701; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100702; GFX11-DL-NEXT: s_waitcnt vmcnt(1)703; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0100704; GFX11-DL-NEXT: s_waitcnt vmcnt(0)705; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)706; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v3707; GFX11-DL-NEXT: global_store_b8 v2, v0, s[4:5]708; GFX11-DL-NEXT: s_endpgm709 ptr addrspace(1) %src2,710 ptr addrspace(1) nocapture %dst) {711entry:712 %idx = call i32 @llvm.amdgcn.workitem.id.x()713 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx714 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1715 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx716 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2717 718 %v1e0 = extractelement <4 x i8> %vec1, i64 0719 %v2e0 = extractelement <4 x i8> %vec2, i64 0720 %mul1 = mul nuw nsw i8 %v1e0, %v2e0721 722 %v1e1 = extractelement <4 x i8> %vec1, i64 1723 %v2e1 = extractelement <4 x i8> %vec2, i64 1724 %mul2 = mul nuw nsw i8 %v1e1, %v2e1725 726 %acc = load i8, ptr addrspace(1) %dst, align 2727 %mad1 = add i8 %mul1, %acc728 %mad2 = add i8 %mad1, %mul2729 store i8 %mad2, ptr addrspace(1) %dst, align 2730 ret void731}732 733define amdgpu_kernel void @udot4_CommutationInsideMAD(ptr addrspace(1) %src1,734; GFX7-LABEL: udot4_CommutationInsideMAD:735; GFX7: ; %bb.0: ; %entry736; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9737; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd738; GFX7-NEXT: s_mov_b32 s7, 0xf000739; GFX7-NEXT: s_mov_b32 s10, 0740; GFX7-NEXT: s_mov_b32 s11, s7741; GFX7-NEXT: s_waitcnt lgkmcnt(0)742; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]743; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0744; GFX7-NEXT: v_mov_b32_e32 v1, 0745; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64746; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]747; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64748; GFX7-NEXT: s_mov_b32 s6, -1749; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0750; GFX7-NEXT: s_waitcnt vmcnt(2)751; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2752; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8753; GFX7-NEXT: s_waitcnt vmcnt(1)754; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0755; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8756; GFX7-NEXT: s_waitcnt vmcnt(0)757; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1758; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8759; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8760; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1761; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2762; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0763; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1764; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1765; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0766; GFX7-NEXT: s_endpgm767;768; GFX8-LABEL: udot4_CommutationInsideMAD:769; GFX8: ; %bb.0: ; %entry770; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24771; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34772; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0773; GFX8-NEXT: s_waitcnt lgkmcnt(0)774; GFX8-NEXT: v_mov_b32_e32 v1, s1775; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2776; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc777; GFX8-NEXT: flat_load_dword v3, v[0:1]778; GFX8-NEXT: v_mov_b32_e32 v1, s3779; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2780; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc781; GFX8-NEXT: flat_load_dword v2, v[0:1]782; GFX8-NEXT: v_mov_b32_e32 v0, s4783; GFX8-NEXT: v_mov_b32_e32 v1, s5784; GFX8-NEXT: flat_load_ubyte v4, v[0:1]785; GFX8-NEXT: s_waitcnt vmcnt(2)786; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3787; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3788; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3789; GFX8-NEXT: s_waitcnt vmcnt(1)790; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2791; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2792; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2793; GFX8-NEXT: s_waitcnt vmcnt(0)794; GFX8-NEXT: v_mad_u16 v2, v2, v3, v4795; GFX8-NEXT: v_mad_u16 v2, v8, v7, v2796; GFX8-NEXT: v_mad_u16 v2, v6, v5, v2797; GFX8-NEXT: v_mad_u16 v2, v10, v9, v2798; GFX8-NEXT: flat_store_byte v[0:1], v2799; GFX8-NEXT: s_endpgm800;801; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:802; GFX9-NODL: ; %bb.0: ; %entry803; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24804; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34805; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0806; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)807; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]808; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]809; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0810; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[6:7]811; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)812; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1813; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1814; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)815; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2816; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1817; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)818; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v2, v1, v3819; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2820; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v7, v6, v1821; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2822; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v5, v4, v1823; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v9, v8, v1824; GFX9-NODL-NEXT: global_store_byte v0, v1, s[6:7]825; GFX9-NODL-NEXT: s_endpgm826;827; GFX9-DL-LABEL: udot4_CommutationInsideMAD:828; GFX9-DL: ; %bb.0: ; %entry829; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24830; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34831; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0832; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0833; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)834; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]835; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]836; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[6:7]837; GFX9-DL-NEXT: s_waitcnt vmcnt(0)838; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4839; GFX9-DL-NEXT: global_store_byte v1, v0, s[6:7]840; GFX9-DL-NEXT: s_endpgm841;842; GFX10-DL-LABEL: udot4_CommutationInsideMAD:843; GFX10-DL: ; %bb.0: ; %entry844; GFX10-DL-NEXT: s_clause 0x1845; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24846; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34847; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0848; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0849; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)850; GFX10-DL-NEXT: s_clause 0x1851; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]852; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]853; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]854; GFX10-DL-NEXT: s_waitcnt vmcnt(0)855; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4856; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]857; GFX10-DL-NEXT: s_endpgm858;859; GFX11-DL-LABEL: udot4_CommutationInsideMAD:860; GFX11-DL: ; %bb.0: ; %entry861; GFX11-DL-NEXT: s_clause 0x1862; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24863; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34864; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0865; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)866; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0867; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)868; GFX11-DL-NEXT: s_clause 0x1869; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]870; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]871; GFX11-DL-NEXT: global_load_u8 v3, v1, s[4:5]872; GFX11-DL-NEXT: s_waitcnt vmcnt(0)873; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v3874; GFX11-DL-NEXT: global_store_b8 v1, v0, s[4:5]875; GFX11-DL-NEXT: s_endpgm876 ptr addrspace(1) %src2,877 ptr addrspace(1) nocapture %dst) {878entry:879 %idx = call i32 @llvm.amdgcn.workitem.id.x()880 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx881 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1882 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx883 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2884 885 %v1e0 = extractelement <4 x i8> %vec1, i64 0886 %v2e0 = extractelement <4 x i8> %vec2, i64 0887 %mul1 = mul nuw nsw i8 %v2e0, %v1e0888 889 %v1e1 = extractelement <4 x i8> %vec1, i64 1890 %v2e1 = extractelement <4 x i8> %vec2, i64 1891 %mul2 = mul nuw nsw i8 %v2e1, %v1e1892 893 %v1e2 = extractelement <4 x i8> %vec1, i64 2894 %v2e2 = extractelement <4 x i8> %vec2, i64 2895 %mul3 = mul nuw nsw i8 %v2e2, %v1e2896 897 %v1e3 = extractelement <4 x i8> %vec1, i64 3898 %v2e3 = extractelement <4 x i8> %vec2, i64 3899 %mul4 = mul nuw nsw i8 %v2e3, %v1e3900 901 %acc = load i8, ptr addrspace(1) %dst, align 2902 %mad1 = add i8 %acc, %mul1903 %mad2 = add i8 %mul2, %mad1904 %mad3 = add i8 %mul3, %mad2905 %mad4 = add i8 %mul4, %mad3906 907 store i8 %mad4, ptr addrspace(1) %dst, align 2908 ret void909}910 911define amdgpu_kernel void @udot4_CommutationAccrossMADs(ptr addrspace(1) %src1,912; GFX7-LABEL: udot4_CommutationAccrossMADs:913; GFX7: ; %bb.0: ; %entry914; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9915; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd916; GFX7-NEXT: s_mov_b32 s7, 0xf000917; GFX7-NEXT: s_mov_b32 s10, 0918; GFX7-NEXT: s_mov_b32 s11, s7919; GFX7-NEXT: s_waitcnt lgkmcnt(0)920; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]921; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0922; GFX7-NEXT: v_mov_b32_e32 v1, 0923; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64924; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]925; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64926; GFX7-NEXT: s_mov_b32 s6, -1927; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0928; GFX7-NEXT: s_waitcnt vmcnt(2)929; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8930; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2931; GFX7-NEXT: s_waitcnt vmcnt(1)932; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8933; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0934; GFX7-NEXT: s_waitcnt vmcnt(0)935; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1936; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8937; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8938; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1939; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2940; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0941; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1942; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1943; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0944; GFX7-NEXT: s_endpgm945;946; GFX8-LABEL: udot4_CommutationAccrossMADs:947; GFX8: ; %bb.0: ; %entry948; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24949; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34950; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0951; GFX8-NEXT: s_waitcnt lgkmcnt(0)952; GFX8-NEXT: v_mov_b32_e32 v1, s1953; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2954; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc955; GFX8-NEXT: flat_load_dword v3, v[0:1]956; GFX8-NEXT: v_mov_b32_e32 v1, s3957; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2958; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc959; GFX8-NEXT: flat_load_dword v2, v[0:1]960; GFX8-NEXT: v_mov_b32_e32 v0, s4961; GFX8-NEXT: v_mov_b32_e32 v1, s5962; GFX8-NEXT: flat_load_ubyte v4, v[0:1]963; GFX8-NEXT: s_waitcnt vmcnt(2)964; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3965; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3966; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3967; GFX8-NEXT: s_waitcnt vmcnt(1)968; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2969; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2970; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2971; GFX8-NEXT: s_waitcnt vmcnt(0)972; GFX8-NEXT: v_mad_u16 v4, v8, v7, v4973; GFX8-NEXT: v_mad_u16 v2, v2, v3, v4974; GFX8-NEXT: v_mad_u16 v2, v6, v5, v2975; GFX8-NEXT: v_mad_u16 v2, v10, v9, v2976; GFX8-NEXT: flat_store_byte v[0:1], v2977; GFX8-NEXT: s_endpgm978;979; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:980; GFX9-NODL: ; %bb.0: ; %entry981; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24982; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34983; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0984; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)985; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]986; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]987; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0988; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[6:7]989; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)990; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1991; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)992; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2993; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1994; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)995; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v6, v3996; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2997; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1998; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v2, v1, v3999; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v21000; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v5, v4, v11001; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v9, v8, v11002; GFX9-NODL-NEXT: global_store_byte v0, v1, s[6:7]1003; GFX9-NODL-NEXT: s_endpgm1004;1005; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:1006; GFX9-DL: ; %bb.0: ; %entry1007; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241008; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341009; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01010; GFX9-DL-NEXT: v_mov_b32_e32 v1, 01011; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1012; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]1013; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]1014; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[6:7]1015; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1016; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v41017; GFX9-DL-NEXT: global_store_byte v1, v0, s[6:7]1018; GFX9-DL-NEXT: s_endpgm1019;1020; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:1021; GFX10-DL: ; %bb.0: ; %entry1022; GFX10-DL-NEXT: s_clause 0x11023; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241024; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341025; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01026; GFX10-DL-NEXT: v_mov_b32_e32 v1, 01027; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1028; GFX10-DL-NEXT: s_clause 0x11029; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1030; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]1031; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]1032; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1033; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v41034; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]1035; GFX10-DL-NEXT: s_endpgm1036;1037; GFX11-DL-LABEL: udot4_CommutationAccrossMADs:1038; GFX11-DL: ; %bb.0: ; %entry1039; GFX11-DL-NEXT: s_clause 0x11040; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241041; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341042; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v01043; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1044; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01045; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1046; GFX11-DL-NEXT: s_clause 0x11047; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]1048; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]1049; GFX11-DL-NEXT: global_load_u8 v3, v1, s[4:5]1050; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1051; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v31052; GFX11-DL-NEXT: global_store_b8 v1, v0, s[4:5]1053; GFX11-DL-NEXT: s_endpgm1054 ptr addrspace(1) %src2,1055 ptr addrspace(1) nocapture %dst) {1056entry:1057 %idx = call i32 @llvm.amdgcn.workitem.id.x()1058 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1059 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11060 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1061 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21062 1063 %v1e0 = extractelement <4 x i8> %vec1, i64 01064 %v2e0 = extractelement <4 x i8> %vec2, i64 01065 %mul1 = mul nuw nsw i8 %v2e0, %v1e01066 1067 %v1e1 = extractelement <4 x i8> %vec1, i64 11068 %v2e1 = extractelement <4 x i8> %vec2, i64 11069 %mul2 = mul nuw nsw i8 %v2e1, %v1e11070 1071 %v1e2 = extractelement <4 x i8> %vec1, i64 21072 %v2e2 = extractelement <4 x i8> %vec2, i64 21073 %mul3 = mul nuw nsw i8 %v2e2, %v1e21074 1075 %v1e3 = extractelement <4 x i8> %vec1, i64 31076 %v2e3 = extractelement <4 x i8> %vec2, i64 31077 %mul4 = mul nuw nsw i8 %v2e3, %v1e31078 1079 %acc = load i8, ptr addrspace(1) %dst, align 21080 %mad1 = add i8 %acc, %mul21081 %mad2 = add i8 %mad1, %mul11082 %mad3 = add i8 %mad2, %mul31083 %mad4 = add i8 %mad3, %mul41084 1085 store i8 %mad4, ptr addrspace(1) %dst, align 21086 ret void1087}1088 1089define amdgpu_kernel void @udot4_multiuse_mul1(ptr addrspace(1) %src1,1090; GFX7-LABEL: udot4_multiuse_mul1:1091; GFX7: ; %bb.0: ; %entry1092; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91093; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1094; GFX7-NEXT: s_mov_b32 s3, 0xf0001095; GFX7-NEXT: s_mov_b32 s6, 01096; GFX7-NEXT: s_mov_b32 s7, s31097; GFX7-NEXT: s_waitcnt lgkmcnt(0)1098; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1099; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01100; GFX7-NEXT: v_mov_b32_e32 v1, 01101; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641102; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1103; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641104; GFX7-NEXT: s_load_dword s4, s[0:1], 0x01105; GFX7-NEXT: s_mov_b32 s2, -11106; GFX7-NEXT: s_waitcnt vmcnt(1)1107; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v21108; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 81109; GFX7-NEXT: s_waitcnt vmcnt(0)1110; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v01111; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 81112; GFX7-NEXT: s_waitcnt lgkmcnt(0)1113; GFX7-NEXT: v_mad_u32_u24 v8, v1, v5, s41114; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v81115; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 81116; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 81117; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v31118; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v21119; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v01120; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v11121; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11122; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 01123; GFX7-NEXT: s_endpgm1124;1125; GFX8-LABEL: udot4_multiuse_mul1:1126; GFX8: ; %bb.0: ; %entry1127; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241128; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341129; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01130; GFX8-NEXT: s_waitcnt lgkmcnt(0)1131; GFX8-NEXT: v_mov_b32_e32 v1, s11132; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21133; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1134; GFX8-NEXT: flat_load_dword v3, v[0:1]1135; GFX8-NEXT: v_mov_b32_e32 v1, s31136; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21137; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1138; GFX8-NEXT: flat_load_dword v0, v[0:1]1139; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01140; GFX8-NEXT: s_waitcnt vmcnt(1)1141; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v31142; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 81143; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 81144; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v31145; GFX8-NEXT: s_waitcnt vmcnt(0)1146; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v01147; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 81148; GFX8-NEXT: s_waitcnt lgkmcnt(0)1149; GFX8-NEXT: v_mad_u32_u24 v8, v1, v2, s01150; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v81151; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 81152; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v41153; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v01154; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v11155; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v11156; GFX8-NEXT: v_mov_b32_e32 v0, s41157; GFX8-NEXT: v_mov_b32_e32 v1, s51158; GFX8-NEXT: flat_store_dword v[0:1], v21159; GFX8-NEXT: s_endpgm1160;1161; GFX9-NODL-LABEL: udot4_multiuse_mul1:1162; GFX9-NODL: ; %bb.0: ; %entry1163; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241164; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341165; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01166; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1167; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1168; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1169; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01170; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01171; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1172; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v11173; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1174; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v21175; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11176; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_21177; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_31178; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v3, v41179; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1180; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v3, v4, s01181; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v21182; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v11183; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1184; GFX9-NODL-NEXT: s_endpgm1185;1186; GFX9-DL-LABEL: udot4_multiuse_mul1:1187; GFX9-DL: ; %bb.0: ; %entry1188; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241189; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341190; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01191; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1192; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1193; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1194; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01195; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01196; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1197; GFX9-DL-NEXT: v_and_b32_e32 v3, 0xff, v11198; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1199; GFX9-DL-NEXT: v_and_b32_e32 v4, 0xff, v21200; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1201; GFX9-DL-NEXT: v_mad_u32_u24 v3, v3, v4, s01202; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v31203; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1204; GFX9-DL-NEXT: s_endpgm1205;1206; GFX10-DL-LABEL: udot4_multiuse_mul1:1207; GFX10-DL: ; %bb.0: ; %entry1208; GFX10-DL-NEXT: s_clause 0x11209; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241210; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341211; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01212; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1213; GFX10-DL-NEXT: s_clause 0x11214; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1215; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1216; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1217; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01218; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1219; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xff, v11220; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1221; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xff, v21222; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1223; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v3, s01224; GFX10-DL-NEXT: v_mov_b32_e32 v3, 01225; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v2, v01226; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]1227; GFX10-DL-NEXT: s_endpgm1228;1229; GFX11-DL-LABEL: udot4_multiuse_mul1:1230; GFX11-DL: ; %bb.0: ; %entry1231; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241232; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01233; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341234; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1235; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01236; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1237; GFX11-DL-NEXT: s_clause 0x11238; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]1239; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]1240; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x01241; GFX11-DL-NEXT: s_waitcnt vmcnt(1)1242; GFX11-DL-NEXT: v_and_b32_e32 v2, 0xff, v11243; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1244; GFX11-DL-NEXT: v_and_b32_e32 v3, 0xff, v01245; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1246; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1247; GFX11-DL-NEXT: v_mad_u32_u24 v2, v2, v3, s01248; GFX11-DL-NEXT: v_mov_b32_e32 v3, 01249; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v21250; GFX11-DL-NEXT: global_store_b32 v3, v0, s[4:5]1251; GFX11-DL-NEXT: s_endpgm1252 ptr addrspace(1) %src2,1253 ptr addrspace(1) nocapture %dst) {1254entry:1255 %idx = call i32 @llvm.amdgcn.workitem.id.x()1256 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1257 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11258 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1259 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21260 1261 %v1e0 = extractelement <4 x i8> %vec1, i64 01262 %cv1e0 = zext i8 %v1e0 to i321263 %v2e0 = extractelement <4 x i8> %vec2, i64 01264 %cv2e0 = zext i8 %v2e0 to i321265 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01266 1267 %v1e1 = extractelement <4 x i8> %vec1, i64 11268 %cv1e1 = zext i8 %v1e1 to i321269 %v2e1 = extractelement <4 x i8> %vec2, i64 11270 %cv2e1 = zext i8 %v2e1 to i321271 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11272 1273 %v1e2 = extractelement <4 x i8> %vec1, i64 21274 %cv1e2 = zext i8 %v1e2 to i321275 %v2e2 = extractelement <4 x i8> %vec2, i64 21276 %cv2e2 = zext i8 %v2e2 to i321277 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e21278 1279 %v1e3 = extractelement <4 x i8> %vec1, i64 31280 %cv1e3 = zext i8 %v1e3 to i321281 %v2e3 = extractelement <4 x i8> %vec2, i64 31282 %cv2e3 = zext i8 %v2e3 to i321283 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e31284 1285 %acc = load i32, ptr addrspace(1) %dst, align 41286 %add = add i32 %mul1, %acc1287 %add1 = add i32 %mul2, %add1288 %add2 = add i32 %add1, %mul11289 %add3 = add i32 %add2, %mul31290 %add4 = add i32 %add3, %mul41291 1292 store i32 %add4, ptr addrspace(1) %dst, align 41293 ret void1294}1295 1296define amdgpu_kernel void @udot4_multiuse_add1(ptr addrspace(1) %src1,1297; GFX7-LABEL: udot4_multiuse_add1:1298; GFX7: ; %bb.0: ; %entry1299; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91300; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1301; GFX7-NEXT: s_mov_b32 s3, 0xf0001302; GFX7-NEXT: s_mov_b32 s6, 01303; GFX7-NEXT: s_mov_b32 s7, s31304; GFX7-NEXT: s_waitcnt lgkmcnt(0)1305; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1306; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01307; GFX7-NEXT: v_mov_b32_e32 v1, 01308; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641309; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1310; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641311; GFX7-NEXT: s_load_dword s4, s[0:1], 0x01312; GFX7-NEXT: s_mov_b32 s2, -11313; GFX7-NEXT: s_waitcnt vmcnt(1)1314; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 81315; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v21316; GFX7-NEXT: s_waitcnt vmcnt(0)1317; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 81318; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v01319; GFX7-NEXT: s_waitcnt lgkmcnt(0)1320; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, s41321; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 81322; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 81323; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v31324; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v21325; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v01326; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v11327; GFX7-NEXT: v_add_i32_e32 v6, vcc, s4, v31328; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11329; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v61330; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 01331; GFX7-NEXT: s_endpgm1332;1333; GFX8-LABEL: udot4_multiuse_add1:1334; GFX8: ; %bb.0: ; %entry1335; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241336; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341337; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01338; GFX8-NEXT: s_waitcnt lgkmcnt(0)1339; GFX8-NEXT: v_mov_b32_e32 v1, s11340; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21341; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1342; GFX8-NEXT: flat_load_dword v3, v[0:1]1343; GFX8-NEXT: v_mov_b32_e32 v1, s31344; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21345; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1346; GFX8-NEXT: flat_load_dword v0, v[0:1]1347; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01348; GFX8-NEXT: s_waitcnt vmcnt(1)1349; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 81350; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v31351; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 81352; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v31353; GFX8-NEXT: s_waitcnt vmcnt(0)1354; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 81355; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v01356; GFX8-NEXT: s_waitcnt lgkmcnt(0)1357; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, s01358; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 81359; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v41360; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v01361; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v11362; GFX8-NEXT: v_add_u32_e32 v5, vcc, s0, v41363; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, v11364; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v51365; GFX8-NEXT: v_mov_b32_e32 v0, s41366; GFX8-NEXT: v_mov_b32_e32 v1, s51367; GFX8-NEXT: flat_store_dword v[0:1], v21368; GFX8-NEXT: s_endpgm1369;1370; GFX9-NODL-LABEL: udot4_multiuse_add1:1371; GFX9-NODL: ; %bb.0: ; %entry1372; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241373; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341374; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01375; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1376; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1377; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1378; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01379; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01380; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1381; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 81382; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1383; GFX9-NODL-NEXT: v_bfe_u32 v5, v2, 8, 81384; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_01385; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_21386; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_31387; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1388; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v4, v5, s01389; GFX9-NODL-NEXT: v_add_u32_e32 v4, s0, v21390; GFX9-NODL-NEXT: v_add3_u32 v2, v2, v3, v61391; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v1, v41392; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1393; GFX9-NODL-NEXT: s_endpgm1394;1395; GFX9-DL-LABEL: udot4_multiuse_add1:1396; GFX9-DL: ; %bb.0: ; %entry1397; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241398; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341399; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01400; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1401; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1402; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1403; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01404; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01405; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1406; GFX9-DL-NEXT: s_add_i32 s1, s0, s01407; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1408; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11409; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s01410; GFX9-DL-NEXT: v_add3_u32 v1, s1, v3, v11411; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1412; GFX9-DL-NEXT: s_endpgm1413;1414; GFX10-DL-LABEL: udot4_multiuse_add1:1415; GFX10-DL: ; %bb.0: ; %entry1416; GFX10-DL-NEXT: s_clause 0x11417; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241418; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341419; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01420; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1421; GFX10-DL-NEXT: s_clause 0x11422; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1423; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1424; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1425; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01426; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1427; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11428; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1429; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s01430; GFX10-DL-NEXT: s_add_i32 s0, s0, s01431; GFX10-DL-NEXT: v_mov_b32_e32 v2, 01432; GFX10-DL-NEXT: v_add3_u32 v0, s0, v0, v11433; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]1434; GFX10-DL-NEXT: s_endpgm1435;1436; GFX11-DL-LABEL: udot4_multiuse_add1:1437; GFX11-DL: ; %bb.0: ; %entry1438; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241439; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01440; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341441; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1442; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01443; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1444; GFX11-DL-NEXT: s_clause 0x11445; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]1446; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]1447; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x01448; GFX11-DL-NEXT: s_waitcnt vmcnt(1)1449; GFX11-DL-NEXT: v_bfe_u32 v2, v1, 8, 81450; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1451; GFX11-DL-NEXT: v_bfe_u32 v3, v0, 8, 81452; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1453; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s01454; GFX11-DL-NEXT: s_add_i32 s0, s0, s01455; GFX11-DL-NEXT: v_mov_b32_e32 v1, 01456; GFX11-DL-NEXT: v_mul_u32_u24_e32 v2, v2, v31457; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1458; GFX11-DL-NEXT: v_add3_u32 v0, s0, v2, v01459; GFX11-DL-NEXT: global_store_b32 v1, v0, s[4:5]1460; GFX11-DL-NEXT: s_endpgm1461 ptr addrspace(1) %src2,1462 ptr addrspace(1) nocapture %dst) {1463entry:1464 %idx = call i32 @llvm.amdgcn.workitem.id.x()1465 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1466 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11467 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1468 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21469 1470 %v1e0 = extractelement <4 x i8> %vec1, i64 01471 %cv1e0 = zext i8 %v1e0 to i321472 %v2e0 = extractelement <4 x i8> %vec2, i64 01473 %cv2e0 = zext i8 %v2e0 to i321474 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01475 1476 %v1e1 = extractelement <4 x i8> %vec1, i64 11477 %cv1e1 = zext i8 %v1e1 to i321478 %v2e1 = extractelement <4 x i8> %vec2, i64 11479 %cv2e1 = zext i8 %v2e1 to i321480 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11481 1482 %v1e2 = extractelement <4 x i8> %vec1, i64 21483 %cv1e2 = zext i8 %v1e2 to i321484 %v2e2 = extractelement <4 x i8> %vec2, i64 21485 %cv2e2 = zext i8 %v2e2 to i321486 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e21487 1488 %v1e3 = extractelement <4 x i8> %vec1, i64 31489 %cv1e3 = zext i8 %v1e3 to i321490 %v2e3 = extractelement <4 x i8> %vec2, i64 31491 %cv2e3 = zext i8 %v2e3 to i321492 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e31493 1494 %acc = load i32, ptr addrspace(1) %dst, align 41495 %add1 = add i32 %mul2, %acc1496 %add = add i32 %add1, %acc1497 %add2 = add i32 %add1, %mul11498 %add3 = add i32 %add2, %mul31499 %add4 = add i32 %add3, %mul41500 %res = add i32 %add4, %add1501 store i32 %res, ptr addrspace(1) %dst, align 41502 ret void1503}1504 1505define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,1506; GFX7-LABEL: notdot4_mixedtypes:1507; GFX7: ; %bb.0: ; %entry1508; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91509; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1510; GFX7-NEXT: s_mov_b32 s3, 0xf0001511; GFX7-NEXT: s_mov_b32 s6, 01512; GFX7-NEXT: s_mov_b32 s7, s31513; GFX7-NEXT: s_waitcnt lgkmcnt(0)1514; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1515; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01516; GFX7-NEXT: v_mov_b32_e32 v1, 01517; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641518; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1519; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641520; GFX7-NEXT: s_mov_b32 s2, -11521; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 01522; GFX7-NEXT: s_waitcnt vmcnt(2)1523; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 81524; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 81525; GFX7-NEXT: s_waitcnt vmcnt(1)1526; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 81527; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 81528; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v31529; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v61530; GFX7-NEXT: s_waitcnt vmcnt(0)1531; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v11532; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 81533; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 81534; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v11535; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v21536; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v01537; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v11538; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11539; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 01540; GFX7-NEXT: s_endpgm1541;1542; GFX8-LABEL: notdot4_mixedtypes:1543; GFX8: ; %bb.0: ; %entry1544; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241545; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341546; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01547; GFX8-NEXT: v_mov_b32_e32 v5, 0xff1548; GFX8-NEXT: s_waitcnt lgkmcnt(0)1549; GFX8-NEXT: v_mov_b32_e32 v1, s11550; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21551; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1552; GFX8-NEXT: flat_load_dword v3, v[0:1]1553; GFX8-NEXT: v_mov_b32_e32 v1, s31554; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21555; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1556; GFX8-NEXT: flat_load_dword v2, v[0:1]1557; GFX8-NEXT: v_mov_b32_e32 v0, s41558; GFX8-NEXT: v_mov_b32_e32 v1, s51559; GFX8-NEXT: flat_load_ushort v4, v[0:1]1560; GFX8-NEXT: s_waitcnt vmcnt(2)1561; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v31562; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v81563; GFX8-NEXT: v_bfe_i32 v6, v3, 0, 81564; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1565; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v31566; GFX8-NEXT: s_waitcnt vmcnt(1)1567; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v21568; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v91569; GFX8-NEXT: v_bfe_i32 v7, v2, 0, 81570; GFX8-NEXT: s_waitcnt vmcnt(0)1571; GFX8-NEXT: v_mad_u16 v4, v8, v9, v41572; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1573; GFX8-NEXT: v_mad_u16 v4, v6, v7, v41574; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v21575; GFX8-NEXT: v_mad_u16 v4, v10, v5, v41576; GFX8-NEXT: v_mad_u16 v2, v3, v2, v41577; GFX8-NEXT: flat_store_short v[0:1], v21578; GFX8-NEXT: s_endpgm1579;1580; GFX9-NODL-LABEL: notdot4_mixedtypes:1581; GFX9-NODL: ; %bb.0: ; %entry1582; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241583; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341584; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01585; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1586; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1587; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1588; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01589; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]1590; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff1591; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)1592; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v11593; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1594; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v21595; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v61596; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v71597; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 81598; GFX9-NODL-NEXT: v_bfe_i32 v5, v2, 0, 81599; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1600; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v6, v7, v31601; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1602; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1603; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v31604; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v11605; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v21606; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v9, v31607; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v31608; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]1609; GFX9-NODL-NEXT: s_endpgm1610;1611; GFX9-DL-LABEL: notdot4_mixedtypes:1612; GFX9-DL: ; %bb.0: ; %entry1613; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241614; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341615; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01616; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1617; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1618; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1619; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01620; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]1621; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c03021622; GFX9-DL-NEXT: s_waitcnt vmcnt(2)1623; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v11624; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1625; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v21626; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v61627; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v71628; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 81629; GFX9-DL-NEXT: v_bfe_i32 v5, v2, 0, 81630; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1631; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v7, v31632; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s01633; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v5, v31634; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s01635; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v31636; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]1637; GFX9-DL-NEXT: s_endpgm1638;1639; GFX10-DL-LABEL: notdot4_mixedtypes:1640; GFX10-DL: ; %bb.0: ; %entry1641; GFX10-DL-NEXT: s_clause 0x11642; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241643; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341644; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01645; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1646; GFX10-DL-NEXT: s_clause 0x11647; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1648; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1649; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01650; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]1651; GFX10-DL-NEXT: s_waitcnt vmcnt(2)1652; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v11653; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1654; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v21655; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 81656; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 81657; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c03021658; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v41659; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xff, v51660; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c03021661; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1662; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v31663; GFX10-DL-NEXT: v_mad_u16 v3, v6, v7, v31664; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v31665; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v31666; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]1667; GFX10-DL-NEXT: s_endpgm1668;1669; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes:1670; GFX11-DL-TRUE16: ; %bb.0: ; %entry1671; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x241672; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v01673; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x341674; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v6, 01675; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1676; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01677; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1678; GFX11-DL-TRUE16-NEXT: s_clause 0x11679; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1]1680; GFX11-DL-TRUE16-NEXT: global_load_b32 v5, v0, s[2:3]1681; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v6, s[4:5]1682; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)1683; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v41684; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)1685; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v51686; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 81687; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v5, 0, 81688; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)1689; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l1690; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l1691; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l1692; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l1693; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)1694; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l1695; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 01696; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v5, v5, 0xc0c03021697; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v3.l, v0.l1698; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v4, v4, 0xc0c03021699; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1700; GFX11-DL-TRUE16-NEXT: v_dot4_u32_u8 v0, v2, v1, v01701; GFX11-DL-TRUE16-NEXT: global_store_b16 v6, v0, s[4:5]1702; GFX11-DL-TRUE16-NEXT: s_endpgm1703;1704; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes:1705; GFX11-DL-FAKE16: ; %bb.0: ; %entry1706; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x241707; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v01708; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x341709; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1710; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01711; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1712; GFX11-DL-FAKE16-NEXT: s_clause 0x11713; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]1714; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]1715; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)1716; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v11717; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)1718; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v01719; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 01720; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 81721; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v7, v0, 0, 81722; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v41723; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v51724; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]1725; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0xc0c03021726; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0xc0c03021727; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)1728; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v31729; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1730; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v31731; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v31732; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1733; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v1, v0, v31734; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]1735; GFX11-DL-FAKE16-NEXT: s_endpgm1736 ptr addrspace(1) %src2,1737 ptr addrspace(1) nocapture %dst) {1738entry:1739 %idx = call i32 @llvm.amdgcn.workitem.id.x()1740 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1741 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11742 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1743 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21744 1745 %v1e0 = extractelement <4 x i8> %vec1, i64 01746 %cv1e0 = sext i8 %v1e0 to i161747 %v2e0 = extractelement <4 x i8> %vec2, i64 01748 %cv2e0 = sext i8 %v2e0 to i161749 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e01750 1751 %v1e1 = extractelement <4 x i8> %vec1, i64 11752 %cv1e1 = zext i8 %v1e1 to i161753 %v2e1 = extractelement <4 x i8> %vec2, i64 11754 %cv2e1 = zext i8 %v2e1 to i161755 %mul2 = mul nuw nsw i16 %cv1e1, %cv2e11756 1757 %v1e2 = extractelement <4 x i8> %vec1, i64 21758 %cv1e2 = zext i8 %v1e2 to i161759 %v2e2 = extractelement <4 x i8> %vec2, i64 21760 %cv2e2 = zext i8 %v2e2 to i161761 %mul3 = mul nuw nsw i16 %cv1e2, %cv2e21762 1763 %v1e3 = extractelement <4 x i8> %vec1, i64 31764 %cv1e3 = zext i8 %v1e3 to i161765 %v2e3 = extractelement <4 x i8> %vec2, i64 31766 %cv2e3 = zext i8 %v2e3 to i161767 %mul4 = mul nuw nsw i16 %cv1e3, %cv2e31768 1769 %acc = load i16, ptr addrspace(1) %dst, align 21770 %add1 = add i16 %mul2, %acc1771 %add2 = add i16 %add1, %mul11772 %add3 = add i16 %add2, %mul31773 %add4 = add i16 %add3, %mul41774 1775 store i16 %add4, ptr addrspace(1) %dst, align 21776 ret void1777}1778 1779 1780define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,1781; GFX7-LABEL: notdot4_mixedtypes2:1782; GFX7: ; %bb.0: ; %entry1783; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91784; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1785; GFX7-NEXT: s_mov_b32 s3, 0xf0001786; GFX7-NEXT: s_mov_b32 s6, 01787; GFX7-NEXT: s_mov_b32 s7, s31788; GFX7-NEXT: s_waitcnt lgkmcnt(0)1789; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1790; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01791; GFX7-NEXT: v_mov_b32_e32 v1, 01792; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641793; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1794; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641795; GFX7-NEXT: s_mov_b32 s2, -11796; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 01797; GFX7-NEXT: s_waitcnt vmcnt(2)1798; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 81799; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 81800; GFX7-NEXT: s_waitcnt vmcnt(1)1801; GFX7-NEXT: v_bfe_i32 v7, v0, 8, 81802; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v71803; GFX7-NEXT: v_bfe_i32 v5, v2, 16, 81804; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v31805; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v01806; GFX7-NEXT: s_waitcnt vmcnt(0)1807; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v11808; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v51809; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 81810; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v01811; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v11812; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v21813; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v01814; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v11815; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11816; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 01817; GFX7-NEXT: s_endpgm1818;1819; GFX8-LABEL: notdot4_mixedtypes2:1820; GFX8: ; %bb.0: ; %entry1821; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241822; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341823; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01824; GFX8-NEXT: v_mov_b32_e32 v5, 0xff1825; GFX8-NEXT: s_waitcnt lgkmcnt(0)1826; GFX8-NEXT: v_mov_b32_e32 v1, s11827; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21828; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1829; GFX8-NEXT: flat_load_dword v3, v[0:1]1830; GFX8-NEXT: v_mov_b32_e32 v1, s31831; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21832; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1833; GFX8-NEXT: flat_load_dword v2, v[0:1]1834; GFX8-NEXT: v_mov_b32_e32 v0, s41835; GFX8-NEXT: v_mov_b32_e32 v1, s51836; GFX8-NEXT: flat_load_ushort v4, v[0:1]1837; GFX8-NEXT: s_waitcnt vmcnt(2)1838; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v31839; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v91840; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v31841; GFX8-NEXT: v_bfe_i32 v7, v3, 0, 81842; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 81843; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v31844; GFX8-NEXT: s_waitcnt vmcnt(1)1845; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v21846; GFX8-NEXT: v_bfe_i32 v10, v10, 0, 81847; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v21848; GFX8-NEXT: s_waitcnt vmcnt(0)1849; GFX8-NEXT: v_mad_u16 v4, v9, v10, v41850; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1851; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v21852; GFX8-NEXT: v_mad_u16 v4, v7, v8, v41853; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 81854; GFX8-NEXT: v_mad_u16 v4, v6, v5, v41855; GFX8-NEXT: v_mad_u16 v2, v3, v2, v41856; GFX8-NEXT: flat_store_short v[0:1], v21857; GFX8-NEXT: s_endpgm1858;1859; GFX9-NODL-LABEL: notdot4_mixedtypes2:1860; GFX9-NODL: ; %bb.0: ; %entry1861; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241862; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341863; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01864; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1865; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1866; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1867; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01868; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]1869; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff1870; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)1871; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v11872; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1873; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v21874; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v71875; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 81876; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v11877; GFX9-NODL-NEXT: v_bfe_i32 v5, v1, 0, 81878; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v21879; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1880; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v8, v31881; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1882; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v21883; GFX9-NODL-NEXT: v_bfe_i32 v4, v4, 0, 81884; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v5, v6, v31885; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v11886; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 81887; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v9, v31888; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v31889; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]1890; GFX9-NODL-NEXT: s_endpgm1891;1892; GFX9-DL-LABEL: notdot4_mixedtypes2:1893; GFX9-DL: ; %bb.0: ; %entry1894; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241895; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341896; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01897; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1898; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1899; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1900; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01901; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]1902; GFX9-DL-NEXT: s_movk_i32 s0, 0xff1903; GFX9-DL-NEXT: s_waitcnt vmcnt(2)1904; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v11905; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1906; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v21907; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v71908; GFX9-DL-NEXT: v_bfe_i32 v8, v8, 0, 81909; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v11910; GFX9-DL-NEXT: v_bfe_i32 v5, v1, 0, 81911; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v21912; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1913; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v8, v31914; GFX9-DL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1915; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v21916; GFX9-DL-NEXT: v_bfe_i32 v4, v4, 0, 81917; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v5, v6, v31918; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v11919; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 81920; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v9, v31921; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v31922; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]1923; GFX9-DL-NEXT: s_endpgm1924;1925; GFX10-DL-LABEL: notdot4_mixedtypes2:1926; GFX10-DL: ; %bb.0: ; %entry1927; GFX10-DL-NEXT: s_clause 0x11928; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241929; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341930; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01931; GFX10-DL-NEXT: v_mov_b32_e32 v8, 0xff1932; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1933; GFX10-DL-NEXT: s_clause 0x11934; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1935; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1936; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01937; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]1938; GFX10-DL-NEXT: s_waitcnt vmcnt(2)1939; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v11940; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1941; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v21942; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 16, v11943; GFX10-DL-NEXT: v_bfe_i32 v7, v1, 0, 81944; GFX10-DL-NEXT: v_and_b32_e32 v9, 0xff, v21945; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v41946; GFX10-DL-NEXT: v_bfe_i32 v5, v5, 0, 81947; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v11948; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1949; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v31950; GFX10-DL-NEXT: v_bfe_i32 v4, v6, 0, 81951; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD1952; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v21953; GFX10-DL-NEXT: v_mad_u16 v3, v7, v9, v31954; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 81955; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v31956; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v31957; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]1958; GFX10-DL-NEXT: s_endpgm1959;1960; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes2:1961; GFX11-DL-TRUE16: ; %bb.0: ; %entry1962; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x241963; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v01964; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x341965; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1966; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01967; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1968; GFX11-DL-TRUE16-NEXT: s_clause 0x11969; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[2:3]1970; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[0:1]1971; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v5, s[4:5]1972; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)1973; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v31974; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)1975; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v41976; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v4, 0, 81977; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h1978; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 81979; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)1980; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l1981; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l1982; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 81983; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l1984; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v31985; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l1986; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.h1987; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l1988; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v41989; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)1990; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v1.h, v0.l1991; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v2, v6, 0, 81992; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v0.h, v0.l1993; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1994; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l1995; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.l, v1.l, v0.l1996; GFX11-DL-TRUE16-NEXT: global_store_b16 v5, v0, s[4:5]1997; GFX11-DL-TRUE16-NEXT: s_endpgm1998;1999; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes2:2000; GFX11-DL-FAKE16: ; %bb.0: ; %entry2001; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x242002; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v02003; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x342004; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2005; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v02006; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2007; GFX11-DL-FAKE16-NEXT: s_clause 0x12008; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]2009; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]2010; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 02011; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)2012; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v12013; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)2014; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v02015; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]2016; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v02017; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v12018; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v42019; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v02020; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v1, 0, 82021; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v5, v5, 0, 82022; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v02023; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v12024; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)2025; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 82026; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)2027; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v32028; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v6, 0, 82029; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v72030; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v8, v9, v32031; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2032; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v32033; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v32034; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]2035; GFX11-DL-FAKE16-NEXT: s_endpgm2036 ptr addrspace(1) %src2,2037 ptr addrspace(1) nocapture %dst) {2038entry:2039 %idx = call i32 @llvm.amdgcn.workitem.id.x()2040 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2041 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12042 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2043 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22044 2045 %v1e0 = extractelement <4 x i8> %vec1, i64 02046 %cv1e0 = sext i8 %v1e0 to i162047 %v2e0 = extractelement <4 x i8> %vec2, i64 02048 %cv2e0 = zext i8 %v2e0 to i162049 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e02050 2051 %v1e1 = extractelement <4 x i8> %vec1, i64 12052 %cv1e1 = zext i8 %v1e1 to i162053 %v2e1 = extractelement <4 x i8> %vec2, i64 12054 %cv2e1 = sext i8 %v2e1 to i162055 %mul2 = mul nuw nsw i16 %cv1e1, %cv2e12056 2057 %v1e2 = extractelement <4 x i8> %vec1, i64 22058 %cv1e2 = sext i8 %v1e2 to i162059 %v2e2 = extractelement <4 x i8> %vec2, i64 22060 %cv2e2 = zext i8 %v2e2 to i162061 %mul3 = mul nuw nsw i16 %cv1e2, %cv2e22062 2063 %v1e3 = extractelement <4 x i8> %vec1, i64 32064 %cv1e3 = zext i8 %v1e3 to i162065 %v2e3 = extractelement <4 x i8> %vec2, i64 32066 %cv2e3 = sext i8 %v2e3 to i162067 %mul4 = mul nuw nsw i16 %cv1e3, %cv2e32068 2069 %acc = load i16, ptr addrspace(1) %dst, align 22070 %add1 = add i16 %mul2, %acc2071 %add2 = add i16 %add1, %mul12072 %add3 = add i16 %add2, %mul32073 %add4 = add i16 %add3, %mul42074 2075 store i16 %add4, ptr addrspace(1) %dst, align 22076 ret void2077}2078 2079; TODO: cleanup s_lshr_b322080define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,2081; GFX7-LABEL: udot4_acc32_vecMul:2082; GFX7: ; %bb.0: ; %entry2083; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x92084; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2085; GFX7-NEXT: s_mov_b32 s3, 0xf0002086; GFX7-NEXT: s_mov_b32 s6, 02087; GFX7-NEXT: s_mov_b32 s7, s32088; GFX7-NEXT: s_waitcnt lgkmcnt(0)2089; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]2090; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02091; GFX7-NEXT: v_mov_b32_e32 v1, 02092; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr642093; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]2094; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr642095; GFX7-NEXT: s_load_dword s4, s[0:1], 0x02096; GFX7-NEXT: s_mov_b32 s2, -12097; GFX7-NEXT: s_waitcnt vmcnt(1)2098; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v22099; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 82100; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 82101; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v22102; GFX7-NEXT: s_waitcnt vmcnt(0)2103; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v02104; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 82105; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 82106; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v02107; GFX7-NEXT: s_waitcnt lgkmcnt(0)2108; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s42109; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v02110; GFX7-NEXT: v_mad_u32_u24 v0, v4, v7, v02111; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v02112; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 02113; GFX7-NEXT: s_endpgm2114;2115; GFX8-LABEL: udot4_acc32_vecMul:2116; GFX8: ; %bb.0: ; %entry2117; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242118; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342119; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02120; GFX8-NEXT: s_waitcnt lgkmcnt(0)2121; GFX8-NEXT: v_mov_b32_e32 v1, s12122; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22123; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2124; GFX8-NEXT: flat_load_dword v3, v[0:1]2125; GFX8-NEXT: v_mov_b32_e32 v1, s32126; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22127; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2128; GFX8-NEXT: flat_load_dword v0, v[0:1]2129; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02130; GFX8-NEXT: s_waitcnt vmcnt(1)2131; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v32132; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 82133; GFX8-NEXT: v_lshrrev_b16_e32 v5, 8, v32134; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v32135; GFX8-NEXT: s_waitcnt vmcnt(0)2136; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v02137; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 82138; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v02139; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v02140; GFX8-NEXT: s_waitcnt lgkmcnt(0)2141; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s02142; GFX8-NEXT: v_mad_u32_u24 v0, v5, v7, v02143; GFX8-NEXT: v_mad_u32_u24 v0, v4, v6, v02144; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v02145; GFX8-NEXT: v_mov_b32_e32 v0, s42146; GFX8-NEXT: v_mov_b32_e32 v1, s52147; GFX8-NEXT: flat_store_dword v[0:1], v22148; GFX8-NEXT: s_endpgm2149;2150; GFX9-NODL-LABEL: udot4_acc32_vecMul:2151; GFX9-NODL: ; %bb.0: ; %entry2152; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242153; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342154; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02155; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2156; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2157; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2158; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02159; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02160; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2161; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_02162; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12163; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22164; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_32165; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2166; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v42167; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v12168; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2169; GFX9-NODL-NEXT: s_endpgm2170;2171; GFX9-DL-LABEL: udot4_acc32_vecMul:2172; GFX9-DL: ; %bb.0: ; %entry2173; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242174; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342175; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02176; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2177; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2178; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2179; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02180; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02181; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2182; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s02183; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2184; GFX9-DL-NEXT: s_endpgm2185;2186; GFX10-DL-LABEL: udot4_acc32_vecMul:2187; GFX10-DL: ; %bb.0: ; %entry2188; GFX10-DL-NEXT: s_clause 0x12189; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242190; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342191; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02192; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2193; GFX10-DL-NEXT: s_clause 0x12194; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2195; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2196; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2197; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02198; GFX10-DL-NEXT: v_mov_b32_e32 v0, 02199; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2200; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s02201; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]2202; GFX10-DL-NEXT: s_endpgm2203;2204; GFX11-DL-LABEL: udot4_acc32_vecMul:2205; GFX11-DL: ; %bb.0: ; %entry2206; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x242207; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02208; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x342209; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02210; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)2211; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02212; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2213; GFX11-DL-NEXT: s_clause 0x12214; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]2215; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]2216; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x02217; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2218; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s02219; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]2220; GFX11-DL-NEXT: s_endpgm2221 ptr addrspace(1) %src2,2222 ptr addrspace(1) nocapture %dst) {2223entry:2224 %idx = call i32 @llvm.amdgcn.workitem.id.x()2225 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2226 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12227 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2228 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22229 2230 %cvec1 = zext <4 x i8> %vec1 to <4 x i32>2231 %cvec2 = zext <4 x i8> %vec2 to <4 x i32>2232 2233 %mul = mul <4 x i32> %cvec1, %cvec22234 %mul0 = extractelement <4 x i32> %mul, i64 02235 %mul1 = extractelement <4 x i32> %mul, i64 12236 %mul2 = extractelement <4 x i32> %mul, i64 22237 %mul3 = extractelement <4 x i32> %mul, i64 32238 2239 %acc = load i32, ptr addrspace(1) %dst, align 42240 %add1 = add i32 %mul0, %acc2241 %add2 = add i32 %add1, %mul12242 %add3 = add i32 %add2, %mul22243 %add4 = add i32 %add3, %mul32244 2245 store i32 %add4, ptr addrspace(1) %dst, align 42246 ret void2247}2248 2249; TODO: This pattern should be recognized.2250define amdgpu_kernel void @udot4_acc16_vecMul(ptr addrspace(1) %src1,2251; GFX7-LABEL: udot4_acc16_vecMul:2252; GFX7: ; %bb.0: ; %entry2253; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x92254; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2255; GFX7-NEXT: s_mov_b32 s3, 0xf0002256; GFX7-NEXT: s_mov_b32 s6, 02257; GFX7-NEXT: s_mov_b32 s7, s32258; GFX7-NEXT: s_waitcnt lgkmcnt(0)2259; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]2260; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02261; GFX7-NEXT: v_mov_b32_e32 v1, 02262; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr642263; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]2264; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr642265; GFX7-NEXT: s_mov_b32 s2, -12266; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 02267; GFX7-NEXT: s_waitcnt vmcnt(2)2268; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v22269; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v22270; GFX7-NEXT: s_waitcnt vmcnt(1)2271; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v02272; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v02273; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 82274; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 162275; GFX7-NEXT: v_bfe_u32 v3, v0, 8, 82276; GFX7-NEXT: v_alignbit_b32 v0, v6, v0, 162277; GFX7-NEXT: s_waitcnt vmcnt(0)2278; GFX7-NEXT: v_mad_u32_u24 v1, v5, v7, v12279; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v22280; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v22281; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v02282; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v02283; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v12284; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12285; GFX7-NEXT: v_mad_u32_u24 v0, v6, v5, v02286; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 02287; GFX7-NEXT: s_endpgm2288;2289; GFX8-LABEL: udot4_acc16_vecMul:2290; GFX8: ; %bb.0: ; %entry2291; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242292; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342293; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02294; GFX8-NEXT: v_mov_b32_e32 v5, 0xff2295; GFX8-NEXT: s_waitcnt lgkmcnt(0)2296; GFX8-NEXT: v_mov_b32_e32 v1, s12297; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22298; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2299; GFX8-NEXT: flat_load_dword v3, v[0:1]2300; GFX8-NEXT: v_mov_b32_e32 v1, s32301; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22302; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2303; GFX8-NEXT: flat_load_dword v2, v[0:1]2304; GFX8-NEXT: v_mov_b32_e32 v0, s42305; GFX8-NEXT: v_mov_b32_e32 v1, s52306; GFX8-NEXT: flat_load_ushort v4, v[0:1]2307; GFX8-NEXT: s_waitcnt vmcnt(2)2308; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v32309; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v32310; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2311; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v32312; GFX8-NEXT: s_waitcnt vmcnt(1)2313; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v22314; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v22315; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2316; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v22317; GFX8-NEXT: s_waitcnt vmcnt(0)2318; GFX8-NEXT: v_mad_u16 v2, v3, v2, v42319; GFX8-NEXT: v_mad_u16 v2, v7, v9, v22320; GFX8-NEXT: v_mad_u16 v2, v10, v5, v22321; GFX8-NEXT: v_mad_u16 v2, v6, v8, v22322; GFX8-NEXT: flat_store_short v[0:1], v22323; GFX8-NEXT: s_endpgm2324;2325; GFX9-NODL-LABEL: udot4_acc16_vecMul:2326; GFX9-NODL: ; %bb.0: ; %entry2327; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242328; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342329; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02330; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2331; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2332; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2333; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02334; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]2335; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff2336; GFX9-NODL-NEXT: s_mov_b32 s1, 0x50401002337; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2338; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v12339; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 24, v12340; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2341; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v22342; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 24, v22343; GFX9-NODL-NEXT: v_and_b32_e32 v8, 0xff, v12344; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2345; GFX9-NODL-NEXT: v_and_b32_e32 v9, 0xff, v22346; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2347; GFX9-NODL-NEXT: v_perm_b32 v2, v7, v2, s12348; GFX9-NODL-NEXT: v_perm_b32 v1, v5, v1, s12349; GFX9-NODL-NEXT: v_perm_b32 v5, v6, v9, s12350; GFX9-NODL-NEXT: v_perm_b32 v4, v4, v8, s12351; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v22352; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v52353; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2354; GFX9-NODL-NEXT: v_add_u16_e32 v3, v2, v32355; GFX9-NODL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12356; GFX9-NODL-NEXT: v_add_u16_e32 v2, v2, v12357; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12358; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]2359; GFX9-NODL-NEXT: s_endpgm2360;2361; GFX9-DL-LABEL: udot4_acc16_vecMul:2362; GFX9-DL: ; %bb.0: ; %entry2363; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242364; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342365; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02366; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2367; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2368; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2369; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02370; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]2371; GFX9-DL-NEXT: s_movk_i32 s0, 0xff2372; GFX9-DL-NEXT: s_mov_b32 s1, 0x50401002373; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2374; GFX9-DL-NEXT: v_lshrrev_b16_e32 v4, 8, v12375; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v12376; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2377; GFX9-DL-NEXT: v_lshrrev_b16_e32 v6, 8, v22378; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v22379; GFX9-DL-NEXT: v_and_b32_e32 v8, 0xff, v12380; GFX9-DL-NEXT: v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2381; GFX9-DL-NEXT: v_and_b32_e32 v9, 0xff, v22382; GFX9-DL-NEXT: v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2383; GFX9-DL-NEXT: v_perm_b32 v2, v7, v2, s12384; GFX9-DL-NEXT: v_perm_b32 v1, v5, v1, s12385; GFX9-DL-NEXT: v_perm_b32 v5, v6, v9, s12386; GFX9-DL-NEXT: v_perm_b32 v4, v4, v8, s12387; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v22388; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v52389; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2390; GFX9-DL-NEXT: v_add_u16_e32 v3, v2, v32391; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12392; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v12393; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12394; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]2395; GFX9-DL-NEXT: s_endpgm2396;2397; GFX10-DL-LABEL: udot4_acc16_vecMul:2398; GFX10-DL: ; %bb.0: ; %entry2399; GFX10-DL-NEXT: s_clause 0x12400; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242401; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342402; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02403; GFX10-DL-NEXT: v_mov_b32_e32 v8, 0xff2404; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2405; GFX10-DL-NEXT: s_clause 0x12406; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2407; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2408; GFX10-DL-NEXT: v_mov_b32_e32 v0, 02409; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]2410; GFX10-DL-NEXT: s_waitcnt vmcnt(2)2411; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v12412; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2413; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v22414; GFX10-DL-NEXT: v_and_b32_e32 v6, 0xff, v22415; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xff, v12416; GFX10-DL-NEXT: v_perm_b32 v5, v5, v6, 0x50401002417; GFX10-DL-NEXT: v_perm_b32 v4, v4, v7, 0x50401002418; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v12419; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v22420; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2421; GFX10-DL-NEXT: v_and_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD2422; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v52423; GFX10-DL-NEXT: v_perm_b32 v2, v7, v2, 0x50401002424; GFX10-DL-NEXT: v_perm_b32 v1, v6, v1, 0x50401002425; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v42426; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2427; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v32428; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v22429; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v52430; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v12431; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v12432; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v32433; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]2434; GFX10-DL-NEXT: s_endpgm2435;2436; GFX11-DL-TRUE16-LABEL: udot4_acc16_vecMul:2437; GFX11-DL-TRUE16: ; %bb.0: ; %entry2438; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x242439; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v02440; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x342441; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2442; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v02443; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2444; GFX11-DL-TRUE16-NEXT: s_clause 0x12445; GFX11-DL-TRUE16-NEXT: global_load_b32 v1, v0, s[0:1]2446; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3]2447; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[4:5]2448; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)2449; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v4.h, 8, v1.l2450; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)2451; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v5.h, 8, v2.l2452; GFX11-DL-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v1.l2453; GFX11-DL-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v2.l2454; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v22455; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12456; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.h2457; GFX11-DL-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.h2458; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v4, v4, v52459; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l2460; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v7.l2461; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)2462; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2463; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v4.l, v0.l2464; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v22465; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2466; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v4.h2467; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l2468; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2469; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h2470; GFX11-DL-TRUE16-NEXT: global_store_b16 v3, v0, s[4:5]2471; GFX11-DL-TRUE16-NEXT: s_endpgm2472;2473; GFX11-DL-FAKE16-LABEL: udot4_acc16_vecMul:2474; GFX11-DL-FAKE16: ; %bb.0: ; %entry2475; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x242476; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v02477; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x342478; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2479; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v02480; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2481; GFX11-DL-FAKE16-NEXT: s_clause 0x12482; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]2483; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]2484; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)2485; GFX11-DL-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v7, 0xff, v12486; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]2487; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v12488; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)2489; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v02490; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v02491; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v12492; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v02493; GFX11-DL-FAKE16-NEXT: v_perm_b32 v4, v4, v7, 0x50401002494; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v12495; GFX11-DL-FAKE16-NEXT: v_perm_b32 v5, v5, v6, 0x50401002496; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v02497; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v92498; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v82499; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)2500; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v4, v4, v52501; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v6, 0x50401002502; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)2503; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v7, 0x50401002504; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v42505; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)2506; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v1, v02507; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)2508; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v3, v4, v32509; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v1, v3, v52510; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2511; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v02512; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v1, v02513; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2514; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v32515; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]2516; GFX11-DL-FAKE16-NEXT: s_endpgm2517 ptr addrspace(1) %src2,2518 ptr addrspace(1) nocapture %dst) {2519entry:2520 %idx = call i32 @llvm.amdgcn.workitem.id.x()2521 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2522 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12523 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2524 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22525 2526 %cvec1 = zext <4 x i8> %vec1 to <4 x i16>2527 %cvec2 = zext <4 x i8> %vec2 to <4 x i16>2528 2529 %mul = mul <4 x i16> %cvec1, %cvec22530 %mul0 = extractelement <4 x i16> %mul, i64 02531 %mul1 = extractelement <4 x i16> %mul, i64 12532 %mul2 = extractelement <4 x i16> %mul, i64 22533 %mul3 = extractelement <4 x i16> %mul, i64 32534 2535 %acc = load i16, ptr addrspace(1) %dst, align 42536 %add1 = add i16 %mul0, %acc2537 %add2 = add i16 %add1, %mul12538 %add3 = add i16 %add2, %mul22539 %add4 = add i16 %add3, %mul32540 2541 store i16 %add4, ptr addrspace(1) %dst, align 42542 ret void2543}2544 2545; TODO: Support this pattern.2546define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,2547; GFX7-LABEL: udot4_acc8_vecMul:2548; GFX7: ; %bb.0: ; %entry2549; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92550; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2551; GFX7-NEXT: s_mov_b32 s7, 0xf0002552; GFX7-NEXT: s_mov_b32 s10, 02553; GFX7-NEXT: s_mov_b32 s11, s72554; GFX7-NEXT: s_waitcnt lgkmcnt(0)2555; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2556; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02557; GFX7-NEXT: v_mov_b32_e32 v1, 02558; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642559; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2560; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642561; GFX7-NEXT: s_mov_b32 s6, -12562; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 02563; GFX7-NEXT: s_waitcnt vmcnt(2)2564; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v22565; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 82566; GFX7-NEXT: s_waitcnt vmcnt(1)2567; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v02568; GFX7-NEXT: v_bfe_u32 v8, v0, 8, 82569; GFX7-NEXT: s_waitcnt vmcnt(0)2570; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v12571; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v22572; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 82573; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v02574; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 82575; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v12576; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12577; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v02578; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 02579; GFX7-NEXT: s_endpgm2580;2581; GFX8-LABEL: udot4_acc8_vecMul:2582; GFX8: ; %bb.0: ; %entry2583; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242584; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342585; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02586; GFX8-NEXT: s_waitcnt lgkmcnt(0)2587; GFX8-NEXT: v_mov_b32_e32 v1, s12588; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22589; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2590; GFX8-NEXT: flat_load_dword v3, v[0:1]2591; GFX8-NEXT: v_mov_b32_e32 v1, s32592; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22593; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2594; GFX8-NEXT: flat_load_dword v2, v[0:1]2595; GFX8-NEXT: v_mov_b32_e32 v0, s42596; GFX8-NEXT: v_mov_b32_e32 v1, s52597; GFX8-NEXT: flat_load_ubyte v4, v[0:1]2598; GFX8-NEXT: s_waitcnt vmcnt(2)2599; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v32600; GFX8-NEXT: s_waitcnt vmcnt(1)2601; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v22602; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_32603; GFX8-NEXT: v_mul_lo_u16_e32 v9, v5, v62604; GFX8-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2605; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12606; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v72607; GFX8-NEXT: v_or_b32_e32 v8, v8, v92608; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v82609; GFX8-NEXT: s_waitcnt vmcnt(0)2610; GFX8-NEXT: v_mad_u16 v2, v3, v2, v42611; GFX8-NEXT: v_add_u16_e32 v2, v2, v82612; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v72613; GFX8-NEXT: v_mad_u16 v2, v5, v6, v22614; GFX8-NEXT: v_add_u16_e32 v2, v2, v72615; GFX8-NEXT: flat_store_byte v[0:1], v22616; GFX8-NEXT: s_endpgm2617;2618; GFX9-NODL-LABEL: udot4_acc8_vecMul:2619; GFX9-NODL: ; %bb.0: ; %entry2620; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242621; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342622; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02623; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2624; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2625; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2626; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02627; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[6:7]2628; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2629; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v12630; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2631; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v22632; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_32633; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v7, v4, v52634; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v62635; GFX9-NODL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2636; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:BYTE_1 src1_sel:BYTE_12637; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v62638; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2639; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v32640; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v62641; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v12642; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v82643; GFX9-NODL-NEXT: global_store_byte v0, v1, s[6:7]2644; GFX9-NODL-NEXT: s_endpgm2645;2646; GFX9-DL-LABEL: udot4_acc8_vecMul:2647; GFX9-DL: ; %bb.0: ; %entry2648; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242649; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342650; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02651; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2652; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2653; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2654; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02655; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]2656; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2657; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v12658; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2659; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v22660; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_32661; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v4, v52662; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v62663; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2664; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:BYTE_1 src1_sel:BYTE_12665; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v62666; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2667; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v32668; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v62669; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v5, v12670; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v82671; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]2672; GFX9-DL-NEXT: s_endpgm2673;2674; GFX10-DL-LABEL: udot4_acc8_vecMul:2675; GFX10-DL: ; %bb.0: ; %entry2676; GFX10-DL-NEXT: s_clause 0x12677; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242678; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342679; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02680; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2681; GFX10-DL-NEXT: s_clause 0x12682; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2683; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2684; GFX10-DL-NEXT: v_mov_b32_e32 v0, 02685; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]2686; GFX10-DL-NEXT: s_waitcnt vmcnt(2)2687; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v12688; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2689; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v22690; GFX10-DL-NEXT: v_lshrrev_b16 v6, 8, v12691; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v12692; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v22693; GFX10-DL-NEXT: v_lshrrev_b16 v9, 8, v22694; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v52695; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2696; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v32697; GFX10-DL-NEXT: v_mul_lo_u16 v5, v7, v82698; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v92699; GFX10-DL-NEXT: v_lshlrev_b16 v4, 8, v42700; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v62701; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2702; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v42703; GFX10-DL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2704; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v52705; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v52706; GFX10-DL-NEXT: v_mad_u16 v1, v7, v8, v12707; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v22708; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]2709; GFX10-DL-NEXT: s_endpgm2710;2711; GFX11-DL-TRUE16-LABEL: udot4_acc8_vecMul:2712; GFX11-DL-TRUE16: ; %bb.0: ; %entry2713; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x242714; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v02715; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x342716; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2717; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v02718; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2719; GFX11-DL-TRUE16-NEXT: s_clause 0x12720; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[0:1]2721; GFX11-DL-TRUE16-NEXT: global_load_b32 v4, v0, s[2:3]2722; GFX11-DL-TRUE16-NEXT: global_load_d16_u8 v0, v5, s[4:5]2723; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)2724; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 24, v32725; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)2726; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v42727; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)2728; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v3.l2729; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v3.h, v4.h2730; GFX11-DL-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v4.l2731; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l2732; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v6.l2733; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)2734; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l2735; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h2736; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2737; GFX11-DL-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l2738; GFX11-DL-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.h2739; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2740; GFX11-DL-TRUE16-NEXT: v_or_b16 v6.h, v1.l, v2.l2741; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v62742; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)2743; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l2744; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v22745; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v3.h, v4.h, v0.l2746; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2747; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l2748; GFX11-DL-TRUE16-NEXT: global_store_b8 v5, v0, s[4:5]2749; GFX11-DL-TRUE16-NEXT: s_endpgm2750;2751; GFX11-DL-FAKE16-LABEL: udot4_acc8_vecMul:2752; GFX11-DL-FAKE16: ; %bb.0: ; %entry2753; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x242754; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v02755; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x342756; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 02757; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)2758; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v02759; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2760; GFX11-DL-FAKE16-NEXT: s_clause 0x12761; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]2762; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]2763; GFX11-DL-FAKE16-NEXT: global_load_u8 v3, v2, s[4:5]2764; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)2765; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v12766; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 24, v12767; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)2768; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v02769; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v02770; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v8, 8, v12771; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v9, 8, v02772; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)2773; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v32774; GFX11-DL-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v62775; GFX11-DL-FAKE16-NEXT: v_mul_lo_u16 v6, v4, v72776; GFX11-DL-FAKE16-NEXT: v_mul_lo_u16 v8, v8, v92777; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)2778; GFX11-DL-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v52779; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v62780; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)2781; GFX11-DL-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v82782; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v52783; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)2784; GFX11-DL-FAKE16-NEXT: v_or_b32_e32 v6, v6, v52785; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v82786; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2787; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v62788; GFX11-DL-FAKE16-NEXT: v_or_b32_e32 v6, v8, v62789; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2790; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v62791; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v62792; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2793; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v4, v7, v02794; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v12795; GFX11-DL-FAKE16-NEXT: global_store_b8 v2, v0, s[4:5]2796; GFX11-DL-FAKE16-NEXT: s_endpgm2797 ptr addrspace(1) %src2,2798 ptr addrspace(1) nocapture %dst) {2799entry:2800 %idx = call i32 @llvm.amdgcn.workitem.id.x()2801 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2802 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12803 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2804 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22805 2806 %mul = mul <4 x i8> %vec1, %vec22807 %mul0 = extractelement <4 x i8> %mul, i64 02808 %mul1 = extractelement <4 x i8> %mul, i64 12809 %mul2 = extractelement <4 x i8> %mul, i64 22810 %mul3 = extractelement <4 x i8> %mul, i64 32811 2812 %acc = load i8, ptr addrspace(1) %dst, align 42813 %add1 = add i8 %mul0, %acc2814 %add2 = add i8 %add1, %mul12815 %add3 = add i8 %add2, %mul22816 %add4 = add i8 %add3, %mul32817 2818 store i8 %add4, ptr addrspace(1) %dst, align 42819 ret void2820}2821 2822define amdgpu_kernel void @idot4_acc32_2ele(ptr addrspace(1) %src1,2823; GFX7-LABEL: idot4_acc32_2ele:2824; GFX7: ; %bb.0: ; %entry2825; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92826; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2827; GFX7-NEXT: s_mov_b32 s7, 0xf0002828; GFX7-NEXT: s_mov_b32 s10, 02829; GFX7-NEXT: s_mov_b32 s11, s72830; GFX7-NEXT: s_waitcnt lgkmcnt(0)2831; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2832; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02833; GFX7-NEXT: v_mov_b32_e32 v1, 02834; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642835; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2836; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642837; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02838; GFX7-NEXT: s_mov_b32 s6, -12839; GFX7-NEXT: s_waitcnt vmcnt(1)2840; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v22841; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 82842; GFX7-NEXT: s_waitcnt vmcnt(0)2843; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v02844; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 82845; GFX7-NEXT: s_waitcnt lgkmcnt(0)2846; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s02847; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12848; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02849; GFX7-NEXT: s_endpgm2850;2851; GFX8-LABEL: idot4_acc32_2ele:2852; GFX8: ; %bb.0: ; %entry2853; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242854; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342855; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02856; GFX8-NEXT: s_waitcnt lgkmcnt(0)2857; GFX8-NEXT: v_mov_b32_e32 v1, s12858; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22859; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2860; GFX8-NEXT: flat_load_dword v3, v[0:1]2861; GFX8-NEXT: v_mov_b32_e32 v1, s32862; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22863; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2864; GFX8-NEXT: flat_load_dword v0, v[0:1]2865; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02866; GFX8-NEXT: s_waitcnt vmcnt(1)2867; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v32868; GFX8-NEXT: v_bfe_u32 v3, v3, 8, 82869; GFX8-NEXT: s_waitcnt vmcnt(0)2870; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v02871; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 82872; GFX8-NEXT: s_waitcnt lgkmcnt(0)2873; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s02874; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v12875; GFX8-NEXT: v_mov_b32_e32 v0, s42876; GFX8-NEXT: v_mov_b32_e32 v1, s52877; GFX8-NEXT: flat_store_dword v[0:1], v22878; GFX8-NEXT: s_endpgm2879;2880; GFX9-NODL-LABEL: idot4_acc32_2ele:2881; GFX9-NODL: ; %bb.0: ; %entry2882; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242883; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342884; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02885; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2886; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2887; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2888; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02889; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02890; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2891; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_02892; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12893; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2894; GFX9-NODL-NEXT: v_add3_u32 v1, v3, s0, v12895; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2896; GFX9-NODL-NEXT: s_endpgm2897;2898; GFX9-DL-LABEL: idot4_acc32_2ele:2899; GFX9-DL: ; %bb.0: ; %entry2900; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242901; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342902; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02903; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2904; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]2905; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]2906; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02907; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c01002908; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02909; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2910; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s12911; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2912; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s12913; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2914; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s02915; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2916; GFX9-DL-NEXT: s_endpgm2917;2918; GFX10-DL-LABEL: idot4_acc32_2ele:2919; GFX10-DL: ; %bb.0: ; %entry2920; GFX10-DL-NEXT: s_clause 0x12921; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242922; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342923; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02924; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2925; GFX10-DL-NEXT: s_clause 0x12926; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]2927; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]2928; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2929; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02930; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2931; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0c01002932; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2933; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0c01002934; GFX10-DL-NEXT: v_mov_b32_e32 v2, 02935; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2936; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s02937; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]2938; GFX10-DL-NEXT: s_endpgm2939;2940; GFX11-DL-LABEL: idot4_acc32_2ele:2941; GFX11-DL: ; %bb.0: ; %entry2942; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x242943; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02944; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x342945; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02946; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)2947; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02948; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2949; GFX11-DL-NEXT: s_clause 0x12950; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]2951; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]2952; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x02953; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2954; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c01002955; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2956; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c01002957; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2958; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2959; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s02960; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]2961; GFX11-DL-NEXT: s_endpgm2962 ptr addrspace(1) %src2,2963 ptr addrspace(1) nocapture %dst) {2964entry:2965 %idx = call i32 @llvm.amdgcn.workitem.id.x()2966 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2967 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12968 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2969 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22970 2971 %v1e0 = extractelement <4 x i8> %vec1, i64 02972 %cv1e0 = zext i8 %v1e0 to i322973 %v2e0 = extractelement <4 x i8> %vec2, i64 02974 %cv2e0 = zext i8 %v2e0 to i322975 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e02976 2977 %v1e1 = extractelement <4 x i8> %vec1, i64 12978 %cv1e1 = zext i8 %v1e1 to i322979 %v2e1 = extractelement <4 x i8> %vec2, i64 12980 %cv2e1 = zext i8 %v2e1 to i322981 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12982 2983 %acc = load i32, ptr addrspace(1) %dst, align 42984 %add1 = add i32 %mul1, %acc2985 %add2 = add i32 %add1, %mul22986 store i32 %add2, ptr addrspace(1) %dst, align 42987 ret void2988}2989 2990define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,2991; GFX7-LABEL: idot4_acc32_3ele:2992; GFX7: ; %bb.0: ; %entry2993; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92994; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2995; GFX7-NEXT: s_mov_b32 s7, 0xf0002996; GFX7-NEXT: s_mov_b32 s10, 02997; GFX7-NEXT: s_mov_b32 s11, s72998; GFX7-NEXT: s_waitcnt lgkmcnt(0)2999; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]3000; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03001; GFX7-NEXT: v_mov_b32_e32 v1, 03002; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr643003; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]3004; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr643005; GFX7-NEXT: s_load_dword s0, s[4:5], 0x03006; GFX7-NEXT: s_mov_b32 s6, -13007; GFX7-NEXT: s_waitcnt vmcnt(1)3008; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v23009; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 83010; GFX7-NEXT: s_waitcnt vmcnt(0)3011; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v03012; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 83013; GFX7-NEXT: s_waitcnt lgkmcnt(0)3014; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s03015; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 83016; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 83017; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v13018; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13019; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 03020; GFX7-NEXT: s_endpgm3021;3022; GFX8-LABEL: idot4_acc32_3ele:3023; GFX8: ; %bb.0: ; %entry3024; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243025; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343026; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03027; GFX8-NEXT: s_waitcnt lgkmcnt(0)3028; GFX8-NEXT: v_mov_b32_e32 v1, s13029; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23030; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3031; GFX8-NEXT: flat_load_dword v3, v[0:1]3032; GFX8-NEXT: v_mov_b32_e32 v1, s33033; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23034; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3035; GFX8-NEXT: flat_load_dword v0, v[0:1]3036; GFX8-NEXT: s_load_dword s0, s[4:5], 0x03037; GFX8-NEXT: s_waitcnt vmcnt(1)3038; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v33039; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 83040; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 83041; GFX8-NEXT: s_waitcnt vmcnt(0)3042; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v03043; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 83044; GFX8-NEXT: s_waitcnt lgkmcnt(0)3045; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s03046; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 83047; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v13048; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v13049; GFX8-NEXT: v_mov_b32_e32 v0, s43050; GFX8-NEXT: v_mov_b32_e32 v1, s53051; GFX8-NEXT: flat_store_dword v[0:1], v23052; GFX8-NEXT: s_endpgm3053;3054; GFX9-NODL-LABEL: idot4_acc32_3ele:3055; GFX9-NODL: ; %bb.0: ; %entry3056; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243057; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343058; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03059; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3060; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]3061; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]3062; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x03063; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03064; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3065; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v13066; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3067; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v23068; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_13069; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_23070; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3071; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s03072; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v13073; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]3074; GFX9-NODL-NEXT: s_endpgm3075;3076; GFX9-DL-LABEL: idot4_acc32_3ele:3077; GFX9-DL: ; %bb.0: ; %entry3078; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243079; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343080; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03081; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3082; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]3083; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]3084; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x03085; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0201003086; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03087; GFX9-DL-NEXT: s_waitcnt vmcnt(1)3088; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s13089; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3090; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s13091; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3092; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s03093; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]3094; GFX9-DL-NEXT: s_endpgm3095;3096; GFX10-DL-LABEL: idot4_acc32_3ele:3097; GFX10-DL: ; %bb.0: ; %entry3098; GFX10-DL-NEXT: s_clause 0x13099; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243100; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343101; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03102; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3103; GFX10-DL-NEXT: s_clause 0x13104; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]3105; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]3106; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)3107; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x03108; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3109; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0201003110; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3111; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0201003112; GFX10-DL-NEXT: v_mov_b32_e32 v2, 03113; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3114; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s03115; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]3116; GFX10-DL-NEXT: s_endpgm3117;3118; GFX11-DL-LABEL: idot4_acc32_3ele:3119; GFX11-DL: ; %bb.0: ; %entry3120; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x243121; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03122; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x343123; GFX11-DL-NEXT: v_mov_b32_e32 v2, 03124; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)3125; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03126; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3127; GFX11-DL-NEXT: s_clause 0x13128; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]3129; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]3130; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x03131; GFX11-DL-NEXT: s_waitcnt vmcnt(1)3132; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0201003133; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3134; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201003135; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3136; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)3137; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s03138; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]3139; GFX11-DL-NEXT: s_endpgm3140 ptr addrspace(1) %src2,3141 ptr addrspace(1) nocapture %dst) {3142entry:3143 %idx = call i32 @llvm.amdgcn.workitem.id.x()3144 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3145 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13146 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3147 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23148 3149 %v1e0 = extractelement <4 x i8> %vec1, i64 03150 %cv1e0 = zext i8 %v1e0 to i323151 %v2e0 = extractelement <4 x i8> %vec2, i64 03152 %cv2e0 = zext i8 %v2e0 to i323153 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e03154 3155 %v1e1 = extractelement <4 x i8> %vec1, i64 13156 %cv1e1 = zext i8 %v1e1 to i323157 %v2e1 = extractelement <4 x i8> %vec2, i64 13158 %cv2e1 = zext i8 %v2e1 to i323159 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e13160 3161 %v1e2 = extractelement <4 x i8> %vec1, i64 23162 %cv1e2 = zext i8 %v1e2 to i323163 %v2e2 = extractelement <4 x i8> %vec2, i64 23164 %cv2e2 = zext i8 %v2e2 to i323165 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e23166 3167 %acc = load i32, ptr addrspace(1) %dst, align 43168 %add1 = add i32 %mul1, %acc3169 %add2 = add i32 %add1, %mul23170 %add3 = add i32 %add2, %mul33171 store i32 %add3, ptr addrspace(1) %dst, align 43172 ret void3173}3174 3175define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,3176; GFX7-LABEL: idot4_acc32_3ele_permuted:3177; GFX7: ; %bb.0: ; %entry3178; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93179; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3180; GFX7-NEXT: s_mov_b32 s7, 0xf0003181; GFX7-NEXT: s_mov_b32 s10, 03182; GFX7-NEXT: s_mov_b32 s11, s73183; GFX7-NEXT: s_waitcnt lgkmcnt(0)3184; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]3185; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03186; GFX7-NEXT: v_mov_b32_e32 v1, 03187; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr643188; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]3189; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr643190; GFX7-NEXT: s_load_dword s0, s[4:5], 0x03191; GFX7-NEXT: s_mov_b32 s6, -13192; GFX7-NEXT: s_waitcnt vmcnt(1)3193; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v23194; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v23195; GFX7-NEXT: s_waitcnt vmcnt(0)3196; GFX7-NEXT: v_lshrrev_b32_e32 v4, 24, v03197; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v03198; GFX7-NEXT: s_waitcnt lgkmcnt(0)3199; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s03200; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 83201; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 83202; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v13203; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13204; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 03205; GFX7-NEXT: s_endpgm3206;3207; GFX8-LABEL: idot4_acc32_3ele_permuted:3208; GFX8: ; %bb.0: ; %entry3209; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243210; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343211; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03212; GFX8-NEXT: s_waitcnt lgkmcnt(0)3213; GFX8-NEXT: v_mov_b32_e32 v1, s13214; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23215; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3216; GFX8-NEXT: flat_load_dword v3, v[0:1]3217; GFX8-NEXT: v_mov_b32_e32 v1, s33218; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23219; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3220; GFX8-NEXT: flat_load_dword v0, v[0:1]3221; GFX8-NEXT: s_load_dword s0, s[4:5], 0x03222; GFX8-NEXT: s_waitcnt vmcnt(1)3223; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v33224; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v33225; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 83226; GFX8-NEXT: s_waitcnt vmcnt(0)3227; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v03228; GFX8-NEXT: v_and_b32_e32 v5, 0xff, v03229; GFX8-NEXT: s_waitcnt lgkmcnt(0)3230; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s03231; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 83232; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v13233; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v13234; GFX8-NEXT: v_mov_b32_e32 v0, s43235; GFX8-NEXT: v_mov_b32_e32 v1, s53236; GFX8-NEXT: flat_store_dword v[0:1], v23237; GFX8-NEXT: s_endpgm3238;3239; GFX9-NODL-LABEL: idot4_acc32_3ele_permuted:3240; GFX9-NODL: ; %bb.0: ; %entry3241; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243242; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343243; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03244; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3245; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]3246; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]3247; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x03248; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03249; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3250; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v13251; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3252; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 24, v23253; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_03254; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_23255; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3256; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s03257; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v13258; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]3259; GFX9-NODL-NEXT: s_endpgm3260;3261; GFX9-DL-LABEL: idot4_acc32_3ele_permuted:3262; GFX9-DL: ; %bb.0: ; %entry3263; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243264; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343265; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03266; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3267; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]3268; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]3269; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x03270; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0200033271; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03272; GFX9-DL-NEXT: s_waitcnt vmcnt(1)3273; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s13274; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3275; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s13276; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3277; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s03278; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]3279; GFX9-DL-NEXT: s_endpgm3280;3281; GFX10-DL-LABEL: idot4_acc32_3ele_permuted:3282; GFX10-DL: ; %bb.0: ; %entry3283; GFX10-DL-NEXT: s_clause 0x13284; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243285; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343286; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03287; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3288; GFX10-DL-NEXT: s_clause 0x13289; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]3290; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]3291; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)3292; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x03293; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3294; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0200033295; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3296; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0200033297; GFX10-DL-NEXT: v_mov_b32_e32 v2, 03298; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3299; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s03300; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]3301; GFX10-DL-NEXT: s_endpgm3302;3303; GFX11-DL-LABEL: idot4_acc32_3ele_permuted:3304; GFX11-DL: ; %bb.0: ; %entry3305; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x243306; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03307; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x343308; GFX11-DL-NEXT: v_mov_b32_e32 v2, 03309; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)3310; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03311; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3312; GFX11-DL-NEXT: s_clause 0x13313; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]3314; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]3315; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x03316; GFX11-DL-NEXT: s_waitcnt vmcnt(1)3317; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0200033318; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3319; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0200033320; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3321; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)3322; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s03323; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]3324; GFX11-DL-NEXT: s_endpgm3325 ptr addrspace(1) %src2,3326 ptr addrspace(1) nocapture %dst) {3327entry:3328 %idx = call i32 @llvm.amdgcn.workitem.id.x()3329 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3330 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13331 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3332 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23333 3334 %v1e0 = extractelement <4 x i8> %vec1, i64 33335 %cv1e0 = zext i8 %v1e0 to i323336 %v2e0 = extractelement <4 x i8> %vec2, i64 33337 %cv2e0 = zext i8 %v2e0 to i323338 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e03339 3340 %v1e1 = extractelement <4 x i8> %vec1, i64 03341 %cv1e1 = zext i8 %v1e1 to i323342 %v2e1 = extractelement <4 x i8> %vec2, i64 03343 %cv2e1 = zext i8 %v2e1 to i323344 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e13345 3346 %v1e2 = extractelement <4 x i8> %vec1, i64 23347 %cv1e2 = zext i8 %v1e2 to i323348 %v2e2 = extractelement <4 x i8> %vec2, i64 23349 %cv2e2 = zext i8 %v2e2 to i323350 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e23351 3352 %acc = load i32, ptr addrspace(1) %dst, align 43353 %add1 = add i32 %mul1, %acc3354 %add2 = add i32 %add1, %mul23355 %add3 = add i32 %add2, %mul33356 store i32 %add3, ptr addrspace(1) %dst, align 43357 ret void3358}3359 3360 3361define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,3362; GFX7-LABEL: idot4_acc32_opt:3363; GFX7: ; %bb.0: ; %entry3364; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93365; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3366; GFX7-NEXT: s_mov_b32 s7, 0xf0003367; GFX7-NEXT: s_mov_b32 s10, 03368; GFX7-NEXT: s_mov_b32 s11, s73369; GFX7-NEXT: s_waitcnt lgkmcnt(0)3370; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]3371; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03372; GFX7-NEXT: v_mov_b32_e32 v1, 03373; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr643374; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]3375; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr643376; GFX7-NEXT: s_mov_b32 s6, -13377; GFX7-NEXT: s_waitcnt vmcnt(1)3378; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 83379; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v23380; GFX7-NEXT: s_waitcnt vmcnt(0)3381; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 83382; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v03383; GFX7-NEXT: v_mul_u32_u24_e32 v3, v3, v63384; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 83385; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 83386; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v33387; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v23388; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v03389; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v13390; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13391; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 03392; GFX7-NEXT: s_endpgm3393;3394; GFX8-LABEL: idot4_acc32_opt:3395; GFX8: ; %bb.0: ; %entry3396; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243397; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343398; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03399; GFX8-NEXT: s_waitcnt lgkmcnt(0)3400; GFX8-NEXT: v_mov_b32_e32 v1, s13401; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23402; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3403; GFX8-NEXT: flat_load_dword v3, v[0:1]3404; GFX8-NEXT: v_mov_b32_e32 v1, s33405; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23406; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3407; GFX8-NEXT: flat_load_dword v2, v[0:1]3408; GFX8-NEXT: v_mov_b32_e32 v0, s43409; GFX8-NEXT: v_mov_b32_e32 v1, s53410; GFX8-NEXT: s_waitcnt vmcnt(1)3411; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v33412; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 83413; GFX8-NEXT: s_waitcnt vmcnt(0)3414; GFX8-NEXT: v_and_b32_e32 v5, 0xff, v23415; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_13416; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 83417; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v63418; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v33419; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v23420; GFX8-NEXT: v_mad_u32_u24 v4, v7, v8, v43421; GFX8-NEXT: v_mad_u32_u24 v2, v3, v2, v43422; GFX8-NEXT: flat_store_dword v[0:1], v23423; GFX8-NEXT: s_endpgm3424;3425; GFX9-NODL-LABEL: idot4_acc32_opt:3426; GFX9-NODL: ; %bb.0: ; %entry3427; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243428; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343429; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03430; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3431; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]3432; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]3433; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03434; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3435; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v13436; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3437; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v23438; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_13439; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_23440; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_33441; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v53442; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v13443; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]3444; GFX9-NODL-NEXT: s_endpgm3445;3446; GFX9-DL-LABEL: idot4_acc32_opt:3447; GFX9-DL: ; %bb.0: ; %entry3448; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243449; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343450; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03451; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3452; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]3453; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]3454; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03455; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3456; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 03457; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]3458; GFX9-DL-NEXT: s_endpgm3459;3460; GFX10-DL-LABEL: idot4_acc32_opt:3461; GFX10-DL: ; %bb.0: ; %entry3462; GFX10-DL-NEXT: s_clause 0x13463; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243464; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343465; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03466; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3467; GFX10-DL-NEXT: s_clause 0x13468; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]3469; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]3470; GFX10-DL-NEXT: v_mov_b32_e32 v0, 03471; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3472; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 03473; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]3474; GFX10-DL-NEXT: s_endpgm3475;3476; GFX11-DL-LABEL: idot4_acc32_opt:3477; GFX11-DL: ; %bb.0: ; %entry3478; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x243479; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03480; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x343481; GFX11-DL-NEXT: v_mov_b32_e32 v2, 03482; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)3483; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03484; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3485; GFX11-DL-NEXT: s_clause 0x13486; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]3487; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]3488; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3489; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 03490; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]3491; GFX11-DL-NEXT: s_endpgm3492 ptr addrspace(1) %src2,3493 ptr addrspace(1) nocapture %dst) {3494entry:3495 %idx = call i32 @llvm.amdgcn.workitem.id.x()3496 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3497 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13498 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3499 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23500 3501 %v1e0 = extractelement <4 x i8> %vec1, i64 03502 %cv1e0 = zext i8 %v1e0 to i323503 %v2e0 = extractelement <4 x i8> %vec2, i64 03504 %cv2e0 = zext i8 %v2e0 to i323505 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e03506 3507 %v1e1 = extractelement <4 x i8> %vec1, i64 13508 %cv1e1 = zext i8 %v1e1 to i323509 %v2e1 = extractelement <4 x i8> %vec2, i64 13510 %cv2e1 = zext i8 %v2e1 to i323511 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e13512 3513 %v1e2 = extractelement <4 x i8> %vec1, i64 23514 %cv1e2 = zext i8 %v1e2 to i323515 %v2e2 = extractelement <4 x i8> %vec2, i64 23516 %cv2e2 = zext i8 %v2e2 to i323517 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e23518 3519 %v1e3 = extractelement <4 x i8> %vec1, i64 33520 %cv1e3 = zext i8 %v1e3 to i323521 %v2e3 = extractelement <4 x i8> %vec2, i64 33522 %cv2e3 = zext i8 %v2e3 to i323523 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e33524 3525 %add2 = add i32 %mul1, %mul23526 %add3 = add i32 %add2, %mul33527 %add4 = add i32 %add3, %mul43528 store i32 %add4, ptr addrspace(1) %dst, align 43529 ret void3530}3531 3532define amdgpu_kernel void @udot4_acc32_3src(ptr addrspace(1) %src1,3533; GFX7-LABEL: udot4_acc32_3src:3534; GFX7: ; %bb.0: ; %entry3535; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x93536; GFX7-NEXT: s_mov_b32 s11, 0xf0003537; GFX7-NEXT: s_mov_b32 s14, 03538; GFX7-NEXT: s_mov_b32 s15, s113539; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03540; GFX7-NEXT: s_waitcnt lgkmcnt(0)3541; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]3542; GFX7-NEXT: v_mov_b32_e32 v1, 03543; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr643544; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]3545; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr643546; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]3547; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr643548; GFX7-NEXT: s_load_dword s0, s[6:7], 0x03549; GFX7-NEXT: s_mov_b32 s10, -13550; GFX7-NEXT: s_mov_b32 s8, s63551; GFX7-NEXT: s_mov_b32 s9, s73552; GFX7-NEXT: s_waitcnt vmcnt(2)3553; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v23554; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 83555; GFX7-NEXT: s_waitcnt vmcnt(1)3556; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 83557; GFX7-NEXT: s_waitcnt lgkmcnt(0)3558; GFX7-NEXT: v_mad_u32_u24 v1, v1, v1, s03559; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 83560; GFX7-NEXT: s_waitcnt vmcnt(0)3561; GFX7-NEXT: v_bfe_u32 v6, v0, 16, 83562; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v13563; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v23564; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v03565; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v13566; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13567; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 03568; GFX7-NEXT: s_endpgm3569;3570; GFX8-LABEL: udot4_acc32_3src:3571; GFX8: ; %bb.0: ; %entry3572; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x243573; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03574; GFX8-NEXT: s_waitcnt lgkmcnt(0)3575; GFX8-NEXT: v_mov_b32_e32 v1, s13576; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23577; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3578; GFX8-NEXT: flat_load_dword v3, v[0:1]3579; GFX8-NEXT: v_mov_b32_e32 v1, s33580; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23581; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3582; GFX8-NEXT: flat_load_dword v4, v[0:1]3583; GFX8-NEXT: v_mov_b32_e32 v1, s53584; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v23585; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3586; GFX8-NEXT: flat_load_dword v0, v[0:1]3587; GFX8-NEXT: s_load_dword s0, s[6:7], 0x03588; GFX8-NEXT: s_waitcnt vmcnt(2)3589; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v33590; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 83591; GFX8-NEXT: s_waitcnt lgkmcnt(0)3592; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s03593; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 83594; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v33595; GFX8-NEXT: s_waitcnt vmcnt(1)3596; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 83597; GFX8-NEXT: v_mad_u32_u24 v1, v2, v4, v13598; GFX8-NEXT: s_waitcnt vmcnt(0)3599; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 83600; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v03601; GFX8-NEXT: v_mad_u32_u24 v1, v5, v6, v13602; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v13603; GFX8-NEXT: v_mov_b32_e32 v0, s63604; GFX8-NEXT: v_mov_b32_e32 v1, s73605; GFX8-NEXT: flat_store_dword v[0:1], v23606; GFX8-NEXT: s_endpgm3607;3608; GFX9-NODL-LABEL: udot4_acc32_3src:3609; GFX9-NODL: ; %bb.0: ; %entry3610; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243611; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03612; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3613; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]3614; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]3615; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]3616; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x03617; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03618; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)3619; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_03620; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3621; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_13622; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3623; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_23624; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_33625; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3626; GFX9-NODL-NEXT: v_add3_u32 v2, v4, s0, v23627; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v13628; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]3629; GFX9-NODL-NEXT: s_endpgm3630;3631; GFX9-DL-LABEL: udot4_acc32_3src:3632; GFX9-DL: ; %bb.0: ; %entry3633; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243634; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03635; GFX9-DL-NEXT: s_mov_b32 s0, 0x706010c3636; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c0c003637; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3638; GFX9-DL-NEXT: global_load_dword v1, v0, s[10:11]3639; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]3640; GFX9-DL-NEXT: global_load_dword v3, v0, s[8:9]3641; GFX9-DL-NEXT: s_load_dword s1, s[14:15], 0x03642; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03643; GFX9-DL-NEXT: s_waitcnt vmcnt(1)3644; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s03645; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3646; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s23647; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v23648; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3649; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v3, v1, s13650; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]3651; GFX9-DL-NEXT: s_endpgm3652;3653; GFX10-DL-LABEL: udot4_acc32_3src:3654; GFX10-DL: ; %bb.0: ; %entry3655; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243656; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03657; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3658; GFX10-DL-NEXT: s_clause 0x23659; GFX10-DL-NEXT: global_load_dword v1, v0, s[10:11]3660; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]3661; GFX10-DL-NEXT: global_load_dword v3, v0, s[8:9]3662; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x03663; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3664; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0x706010c3665; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3666; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c003667; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v13668; GFX10-DL-NEXT: v_mov_b32_e32 v1, 03669; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3670; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v0, s03671; GFX10-DL-NEXT: global_store_dword v1, v0, s[14:15]3672; GFX10-DL-NEXT: s_endpgm3673;3674; GFX11-DL-LABEL: udot4_acc32_3src:3675; GFX11-DL: ; %bb.0: ; %entry3676; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x243677; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03678; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)3679; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03680; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3681; GFX11-DL-NEXT: s_clause 0x23682; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]3683; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]3684; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]3685; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x03686; GFX11-DL-NEXT: s_waitcnt vmcnt(1)3687; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0x706010c3688; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3689; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c003690; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)3691; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v23692; GFX11-DL-NEXT: v_mov_b32_e32 v2, 03693; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3694; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s03695; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]3696; GFX11-DL-NEXT: s_endpgm3697 ptr addrspace(1) %src2,3698 ptr addrspace(1) %src3,3699 ptr addrspace(1) nocapture %dst) {3700entry:3701 %idx = call i32 @llvm.amdgcn.workitem.id.x()3702 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3703 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13704 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3705 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23706 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx3707 %vec3 = load <4 x i8>, ptr addrspace(1) %gep33708 3709 %v1e0 = extractelement <4 x i8> %vec1, i64 03710 %cv1e0 = zext i8 %v1e0 to i323711 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e03712 3713 %v1e1 = extractelement <4 x i8> %vec1, i64 13714 %cv1e1 = zext i8 %v1e1 to i323715 %v2e1 = extractelement <4 x i8> %vec2, i64 13716 %cv2e1 = zext i8 %v2e1 to i323717 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e13718 3719 %v1e2 = extractelement <4 x i8> %vec1, i64 23720 %cv1e2 = zext i8 %v1e2 to i323721 %v3e2 = extractelement <4 x i8> %vec3, i64 23722 %cv3e2 = zext i8 %v3e2 to i323723 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e23724 3725 %v1e3 = extractelement <4 x i8> %vec1, i64 33726 %cv1e3 = zext i8 %v1e3 to i323727 %v3e3 = extractelement <4 x i8> %vec3, i64 33728 %cv3e3 = zext i8 %v3e3 to i323729 %mul4 = mul nuw nsw i32 %cv1e3, %cv3e33730 3731 %acc = load i32, ptr addrspace(1) %dst, align 43732 %mad1 = add i32 %mul1, %acc3733 %mad2 = add i32 %mad1, %mul23734 %mad3 = add i32 %mad2, %mul33735 %mad4 = add i32 %mad3, %mul43736 3737 store i32 %mad4, ptr addrspace(1) %dst, align 43738 ret void3739}3740 3741define amdgpu_kernel void @udot4_acc32_3src_3ele(ptr addrspace(1) %src1,3742; GFX7-LABEL: udot4_acc32_3src_3ele:3743; GFX7: ; %bb.0: ; %entry3744; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x93745; GFX7-NEXT: s_mov_b32 s11, 0xf0003746; GFX7-NEXT: s_mov_b32 s14, 03747; GFX7-NEXT: s_mov_b32 s15, s113748; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03749; GFX7-NEXT: s_waitcnt lgkmcnt(0)3750; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]3751; GFX7-NEXT: v_mov_b32_e32 v1, 03752; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr643753; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]3754; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr643755; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]3756; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr643757; GFX7-NEXT: s_load_dword s0, s[6:7], 0x03758; GFX7-NEXT: s_mov_b32 s10, -13759; GFX7-NEXT: s_mov_b32 s8, s63760; GFX7-NEXT: s_mov_b32 s9, s73761; GFX7-NEXT: s_waitcnt vmcnt(2)3762; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v23763; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 83764; GFX7-NEXT: s_waitcnt vmcnt(1)3765; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 83766; GFX7-NEXT: s_waitcnt lgkmcnt(0)3767; GFX7-NEXT: v_mad_u32_u24 v1, v1, v1, s03768; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 83769; GFX7-NEXT: s_waitcnt vmcnt(0)3770; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 83771; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v13772; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13773; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 03774; GFX7-NEXT: s_endpgm3775;3776; GFX8-LABEL: udot4_acc32_3src_3ele:3777; GFX8: ; %bb.0: ; %entry3778; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x243779; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03780; GFX8-NEXT: s_waitcnt lgkmcnt(0)3781; GFX8-NEXT: v_mov_b32_e32 v1, s13782; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23783; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3784; GFX8-NEXT: flat_load_dword v3, v[0:1]3785; GFX8-NEXT: v_mov_b32_e32 v1, s33786; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23787; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3788; GFX8-NEXT: flat_load_dword v4, v[0:1]3789; GFX8-NEXT: v_mov_b32_e32 v1, s53790; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v23791; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3792; GFX8-NEXT: flat_load_dword v0, v[0:1]3793; GFX8-NEXT: s_load_dword s0, s[6:7], 0x03794; GFX8-NEXT: s_waitcnt vmcnt(2)3795; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v33796; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 83797; GFX8-NEXT: s_waitcnt lgkmcnt(0)3798; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s03799; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 83800; GFX8-NEXT: s_waitcnt vmcnt(1)3801; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 83802; GFX8-NEXT: v_mad_u32_u24 v1, v2, v4, v13803; GFX8-NEXT: s_waitcnt vmcnt(0)3804; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 83805; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v13806; GFX8-NEXT: v_mov_b32_e32 v0, s63807; GFX8-NEXT: v_mov_b32_e32 v1, s73808; GFX8-NEXT: flat_store_dword v[0:1], v23809; GFX8-NEXT: s_endpgm3810;3811; GFX9-NODL-LABEL: udot4_acc32_3src_3ele:3812; GFX9-NODL: ; %bb.0: ; %entry3813; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243814; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03815; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3816; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]3817; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]3818; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]3819; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x03820; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03821; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)3822; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v13823; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3824; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_13825; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3826; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_23827; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3828; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v4, s03829; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v13830; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]3831; GFX9-NODL-NEXT: s_endpgm3832;3833; GFX9-DL-LABEL: udot4_acc32_3src_3ele:3834; GFX9-DL: ; %bb.0: ; %entry3835; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243836; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03837; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c3838; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c003839; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0201003840; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3841; GFX9-DL-NEXT: global_load_dword v1, v0, s[10:11]3842; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]3843; GFX9-DL-NEXT: global_load_dword v3, v0, s[8:9]3844; GFX9-DL-NEXT: s_load_dword s3, s[14:15], 0x03845; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03846; GFX9-DL-NEXT: s_waitcnt vmcnt(1)3847; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s03848; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3849; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s13850; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v23851; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s23852; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3853; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s33854; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]3855; GFX9-DL-NEXT: s_endpgm3856;3857; GFX10-DL-LABEL: udot4_acc32_3src_3ele:3858; GFX10-DL: ; %bb.0: ; %entry3859; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243860; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03861; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3862; GFX10-DL-NEXT: s_clause 0x23863; GFX10-DL-NEXT: global_load_dword v1, v0, s[10:11]3864; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]3865; GFX10-DL-NEXT: global_load_dword v3, v0, s[8:9]3866; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x03867; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3868; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc06010c3869; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3870; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c003871; GFX10-DL-NEXT: v_mov_b32_e32 v2, 03872; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v13873; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0201003874; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3875; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s03876; GFX10-DL-NEXT: global_store_dword v2, v0, s[14:15]3877; GFX10-DL-NEXT: s_endpgm3878;3879; GFX11-DL-LABEL: udot4_acc32_3src_3ele:3880; GFX11-DL: ; %bb.0: ; %entry3881; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x243882; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03883; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)3884; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03885; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3886; GFX11-DL-NEXT: s_clause 0x23887; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]3888; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]3889; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]3890; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x03891; GFX11-DL-NEXT: s_waitcnt vmcnt(1)3892; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc06010c3893; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3894; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c003895; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201003896; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)3897; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v23898; GFX11-DL-NEXT: v_mov_b32_e32 v2, 03899; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3900; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s03901; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]3902; GFX11-DL-NEXT: s_endpgm3903 ptr addrspace(1) %src2,3904 ptr addrspace(1) %src3,3905 ptr addrspace(1) nocapture %dst) {3906entry:3907 %idx = call i32 @llvm.amdgcn.workitem.id.x()3908 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3909 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13910 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3911 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23912 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx3913 %vec3 = load <4 x i8>, ptr addrspace(1) %gep33914 3915 %v1e0 = extractelement <4 x i8> %vec1, i64 03916 %cv1e0 = zext i8 %v1e0 to i323917 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e03918 3919 %v1e1 = extractelement <4 x i8> %vec1, i64 13920 %cv1e1 = zext i8 %v1e1 to i323921 %v2e1 = extractelement <4 x i8> %vec2, i64 13922 %cv2e1 = zext i8 %v2e1 to i323923 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e13924 3925 %v1e2 = extractelement <4 x i8> %vec1, i64 23926 %cv1e2 = zext i8 %v1e2 to i323927 %v3e2 = extractelement <4 x i8> %vec3, i64 23928 %cv3e2 = zext i8 %v3e2 to i323929 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e23930 3931 3932 %acc = load i32, ptr addrspace(1) %dst, align 43933 %mad1 = add i32 %mul1, %acc3934 %mad2 = add i32 %mad1, %mul23935 %mad3 = add i32 %mad2, %mul33936 3937 store i32 %mad3, ptr addrspace(1) %dst, align 43938 ret void3939}3940 3941define amdgpu_kernel void @udot4_bad_source(ptr addrspace(1) %src1,3942; GFX7-LABEL: udot4_bad_source:3943; GFX7: ; %bb.0: ; %entry3944; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93945; GFX7-NEXT: s_load_dword s12, s[4:5], 0xf3946; GFX7-NEXT: s_mov_b32 s7, 0xf0003947; GFX7-NEXT: s_mov_b32 s10, 03948; GFX7-NEXT: s_mov_b32 s11, s73949; GFX7-NEXT: s_waitcnt lgkmcnt(0)3950; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]3951; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03952; GFX7-NEXT: v_mov_b32_e32 v1, 03953; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr643954; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]3955; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr643956; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x113957; GFX7-NEXT: s_and_b32 s1, s12, 0xffff3958; GFX7-NEXT: s_mov_b32 s6, -13959; GFX7-NEXT: s_waitcnt lgkmcnt(0)3960; GFX7-NEXT: s_load_dword s0, s[4:5], 0x03961; GFX7-NEXT: s_waitcnt lgkmcnt(0)3962; GFX7-NEXT: v_mov_b32_e32 v1, s03963; GFX7-NEXT: s_waitcnt vmcnt(1)3964; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v23965; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 83966; GFX7-NEXT: s_waitcnt vmcnt(0)3967; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 83968; GFX7-NEXT: v_mad_u32_u24 v1, v3, s1, v13969; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 83970; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 83971; GFX7-NEXT: v_mad_u32_u24 v1, v4, v5, v13972; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13973; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 03974; GFX7-NEXT: s_endpgm3975;3976; GFX8-LABEL: udot4_bad_source:3977; GFX8: ; %bb.0: ; %entry3978; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243979; GFX8-NEXT: s_load_dword s6, s[4:5], 0x3c3980; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03981; GFX8-NEXT: s_waitcnt lgkmcnt(0)3982; GFX8-NEXT: v_mov_b32_e32 v1, s13983; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23984; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3985; GFX8-NEXT: flat_load_dword v3, v[0:1]3986; GFX8-NEXT: v_mov_b32_e32 v1, s33987; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23988; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3989; GFX8-NEXT: flat_load_dword v0, v[0:1]3990; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x443991; GFX8-NEXT: s_and_b32 s3, s6, 0xffff3992; GFX8-NEXT: s_waitcnt lgkmcnt(0)3993; GFX8-NEXT: s_load_dword s2, s[0:1], 0x03994; GFX8-NEXT: s_waitcnt lgkmcnt(0)3995; GFX8-NEXT: v_mov_b32_e32 v1, s23996; GFX8-NEXT: s_waitcnt vmcnt(1)3997; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v33998; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 83999; GFX8-NEXT: v_mad_u32_u24 v1, v2, s3, v14000; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 84001; GFX8-NEXT: s_waitcnt vmcnt(0)4002; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 84003; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 84004; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v14005; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v14006; GFX8-NEXT: v_mov_b32_e32 v0, s04007; GFX8-NEXT: v_mov_b32_e32 v1, s14008; GFX8-NEXT: flat_store_dword v[0:1], v24009; GFX8-NEXT: s_endpgm4010;4011; GFX9-NODL-LABEL: udot4_bad_source:4012; GFX9-NODL: ; %bb.0: ; %entry4013; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244014; GFX9-NODL-NEXT: s_load_dword s6, s[4:5], 0x3c4015; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v04016; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4017; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]4018; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]4019; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444020; GFX9-NODL-NEXT: s_and_b32 s3, s6, 0xffff4021; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 04022; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4023; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x04024; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)4025; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v14026; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)4027; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_14028; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_24029; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4030; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s24031; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, s3, v24032; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v4, v14033; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]4034; GFX9-NODL-NEXT: s_endpgm4035;4036; GFX9-DL-LABEL: udot4_bad_source:4037; GFX9-DL: ; %bb.0: ; %entry4038; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244039; GFX9-DL-NEXT: s_load_dword s6, s[4:5], 0x3c4040; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04041; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4042; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]4043; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]4044; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444045; GFX9-DL-NEXT: s_mov_b32 s3, 0xc0c02014046; GFX9-DL-NEXT: s_and_b32 s4, s6, 0xffff4047; GFX9-DL-NEXT: v_mov_b32_e32 v0, 04048; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4049; GFX9-DL-NEXT: s_load_dword s2, s[0:1], 0x04050; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4051; GFX9-DL-NEXT: v_mov_b32_e32 v3, s24052; GFX9-DL-NEXT: s_waitcnt vmcnt(1)4053; GFX9-DL-NEXT: v_and_b32_e32 v4, 0xff, v14054; GFX9-DL-NEXT: s_waitcnt vmcnt(0)4055; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s34056; GFX9-DL-NEXT: v_mad_u32_u24 v3, v4, s4, v34057; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s34058; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v34059; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]4060; GFX9-DL-NEXT: s_endpgm4061;4062; GFX10-DL-LABEL: udot4_bad_source:4063; GFX10-DL: ; %bb.0: ; %entry4064; GFX10-DL-NEXT: s_clause 0x14065; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244066; GFX10-DL-NEXT: s_load_dword s6, s[4:5], 0x3c4067; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04068; GFX10-DL-NEXT: v_mov_b32_e32 v3, 04069; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4070; GFX10-DL-NEXT: s_clause 0x14071; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]4072; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]4073; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4074; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444075; GFX10-DL-NEXT: s_and_b32 s3, s6, 0xffff4076; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4077; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x04078; GFX10-DL-NEXT: s_waitcnt vmcnt(1)4079; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xff, v14080; GFX10-DL-NEXT: s_waitcnt vmcnt(0)4081; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c02014082; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c02014083; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4084; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, s3, s24085; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v2, v04086; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]4087; GFX10-DL-NEXT: s_endpgm4088;4089; GFX11-DL-LABEL: udot4_bad_source:4090; GFX11-DL: ; %bb.0: ; %entry4091; GFX11-DL-NEXT: s_clause 0x14092; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x244093; GFX11-DL-NEXT: s_load_b32 s6, s[4:5], 0x3c4094; GFX11-DL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v04095; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4096; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04097; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4098; GFX11-DL-NEXT: s_clause 0x14099; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]4100; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]4101; GFX11-DL-NEXT: s_load_b64 s[0:1], s[4:5], 0x444102; GFX11-DL-NEXT: s_and_b32 s3, s6, 0xffff4103; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4104; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x04105; GFX11-DL-NEXT: s_waitcnt vmcnt(1)4106; GFX11-DL-NEXT: v_and_b32_e32 v2, 0xff, v14107; GFX11-DL-NEXT: s_waitcnt vmcnt(0)4108; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c02014109; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c02014110; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4111; GFX11-DL-NEXT: v_mad_u32_u24 v2, v2, s3, s24112; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4113; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v24114; GFX11-DL-NEXT: global_store_b32 v3, v0, s[0:1]4115; GFX11-DL-NEXT: s_endpgm4116 ptr addrspace(1) %src2,4117 ptr addrspace(1) %src3,4118 i16 %badsource,4119 ptr addrspace(1) nocapture %dst) {4120entry:4121 %idx = call i32 @llvm.amdgcn.workitem.id.x()4122 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx4123 %vec1 = load <4 x i8>, ptr addrspace(1) %gep14124 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx4125 %vec2 = load <4 x i8>, ptr addrspace(1) %gep24126 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx4127 %vec3 = load <4 x i8>, ptr addrspace(1) %gep34128 4129 %v1e0 = extractelement <4 x i8> %vec1, i64 04130 %cv1e0 = zext i8 %v1e0 to i324131 %v2e0 = extractelement <4 x i8> %vec2, i64 04132 %other = zext i16 %badsource to i324133 %mul1 = mul nuw nsw i32 %cv1e0, %other4134 4135 %v1e1 = extractelement <4 x i8> %vec1, i64 14136 %cv1e1 = zext i8 %v1e1 to i324137 %v2e1 = extractelement <4 x i8> %vec2, i64 14138 %cv2e1 = zext i8 %v2e1 to i324139 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e14140 4141 %v2e2 = extractelement <4 x i8> %vec2, i64 24142 %cv2e2 = zext i8 %v2e2 to i324143 %v1e2 = extractelement <4 x i8> %vec1, i64 24144 %cv1e2 = zext i8 %v1e2 to i324145 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e24146 4147 4148 %acc = load i32, ptr addrspace(1) %dst, align 44149 %mad1 = add i32 %mul1, %acc4150 %mad2 = add i32 %mad1, %mul24151 %mad3 = add i32 %mad2, %mul34152 4153 store i32 %mad3, ptr addrspace(1) %dst, align 44154 ret void4155}4156 4157 4158define amdgpu_kernel void @udot4_commutative(ptr addrspace(1) %src1,4159; GFX7-LABEL: udot4_commutative:4160; GFX7: ; %bb.0: ; %entry4161; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94162; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf4163; GFX7-NEXT: s_mov_b32 s7, 0xf0004164; GFX7-NEXT: s_mov_b32 s10, 04165; GFX7-NEXT: s_mov_b32 s11, s74166; GFX7-NEXT: s_waitcnt lgkmcnt(0)4167; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]4168; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v04169; GFX7-NEXT: v_mov_b32_e32 v1, 04170; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr644171; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]4172; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr644173; GFX7-NEXT: s_load_dword s0, s[4:5], 0x04174; GFX7-NEXT: s_mov_b32 s6, -14175; GFX7-NEXT: s_waitcnt vmcnt(1)4176; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v24177; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 84178; GFX7-NEXT: s_waitcnt vmcnt(0)4179; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v04180; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 84181; GFX7-NEXT: s_waitcnt lgkmcnt(0)4182; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s04183; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 84184; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 84185; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v14186; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v14187; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 04188; GFX7-NEXT: s_endpgm4189;4190; GFX8-LABEL: udot4_commutative:4191; GFX8: ; %bb.0: ; %entry4192; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244193; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3c4194; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v04195; GFX8-NEXT: s_waitcnt lgkmcnt(0)4196; GFX8-NEXT: v_mov_b32_e32 v1, s14197; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v24198; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4199; GFX8-NEXT: flat_load_dword v3, v[0:1]4200; GFX8-NEXT: v_mov_b32_e32 v1, s34201; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v24202; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4203; GFX8-NEXT: flat_load_dword v0, v[0:1]4204; GFX8-NEXT: s_load_dword s0, s[4:5], 0x04205; GFX8-NEXT: s_waitcnt vmcnt(1)4206; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v34207; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 84208; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 84209; GFX8-NEXT: s_waitcnt vmcnt(0)4210; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v04211; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 84212; GFX8-NEXT: s_waitcnt lgkmcnt(0)4213; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s04214; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 84215; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v14216; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v14217; GFX8-NEXT: v_mov_b32_e32 v0, s44218; GFX8-NEXT: v_mov_b32_e32 v1, s54219; GFX8-NEXT: flat_store_dword v[0:1], v24220; GFX8-NEXT: s_endpgm4221;4222; GFX9-NODL-LABEL: udot4_commutative:4223; GFX9-NODL: ; %bb.0: ; %entry4224; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244225; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4226; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v04227; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4228; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]4229; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]4230; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x04231; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 04232; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)4233; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v14234; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)4235; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v24236; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_14237; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_24238; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4239; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s04240; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v14241; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]4242; GFX9-NODL-NEXT: s_endpgm4243;4244; GFX9-DL-LABEL: udot4_commutative:4245; GFX9-DL: ; %bb.0: ; %entry4246; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244247; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4248; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04249; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4250; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]4251; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]4252; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x04253; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0201004254; GFX9-DL-NEXT: v_mov_b32_e32 v0, 04255; GFX9-DL-NEXT: s_waitcnt vmcnt(1)4256; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s14257; GFX9-DL-NEXT: s_waitcnt vmcnt(0)4258; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s14259; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4260; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s04261; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]4262; GFX9-DL-NEXT: s_endpgm4263;4264; GFX10-DL-LABEL: udot4_commutative:4265; GFX10-DL: ; %bb.0: ; %entry4266; GFX10-DL-NEXT: s_clause 0x14267; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244268; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4269; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04270; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4271; GFX10-DL-NEXT: s_clause 0x14272; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]4273; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]4274; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4275; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x04276; GFX10-DL-NEXT: s_waitcnt vmcnt(1)4277; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0201004278; GFX10-DL-NEXT: s_waitcnt vmcnt(0)4279; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0201004280; GFX10-DL-NEXT: v_mov_b32_e32 v2, 04281; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4282; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s04283; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]4284; GFX10-DL-NEXT: s_endpgm4285;4286; GFX11-DL-LABEL: udot4_commutative:4287; GFX11-DL: ; %bb.0: ; %entry4288; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x244289; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v04290; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x3c4291; GFX11-DL-NEXT: v_mov_b32_e32 v2, 04292; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)4293; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04294; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4295; GFX11-DL-NEXT: s_clause 0x14296; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]4297; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]4298; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x04299; GFX11-DL-NEXT: s_waitcnt vmcnt(1)4300; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0201004301; GFX11-DL-NEXT: s_waitcnt vmcnt(0)4302; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201004303; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4304; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4305; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s04306; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]4307; GFX11-DL-NEXT: s_endpgm4308 ptr addrspace(1) %src2,4309 ptr addrspace(1) %src3,4310 ptr addrspace(1) nocapture %dst) {4311entry:4312 %idx = call i32 @llvm.amdgcn.workitem.id.x()4313 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx4314 %vec1 = load <4 x i8>, ptr addrspace(1) %gep14315 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx4316 %vec2 = load <4 x i8>, ptr addrspace(1) %gep24317 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx4318 %vec3 = load <4 x i8>, ptr addrspace(1) %gep34319 4320 %v1e0 = extractelement <4 x i8> %vec1, i64 04321 %cv1e0 = zext i8 %v1e0 to i324322 %v2e0 = extractelement <4 x i8> %vec2, i64 04323 %cv2e0 = zext i8 %v2e0 to i324324 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e04325 4326 %v1e1 = extractelement <4 x i8> %vec1, i64 14327 %cv1e1 = zext i8 %v1e1 to i324328 %v2e1 = extractelement <4 x i8> %vec2, i64 14329 %cv2e1 = zext i8 %v2e1 to i324330 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e14331 4332 %v2e2 = extractelement <4 x i8> %vec2, i64 24333 %cv2e2 = zext i8 %v2e2 to i324334 %v1e2 = extractelement <4 x i8> %vec1, i64 24335 %cv1e2 = zext i8 %v1e2 to i324336 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e24337 4338 4339 %acc = load i32, ptr addrspace(1) %dst, align 44340 %mad1 = add i32 %mul1, %acc4341 %mad2 = add i32 %mad1, %mul24342 %mad3 = add i32 %mad2, %mul34343 4344 store i32 %mad3, ptr addrspace(1) %dst, align 44345 ret void4346}4347 4348define amdgpu_kernel void @udot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,4349; GFX7-LABEL: udot4_acc32_3src_3ele_src0:4350; GFX7: ; %bb.0: ; %entry4351; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x94352; GFX7-NEXT: s_mov_b32 s11, 0xf0004353; GFX7-NEXT: s_mov_b32 s14, 04354; GFX7-NEXT: s_mov_b32 s15, s114355; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v04356; GFX7-NEXT: s_waitcnt lgkmcnt(0)4357; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]4358; GFX7-NEXT: v_mov_b32_e32 v1, 04359; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr644360; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]4361; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr644362; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]4363; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr644364; GFX7-NEXT: s_load_dword s0, s[6:7], 0x04365; GFX7-NEXT: s_mov_b32 s10, -14366; GFX7-NEXT: s_mov_b32 s8, s64367; GFX7-NEXT: s_mov_b32 s9, s74368; GFX7-NEXT: s_waitcnt vmcnt(2)4369; GFX7-NEXT: v_bfe_u32 v1, v2, 8, 84370; GFX7-NEXT: s_waitcnt vmcnt(1)4371; GFX7-NEXT: v_bfe_u32 v2, v3, 8, 84372; GFX7-NEXT: s_waitcnt lgkmcnt(0)4373; GFX7-NEXT: v_mad_u32_u24 v4, v2, v2, s04374; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 84375; GFX7-NEXT: s_waitcnt vmcnt(0)4376; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 84377; GFX7-NEXT: v_mad_u32_u24 v1, v1, v2, v44378; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v14379; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 04380; GFX7-NEXT: s_endpgm4381;4382; GFX8-LABEL: udot4_acc32_3src_3ele_src0:4383; GFX8: ; %bb.0: ; %entry4384; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x244385; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v04386; GFX8-NEXT: s_waitcnt lgkmcnt(0)4387; GFX8-NEXT: v_mov_b32_e32 v1, s14388; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v24389; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4390; GFX8-NEXT: flat_load_dword v3, v[0:1]4391; GFX8-NEXT: v_mov_b32_e32 v1, s34392; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v24393; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4394; GFX8-NEXT: flat_load_dword v4, v[0:1]4395; GFX8-NEXT: v_mov_b32_e32 v1, s54396; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v24397; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4398; GFX8-NEXT: flat_load_dword v0, v[0:1]4399; GFX8-NEXT: s_load_dword s0, s[6:7], 0x04400; GFX8-NEXT: s_waitcnt vmcnt(2)4401; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 84402; GFX8-NEXT: s_waitcnt vmcnt(1)4403; GFX8-NEXT: v_bfe_u32 v1, v4, 8, 84404; GFX8-NEXT: v_bfe_u32 v3, v4, 16, 84405; GFX8-NEXT: s_waitcnt lgkmcnt(0)4406; GFX8-NEXT: v_mad_u32_u24 v4, v1, v1, s04407; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, v44408; GFX8-NEXT: s_waitcnt vmcnt(0)4409; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 84410; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v14411; GFX8-NEXT: v_mov_b32_e32 v0, s64412; GFX8-NEXT: v_mov_b32_e32 v1, s74413; GFX8-NEXT: flat_store_dword v[0:1], v24414; GFX8-NEXT: s_endpgm4415;4416; GFX9-NODL-LABEL: udot4_acc32_3src_3ele_src0:4417; GFX9-NODL: ; %bb.0: ; %entry4418; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244419; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v04420; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4421; GFX9-NODL-NEXT: global_load_dword v1, v0, s[10:11]4422; GFX9-NODL-NEXT: global_load_dword v2, v0, s[12:13]4423; GFX9-NODL-NEXT: global_load_dword v3, v0, s[8:9]4424; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x04425; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 04426; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)4427; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 84428; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)4429; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_24430; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)4431; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD4432; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4433; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v4, s04434; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v14435; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]4436; GFX9-NODL-NEXT: s_endpgm4437;4438; GFX9-DL-LABEL: udot4_acc32_3src_3ele_src0:4439; GFX9-DL: ; %bb.0: ; %entry4440; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244441; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04442; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c4443; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c014444; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0201014445; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4446; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]4447; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]4448; GFX9-DL-NEXT: global_load_dword v3, v0, s[10:11]4449; GFX9-DL-NEXT: s_load_dword s3, s[14:15], 0x04450; GFX9-DL-NEXT: v_mov_b32_e32 v0, 04451; GFX9-DL-NEXT: s_waitcnt vmcnt(1)4452; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s04453; GFX9-DL-NEXT: s_waitcnt vmcnt(0)4454; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s14455; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v24456; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s24457; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4458; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s34459; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]4460; GFX9-DL-NEXT: s_endpgm4461;4462; GFX10-DL-LABEL: udot4_acc32_3src_3ele_src0:4463; GFX10-DL: ; %bb.0: ; %entry4464; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244465; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04466; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4467; GFX10-DL-NEXT: s_clause 0x24468; GFX10-DL-NEXT: global_load_dword v1, v0, s[8:9]4469; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]4470; GFX10-DL-NEXT: global_load_dword v3, v0, s[10:11]4471; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x04472; GFX10-DL-NEXT: s_waitcnt vmcnt(1)4473; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc06010c4474; GFX10-DL-NEXT: s_waitcnt vmcnt(0)4475; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c014476; GFX10-DL-NEXT: v_mov_b32_e32 v2, 04477; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v14478; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0201014479; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4480; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s04481; GFX10-DL-NEXT: global_store_dword v2, v0, s[14:15]4482; GFX10-DL-NEXT: s_endpgm4483;4484; GFX11-DL-LABEL: udot4_acc32_3src_3ele_src0:4485; GFX11-DL: ; %bb.0: ; %entry4486; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x244487; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v04488; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4489; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04490; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4491; GFX11-DL-NEXT: s_clause 0x24492; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]4493; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]4494; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]4495; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x04496; GFX11-DL-NEXT: s_waitcnt vmcnt(1)4497; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc06010c4498; GFX11-DL-NEXT: s_waitcnt vmcnt(0)4499; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c014500; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201014501; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)4502; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v24503; GFX11-DL-NEXT: v_mov_b32_e32 v2, 04504; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4505; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s04506; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]4507; GFX11-DL-NEXT: s_endpgm4508 ptr addrspace(1) %src2,4509 ptr addrspace(1) %src3,4510 ptr addrspace(1) nocapture %dst) {4511entry:4512 %idx = call i32 @llvm.amdgcn.workitem.id.x()4513 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx4514 %vec1 = load <4 x i8>, ptr addrspace(1) %gep14515 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx4516 %vec2 = load <4 x i8>, ptr addrspace(1) %gep24517 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx4518 %vec3 = load <4 x i8>, ptr addrspace(1) %gep34519 4520 %v2e0 = extractelement <4 x i8> %vec2, i64 14521 %cv2e0 = zext i8 %v2e0 to i324522 %mul1 = mul nuw nsw i32 %cv2e0, %cv2e04523 4524 %v1e1 = extractelement <4 x i8> %vec1, i64 14525 %cv1e1 = zext i8 %v1e1 to i324526 %v2e1 = extractelement <4 x i8> %vec2, i64 14527 %cv2e1 = zext i8 %v2e1 to i324528 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e14529 4530 %v3e2 = extractelement <4 x i8> %vec3, i64 24531 %cv3e2 = zext i8 %v3e2 to i324532 %v2e2 = extractelement <4 x i8> %vec2, i64 24533 %cv2e2 = zext i8 %v2e2 to i324534 %mul3 = mul nuw nsw i32 %cv2e2, %cv3e24535 4536 4537 %acc = load i32, ptr addrspace(1) %dst, align 44538 %mad1 = add i32 %mul1, %acc4539 %mad2 = add i32 %mad1, %mul24540 %mad3 = add i32 %mad2, %mul34541 4542 store i32 %mad3, ptr addrspace(1) %dst, align 44543 ret void4544}4545 4546define amdgpu_kernel void @udot4_4src(ptr addrspace(1) %src1,4547; GFX7-LABEL: udot4_4src:4548; GFX7: ; %bb.0: ; %entry4549; GFX7-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x94550; GFX7-NEXT: s_mov_b32 s3, 0xf0004551; GFX7-NEXT: s_mov_b32 s18, 04552; GFX7-NEXT: s_mov_b32 s19, s34553; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v04554; GFX7-NEXT: s_waitcnt lgkmcnt(0)4555; GFX7-NEXT: s_mov_b64 s[16:17], s[8:9]4556; GFX7-NEXT: v_mov_b32_e32 v1, 04557; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[16:19], 0 addr644558; GFX7-NEXT: s_mov_b64 s[16:17], s[10:11]4559; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[16:19], 0 addr644560; GFX7-NEXT: s_mov_b64 s[16:17], s[12:13]4561; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[16:19], 0 addr644562; GFX7-NEXT: s_mov_b64 s[16:17], s[14:15]4563; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[16:19], 0 addr644564; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x114565; GFX7-NEXT: s_mov_b32 s2, -14566; GFX7-NEXT: s_waitcnt lgkmcnt(0)4567; GFX7-NEXT: s_load_dword s4, s[0:1], 0x04568; GFX7-NEXT: s_waitcnt vmcnt(3)4569; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v24570; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 84571; GFX7-NEXT: s_waitcnt vmcnt(2)4572; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v34573; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 84574; GFX7-NEXT: s_waitcnt lgkmcnt(0)4575; GFX7-NEXT: v_mad_u32_u24 v1, v1, v2, s44576; GFX7-NEXT: s_waitcnt vmcnt(1)4577; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v44578; GFX7-NEXT: v_bfe_u32 v4, v4, 8, 84579; GFX7-NEXT: v_mad_u32_u24 v1, v5, v3, v14580; GFX7-NEXT: s_waitcnt vmcnt(0)4581; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v04582; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 84583; GFX7-NEXT: v_mad_u32_u24 v1, v2, v4, v14584; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v14585; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 04586; GFX7-NEXT: s_endpgm4587;4588; GFX8-LABEL: udot4_4src:4589; GFX8: ; %bb.0: ; %entry4590; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244591; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v04592; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444593; GFX8-NEXT: s_waitcnt lgkmcnt(0)4594; GFX8-NEXT: v_mov_b32_e32 v1, s94595; GFX8-NEXT: v_add_u32_e32 v0, vcc, s8, v24596; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4597; GFX8-NEXT: flat_load_dword v3, v[0:1]4598; GFX8-NEXT: v_mov_b32_e32 v1, s114599; GFX8-NEXT: v_add_u32_e32 v0, vcc, s10, v24600; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4601; GFX8-NEXT: flat_load_dword v4, v[0:1]4602; GFX8-NEXT: v_mov_b32_e32 v1, s134603; GFX8-NEXT: v_add_u32_e32 v0, vcc, s12, v24604; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4605; GFX8-NEXT: flat_load_dword v5, v[0:1]4606; GFX8-NEXT: v_mov_b32_e32 v1, s154607; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v24608; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4609; GFX8-NEXT: flat_load_dword v0, v[0:1]4610; GFX8-NEXT: s_load_dword s2, s[0:1], 0x04611; GFX8-NEXT: s_waitcnt vmcnt(3)4612; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v34613; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 84614; GFX8-NEXT: s_waitcnt lgkmcnt(0)4615; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s24616; GFX8-NEXT: s_waitcnt vmcnt(2)4617; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v44618; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 84619; GFX8-NEXT: v_mad_u32_u24 v1, v3, v4, v14620; GFX8-NEXT: s_waitcnt vmcnt(1)4621; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v54622; GFX8-NEXT: v_bfe_u32 v5, v5, 8, 84623; GFX8-NEXT: v_mad_u32_u24 v1, v6, v5, v14624; GFX8-NEXT: s_waitcnt vmcnt(0)4625; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v04626; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 84627; GFX8-NEXT: v_mad_u32_u24 v2, v7, v0, v14628; GFX8-NEXT: v_mov_b32_e32 v0, s04629; GFX8-NEXT: v_mov_b32_e32 v1, s14630; GFX8-NEXT: flat_store_dword v[0:1], v24631; GFX8-NEXT: s_endpgm4632;4633; GFX9-NODL-LABEL: udot4_4src:4634; GFX9-NODL: ; %bb.0: ; %entry4635; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244636; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v04637; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444638; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4639; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]4640; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]4641; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]4642; GFX9-NODL-NEXT: global_load_dword v4, v0, s[14:15]4643; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 04644; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x04645; GFX9-NODL-NEXT: s_waitcnt vmcnt(3)4646; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_14647; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)4648; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_14649; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)4650; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_14651; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)4652; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_14653; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4654; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s2, v24655; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v44656; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]4657; GFX9-NODL-NEXT: s_endpgm4658;4659; GFX9-DL-LABEL: udot4_4src:4660; GFX9-DL: ; %bb.0: ; %entry4661; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244662; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04663; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444664; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c05014665; GFX9-DL-NEXT: s_mov_b32 s3, 0x5010c0c4666; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4667; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]4668; GFX9-DL-NEXT: global_load_dword v2, v0, s[10:11]4669; GFX9-DL-NEXT: global_load_dword v3, v0, s[12:13]4670; GFX9-DL-NEXT: global_load_dword v4, v0, s[14:15]4671; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c04004672; GFX9-DL-NEXT: s_load_dword s6, s[0:1], 0x04673; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c4674; GFX9-DL-NEXT: v_mov_b32_e32 v0, 04675; GFX9-DL-NEXT: s_waitcnt vmcnt(2)4676; GFX9-DL-NEXT: v_perm_b32 v5, v2, v1, s24677; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s44678; GFX9-DL-NEXT: s_waitcnt vmcnt(0)4679; GFX9-DL-NEXT: v_perm_b32 v6, v4, v3, s34680; GFX9-DL-NEXT: v_perm_b32 v2, v4, v3, s54681; GFX9-DL-NEXT: v_or_b32_e32 v3, v6, v54682; GFX9-DL-NEXT: v_or_b32_e32 v1, v2, v14683; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4684; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v3, s64685; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]4686; GFX9-DL-NEXT: s_endpgm4687;4688; GFX10-DL-LABEL: udot4_4src:4689; GFX10-DL: ; %bb.0: ; %entry4690; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244691; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04692; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x444693; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4694; GFX10-DL-NEXT: s_clause 0x34695; GFX10-DL-NEXT: global_load_dword v1, v0, s[8:9]4696; GFX10-DL-NEXT: global_load_dword v2, v0, s[10:11]4697; GFX10-DL-NEXT: global_load_dword v3, v0, s[12:13]4698; GFX10-DL-NEXT: global_load_dword v4, v0, s[14:15]4699; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x04700; GFX10-DL-NEXT: s_waitcnt vmcnt(2)4701; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc0c05014702; GFX10-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c04004703; GFX10-DL-NEXT: s_waitcnt vmcnt(0)4704; GFX10-DL-NEXT: v_perm_b32 v5, v4, v3, 0x5010c0c4705; GFX10-DL-NEXT: v_perm_b32 v2, v4, v3, 0x4000c0c4706; GFX10-DL-NEXT: v_or_b32_e32 v0, v5, v04707; GFX10-DL-NEXT: v_or_b32_e32 v1, v2, v14708; GFX10-DL-NEXT: v_mov_b32_e32 v2, 04709; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4710; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s24711; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]4712; GFX10-DL-NEXT: s_endpgm4713;4714; GFX11-DL-LABEL: udot4_4src:4715; GFX11-DL: ; %bb.0: ; %entry4716; GFX11-DL-NEXT: s_load_b256 s[8:15], s[4:5], 0x244717; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v04718; GFX11-DL-NEXT: s_load_b64 s[0:1], s[4:5], 0x444719; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4720; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v04721; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4722; GFX11-DL-NEXT: s_clause 0x34723; GFX11-DL-NEXT: global_load_b32 v1, v0, s[8:9]4724; GFX11-DL-NEXT: global_load_b32 v2, v0, s[10:11]4725; GFX11-DL-NEXT: global_load_b32 v3, v0, s[12:13]4726; GFX11-DL-NEXT: global_load_b32 v0, v0, s[14:15]4727; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x04728; GFX11-DL-NEXT: s_waitcnt vmcnt(2)4729; GFX11-DL-NEXT: v_perm_b32 v4, v2, v1, 0xc0c05014730; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c04004731; GFX11-DL-NEXT: s_waitcnt vmcnt(0)4732; GFX11-DL-NEXT: v_perm_b32 v5, v0, v3, 0x5010c0c4733; GFX11-DL-NEXT: v_perm_b32 v0, v0, v3, 0x4000c0c4734; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)4735; GFX11-DL-NEXT: v_or_b32_e32 v2, v5, v44736; GFX11-DL-NEXT: v_or_b32_e32 v0, v0, v14737; GFX11-DL-NEXT: v_mov_b32_e32 v1, 04738; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4739; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)4740; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, s24741; GFX11-DL-NEXT: global_store_b32 v1, v0, s[0:1]4742; GFX11-DL-NEXT: s_endpgm4743 ptr addrspace(1) %src2,4744 ptr addrspace(1) %src3,4745 ptr addrspace(1) %src4,4746 ptr addrspace(1) nocapture %dst) {4747entry:4748 %idx = call i32 @llvm.amdgcn.workitem.id.x()4749 4750 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx4751 %vec1 = load <4 x i8>, ptr addrspace(1) %gep14752 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx4753 %vec2 = load <4 x i8>, ptr addrspace(1) %gep24754 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx4755 %vec3 = load <4 x i8>, ptr addrspace(1) %gep34756 %gep4 = getelementptr <4 x i8>, ptr addrspace(1) %src4, i32 %idx4757 %vec4 = load <4 x i8>, ptr addrspace(1) %gep44758 4759 4760 %v1e0 = extractelement <4 x i8> %vec1, i64 04761 %cv1e0 = zext i8 %v1e0 to i324762 %v1e1 = extractelement <4 x i8> %vec1, i64 14763 %cv1e1 = zext i8 %v1e1 to i324764 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e14765 4766 %v2e0 = extractelement <4 x i8> %vec2, i64 04767 %cv2e0 = zext i8 %v2e0 to i324768 %v2e1 = extractelement <4 x i8> %vec2, i64 14769 %cv2e1 = zext i8 %v2e1 to i324770 %mul2 = mul nuw nsw i32 %cv2e0, %cv2e14771 4772 %v3e0 = extractelement <4 x i8> %vec3, i64 04773 %cv3e0 = zext i8 %v3e0 to i324774 %v3e1 = extractelement <4 x i8> %vec3, i64 14775 %cv3e1 = zext i8 %v3e1 to i324776 %mul3 = mul nuw nsw i32 %cv3e0, %cv3e14777 4778 %v4e0 = extractelement <4 x i8> %vec4, i64 04779 %cv4e0 = zext i8 %v4e0 to i324780 %v4e1 = extractelement <4 x i8> %vec4, i64 14781 %cv4e1 = zext i8 %v4e1 to i324782 %mul4 = mul nuw nsw i32 %cv4e0, %cv4e14783 4784 4785 %acc = load i32, ptr addrspace(1) %dst, align 44786 %mad1 = add i32 %mul1, %acc4787 %mad2 = add i32 %mad1, %mul24788 %mad3 = add i32 %mad2, %mul34789 %mad4 = add i32 %mad3, %mul44790 4791 store i32 %mad4, ptr addrspace(1) %dst, align 44792 ret void4793}4794 4795define amdgpu_kernel void @udot4_acc32_multi(ptr addrspace(1) %src1,4796; GFX7-LABEL: udot4_acc32_multi:4797; GFX7: ; %bb.0: ; %entry4798; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x94799; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd4800; GFX7-NEXT: s_mov_b32 s3, 0xf0004801; GFX7-NEXT: s_mov_b32 s6, 04802; GFX7-NEXT: s_mov_b32 s7, s34803; GFX7-NEXT: s_waitcnt lgkmcnt(0)4804; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]4805; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v04806; GFX7-NEXT: v_mov_b32_e32 v1, 04807; GFX7-NEXT: s_mov_b64 s[8:9], s[10:11]4808; GFX7-NEXT: s_mov_b64 s[10:11], s[6:7]4809; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr644810; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr644811; GFX7-NEXT: s_load_dword s4, s[0:1], 0x04812; GFX7-NEXT: s_mov_b32 s2, -14813; GFX7-NEXT: s_waitcnt vmcnt(1)4814; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v24815; GFX7-NEXT: s_waitcnt vmcnt(0)4816; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v04817; GFX7-NEXT: v_bfe_u32 v7, v2, 16, 84818; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 84819; GFX7-NEXT: s_waitcnt lgkmcnt(0)4820; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s44821; GFX7-NEXT: v_and_b32_e32 v9, 0xff, v34822; GFX7-NEXT: v_mad_u32_u24 v1, v7, v8, v14823; GFX7-NEXT: v_bfe_u32 v11, v3, 16, 84824; GFX7-NEXT: v_mad_u32_u24 v1, v9, v4, v14825; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 84826; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 84827; GFX7-NEXT: v_mad_u32_u24 v1, v11, v8, v14828; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v24829; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v04830; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v14831; GFX7-NEXT: v_bfe_u32 v10, v3, 8, 84832; GFX7-NEXT: v_mad_u32_u24 v1, v2, v0, v14833; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v34834; GFX7-NEXT: v_mad_u32_u24 v1, v10, v6, v14835; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v14836; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 04837; GFX7-NEXT: s_endpgm4838;4839; GFX8-LABEL: udot4_acc32_multi:4840; GFX8: ; %bb.0: ; %entry4841; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244842; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344843; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v04844; GFX8-NEXT: s_waitcnt lgkmcnt(0)4845; GFX8-NEXT: v_mov_b32_e32 v1, s14846; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v24847; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4848; GFX8-NEXT: v_mov_b32_e32 v3, s34849; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v24850; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc4851; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]4852; GFX8-NEXT: flat_load_dword v2, v[2:3]4853; GFX8-NEXT: s_load_dword s0, s[4:5], 0x04854; GFX8-NEXT: s_waitcnt vmcnt(1)4855; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v04856; GFX8-NEXT: s_waitcnt vmcnt(0)4857; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v24858; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 84859; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 84860; GFX8-NEXT: s_waitcnt lgkmcnt(0)4861; GFX8-NEXT: v_mad_u32_u24 v3, v3, v4, s04862; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v14863; GFX8-NEXT: v_mad_u32_u24 v3, v7, v8, v34864; GFX8-NEXT: v_bfe_u32 v11, v1, 16, 84865; GFX8-NEXT: v_mad_u32_u24 v3, v9, v4, v34866; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 84867; GFX8-NEXT: v_bfe_u32 v6, v2, 8, 84868; GFX8-NEXT: v_mad_u32_u24 v3, v11, v8, v34869; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v04870; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v24871; GFX8-NEXT: v_mad_u32_u24 v3, v5, v6, v34872; GFX8-NEXT: v_bfe_u32 v10, v1, 8, 84873; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v34874; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v14875; GFX8-NEXT: v_mad_u32_u24 v0, v10, v6, v04876; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v04877; GFX8-NEXT: v_mov_b32_e32 v0, s44878; GFX8-NEXT: v_mov_b32_e32 v1, s54879; GFX8-NEXT: flat_store_dword v[0:1], v24880; GFX8-NEXT: s_endpgm4881;4882; GFX9-NODL-LABEL: udot4_acc32_multi:4883; GFX9-NODL: ; %bb.0: ; %entry4884; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244885; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344886; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v2, 3, v04887; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4888; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]4889; GFX9-NODL-NEXT: global_load_dword v3, v2, s[2:3]4890; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x04891; GFX9-NODL-NEXT: v_mov_b32_e32 v2, 04892; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)4893; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_04894; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_24895; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_14896; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_34897; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v7, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_04898; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v8, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_14899; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v9, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_24900; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_34901; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)4902; GFX9-NODL-NEXT: v_add3_u32 v3, v4, s0, v64903; GFX9-NODL-NEXT: v_add3_u32 v3, v3, v7, v94904; GFX9-NODL-NEXT: v_add3_u32 v0, v5, v3, v04905; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v8, v14906; GFX9-NODL-NEXT: global_store_dword v2, v0, s[6:7]4907; GFX9-NODL-NEXT: s_endpgm4908;4909; GFX9-DL-LABEL: udot4_acc32_multi:4910; GFX9-DL: ; %bb.0: ; %entry4911; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244912; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344913; GFX9-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v04914; GFX9-DL-NEXT: s_mov_b32 s4, 0x30103014915; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4916; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]4917; GFX9-DL-NEXT: global_load_dword v3, v2, s[2:3]4918; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x04919; GFX9-DL-NEXT: s_mov_b32 s0, 0x60402004920; GFX9-DL-NEXT: s_mov_b32 s1, 0x20002004921; GFX9-DL-NEXT: s_mov_b32 s2, 0x70503014922; GFX9-DL-NEXT: v_mov_b32_e32 v2, 04923; GFX9-DL-NEXT: s_waitcnt vmcnt(1)4924; GFX9-DL-NEXT: v_perm_b32 v4, v1, v0, s04925; GFX9-DL-NEXT: s_waitcnt vmcnt(0)4926; GFX9-DL-NEXT: v_perm_b32 v5, v3, v3, s14927; GFX9-DL-NEXT: v_perm_b32 v0, v1, v0, s24928; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)4929; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v4, v5, s34930; GFX9-DL-NEXT: v_perm_b32 v3, v3, v3, s44931; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v3, v14932; GFX9-DL-NEXT: global_store_dword v2, v0, s[6:7]4933; GFX9-DL-NEXT: s_endpgm4934;4935; GFX10-DL-LABEL: udot4_acc32_multi:4936; GFX10-DL: ; %bb.0: ; %entry4937; GFX10-DL-NEXT: s_clause 0x14938; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244939; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344940; GFX10-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v04941; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4942; GFX10-DL-NEXT: s_clause 0x14943; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]4944; GFX10-DL-NEXT: global_load_dword v3, v2, s[2:3]4945; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4946; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x04947; GFX10-DL-NEXT: s_waitcnt vmcnt(1)4948; GFX10-DL-NEXT: v_perm_b32 v2, v1, v0, 0x60402004949; GFX10-DL-NEXT: s_waitcnt vmcnt(0)4950; GFX10-DL-NEXT: v_perm_b32 v4, v3, v3, 0x20002004951; GFX10-DL-NEXT: v_perm_b32 v0, v1, v0, 0x70503014952; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)4953; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v2, v4, s04954; GFX10-DL-NEXT: v_perm_b32 v2, v3, v3, 0x30103014955; GFX10-DL-NEXT: v_mov_b32_e32 v3, 04956; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v14957; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]4958; GFX10-DL-NEXT: s_endpgm4959;4960; GFX11-DL-LABEL: udot4_acc32_multi:4961; GFX11-DL: ; %bb.0: ; %entry4962; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x244963; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v04964; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x344965; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)4966; GFX11-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v04967; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4968; GFX11-DL-NEXT: s_clause 0x14969; GFX11-DL-NEXT: global_load_b64 v[0:1], v2, s[0:1]4970; GFX11-DL-NEXT: global_load_b32 v2, v2, s[2:3]4971; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x04972; GFX11-DL-NEXT: s_waitcnt vmcnt(1)4973; GFX11-DL-NEXT: v_perm_b32 v3, v1, v0, 0x60402004974; GFX11-DL-NEXT: s_waitcnt vmcnt(0)4975; GFX11-DL-NEXT: v_perm_b32 v4, v2, v2, 0x20002004976; GFX11-DL-NEXT: v_perm_b32 v0, v1, v0, 0x70503014977; GFX11-DL-NEXT: v_perm_b32 v2, v2, v2, 0x30103014978; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)4979; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)4980; GFX11-DL-NEXT: v_dot4_u32_u8 v1, v3, v4, s04981; GFX11-DL-NEXT: v_mov_b32_e32 v3, 04982; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v14983; GFX11-DL-NEXT: global_store_b32 v3, v0, s[4:5]4984; GFX11-DL-NEXT: s_endpgm4985 ptr addrspace(1) %src2,4986 ptr addrspace(1) nocapture %dst) {4987entry:4988 %idx = call i32 @llvm.amdgcn.workitem.id.x()4989 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx4990 %vec1 = load <8 x i8>, ptr addrspace(1) %gep14991 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx4992 %vec2 = load <8 x i8>, ptr addrspace(1) %gep24993 4994 %v1e0 = extractelement <8 x i8> %vec1, i64 04995 %cv1e0 = zext i8 %v1e0 to i324996 %v2e0 = extractelement <8 x i8> %vec2, i64 04997 %cv2e0 = zext i8 %v2e0 to i324998 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e04999 5000 %v1e1 = extractelement <8 x i8> %vec1, i64 15001 %cv1e1 = zext i8 %v1e1 to i325002 %v2e1 = extractelement <8 x i8> %vec2, i64 15003 %cv2e1 = zext i8 %v2e1 to i325004 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15005 5006 %v1e2 = extractelement <8 x i8> %vec1, i64 25007 %cv1e2 = zext i8 %v1e2 to i325008 %v2e2 = extractelement <8 x i8> %vec2, i64 25009 %cv2e2 = zext i8 %v2e2 to i325010 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25011 5012 %v1e3 = extractelement <8 x i8> %vec1, i64 35013 %cv1e3 = zext i8 %v1e3 to i325014 %v2e3 = extractelement <8 x i8> %vec2, i64 35015 %cv2e3 = zext i8 %v2e3 to i325016 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35017 5018 %v1e4 = extractelement <8 x i8> %vec1, i64 45019 %cv1e4 = zext i8 %v1e4 to i325020 %v2e4 = extractelement <8 x i8> %vec2, i64 45021 %cv2e4 = zext i8 %v2e4 to i325022 %mul5 = mul nuw nsw i32 %cv1e4, %cv2e05023 5024 %v1e5 = extractelement <8 x i8> %vec1, i64 55025 %cv1e5 = zext i8 %v1e5 to i325026 %v2e5 = extractelement <8 x i8> %vec2, i64 55027 %cv2e5 = zext i8 %v2e5 to i325028 %mul6 = mul nuw nsw i32 %cv1e5, %cv2e15029 5030 %v1e6 = extractelement <8 x i8> %vec1, i64 65031 %cv1e6 = zext i8 %v1e6 to i325032 %v2e6 = extractelement <8 x i8> %vec2, i64 65033 %cv2e6 = zext i8 %v2e6 to i325034 %mul7 = mul nuw nsw i32 %cv1e6, %cv2e25035 5036 %v1e7 = extractelement <8 x i8> %vec1, i64 75037 %cv1e7 = zext i8 %v1e7 to i325038 %v2e7 = extractelement <8 x i8> %vec2, i64 75039 %cv2e7 = zext i8 %v2e7 to i325040 %mul8 = mul nuw nsw i32 %cv1e7, %cv2e35041 5042 %acc = load i32, ptr addrspace(1) %dst, align 45043 %mad11 = add i32 %mul1, %acc5044 %mad21 = add i32 %mad11, %mul35045 %mad31 = add i32 %mad21, %mul55046 %mad41 = add i32 %mad31, %mul75047 %mad12 = add i32 %mul2, %mad415048 %mad22 = add i32 %mad12, %mul45049 %mad32 = add i32 %mad22, %mul65050 %mad42 = add i32 %mad32, %mul85051 5052 store i32 %mad42, ptr addrspace(1) %dst, align 45053 ret void5054}5055 5056define amdgpu_kernel void @idot4_acc32_hilo(ptr addrspace(1) %src1,5057; GFX7-LABEL: idot4_acc32_hilo:5058; GFX7: ; %bb.0: ; %entry5059; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95060; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5061; GFX7-NEXT: s_mov_b32 s7, 0xf0005062; GFX7-NEXT: s_mov_b32 s10, 05063; GFX7-NEXT: s_mov_b32 s11, s75064; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v05065; GFX7-NEXT: v_mov_b32_e32 v1, 05066; GFX7-NEXT: s_waitcnt lgkmcnt(0)5067; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]5068; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]5069; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 offset:45070; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr645071; GFX7-NEXT: s_mov_b32 s6, -15072; GFX7-NEXT: s_waitcnt vmcnt(1)5073; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 85074; GFX7-NEXT: s_waitcnt vmcnt(0)5075; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 85076; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v25077; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v05078; GFX7-NEXT: v_mul_u32_u24_e32 v4, v4, v55079; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 85080; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 85081; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, v45082; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v25083; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v05084; GFX7-NEXT: v_mad_u32_u24 v1, v6, v7, v15085; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v15086; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05087; GFX7-NEXT: s_endpgm5088;5089; GFX8-LABEL: idot4_acc32_hilo:5090; GFX8: ; %bb.0: ; %entry5091; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245092; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345093; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v05094; GFX8-NEXT: s_waitcnt lgkmcnt(0)5095; GFX8-NEXT: v_mov_b32_e32 v1, s15096; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v05097; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc5098; GFX8-NEXT: v_mov_b32_e32 v1, s35099; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v05100; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5101; GFX8-NEXT: flat_load_dword v4, v[0:1]5102; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v25103; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc5104; GFX8-NEXT: flat_load_dword v2, v[0:1]5105; GFX8-NEXT: v_mov_b32_e32 v0, s45106; GFX8-NEXT: v_mov_b32_e32 v1, s55107; GFX8-NEXT: s_waitcnt vmcnt(1)5108; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v45109; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 85110; GFX8-NEXT: s_waitcnt vmcnt(0)5111; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v25112; GFX8-NEXT: v_mul_u32_u24_sdwa v7, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_15113; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 85114; GFX8-NEXT: v_mad_u32_u24 v3, v6, v3, v75115; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v25116; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v45117; GFX8-NEXT: v_mad_u32_u24 v3, v8, v5, v35118; GFX8-NEXT: v_mad_u32_u24 v2, v2, v4, v35119; GFX8-NEXT: flat_store_dword v[0:1], v25120; GFX8-NEXT: s_endpgm5121;5122; GFX9-NODL-LABEL: idot4_acc32_hilo:5123; GFX9-NODL: ; %bb.0: ; %entry5124; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245125; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345126; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v05127; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)5128; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1] offset:45129; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]5130; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 05131; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)5132; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v15133; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)5134; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v25135; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_15136; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_25137; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_35138; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v55139; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v15140; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]5141; GFX9-NODL-NEXT: s_endpgm5142;5143; GFX9-DL-LABEL: idot4_acc32_hilo:5144; GFX9-DL: ; %bb.0: ; %entry5145; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245146; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345147; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05148; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)5149; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1] offset:45150; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]5151; GFX9-DL-NEXT: v_mov_b32_e32 v0, 05152; GFX9-DL-NEXT: s_waitcnt vmcnt(0)5153; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 05154; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]5155; GFX9-DL-NEXT: s_endpgm5156;5157; GFX10-DL-LABEL: idot4_acc32_hilo:5158; GFX10-DL: ; %bb.0: ; %entry5159; GFX10-DL-NEXT: s_clause 0x15160; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245161; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345162; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05163; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)5164; GFX10-DL-NEXT: s_clause 0x15165; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1] offset:45166; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]5167; GFX10-DL-NEXT: v_mov_b32_e32 v0, 05168; GFX10-DL-NEXT: s_waitcnt vmcnt(0)5169; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 05170; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]5171; GFX10-DL-NEXT: s_endpgm5172;5173; GFX11-DL-LABEL: idot4_acc32_hilo:5174; GFX11-DL: ; %bb.0: ; %entry5175; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x245176; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v05177; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x345178; GFX11-DL-NEXT: v_mov_b32_e32 v2, 05179; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)5180; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05181; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)5182; GFX11-DL-NEXT: s_clause 0x15183; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1] offset:45184; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]5185; GFX11-DL-NEXT: s_waitcnt vmcnt(0)5186; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 05187; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]5188; GFX11-DL-NEXT: s_endpgm5189 ptr addrspace(1) %src2,5190 ptr addrspace(1) nocapture %dst) {5191entry:5192 %idx = call i32 @llvm.amdgcn.workitem.id.x()5193 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx5194 %vec1 = load <8 x i8>, ptr addrspace(1) %gep15195 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx5196 %vec2 = load <8 x i8>, ptr addrspace(1) %gep25197 5198 %v1e0 = extractelement <8 x i8> %vec1, i64 45199 %cv1e0 = zext i8 %v1e0 to i325200 %v2e0 = extractelement <8 x i8> %vec2, i64 05201 %cv2e0 = zext i8 %v2e0 to i325202 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e05203 5204 %v1e1 = extractelement <8 x i8> %vec1, i64 55205 %cv1e1 = zext i8 %v1e1 to i325206 %v2e1 = extractelement <8 x i8> %vec2, i64 15207 %cv2e1 = zext i8 %v2e1 to i325208 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15209 5210 %v1e2 = extractelement <8 x i8> %vec1, i64 65211 %cv1e2 = zext i8 %v1e2 to i325212 %v2e2 = extractelement <8 x i8> %vec2, i64 25213 %cv2e2 = zext i8 %v2e2 to i325214 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25215 5216 %v1e3 = extractelement <8 x i8> %vec1, i64 75217 %cv1e3 = zext i8 %v1e3 to i325218 %v2e3 = extractelement <8 x i8> %vec2, i64 35219 %cv2e3 = zext i8 %v2e3 to i325220 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35221 5222 %add1 = add i32 %mul1, 05223 %add2 = add i32 %add1, %mul25224 %add3 = add i32 %add2, %mul35225 %add4 = add i32 %add3, %mul45226 store i32 %add4, ptr addrspace(1) %dst, align 45227 ret void5228}5229 5230define amdgpu_kernel void @idot4_acc32_lohi(ptr addrspace(1) %src1,5231; GFX7-LABEL: idot4_acc32_lohi:5232; GFX7: ; %bb.0: ; %entry5233; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95234; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5235; GFX7-NEXT: s_mov_b32 s7, 0xf0005236; GFX7-NEXT: s_mov_b32 s10, 05237; GFX7-NEXT: s_mov_b32 s11, s75238; GFX7-NEXT: s_waitcnt lgkmcnt(0)5239; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]5240; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v05241; GFX7-NEXT: v_mov_b32_e32 v1, 05242; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr645243; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]5244; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:45245; GFX7-NEXT: s_mov_b32 s6, -15246; GFX7-NEXT: s_waitcnt vmcnt(1)5247; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 85248; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v25249; GFX7-NEXT: s_waitcnt vmcnt(0)5250; GFX7-NEXT: v_bfe_u32 v6, v0, 16, 85251; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v05252; GFX7-NEXT: v_mul_u32_u24_e32 v3, v3, v65253; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 85254; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 85255; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v35256; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v25257; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v05258; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v15259; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v15260; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05261; GFX7-NEXT: s_endpgm5262;5263; GFX8-LABEL: idot4_acc32_lohi:5264; GFX8: ; %bb.0: ; %entry5265; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245266; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345267; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v05268; GFX8-NEXT: s_waitcnt lgkmcnt(0)5269; GFX8-NEXT: v_mov_b32_e32 v1, s15270; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v25271; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5272; GFX8-NEXT: v_mov_b32_e32 v3, s35273; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v25274; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc5275; GFX8-NEXT: flat_load_dword v4, v[0:1]5276; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v25277; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc5278; GFX8-NEXT: flat_load_dword v2, v[0:1]5279; GFX8-NEXT: v_mov_b32_e32 v0, s45280; GFX8-NEXT: v_mov_b32_e32 v1, s55281; GFX8-NEXT: s_waitcnt vmcnt(1)5282; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v45283; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 85284; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v45285; GFX8-NEXT: s_waitcnt vmcnt(0)5286; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v25287; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_25288; GFX8-NEXT: v_bfe_u32 v8, v2, 8, 85289; GFX8-NEXT: v_mad_u32_u24 v3, v3, v7, v45290; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v25291; GFX8-NEXT: v_mad_u32_u24 v3, v5, v8, v35292; GFX8-NEXT: v_mad_u32_u24 v2, v6, v2, v35293; GFX8-NEXT: flat_store_dword v[0:1], v25294; GFX8-NEXT: s_endpgm5295;5296; GFX9-NODL-LABEL: idot4_acc32_lohi:5297; GFX9-NODL: ; %bb.0: ; %entry5298; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245299; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345300; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v05301; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)5302; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]5303; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3] offset:45304; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 05305; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)5306; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v15307; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)5308; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 24, v25309; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_25310; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_15311; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_05312; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v55313; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v15314; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]5315; GFX9-NODL-NEXT: s_endpgm5316;5317; GFX9-DL-LABEL: idot4_acc32_lohi:5318; GFX9-DL: ; %bb.0: ; %entry5319; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245320; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345321; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05322; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)5323; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3] offset:45324; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]5325; GFX9-DL-NEXT: s_mov_b32 s0, 0x103025326; GFX9-DL-NEXT: s_mov_b32 s1, 0x30200015327; GFX9-DL-NEXT: v_mov_b32_e32 v0, 05328; GFX9-DL-NEXT: s_waitcnt vmcnt(1)5329; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s05330; GFX9-DL-NEXT: s_waitcnt vmcnt(0)5331; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s15332; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, 05333; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]5334; GFX9-DL-NEXT: s_endpgm5335;5336; GFX10-DL-LABEL: idot4_acc32_lohi:5337; GFX10-DL: ; %bb.0: ; %entry5338; GFX10-DL-NEXT: s_clause 0x15339; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245340; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345341; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05342; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)5343; GFX10-DL-NEXT: s_clause 0x15344; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3] offset:45345; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]5346; GFX10-DL-NEXT: s_waitcnt vmcnt(1)5347; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0x103025348; GFX10-DL-NEXT: s_waitcnt vmcnt(0)5349; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0x30200015350; GFX10-DL-NEXT: v_mov_b32_e32 v2, 05351; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 05352; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]5353; GFX10-DL-NEXT: s_endpgm5354;5355; GFX11-DL-LABEL: idot4_acc32_lohi:5356; GFX11-DL: ; %bb.0: ; %entry5357; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x245358; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v05359; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x345360; GFX11-DL-NEXT: v_mov_b32_e32 v2, 05361; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)5362; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05363; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)5364; GFX11-DL-NEXT: s_clause 0x15365; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3] offset:45366; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]5367; GFX11-DL-NEXT: s_waitcnt vmcnt(1)5368; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0x103025369; GFX11-DL-NEXT: s_waitcnt vmcnt(0)5370; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0x30200015371; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)5372; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 05373; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]5374; GFX11-DL-NEXT: s_endpgm5375 ptr addrspace(1) %src2,5376 ptr addrspace(1) nocapture %dst) {5377entry:5378 %idx = call i32 @llvm.amdgcn.workitem.id.x()5379 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx5380 %vec1 = load <8 x i8>, ptr addrspace(1) %gep15381 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx5382 %vec2 = load <8 x i8>, ptr addrspace(1) %gep25383 5384 %v1e0 = extractelement <8 x i8> %vec1, i64 05385 %cv1e0 = zext i8 %v1e0 to i325386 %v2e0 = extractelement <8 x i8> %vec2, i64 75387 %cv2e0 = zext i8 %v2e0 to i325388 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e05389 5390 %v1e1 = extractelement <8 x i8> %vec1, i64 15391 %cv1e1 = zext i8 %v1e1 to i325392 %v2e1 = extractelement <8 x i8> %vec2, i64 65393 %cv2e1 = zext i8 %v2e1 to i325394 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15395 5396 %v1e2 = extractelement <8 x i8> %vec1, i64 25397 %cv1e2 = zext i8 %v1e2 to i325398 %v2e2 = extractelement <8 x i8> %vec2, i64 55399 %cv2e2 = zext i8 %v2e2 to i325400 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25401 5402 %v1e3 = extractelement <8 x i8> %vec1, i64 35403 %cv1e3 = zext i8 %v1e3 to i325404 %v2e3 = extractelement <8 x i8> %vec2, i64 45405 %cv2e3 = zext i8 %v2e3 to i325406 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35407 5408 %add1 = add i32 %mul1, 05409 %add2 = add i32 %add1, %mul25410 %add3 = add i32 %add2, %mul35411 %add4 = add i32 %add3, %mul45412 store i32 %add4, ptr addrspace(1) %dst, align 45413 ret void5414}5415 5416define amdgpu_kernel void @idot4_acc32_hihi(ptr addrspace(1) %src1,5417; GFX7-LABEL: idot4_acc32_hihi:5418; GFX7: ; %bb.0: ; %entry5419; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95420; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5421; GFX7-NEXT: s_mov_b32 s7, 0xf0005422; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v05423; GFX7-NEXT: v_mov_b32_e32 v1, 05424; GFX7-NEXT: s_mov_b32 s10, 05425; GFX7-NEXT: s_mov_b32 s11, s75426; GFX7-NEXT: s_waitcnt lgkmcnt(0)5427; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]5428; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 offset:45429; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]5430; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 offset:45431; GFX7-NEXT: s_mov_b32 s6, -15432; GFX7-NEXT: s_waitcnt vmcnt(1)5433; GFX7-NEXT: v_bfe_u32 v3, v2, 16, 85434; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v25435; GFX7-NEXT: s_waitcnt vmcnt(0)5436; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v05437; GFX7-NEXT: v_bfe_u32 v5, v0, 16, 85438; GFX7-NEXT: v_mul_u32_u24_e32 v3, v3, v65439; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 85440; GFX7-NEXT: v_lshrrev_b32_e32 v7, 24, v05441; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v35442; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v25443; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 85444; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v15445; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v15446; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05447; GFX7-NEXT: s_endpgm5448;5449; GFX8-LABEL: idot4_acc32_hihi:5450; GFX8: ; %bb.0: ; %entry5451; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245452; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345453; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v05454; GFX8-NEXT: s_waitcnt lgkmcnt(0)5455; GFX8-NEXT: v_mov_b32_e32 v1, s15456; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v05457; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5458; GFX8-NEXT: v_mov_b32_e32 v3, s35459; GFX8-NEXT: v_add_u32_e32 v4, vcc, s2, v05460; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc5461; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v25462; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5463; GFX8-NEXT: flat_load_dword v2, v[0:1]5464; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v45465; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc5466; GFX8-NEXT: flat_load_dword v3, v[0:1]5467; GFX8-NEXT: v_mov_b32_e32 v0, s45468; GFX8-NEXT: v_mov_b32_e32 v1, s55469; GFX8-NEXT: s_waitcnt vmcnt(1)5470; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v25471; GFX8-NEXT: v_bfe_u32 v7, v2, 8, 85472; GFX8-NEXT: s_waitcnt vmcnt(0)5473; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 85474; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_05475; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v35476; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v65477; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v25478; GFX8-NEXT: v_bfe_u32 v3, v3, 8, 85479; GFX8-NEXT: v_mad_u32_u24 v4, v7, v8, v45480; GFX8-NEXT: v_mad_u32_u24 v2, v2, v3, v45481; GFX8-NEXT: flat_store_dword v[0:1], v25482; GFX8-NEXT: s_endpgm5483;5484; GFX9-NODL-LABEL: idot4_acc32_hihi:5485; GFX9-NODL: ; %bb.0: ; %entry5486; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245487; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345488; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v05489; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)5490; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1] offset:45491; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3] offset:45492; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 05493; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)5494; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v15495; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)5496; GFX9-NODL-NEXT: v_bfe_u32 v4, v2, 16, 85497; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_05498; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_35499; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_15500; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v55501; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v15502; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]5503; GFX9-NODL-NEXT: s_endpgm5504;5505; GFX9-DL-LABEL: idot4_acc32_hihi:5506; GFX9-DL: ; %bb.0: ; %entry5507; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245508; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345509; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05510; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)5511; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3] offset:45512; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1] offset:45513; GFX9-DL-NEXT: s_mov_b32 s0, 0x10302005514; GFX9-DL-NEXT: s_mov_b32 s1, 0x30100025515; GFX9-DL-NEXT: v_mov_b32_e32 v0, 05516; GFX9-DL-NEXT: s_waitcnt vmcnt(1)5517; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s05518; GFX9-DL-NEXT: s_waitcnt vmcnt(0)5519; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s15520; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, 05521; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]5522; GFX9-DL-NEXT: s_endpgm5523;5524; GFX10-DL-LABEL: idot4_acc32_hihi:5525; GFX10-DL: ; %bb.0: ; %entry5526; GFX10-DL-NEXT: s_clause 0x15527; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245528; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345529; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05530; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)5531; GFX10-DL-NEXT: s_clause 0x15532; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3] offset:45533; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1] offset:45534; GFX10-DL-NEXT: s_waitcnt vmcnt(1)5535; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0x10302005536; GFX10-DL-NEXT: s_waitcnt vmcnt(0)5537; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0x30100025538; GFX10-DL-NEXT: v_mov_b32_e32 v2, 05539; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 05540; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]5541; GFX10-DL-NEXT: s_endpgm5542;5543; GFX11-DL-LABEL: idot4_acc32_hihi:5544; GFX11-DL: ; %bb.0: ; %entry5545; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x245546; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v05547; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x345548; GFX11-DL-NEXT: v_mov_b32_e32 v2, 05549; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)5550; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05551; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)5552; GFX11-DL-NEXT: s_clause 0x15553; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3] offset:45554; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1] offset:45555; GFX11-DL-NEXT: s_waitcnt vmcnt(1)5556; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0x10302005557; GFX11-DL-NEXT: s_waitcnt vmcnt(0)5558; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0x30100025559; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)5560; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 05561; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]5562; GFX11-DL-NEXT: s_endpgm5563 ptr addrspace(1) %src2,5564 ptr addrspace(1) nocapture %dst) {5565entry:5566 %idx = call i32 @llvm.amdgcn.workitem.id.x()5567 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx5568 %vec1 = load <8 x i8>, ptr addrspace(1) %gep15569 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx5570 %vec2 = load <8 x i8>, ptr addrspace(1) %gep25571 5572 %v1e0 = extractelement <8 x i8> %vec1, i64 45573 %cv1e0 = zext i8 %v1e0 to i325574 %v2e0 = extractelement <8 x i8> %vec2, i64 65575 %cv2e0 = zext i8 %v2e0 to i325576 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e05577 5578 %v1e1 = extractelement <8 x i8> %vec1, i64 65579 %cv1e1 = zext i8 %v1e1 to i325580 %v2e1 = extractelement <8 x i8> %vec2, i64 45581 %cv2e1 = zext i8 %v2e1 to i325582 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15583 5584 %v1e2 = extractelement <8 x i8> %vec1, i64 55585 %cv1e2 = zext i8 %v1e2 to i325586 %v2e2 = extractelement <8 x i8> %vec2, i64 75587 %cv2e2 = zext i8 %v2e2 to i325588 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25589 5590 %v1e3 = extractelement <8 x i8> %vec1, i64 75591 %cv1e3 = zext i8 %v1e3 to i325592 %v2e3 = extractelement <8 x i8> %vec2, i64 55593 %cv2e3 = zext i8 %v2e3 to i325594 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35595 5596 %add1 = add i32 %mul1, 05597 %add2 = add i32 %add1, %mul25598 %add3 = add i32 %add2, %mul35599 %add4 = add i32 %add3, %mul45600 store i32 %add4, ptr addrspace(1) %dst, align 45601 ret void5602}5603 5604define amdgpu_kernel void @idot4_acc32_v8i8(ptr addrspace(1) %src1,5605; GFX7-LABEL: idot4_acc32_v8i8:5606; GFX7: ; %bb.0: ; %entry5607; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x95608; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5609; GFX7-NEXT: s_mov_b32 s7, 0xf0005610; GFX7-NEXT: s_mov_b32 s2, 05611; GFX7-NEXT: s_mov_b32 s3, s75612; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v05613; GFX7-NEXT: v_mov_b32_e32 v1, 05614; GFX7-NEXT: s_waitcnt lgkmcnt(0)5615; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr645616; GFX7-NEXT: s_mov_b32 s6, -15617; GFX7-NEXT: s_waitcnt vmcnt(0)5618; GFX7-NEXT: v_bfe_u32 v4, v0, 8, 85619; GFX7-NEXT: v_bfe_u32 v5, v1, 8, 85620; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v05621; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v15622; GFX7-NEXT: v_mul_u32_u24_e32 v4, v4, v55623; GFX7-NEXT: v_bfe_u32 v6, v0, 16, 85624; GFX7-NEXT: v_bfe_u32 v7, v1, 16, 85625; GFX7-NEXT: v_mad_u32_u24 v2, v2, v3, v45626; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v05627; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v15628; GFX7-NEXT: v_mad_u32_u24 v2, v6, v7, v25629; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, v25630; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05631; GFX7-NEXT: s_endpgm5632;5633; GFX8-LABEL: idot4_acc32_v8i8:5634; GFX8: ; %bb.0: ; %entry5635; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245636; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x345637; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v05638; GFX8-NEXT: s_waitcnt lgkmcnt(0)5639; GFX8-NEXT: v_mov_b32_e32 v1, s15640; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v05641; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5642; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]5643; GFX8-NEXT: s_waitcnt vmcnt(0)5644; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v05645; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v15646; GFX8-NEXT: v_mul_u32_u24_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_15647; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 85648; GFX8-NEXT: v_bfe_u32 v6, v1, 16, 85649; GFX8-NEXT: v_mad_u32_u24 v2, v2, v3, v45650; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v05651; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v15652; GFX8-NEXT: v_mad_u32_u24 v2, v5, v6, v25653; GFX8-NEXT: v_mad_u32_u24 v2, v0, v1, v25654; GFX8-NEXT: v_mov_b32_e32 v0, s25655; GFX8-NEXT: v_mov_b32_e32 v1, s35656; GFX8-NEXT: flat_store_dword v[0:1], v25657; GFX8-NEXT: s_endpgm5658;5659; GFX9-NODL-LABEL: idot4_acc32_v8i8:5660; GFX9-NODL: ; %bb.0: ; %entry5661; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245662; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x345663; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v05664; GFX9-NODL-NEXT: v_mov_b32_e32 v2, 05665; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)5666; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]5667; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)5668; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v05669; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v15670; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_15671; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_25672; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_35673; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v3, v4, v55674; GFX9-NODL-NEXT: v_add3_u32 v0, v1, v6, v05675; GFX9-NODL-NEXT: global_store_dword v2, v0, s[2:3]5676; GFX9-NODL-NEXT: s_endpgm5677;5678; GFX9-DL-LABEL: idot4_acc32_v8i8:5679; GFX9-DL: ; %bb.0: ; %entry5680; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245681; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x345682; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05683; GFX9-DL-NEXT: v_mov_b32_e32 v2, 05684; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)5685; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]5686; GFX9-DL-NEXT: s_waitcnt vmcnt(0)5687; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 05688; GFX9-DL-NEXT: global_store_dword v2, v0, s[2:3]5689; GFX9-DL-NEXT: s_endpgm5690;5691; GFX10-DL-LABEL: idot4_acc32_v8i8:5692; GFX10-DL: ; %bb.0: ; %entry5693; GFX10-DL-NEXT: s_clause 0x15694; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245695; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x345696; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05697; GFX10-DL-NEXT: v_mov_b32_e32 v2, 05698; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)5699; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]5700; GFX10-DL-NEXT: s_waitcnt vmcnt(0)5701; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 05702; GFX10-DL-NEXT: global_store_dword v2, v0, s[2:3]5703; GFX10-DL-NEXT: s_endpgm5704;5705; GFX11-DL-LABEL: idot4_acc32_v8i8:5706; GFX11-DL: ; %bb.0: ; %entry5707; GFX11-DL-NEXT: s_clause 0x15708; GFX11-DL-NEXT: s_load_b64 s[0:1], s[4:5], 0x245709; GFX11-DL-NEXT: s_load_b64 s[2:3], s[4:5], 0x345710; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v05711; GFX11-DL-NEXT: v_mov_b32_e32 v2, 05712; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)5713; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v05714; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)5715; GFX11-DL-NEXT: global_load_b64 v[0:1], v0, s[0:1]5716; GFX11-DL-NEXT: s_waitcnt vmcnt(0)5717; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 05718; GFX11-DL-NEXT: global_store_b32 v2, v0, s[2:3]5719; GFX11-DL-NEXT: s_endpgm5720 ptr addrspace(1) %src2,5721 ptr addrspace(1) nocapture %dst) {5722entry:5723 %idx = call i32 @llvm.amdgcn.workitem.id.x()5724 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx5725 %vec1 = load <8 x i8>, ptr addrspace(1) %gep15726 5727 5728 %v1e0 = extractelement <8 x i8> %vec1, i64 05729 %cv1e0 = zext i8 %v1e0 to i325730 %v2e0 = extractelement <8 x i8> %vec1, i64 45731 %cv2e0 = zext i8 %v2e0 to i325732 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e05733 5734 %v1e1 = extractelement <8 x i8> %vec1, i64 15735 %cv1e1 = zext i8 %v1e1 to i325736 %v2e1 = extractelement <8 x i8> %vec1, i64 55737 %cv2e1 = zext i8 %v2e1 to i325738 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15739 5740 %v1e2 = extractelement <8 x i8> %vec1, i64 25741 %cv1e2 = zext i8 %v1e2 to i325742 %v2e2 = extractelement <8 x i8> %vec1, i64 65743 %cv2e2 = zext i8 %v2e2 to i325744 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25745 5746 %v1e3 = extractelement <8 x i8> %vec1, i64 35747 %cv1e3 = zext i8 %v1e3 to i325748 %v2e3 = extractelement <8 x i8> %vec1, i64 75749 %cv2e3 = zext i8 %v2e3 to i325750 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35751 5752 %add1 = add i32 %mul1, 05753 %add2 = add i32 %add1, %mul25754 %add3 = add i32 %add2, %mul35755 %add4 = add i32 %add3, %mul45756 store i32 %add4, ptr addrspace(1) %dst, align 45757 ret void5758}5759 5760define amdgpu_kernel void @idot4_acc32_v16i8(ptr addrspace(1) %src1,5761; GFX7-LABEL: idot4_acc32_v16i8:5762; GFX7: ; %bb.0: ; %entry5763; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95764; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5765; GFX7-NEXT: s_mov_b32 s7, 0xf0005766; GFX7-NEXT: s_mov_b32 s10, 05767; GFX7-NEXT: s_mov_b32 s11, s75768; GFX7-NEXT: s_waitcnt lgkmcnt(0)5769; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]5770; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v05771; GFX7-NEXT: v_mov_b32_e32 v2, 05772; GFX7-NEXT: s_mov_b64 s[0:1], s[2:3]5773; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]5774; GFX7-NEXT: v_lshlrev_b32_e32 v4, 3, v05775; GFX7-NEXT: v_mov_b32_e32 v5, v25776; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[1:2], s[8:11], 0 addr645777; GFX7-NEXT: buffer_load_dword v0, v[4:5], s[0:3], 0 addr645778; GFX7-NEXT: s_mov_b32 s6, -15779; GFX7-NEXT: s_waitcnt vmcnt(1)5780; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v25781; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 85782; GFX7-NEXT: s_waitcnt vmcnt(0)5783; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 85784; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v05785; GFX7-NEXT: v_mul_u32_u24_e32 v2, v2, v55786; GFX7-NEXT: v_bfe_u32 v6, v3, 8, 85787; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 85788; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, v25789; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v35790; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v05791; GFX7-NEXT: v_mad_u32_u24 v1, v6, v7, v15792; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v15793; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05794; GFX7-NEXT: s_endpgm5795;5796; GFX8-LABEL: idot4_acc32_v16i8:5797; GFX8: ; %bb.0: ; %entry5798; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245799; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345800; GFX8-NEXT: v_lshlrev_b32_e32 v1, 4, v05801; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v05802; GFX8-NEXT: s_waitcnt lgkmcnt(0)5803; GFX8-NEXT: v_mov_b32_e32 v2, s15804; GFX8-NEXT: v_add_u32_e32 v1, vcc, s0, v15805; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc5806; GFX8-NEXT: v_mov_b32_e32 v3, s35807; GFX8-NEXT: v_add_u32_e32 v4, vcc, s2, v05808; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v3, vcc5809; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[1:2]5810; GFX8-NEXT: flat_load_dword v4, v[4:5]5811; GFX8-NEXT: s_waitcnt vmcnt(1)5812; GFX8-NEXT: v_mov_b32_e32 v0, s45813; GFX8-NEXT: v_mov_b32_e32 v1, s55814; GFX8-NEXT: v_and_b32_e32 v5, 0xff, v25815; GFX8-NEXT: s_waitcnt vmcnt(0)5816; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v45817; GFX8-NEXT: v_mul_u32_u24_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_15818; GFX8-NEXT: v_bfe_u32 v7, v3, 8, 85819; GFX8-NEXT: v_bfe_u32 v8, v4, 16, 85820; GFX8-NEXT: v_mad_u32_u24 v2, v5, v6, v25821; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v35822; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v45823; GFX8-NEXT: v_mad_u32_u24 v2, v7, v8, v25824; GFX8-NEXT: v_mad_u32_u24 v2, v3, v4, v25825; GFX8-NEXT: flat_store_dword v[0:1], v25826; GFX8-NEXT: s_endpgm5827;5828; GFX9-NODL-LABEL: idot4_acc32_v16i8:5829; GFX9-NODL: ; %bb.0: ; %entry5830; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245831; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345832; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v4, 4, v05833; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v5, 3, v05834; GFX9-NODL-NEXT: ; kill: killed $vgpr45835; GFX9-NODL-NEXT: ; kill: killed $vgpr55836; GFX9-NODL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr35837; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)5838; GFX9-NODL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]5839; GFX9-NODL-NEXT: global_load_dword v0, v5, s[2:3]5840; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)5841; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 05842; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v25843; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)5844; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v05845; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_15846; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_25847; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_35848; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v4, v5, v25849; GFX9-NODL-NEXT: v_add3_u32 v0, v2, v6, v05850; GFX9-NODL-NEXT: global_store_dword v1, v0, s[6:7]5851; GFX9-NODL-NEXT: s_endpgm5852;5853; GFX9-DL-LABEL: idot4_acc32_v16i8:5854; GFX9-DL: ; %bb.0: ; %entry5855; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245856; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345857; GFX9-DL-NEXT: v_lshlrev_b32_e32 v4, 4, v05858; GFX9-DL-NEXT: v_lshlrev_b32_e32 v5, 3, v05859; GFX9-DL-NEXT: ; kill: killed $vgpr45860; GFX9-DL-NEXT: ; kill: killed $vgpr55861; GFX9-DL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr35862; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)5863; GFX9-DL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]5864; GFX9-DL-NEXT: global_load_dword v0, v5, s[2:3]5865; GFX9-DL-NEXT: s_mov_b32 s0, 0x70500025866; GFX9-DL-NEXT: s_mov_b32 s1, 0x30200015867; GFX9-DL-NEXT: s_waitcnt vmcnt(1)5868; GFX9-DL-NEXT: v_mov_b32_e32 v1, 05869; GFX9-DL-NEXT: v_perm_b32 v2, v3, v2, s05870; GFX9-DL-NEXT: s_waitcnt vmcnt(0)5871; GFX9-DL-NEXT: v_perm_b32 v0, v0, v0, s15872; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, 05873; GFX9-DL-NEXT: global_store_dword v1, v0, s[6:7]5874; GFX9-DL-NEXT: s_endpgm5875;5876; GFX10-DL-LABEL: idot4_acc32_v16i8:5877; GFX10-DL: ; %bb.0: ; %entry5878; GFX10-DL-NEXT: s_clause 0x15879; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245880; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345881; GFX10-DL-NEXT: v_lshlrev_b32_e32 v4, 4, v05882; GFX10-DL-NEXT: v_lshlrev_b32_e32 v5, 3, v05883; GFX10-DL-NEXT: ; kill: killed $vgpr45884; GFX10-DL-NEXT: ; kill: killed $sgpr0_sgpr1_sgpr2 killed $sgpr35885; GFX10-DL-NEXT: ; kill: killed $vgpr55886; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)5887; GFX10-DL-NEXT: global_load_dwordx4 v[0:3], v4, s[0:1]5888; GFX10-DL-NEXT: global_load_dword v0, v5, s[2:3]5889; GFX10-DL-NEXT: s_waitcnt vmcnt(1)5890; GFX10-DL-NEXT: v_perm_b32 v1, v3, v2, 0x70500025891; GFX10-DL-NEXT: s_waitcnt vmcnt(0)5892; GFX10-DL-NEXT: v_perm_b32 v0, v0, v0, 0x30200015893; GFX10-DL-NEXT: v_mov_b32_e32 v2, 05894; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 05895; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]5896; GFX10-DL-NEXT: s_endpgm5897;5898; GFX11-DL-LABEL: idot4_acc32_v16i8:5899; GFX11-DL: ; %bb.0: ; %entry5900; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x245901; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v05902; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x345903; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)5904; GFX11-DL-NEXT: v_lshlrev_b32_e32 v1, 4, v05905; GFX11-DL-NEXT: v_lshlrev_b32_e32 v4, 3, v05906; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)5907; GFX11-DL-NEXT: global_load_b128 v[0:3], v1, s[0:1]5908; GFX11-DL-NEXT: global_load_b32 v0, v4, s[2:3]5909; GFX11-DL-NEXT: s_waitcnt vmcnt(1)5910; GFX11-DL-NEXT: v_perm_b32 v1, v3, v2, 0x70500025911; GFX11-DL-NEXT: s_waitcnt vmcnt(0)5912; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0x30200015913; GFX11-DL-NEXT: v_mov_b32_e32 v2, 05914; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)5915; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 05916; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]5917; GFX11-DL-NEXT: s_endpgm5918 ptr addrspace(1) %src2,5919 ptr addrspace(1) nocapture %dst) {5920entry:5921 %idx = call i32 @llvm.amdgcn.workitem.id.x()5922 %gep1 = getelementptr <16 x i8>, ptr addrspace(1) %src1, i32 %idx5923 %vec1 = load <16 x i8>, ptr addrspace(1) %gep15924 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx5925 %vec2 = load <8 x i8>, ptr addrspace(1) %gep25926 5927 %v1e0 = extractelement <16 x i8> %vec1, i64 85928 %cv1e0 = zext i8 %v1e0 to i325929 %v2e0 = extractelement <8 x i8> %vec2, i64 05930 %cv2e0 = zext i8 %v2e0 to i325931 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e05932 5933 %v1e1 = extractelement <16 x i8> %vec1, i64 105934 %cv1e1 = zext i8 %v1e1 to i325935 %v2e1 = extractelement <8 x i8> %vec2, i64 15936 %cv2e1 = zext i8 %v2e1 to i325937 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e15938 5939 %v1e2 = extractelement <16 x i8> %vec1, i64 135940 %cv1e2 = zext i8 %v1e2 to i325941 %v2e2 = extractelement <8 x i8> %vec2, i64 25942 %cv2e2 = zext i8 %v2e2 to i325943 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e25944 5945 %v1e3 = extractelement <16 x i8> %vec1, i64 155946 %cv1e3 = zext i8 %v1e3 to i325947 %v2e3 = extractelement <8 x i8> %vec2, i64 35948 %cv2e3 = zext i8 %v2e3 to i325949 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e35950 5951 %add1 = add i32 %mul1, 05952 %add2 = add i32 %add1, %mul25953 %add3 = add i32 %add2, %mul35954 %add4 = add i32 %add3, %mul45955 store i32 %add4, ptr addrspace(1) %dst, align 45956 ret void5957}5958 5959define amdgpu_kernel void @idot4_acc32_v256i8(ptr addrspace(1) %src1,5960; GFX7-LABEL: idot4_acc32_v256i8:5961; GFX7: ; %bb.0: ; %entry5962; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95963; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5964; GFX7-NEXT: s_mov_b32 s7, 0xf0005965; GFX7-NEXT: s_mov_b32 s10, 05966; GFX7-NEXT: s_mov_b32 s11, s75967; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v05968; GFX7-NEXT: v_mov_b32_e32 v2, 05969; GFX7-NEXT: s_waitcnt lgkmcnt(0)5970; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]5971; GFX7-NEXT: s_mov_b64 s[2:3], s[10:11]5972; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v05973; GFX7-NEXT: v_mov_b32_e32 v4, v25974; GFX7-NEXT: buffer_load_dword v0, v[1:2], s[0:3], 0 addr64 offset:2525975; GFX7-NEXT: buffer_load_dword v1, v[3:4], s[8:11], 0 addr645976; GFX7-NEXT: s_mov_b32 s6, -15977; GFX7-NEXT: s_waitcnt vmcnt(1)5978; GFX7-NEXT: v_bfe_u32 v4, v0, 16, 85979; GFX7-NEXT: s_waitcnt vmcnt(0)5980; GFX7-NEXT: v_bfe_u32 v5, v1, 8, 85981; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v05982; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v15983; GFX7-NEXT: v_mul_u32_u24_e32 v4, v4, v55984; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v05985; GFX7-NEXT: v_bfe_u32 v7, v1, 16, 85986; GFX7-NEXT: v_mad_u32_u24 v2, v2, v3, v45987; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 85988; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v15989; GFX7-NEXT: v_mad_u32_u24 v2, v6, v7, v25990; GFX7-NEXT: v_mad_u32_u24 v0, v0, v1, v25991; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 05992; GFX7-NEXT: s_endpgm5993;5994; GFX8-LABEL: idot4_acc32_v256i8:5995; GFX8: ; %bb.0: ; %entry5996; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245997; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345998; GFX8-NEXT: v_lshlrev_b32_e32 v1, 8, v05999; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v06000; GFX8-NEXT: s_waitcnt lgkmcnt(0)6001; GFX8-NEXT: v_mov_b32_e32 v2, s16002; GFX8-NEXT: v_add_u32_e32 v3, vcc, s0, v16003; GFX8-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc6004; GFX8-NEXT: v_mov_b32_e32 v1, s36005; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v06006; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6007; GFX8-NEXT: s_movk_i32 s0, 0xfc6008; GFX8-NEXT: flat_load_dword v4, v[0:1]6009; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v36010; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc6011; GFX8-NEXT: flat_load_dword v2, v[0:1]6012; GFX8-NEXT: v_mov_b32_e32 v0, s46013; GFX8-NEXT: v_mov_b32_e32 v1, s56014; GFX8-NEXT: s_waitcnt vmcnt(1)6015; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v46016; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 86017; GFX8-NEXT: s_waitcnt vmcnt(0)6018; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v26019; GFX8-NEXT: v_mul_u32_u24_sdwa v7, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_16020; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v26021; GFX8-NEXT: v_mad_u32_u24 v3, v6, v3, v76022; GFX8-NEXT: v_bfe_u32 v2, v2, 8, 86023; GFX8-NEXT: v_lshrrev_b32_e32 v4, 24, v46024; GFX8-NEXT: v_mad_u32_u24 v3, v8, v5, v36025; GFX8-NEXT: v_mad_u32_u24 v2, v2, v4, v36026; GFX8-NEXT: flat_store_dword v[0:1], v26027; GFX8-NEXT: s_endpgm6028;6029; GFX9-NODL-LABEL: idot4_acc32_v256i8:6030; GFX9-NODL: ; %bb.0: ; %entry6031; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246032; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346033; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v1, 8, v06034; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 3, v06035; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)6036; GFX9-NODL-NEXT: global_load_dword v2, v1, s[0:1] offset:2526037; GFX9-NODL-NEXT: global_load_dword v3, v0, s[2:3]6038; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 06039; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)6040; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v26041; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)6042; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v36043; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_16044; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_26045; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_36046; GFX9-NODL-NEXT: v_mad_u32_u24 v1, v1, v4, v56047; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v6, v26048; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]6049; GFX9-NODL-NEXT: s_endpgm6050;6051; GFX9-DL-LABEL: idot4_acc32_v256i8:6052; GFX9-DL: ; %bb.0: ; %entry6053; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246054; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346055; GFX9-DL-NEXT: v_lshlrev_b32_e32 v1, 8, v06056; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 3, v06057; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)6058; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]6059; GFX9-DL-NEXT: global_load_dword v3, v1, s[0:1] offset:2526060; GFX9-DL-NEXT: s_mov_b32 s0, 0x30200016061; GFX9-DL-NEXT: s_mov_b32 s1, 0x10003026062; GFX9-DL-NEXT: v_mov_b32_e32 v0, 06063; GFX9-DL-NEXT: s_waitcnt vmcnt(1)6064; GFX9-DL-NEXT: v_perm_b32 v1, v2, v2, s06065; GFX9-DL-NEXT: s_waitcnt vmcnt(0)6066; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s16067; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, 06068; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]6069; GFX9-DL-NEXT: s_endpgm6070;6071; GFX10-DL-LABEL: idot4_acc32_v256i8:6072; GFX10-DL: ; %bb.0: ; %entry6073; GFX10-DL-NEXT: s_clause 0x16074; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246075; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346076; GFX10-DL-NEXT: v_lshlrev_b32_e32 v1, 3, v06077; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 8, v06078; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)6079; GFX10-DL-NEXT: global_load_dword v2, v1, s[2:3]6080; GFX10-DL-NEXT: global_load_dword v3, v0, s[0:1] offset:2526081; GFX10-DL-NEXT: s_waitcnt vmcnt(1)6082; GFX10-DL-NEXT: v_perm_b32 v0, v2, v2, 0x30200016083; GFX10-DL-NEXT: s_waitcnt vmcnt(0)6084; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0x10003026085; GFX10-DL-NEXT: v_mov_b32_e32 v2, 06086; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 06087; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]6088; GFX10-DL-NEXT: s_endpgm6089;6090; GFX11-DL-LABEL: idot4_acc32_v256i8:6091; GFX11-DL: ; %bb.0: ; %entry6092; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x246093; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v06094; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x346095; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)6096; GFX11-DL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v1, 3, v06097; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 8, v06098; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)6099; GFX11-DL-NEXT: global_load_b32 v1, v1, s[2:3]6100; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1] offset:2526101; GFX11-DL-NEXT: s_waitcnt vmcnt(1)6102; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0x30200016103; GFX11-DL-NEXT: s_waitcnt vmcnt(0)6104; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0x10003026105; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)6106; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, 06107; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]6108; GFX11-DL-NEXT: s_endpgm6109 ptr addrspace(1) %src2,6110 ptr addrspace(1) nocapture %dst) {6111entry:6112 %idx = call i32 @llvm.amdgcn.workitem.id.x()6113 %gep1 = getelementptr <256 x i8>, ptr addrspace(1) %src1, i32 %idx6114 %vec1 = load <256 x i8>, ptr addrspace(1) %gep16115 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx6116 %vec2 = load <8 x i8>, ptr addrspace(1) %gep26117 6118 %v1e0 = extractelement <256 x i8> %vec1, i64 2556119 %cv1e0 = zext i8 %v1e0 to i326120 %v2e0 = extractelement <8 x i8> %vec2, i64 06121 %cv2e0 = zext i8 %v2e0 to i326122 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e06123 6124 %v1e1 = extractelement <256 x i8> %vec1, i64 2546125 %cv1e1 = zext i8 %v1e1 to i326126 %v2e1 = extractelement <8 x i8> %vec2, i64 16127 %cv2e1 = zext i8 %v2e1 to i326128 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e16129 6130 %v1e2 = extractelement <256 x i8> %vec1, i64 2526131 %cv1e2 = zext i8 %v1e2 to i326132 %v2e2 = extractelement <8 x i8> %vec2, i64 26133 %cv2e2 = zext i8 %v2e2 to i326134 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e26135 6136 %v1e3 = extractelement <256 x i8> %vec1, i64 2536137 %cv1e3 = zext i8 %v1e3 to i326138 %v2e3 = extractelement <8 x i8> %vec2, i64 36139 %cv2e3 = zext i8 %v2e3 to i326140 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e36141 6142 %add1 = add i32 %mul1, 06143 %add2 = add i32 %add1, %mul26144 %add3 = add i32 %add2, %mul36145 %add4 = add i32 %add3, %mul46146 store i32 %add4, ptr addrspace(1) %dst, align 46147 ret void6148}6149 6150define amdgpu_kernel void @idot4_acc32_anyext(ptr addrspace(1) %src1,6151; GFX7-LABEL: idot4_acc32_anyext:6152; GFX7: ; %bb.0: ; %entry6153; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96154; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd6155; GFX7-NEXT: s_mov_b32 s7, 0xf0006156; GFX7-NEXT: s_mov_b32 s10, 06157; GFX7-NEXT: s_mov_b32 s11, s76158; GFX7-NEXT: s_waitcnt lgkmcnt(0)6159; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]6160; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v06161; GFX7-NEXT: v_mov_b32_e32 v1, 06162; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr646163; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]6164; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr646165; GFX7-NEXT: s_load_dword s0, s[4:5], 0x06166; GFX7-NEXT: s_mov_b32 s6, -16167; GFX7-NEXT: s_waitcnt vmcnt(1)6168; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v26169; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 86170; GFX7-NEXT: s_waitcnt vmcnt(0)6171; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 86172; GFX7-NEXT: s_waitcnt lgkmcnt(0)6173; GFX7-NEXT: v_mad_u32_u24 v1, v1, v1, s06174; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v16175; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 06176; GFX7-NEXT: s_endpgm6177;6178; GFX8-LABEL: idot4_acc32_anyext:6179; GFX8: ; %bb.0: ; %entry6180; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246181; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346182; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v06183; GFX8-NEXT: s_waitcnt lgkmcnt(0)6184; GFX8-NEXT: v_mov_b32_e32 v1, s16185; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v26186; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6187; GFX8-NEXT: flat_load_dword v3, v[0:1]6188; GFX8-NEXT: v_mov_b32_e32 v1, s36189; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v26190; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6191; GFX8-NEXT: flat_load_dword v0, v[0:1]6192; GFX8-NEXT: s_load_dword s0, s[4:5], 0x06193; GFX8-NEXT: s_waitcnt vmcnt(1)6194; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v36195; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 86196; GFX8-NEXT: s_waitcnt lgkmcnt(0)6197; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s06198; GFX8-NEXT: s_waitcnt vmcnt(0)6199; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 86200; GFX8-NEXT: v_mad_u32_u24 v2, v2, v0, v16201; GFX8-NEXT: v_mov_b32_e32 v0, s46202; GFX8-NEXT: v_mov_b32_e32 v1, s56203; GFX8-NEXT: flat_store_dword v[0:1], v26204; GFX8-NEXT: s_endpgm6205;6206; GFX9-NODL-LABEL: idot4_acc32_anyext:6207; GFX9-NODL: ; %bb.0: ; %entry6208; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246209; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346210; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v06211; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)6212; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]6213; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]6214; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x06215; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 06216; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)6217; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_06218; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)6219; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_16220; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)6221; GFX9-NODL-NEXT: v_add3_u32 v1, v3, s0, v16222; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]6223; GFX9-NODL-NEXT: s_endpgm6224;6225; GFX9-DL-LABEL: idot4_acc32_anyext:6226; GFX9-DL: ; %bb.0: ; %entry6227; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246228; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346229; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v06230; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)6231; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]6232; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]6233; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x06234; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c05006235; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c01006236; GFX9-DL-NEXT: v_mov_b32_e32 v0, 06237; GFX9-DL-NEXT: s_waitcnt vmcnt(0)6238; GFX9-DL-NEXT: v_perm_b32 v2, v2, v1, s16239; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s26240; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)6241; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s06242; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]6243; GFX9-DL-NEXT: s_endpgm6244;6245; GFX10-DL-LABEL: idot4_acc32_anyext:6246; GFX10-DL: ; %bb.0: ; %entry6247; GFX10-DL-NEXT: s_clause 0x16248; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246249; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346250; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v06251; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)6252; GFX10-DL-NEXT: s_clause 0x16253; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]6254; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]6255; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)6256; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x06257; GFX10-DL-NEXT: s_waitcnt vmcnt(0)6258; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc0c05006259; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c01006260; GFX10-DL-NEXT: v_mov_b32_e32 v2, 06261; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)6262; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s06263; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]6264; GFX10-DL-NEXT: s_endpgm6265;6266; GFX11-DL-LABEL: idot4_acc32_anyext:6267; GFX11-DL: ; %bb.0: ; %entry6268; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x246269; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v06270; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x346271; GFX11-DL-NEXT: v_mov_b32_e32 v2, 06272; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)6273; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v06274; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)6275; GFX11-DL-NEXT: s_clause 0x16276; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]6277; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]6278; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x06279; GFX11-DL-NEXT: s_waitcnt vmcnt(0)6280; GFX11-DL-NEXT: v_perm_b32 v0, v0, v1, 0xc0c05006281; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c01006282; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)6283; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)6284; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s06285; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]6286; GFX11-DL-NEXT: s_endpgm6287 ptr addrspace(1) %src2,6288 ptr addrspace(1) nocapture %dst) {6289entry:6290 %idx = call i32 @llvm.amdgcn.workitem.id.x()6291 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx6292 %vec1 = load <4 x i8>, ptr addrspace(1) %gep16293 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx6294 %vec2 = load <4 x i8>, ptr addrspace(1) %gep26295 6296 %v1e0 = extractelement <4 x i8> %vec1, i64 06297 %cv1e0t = sext i8 %v1e0 to i326298 %cv1e0 = and i32 %cv1e0t, 2556299 %v2e0 = extractelement <4 x i8> %vec2, i64 06300 %cv2e0t = sext i8 %v2e0 to i326301 %cv2e0 = and i32 %cv1e0t, 2556302 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e06303 6304 %v1e1 = extractelement <4 x i8> %vec1, i64 16305 %cv1e1 = zext i8 %v1e1 to i326306 %v2e1 = extractelement <4 x i8> %vec2, i64 16307 %cv2e1t = sext i8 %v2e1 to i326308 %cv2e1 = and i32 %cv2e1t, 2556309 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e16310 6311 %acc = load i32, ptr addrspace(1) %dst, align 46312 %add1 = add i32 %mul1, %acc6313 %add2 = add i32 %add1, %mul26314 store i32 %add2, ptr addrspace(1) %dst, align 46315 ret void6316}6317 6318declare i32 @llvm.amdgcn.workitem.id.x()6319