3540 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9-NODL %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck -check-prefixes=GFX9-DL %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck -check-prefixes=GFX10-DL %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 < %s | FileCheck -check-prefixes=GFX10-DL %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11-DL,GFX11-DL-TRUE16 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11-DL,GFX11-DL-FAKE16 %s10 11define amdgpu_kernel void @idot4_acc32(ptr addrspace(1) %src1,12; GFX7-LABEL: idot4_acc32:13; GFX7: ; %bb.0: ; %entry14; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x915; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd16; GFX7-NEXT: s_mov_b32 s3, 0xf00017; GFX7-NEXT: s_mov_b32 s6, 018; GFX7-NEXT: s_mov_b32 s7, s319; GFX7-NEXT: s_waitcnt lgkmcnt(0)20; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]21; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v022; GFX7-NEXT: v_mov_b32_e32 v1, 023; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr6424; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]25; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr6426; GFX7-NEXT: s_load_dword s4, s[0:1], 0x027; GFX7-NEXT: s_mov_b32 s2, -128; GFX7-NEXT: s_waitcnt vmcnt(1)29; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 830; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 831; GFX7-NEXT: s_waitcnt vmcnt(0)32; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 833; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 834; GFX7-NEXT: s_waitcnt lgkmcnt(0)35; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, s436; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 837; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 838; GFX7-NEXT: v_mad_i32_i24 v1, v3, v6, v139; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v240; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v041; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v142; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v143; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 044; GFX7-NEXT: s_endpgm45;46; GFX8-LABEL: idot4_acc32:47; GFX8: ; %bb.0: ; %entry48; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2449; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3450; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v051; GFX8-NEXT: s_waitcnt lgkmcnt(0)52; GFX8-NEXT: v_mov_b32_e32 v1, s153; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v254; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc55; GFX8-NEXT: flat_load_dword v3, v[0:1]56; GFX8-NEXT: v_mov_b32_e32 v1, s357; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v258; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc59; GFX8-NEXT: flat_load_dword v0, v[0:1]60; GFX8-NEXT: s_load_dword s0, s[4:5], 0x061; GFX8-NEXT: s_waitcnt vmcnt(1)62; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 863; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 864; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 865; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v366; GFX8-NEXT: s_waitcnt vmcnt(0)67; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 868; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 869; GFX8-NEXT: s_waitcnt lgkmcnt(0)70; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s071; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 872; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v173; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v074; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v175; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v176; GFX8-NEXT: v_mov_b32_e32 v0, s477; GFX8-NEXT: v_mov_b32_e32 v1, s578; GFX8-NEXT: flat_store_dword v[0:1], v279; GFX8-NEXT: s_endpgm80;81; GFX9-NODL-LABEL: idot4_acc32:82; GFX9-NODL: ; %bb.0: ; %entry83; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2484; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3485; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v086; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)87; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]88; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]89; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x090; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 091; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)92; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_093; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_194; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_295; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_396; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)97; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v498; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v199; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]100; GFX9-NODL-NEXT: s_endpgm101;102; GFX9-DL-LABEL: idot4_acc32:103; GFX9-DL: ; %bb.0: ; %entry104; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24105; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34106; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0107; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)108; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]109; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]110; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0111; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0112; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)113; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, s0114; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]115; GFX9-DL-NEXT: s_endpgm116;117; GFX10-DL-LABEL: idot4_acc32:118; GFX10-DL: ; %bb.0: ; %entry119; GFX10-DL-NEXT: s_clause 0x1120; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24121; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34122; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0123; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0124; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)125; GFX10-DL-NEXT: s_clause 0x1126; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]127; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]128; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)129; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0130; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)131; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0132; GFX10-DL-NEXT: s_waitcnt vmcnt(0)133; GFX10-DL-NEXT: v_dot4c_i32_i8 v0, v1, v2134; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]135; GFX10-DL-NEXT: s_endpgm136;137; GFX11-DL-LABEL: idot4_acc32:138; GFX11-DL: ; %bb.0: ; %entry139; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24140; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v0141; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34142; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0143; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)144; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0145; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)146; GFX11-DL-NEXT: s_clause 0x1147; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]148; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]149; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x0150; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)151; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v1, v0, s0 neg_lo:[1,1,0]152; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]153; GFX11-DL-NEXT: s_endpgm154 ptr addrspace(1) %src2,155 ptr addrspace(1) nocapture %dst) {156entry:157 %idx = call i32 @llvm.amdgcn.workitem.id.x()158 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx159 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1160 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx161 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2162 163 %v1e0 = extractelement <4 x i8> %vec1, i64 0164 %cv1e0 = sext i8 %v1e0 to i32165 %v2e0 = extractelement <4 x i8> %vec2, i64 0166 %cv2e0 = sext i8 %v2e0 to i32167 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0168 169 %v1e1 = extractelement <4 x i8> %vec1, i64 1170 %cv1e1 = sext i8 %v1e1 to i32171 %v2e1 = extractelement <4 x i8> %vec2, i64 1172 %cv2e1 = sext i8 %v2e1 to i32173 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1174 175 %v1e2 = extractelement <4 x i8> %vec1, i64 2176 %cv1e2 = sext i8 %v1e2 to i32177 %v2e2 = extractelement <4 x i8> %vec2, i64 2178 %cv2e2 = sext i8 %v2e2 to i32179 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2180 181 %v1e3 = extractelement <4 x i8> %vec1, i64 3182 %cv1e3 = sext i8 %v1e3 to i32183 %v2e3 = extractelement <4 x i8> %vec2, i64 3184 %cv2e3 = sext i8 %v2e3 to i32185 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3186 187 %acc = load i32, ptr addrspace(1) %dst, align 4188 %add1 = add i32 %mul1, %acc189 %add2 = add i32 %add1, %mul2190 %add3 = add i32 %add2, %mul3191 %add4 = add i32 %add3, %mul4192 store i32 %add4, ptr addrspace(1) %dst, align 4193 ret void194}195 196define amdgpu_kernel void @idot4_acc16(ptr addrspace(1) %src1,197; GFX7-LABEL: idot4_acc16:198; GFX7: ; %bb.0: ; %entry199; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9200; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd201; GFX7-NEXT: s_mov_b32 s3, 0xf000202; GFX7-NEXT: s_mov_b32 s6, 0203; GFX7-NEXT: s_mov_b32 s7, s3204; GFX7-NEXT: s_waitcnt lgkmcnt(0)205; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]206; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0207; GFX7-NEXT: v_mov_b32_e32 v1, 0208; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64209; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]210; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64211; GFX7-NEXT: s_mov_b32 s2, -1212; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0213; GFX7-NEXT: s_waitcnt vmcnt(2)214; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8215; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 8216; GFX7-NEXT: s_waitcnt vmcnt(1)217; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8218; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3219; GFX7-NEXT: v_bfe_i32 v7, v0, 8, 8220; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6221; GFX7-NEXT: v_bfe_i32 v5, v2, 16, 8222; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4223; GFX7-NEXT: v_bfe_i32 v8, v0, 16, 8224; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7225; GFX7-NEXT: s_waitcnt vmcnt(0)226; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1227; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2228; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5229; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0230; GFX7-NEXT: v_and_b32_e32 v8, 0xffff, v8231; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1232; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2233; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0234; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1235; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1236; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0237; GFX7-NEXT: s_endpgm238;239; GFX8-LABEL: idot4_acc16:240; GFX8: ; %bb.0: ; %entry241; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24242; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34243; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0244; GFX8-NEXT: s_waitcnt lgkmcnt(0)245; GFX8-NEXT: v_mov_b32_e32 v1, s1246; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2247; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc248; GFX8-NEXT: flat_load_dword v3, v[0:1]249; GFX8-NEXT: v_mov_b32_e32 v1, s3250; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2251; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc252; GFX8-NEXT: flat_load_dword v2, v[0:1]253; GFX8-NEXT: v_mov_b32_e32 v0, s4254; GFX8-NEXT: v_mov_b32_e32 v1, s5255; GFX8-NEXT: flat_load_ushort v4, v[0:1]256; GFX8-NEXT: s_waitcnt vmcnt(2)257; GFX8-NEXT: v_bfe_i32 v7, v3, 0, 8258; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v3259; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3260; GFX8-NEXT: v_bfe_i32 v9, v9, 0, 8261; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3262; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 8263; GFX8-NEXT: s_waitcnt vmcnt(1)264; GFX8-NEXT: v_bfe_i32 v8, v2, 0, 8265; GFX8-NEXT: v_lshrrev_b32_e32 v10, 8, v2266; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2267; GFX8-NEXT: v_bfe_i32 v10, v10, 0, 8268; GFX8-NEXT: s_waitcnt vmcnt(0)269; GFX8-NEXT: v_mad_u16 v4, v7, v8, v4270; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2271; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 8272; GFX8-NEXT: v_mad_u16 v4, v9, v10, v4273; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8274; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 8275; GFX8-NEXT: v_mad_u16 v4, v5, v6, v4276; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4277; GFX8-NEXT: flat_store_short v[0:1], v2278; GFX8-NEXT: s_endpgm279;280; GFX9-NODL-LABEL: idot4_acc16:281; GFX9-NODL: ; %bb.0: ; %entry282; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24283; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34284; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0285; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)286; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]287; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]288; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0289; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]290; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)291; GFX9-NODL-NEXT: v_bfe_i32 v6, v1, 0, 8292; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)293; GFX9-NODL-NEXT: v_bfe_i32 v7, v2, 0, 8294; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v1295; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 8, v2296; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1297; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2298; GFX9-NODL-NEXT: v_bfe_i32 v8, v8, 0, 8299; GFX9-NODL-NEXT: v_bfe_i32 v9, v9, 0, 8300; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)301; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v6, v7, v3302; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1303; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2304; GFX9-NODL-NEXT: v_bfe_i32 v4, v4, 0, 8305; GFX9-NODL-NEXT: v_bfe_i32 v5, v5, 0, 8306; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v9, v3307; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 8308; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 8309; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3310; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3311; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]312; GFX9-NODL-NEXT: s_endpgm313;314; GFX9-DL-LABEL: idot4_acc16:315; GFX9-DL: ; %bb.0: ; %entry316; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24317; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34318; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0319; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0320; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)321; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]322; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]323; GFX9-DL-NEXT: global_load_sshort v4, v1, s[6:7]324; GFX9-DL-NEXT: s_waitcnt vmcnt(0)325; GFX9-DL-NEXT: v_dot4_i32_i8 v0, v2, v3, v4326; GFX9-DL-NEXT: global_store_short v1, v0, s[6:7]327; GFX9-DL-NEXT: s_endpgm328;329; GFX10-DL-LABEL: idot4_acc16:330; GFX10-DL: ; %bb.0: ; %entry331; GFX10-DL-NEXT: s_clause 0x1332; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24333; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34334; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0335; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0336; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)337; GFX10-DL-NEXT: s_clause 0x1338; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]339; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]340; GFX10-DL-NEXT: global_load_sshort v4, v1, s[6:7]341; GFX10-DL-NEXT: s_waitcnt vmcnt(0)342; GFX10-DL-NEXT: v_dot4c_i32_i8 v4, v2, v3343; GFX10-DL-NEXT: global_store_short v1, v4, s[6:7]344; GFX10-DL-NEXT: s_endpgm345;346; GFX11-DL-LABEL: idot4_acc16:347; GFX11-DL: ; %bb.0: ; %entry348; GFX11-DL-NEXT: s_clause 0x1349; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24350; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34351; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0352; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)353; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0354; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)355; GFX11-DL-NEXT: s_clause 0x1356; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]357; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]358; GFX11-DL-NEXT: global_load_i16 v3, v1, s[4:5]359; GFX11-DL-NEXT: s_waitcnt vmcnt(0)360; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v2, v0, v3 neg_lo:[1,1,0]361; GFX11-DL-NEXT: global_store_b16 v1, v0, s[4:5]362; GFX11-DL-NEXT: s_endpgm363 ptr addrspace(1) %src2,364 ptr addrspace(1) nocapture %dst) {365entry:366 %idx = call i32 @llvm.amdgcn.workitem.id.x()367 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx368 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1369 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx370 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2371 372 %v1e0 = extractelement <4 x i8> %vec1, i64 0373 %cv1e0 = sext i8 %v1e0 to i16374 %v2e0 = extractelement <4 x i8> %vec2, i64 0375 %cv2e0 = sext i8 %v2e0 to i16376 %mul1 = mul nsw i16 %cv1e0, %cv2e0377 378 %v1e1 = extractelement <4 x i8> %vec1, i64 1379 %cv1e1 = sext i8 %v1e1 to i16380 %v2e1 = extractelement <4 x i8> %vec2, i64 1381 %cv2e1 = sext i8 %v2e1 to i16382 %mul2 = mul nsw i16 %cv1e1, %cv2e1383 384 %v1e2 = extractelement <4 x i8> %vec1, i64 2385 %cv1e2 = sext i8 %v1e2 to i16386 %v2e2 = extractelement <4 x i8> %vec2, i64 2387 %cv2e2 = sext i8 %v2e2 to i16388 %mul3 = mul nsw i16 %cv1e2, %cv2e2389 390 %v1e3 = extractelement <4 x i8> %vec1, i64 3391 %cv1e3 = sext i8 %v1e3 to i16392 %v2e3 = extractelement <4 x i8> %vec2, i64 3393 %cv2e3 = sext i8 %v2e3 to i16394 %mul4 = mul nsw i16 %cv1e3, %cv2e3395 396 %acc = load i16, ptr addrspace(1) %dst, align 2397 %add1 = add i16 %mul1, %acc398 %add2 = add i16 %add1, %mul2399 %add3 = add i16 %add2, %mul3400 %add4 = add i16 %add3, %mul4401 store i16 %add4, ptr addrspace(1) %dst, align 2402 ret void403}404 405define amdgpu_kernel void @idot4_acc8(ptr addrspace(1) %src1,406; GFX7-LABEL: idot4_acc8:407; GFX7: ; %bb.0: ; %entry408; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9409; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd410; GFX7-NEXT: s_mov_b32 s7, 0xf000411; GFX7-NEXT: s_mov_b32 s10, 0412; GFX7-NEXT: s_mov_b32 s11, s7413; GFX7-NEXT: s_waitcnt lgkmcnt(0)414; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]415; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0416; GFX7-NEXT: v_mov_b32_e32 v1, 0417; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64418; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]419; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64420; GFX7-NEXT: s_mov_b32 s6, -1421; GFX7-NEXT: buffer_load_ubyte v1, off, s[4:7], 0422; GFX7-NEXT: s_waitcnt vmcnt(2)423; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2424; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8425; GFX7-NEXT: s_waitcnt vmcnt(1)426; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0427; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8428; GFX7-NEXT: s_waitcnt vmcnt(0)429; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1430; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8431; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8432; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1433; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2434; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0435; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1436; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1437; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0438; GFX7-NEXT: s_endpgm439;440; GFX8-LABEL: idot4_acc8:441; GFX8: ; %bb.0: ; %entry442; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24443; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34444; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0445; GFX8-NEXT: s_waitcnt lgkmcnt(0)446; GFX8-NEXT: v_mov_b32_e32 v1, s1447; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2448; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc449; GFX8-NEXT: flat_load_dword v3, v[0:1]450; GFX8-NEXT: v_mov_b32_e32 v1, s3451; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2452; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc453; GFX8-NEXT: flat_load_dword v2, v[0:1]454; GFX8-NEXT: v_mov_b32_e32 v0, s4455; GFX8-NEXT: v_mov_b32_e32 v1, s5456; GFX8-NEXT: flat_load_ubyte v4, v[0:1]457; GFX8-NEXT: s_waitcnt vmcnt(2)458; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3459; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3460; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3461; GFX8-NEXT: s_waitcnt vmcnt(1)462; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2463; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2464; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2465; GFX8-NEXT: s_waitcnt vmcnt(0)466; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4467; GFX8-NEXT: v_mad_u16 v2, v7, v8, v2468; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2469; GFX8-NEXT: v_mad_u16 v2, v9, v10, v2470; GFX8-NEXT: flat_store_byte v[0:1], v2471; GFX8-NEXT: s_endpgm472;473; GFX9-NODL-LABEL: idot4_acc8:474; GFX9-NODL: ; %bb.0: ; %entry475; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24476; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34477; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0478; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)479; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]480; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]481; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0482; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[6:7]483; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)484; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1485; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1486; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)487; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2488; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1489; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)490; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3491; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2492; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v6, v7, v1493; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2494; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1495; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v8, v9, v1496; GFX9-NODL-NEXT: global_store_byte v0, v1, s[6:7]497; GFX9-NODL-NEXT: s_endpgm498;499; GFX9-DL-LABEL: idot4_acc8:500; GFX9-DL: ; %bb.0: ; %entry501; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24502; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34503; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0504; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0505; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)506; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]507; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]508; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[6:7]509; GFX9-DL-NEXT: s_waitcnt vmcnt(0)510; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4511; GFX9-DL-NEXT: global_store_byte v1, v0, s[6:7]512; GFX9-DL-NEXT: s_endpgm513;514; GFX10-DL-LABEL: idot4_acc8:515; GFX10-DL: ; %bb.0: ; %entry516; GFX10-DL-NEXT: s_clause 0x1517; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24518; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34519; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0520; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0521; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)522; GFX10-DL-NEXT: s_clause 0x1523; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]524; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]525; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]526; GFX10-DL-NEXT: s_waitcnt vmcnt(0)527; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4528; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]529; GFX10-DL-NEXT: s_endpgm530;531; GFX11-DL-LABEL: idot4_acc8:532; GFX11-DL: ; %bb.0: ; %entry533; GFX11-DL-NEXT: s_clause 0x1534; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24535; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34536; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0537; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)538; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0539; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)540; GFX11-DL-NEXT: s_clause 0x1541; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]542; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]543; GFX11-DL-NEXT: global_load_u8 v3, v1, s[4:5]544; GFX11-DL-NEXT: s_waitcnt vmcnt(0)545; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, v3546; GFX11-DL-NEXT: global_store_b8 v1, v0, s[4:5]547; GFX11-DL-NEXT: s_endpgm548 ptr addrspace(1) %src2,549 ptr addrspace(1) nocapture %dst) {550entry:551 %idx = call i32 @llvm.amdgcn.workitem.id.x()552 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx553 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1554 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx555 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2556 557 %v1e0 = extractelement <4 x i8> %vec1, i64 0558 %v2e0 = extractelement <4 x i8> %vec2, i64 0559 %mul1 = mul i8 %v1e0, %v2e0560 561 %v1e1 = extractelement <4 x i8> %vec1, i64 1562 %v2e1 = extractelement <4 x i8> %vec2, i64 1563 %mul2 = mul i8 %v1e1, %v2e1564 565 %v1e2 = extractelement <4 x i8> %vec1, i64 2566 %v2e2 = extractelement <4 x i8> %vec2, i64 2567 %mul3 = mul i8 %v1e2, %v2e2568 569 %v1e3 = extractelement <4 x i8> %vec1, i64 3570 %v2e3 = extractelement <4 x i8> %vec2, i64 3571 %mul4 = mul i8 %v1e3, %v2e3572 573 %acc = load i8, ptr addrspace(1) %dst, align 2574 %add1 = add i8 %mul1, %acc575 %add2 = add i8 %add1, %mul2576 %add3 = add i8 %add2, %mul3577 %add4 = add nsw i8 %add3, %mul4578 store i8 %add4, ptr addrspace(1) %dst, align 2579 ret void580}581 582define amdgpu_kernel void @idot4_multiuse_mul1(ptr addrspace(1) %src1,583; GFX7-LABEL: idot4_multiuse_mul1:584; GFX7: ; %bb.0: ; %entry585; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9586; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd587; GFX7-NEXT: s_mov_b32 s3, 0xf000588; GFX7-NEXT: s_mov_b32 s6, 0589; GFX7-NEXT: s_mov_b32 s7, s3590; GFX7-NEXT: s_waitcnt lgkmcnt(0)591; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]592; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0593; GFX7-NEXT: v_mov_b32_e32 v1, 0594; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64595; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]596; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64597; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0598; GFX7-NEXT: s_mov_b32 s2, -1599; GFX7-NEXT: s_waitcnt vmcnt(1)600; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 8601; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 8602; GFX7-NEXT: s_waitcnt vmcnt(0)603; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 8604; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 8605; GFX7-NEXT: s_waitcnt lgkmcnt(0)606; GFX7-NEXT: v_mad_i32_i24 v8, v1, v5, s4607; GFX7-NEXT: v_mad_i32_i24 v3, v3, v6, v8608; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 8609; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 8610; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, v3611; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v2612; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v0613; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v1614; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v1615; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0616; GFX7-NEXT: s_endpgm617;618; GFX8-LABEL: idot4_multiuse_mul1:619; GFX8: ; %bb.0: ; %entry620; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24621; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34622; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0623; GFX8-NEXT: s_waitcnt lgkmcnt(0)624; GFX8-NEXT: v_mov_b32_e32 v1, s1625; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2626; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc627; GFX8-NEXT: flat_load_dword v3, v[0:1]628; GFX8-NEXT: v_mov_b32_e32 v1, s3629; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2630; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc631; GFX8-NEXT: flat_load_dword v0, v[0:1]632; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0633; GFX8-NEXT: s_waitcnt vmcnt(1)634; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 8635; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 8636; GFX8-NEXT: v_bfe_i32 v6, v3, 16, 8637; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v3638; GFX8-NEXT: s_waitcnt vmcnt(0)639; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 8640; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 8641; GFX8-NEXT: s_waitcnt lgkmcnt(0)642; GFX8-NEXT: v_mad_i32_i24 v8, v1, v2, s0643; GFX8-NEXT: v_mad_i32_i24 v4, v4, v5, v8644; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8645; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, v4646; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v0647; GFX8-NEXT: v_mad_i32_i24 v1, v6, v7, v1648; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v1649; GFX8-NEXT: v_mov_b32_e32 v0, s4650; GFX8-NEXT: v_mov_b32_e32 v1, s5651; GFX8-NEXT: flat_store_dword v[0:1], v2652; GFX8-NEXT: s_endpgm653;654; GFX9-NODL-LABEL: idot4_multiuse_mul1:655; GFX9-NODL: ; %bb.0: ; %entry656; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24657; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34658; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0659; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)660; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]661; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]662; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0663; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0664; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)665; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 8666; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)667; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 8668; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1669; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2670; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3671; GFX9-NODL-NEXT: v_mul_i32_i24_e32 v2, v3, v4672; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)673; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v3, v4, s0674; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2675; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1676; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]677; GFX9-NODL-NEXT: s_endpgm678;679; GFX9-DL-LABEL: idot4_multiuse_mul1:680; GFX9-DL: ; %bb.0: ; %entry681; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24682; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34683; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0684; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)685; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]686; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]687; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0688; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0689; GFX9-DL-NEXT: s_waitcnt vmcnt(1)690; GFX9-DL-NEXT: v_bfe_i32 v3, v1, 0, 8691; GFX9-DL-NEXT: s_waitcnt vmcnt(0)692; GFX9-DL-NEXT: v_bfe_i32 v4, v2, 0, 8693; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)694; GFX9-DL-NEXT: v_mad_i32_i24 v3, v3, v4, s0695; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, v3696; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]697; GFX9-DL-NEXT: s_endpgm698;699; GFX10-DL-LABEL: idot4_multiuse_mul1:700; GFX10-DL: ; %bb.0: ; %entry701; GFX10-DL-NEXT: s_clause 0x1702; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24703; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34704; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0705; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)706; GFX10-DL-NEXT: s_clause 0x1707; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]708; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]709; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)710; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0711; GFX10-DL-NEXT: s_waitcnt vmcnt(1)712; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 8713; GFX10-DL-NEXT: s_waitcnt vmcnt(0)714; GFX10-DL-NEXT: v_bfe_i32 v3, v2, 0, 8715; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)716; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, v3, s0717; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0718; GFX10-DL-NEXT: v_dot4c_i32_i8 v0, v1, v2719; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]720; GFX10-DL-NEXT: s_endpgm721;722; GFX11-DL-LABEL: idot4_multiuse_mul1:723; GFX11-DL: ; %bb.0: ; %entry724; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24725; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v0726; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34727; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)728; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0729; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)730; GFX11-DL-NEXT: s_clause 0x1731; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]732; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]733; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x0734; GFX11-DL-NEXT: s_waitcnt vmcnt(1)735; GFX11-DL-NEXT: v_bfe_i32 v2, v1, 0, 8736; GFX11-DL-NEXT: s_waitcnt vmcnt(0)737; GFX11-DL-NEXT: v_bfe_i32 v3, v0, 0, 8738; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)739; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)740; GFX11-DL-NEXT: v_mad_i32_i24 v2, v2, v3, s0741; GFX11-DL-NEXT: v_mov_b32_e32 v3, 0742; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v1, v0, v2 neg_lo:[1,1,0]743; GFX11-DL-NEXT: global_store_b32 v3, v0, s[4:5]744; GFX11-DL-NEXT: s_endpgm745 ptr addrspace(1) %src2,746 ptr addrspace(1) nocapture %dst) {747entry:748 %idx = call i32 @llvm.amdgcn.workitem.id.x()749 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx750 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1751 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx752 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2753 754 %v1e0 = extractelement <4 x i8> %vec1, i64 0755 %cv1e0 = sext i8 %v1e0 to i32756 %v2e0 = extractelement <4 x i8> %vec2, i64 0757 %cv2e0 = sext i8 %v2e0 to i32758 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0759 760 %v1e1 = extractelement <4 x i8> %vec1, i64 1761 %cv1e1 = sext i8 %v1e1 to i32762 %v2e1 = extractelement <4 x i8> %vec2, i64 1763 %cv2e1 = sext i8 %v2e1 to i32764 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1765 766 %v1e2 = extractelement <4 x i8> %vec1, i64 2767 %cv1e2 = sext i8 %v1e2 to i32768 %v2e2 = extractelement <4 x i8> %vec2, i64 2769 %cv2e2 = sext i8 %v2e2 to i32770 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2771 772 %v1e3 = extractelement <4 x i8> %vec1, i64 3773 %cv1e3 = sext i8 %v1e3 to i32774 %v2e3 = extractelement <4 x i8> %vec2, i64 3775 %cv2e3 = sext i8 %v2e3 to i32776 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3777 778 %acc = load i32, ptr addrspace(1) %dst, align 4779 %add = add i32 %mul1, %acc780 %add1 = add i32 %mul2, %add781 %add2 = add i32 %add1, %mul1782 %add3 = add i32 %add2, %mul3783 %add4 = add i32 %add3, %mul4784 785 store i32 %add4, ptr addrspace(1) %dst, align 4786 ret void787}788 789define amdgpu_kernel void @idot4_acc32_vecMul(ptr addrspace(1) %src1,790; GFX7-LABEL: idot4_acc32_vecMul:791; GFX7: ; %bb.0: ; %entry792; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9793; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd794; GFX7-NEXT: s_mov_b32 s3, 0xf000795; GFX7-NEXT: s_mov_b32 s6, 0796; GFX7-NEXT: s_mov_b32 s7, s3797; GFX7-NEXT: s_waitcnt lgkmcnt(0)798; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]799; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0800; GFX7-NEXT: v_mov_b32_e32 v1, 0801; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64802; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]803; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64804; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0805; GFX7-NEXT: s_mov_b32 s2, -1806; GFX7-NEXT: s_waitcnt vmcnt(1)807; GFX7-NEXT: v_ashrrev_i32_e32 v1, 24, v2808; GFX7-NEXT: v_bfe_i32 v3, v2, 16, 8809; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 8810; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8811; GFX7-NEXT: s_waitcnt vmcnt(0)812; GFX7-NEXT: v_ashrrev_i32_e32 v5, 24, v0813; GFX7-NEXT: v_bfe_i32 v6, v0, 16, 8814; GFX7-NEXT: v_bfe_i32 v7, v0, 8, 8815; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8816; GFX7-NEXT: s_waitcnt lgkmcnt(0)817; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, s4818; GFX7-NEXT: v_mad_i32_i24 v0, v4, v7, v0819; GFX7-NEXT: v_mad_i32_i24 v0, v3, v6, v0820; GFX7-NEXT: v_mad_i32_i24 v0, v1, v5, v0821; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0822; GFX7-NEXT: s_endpgm823;824; GFX8-LABEL: idot4_acc32_vecMul:825; GFX8: ; %bb.0: ; %entry826; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24827; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34828; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0829; GFX8-NEXT: s_waitcnt lgkmcnt(0)830; GFX8-NEXT: v_mov_b32_e32 v1, s1831; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2832; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc833; GFX8-NEXT: flat_load_dword v3, v[0:1]834; GFX8-NEXT: v_mov_b32_e32 v1, s3835; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2836; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc837; GFX8-NEXT: flat_load_dword v0, v[0:1]838; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0839; GFX8-NEXT: s_waitcnt vmcnt(1)840; GFX8-NEXT: v_lshrrev_b16_e32 v1, 8, v3841; GFX8-NEXT: v_ashrrev_i32_e32 v4, 24, v3842; GFX8-NEXT: v_bfe_i32 v5, v3, 16, 8843; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 8844; GFX8-NEXT: v_bfe_i32 v1, v1, 0, 8845; GFX8-NEXT: s_waitcnt vmcnt(0)846; GFX8-NEXT: v_lshrrev_b16_e32 v2, 8, v0847; GFX8-NEXT: v_ashrrev_i32_e32 v6, 24, v0848; GFX8-NEXT: v_bfe_i32 v7, v0, 16, 8849; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 8850; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 8851; GFX8-NEXT: s_waitcnt lgkmcnt(0)852; GFX8-NEXT: v_mad_i32_i24 v0, v3, v0, s0853; GFX8-NEXT: v_mad_i32_i24 v0, v1, v2, v0854; GFX8-NEXT: v_mad_i32_i24 v0, v5, v7, v0855; GFX8-NEXT: v_mad_i32_i24 v2, v4, v6, v0856; GFX8-NEXT: v_mov_b32_e32 v0, s4857; GFX8-NEXT: v_mov_b32_e32 v1, s5858; GFX8-NEXT: flat_store_dword v[0:1], v2859; GFX8-NEXT: s_endpgm860;861; GFX9-NODL-LABEL: idot4_acc32_vecMul:862; GFX9-NODL: ; %bb.0: ; %entry863; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24864; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34865; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0866; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)867; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]868; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]869; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0870; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0871; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)872; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v3, 8, v1873; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)874; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v2875; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0876; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0877; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2878; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3879; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)880; GFX9-NODL-NEXT: v_add3_u32 v2, v5, s0, v3881; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1882; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]883; GFX9-NODL-NEXT: s_endpgm884;885; GFX9-DL-LABEL: idot4_acc32_vecMul:886; GFX9-DL: ; %bb.0: ; %entry887; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24888; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34889; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0890; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)891; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]892; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]893; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0894; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0895; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)896; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, s0897; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]898; GFX9-DL-NEXT: s_endpgm899;900; GFX10-DL-LABEL: idot4_acc32_vecMul:901; GFX10-DL: ; %bb.0: ; %entry902; GFX10-DL-NEXT: s_clause 0x1903; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24904; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34905; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0906; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0907; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)908; GFX10-DL-NEXT: s_clause 0x1909; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]910; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]911; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)912; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0913; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)914; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0915; GFX10-DL-NEXT: s_waitcnt vmcnt(0)916; GFX10-DL-NEXT: v_dot4c_i32_i8 v0, v1, v2917; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]918; GFX10-DL-NEXT: s_endpgm919;920; GFX11-DL-LABEL: idot4_acc32_vecMul:921; GFX11-DL: ; %bb.0: ; %entry922; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24923; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v0924; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34925; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0926; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)927; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0928; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)929; GFX11-DL-NEXT: s_clause 0x1930; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]931; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]932; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x0933; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)934; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v1, v0, s0 neg_lo:[1,1,0]935; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]936; GFX11-DL-NEXT: s_endpgm937 ptr addrspace(1) %src2,938 ptr addrspace(1) nocapture %dst) {939entry:940 %idx = call i32 @llvm.amdgcn.workitem.id.x()941 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx942 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1943 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx944 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2945 946 %cvec1 = sext <4 x i8> %vec1 to <4 x i32>947 %cvec2 = sext <4 x i8> %vec2 to <4 x i32>948 949 %mul = mul <4 x i32> %cvec1, %cvec2950 %mul0 = extractelement <4 x i32> %mul, i64 0951 %mul1 = extractelement <4 x i32> %mul, i64 1952 %mul2 = extractelement <4 x i32> %mul, i64 2953 %mul3 = extractelement <4 x i32> %mul, i64 3954 955 %acc = load i32, ptr addrspace(1) %dst, align 4956 %add1 = add i32 %mul0, %acc957 %add2 = add i32 %add1, %mul1958 %add3 = add i32 %add2, %mul2959 %add4 = add i32 %add3, %mul3960 961 store i32 %add4, ptr addrspace(1) %dst, align 4962 ret void963}964 965define amdgpu_kernel void @idot4_acc16_vecMul(ptr addrspace(1) %src1,966; GFX7-LABEL: idot4_acc16_vecMul:967; GFX7: ; %bb.0: ; %entry968; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9969; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd970; GFX7-NEXT: s_mov_b32 s3, 0xf000971; GFX7-NEXT: s_mov_b32 s6, 0972; GFX7-NEXT: s_mov_b32 s7, s3973; GFX7-NEXT: s_waitcnt lgkmcnt(0)974; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]975; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0976; GFX7-NEXT: v_mov_b32_e32 v1, 0977; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64978; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]979; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64980; GFX7-NEXT: s_mov_b32 s2, -1981; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0982; GFX7-NEXT: s_waitcnt vmcnt(2)983; GFX7-NEXT: v_bfe_i32 v4, v2, 0, 8984; GFX7-NEXT: v_bfe_i32 v3, v2, 16, 8985; GFX7-NEXT: s_waitcnt vmcnt(1)986; GFX7-NEXT: v_bfe_i32 v7, v0, 0, 8987; GFX7-NEXT: v_ashrrev_i32_e32 v5, 24, v2988; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 8989; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4990; GFX7-NEXT: v_bfe_i32 v6, v0, 16, 8991; GFX7-NEXT: v_ashrrev_i32_e32 v8, 24, v0992; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 8993; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7994; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2995; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0996; GFX7-NEXT: s_waitcnt vmcnt(0)997; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1998; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3999; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v61000; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11001; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v51002; GFX7-NEXT: v_and_b32_e32 v8, 0xffff, v81003; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v01004; GFX7-NEXT: v_mad_u32_u24 v0, v5, v8, v01005; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 01006; GFX7-NEXT: s_endpgm1007;1008; GFX8-LABEL: idot4_acc16_vecMul:1009; GFX8: ; %bb.0: ; %entry1010; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241011; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341012; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01013; GFX8-NEXT: s_waitcnt lgkmcnt(0)1014; GFX8-NEXT: v_mov_b32_e32 v1, s11015; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21016; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1017; GFX8-NEXT: flat_load_dword v3, v[0:1]1018; GFX8-NEXT: v_mov_b32_e32 v1, s31019; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21020; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1021; GFX8-NEXT: flat_load_dword v2, v[0:1]1022; GFX8-NEXT: v_mov_b32_e32 v0, s41023; GFX8-NEXT: v_mov_b32_e32 v1, s51024; GFX8-NEXT: flat_load_ushort v4, v[0:1]1025; GFX8-NEXT: s_waitcnt vmcnt(2)1026; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v31027; GFX8-NEXT: v_ashrrev_i16_e32 v7, 8, v31028; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 81029; GFX8-NEXT: v_ashrrev_i16_e32 v9, 8, v51030; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 81031; GFX8-NEXT: s_waitcnt vmcnt(1)1032; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v21033; GFX8-NEXT: v_ashrrev_i16_e32 v8, 8, v21034; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 81035; GFX8-NEXT: s_waitcnt vmcnt(0)1036; GFX8-NEXT: v_mad_u16 v2, v3, v2, v41037; GFX8-NEXT: v_ashrrev_i16_e32 v10, 8, v61038; GFX8-NEXT: v_bfe_i32 v6, v6, 0, 81039; GFX8-NEXT: v_mad_u16 v2, v7, v8, v21040; GFX8-NEXT: v_mad_u16 v2, v5, v6, v21041; GFX8-NEXT: v_mad_u16 v2, v9, v10, v21042; GFX8-NEXT: flat_store_short v[0:1], v21043; GFX8-NEXT: s_endpgm1044;1045; GFX9-NODL-LABEL: idot4_acc16_vecMul:1046; GFX9-NODL: ; %bb.0: ; %entry1047; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241048; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341049; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01050; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1051; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1052; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1053; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01054; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[6:7]1055; GFX9-NODL-NEXT: s_mov_b32 s0, 0x50401001056; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)1057; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v11058; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1059; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v21060; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v6, 8, v11061; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 81062; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v7, 8, v21063; GFX9-NODL-NEXT: v_bfe_i32 v2, v2, 0, 81064; GFX9-NODL-NEXT: v_perm_b32 v2, v7, v2, s01065; GFX9-NODL-NEXT: v_perm_b32 v1, v6, v1, s01066; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v8, 8, v41067; GFX9-NODL-NEXT: v_bfe_i32 v4, v4, 0, 81068; GFX9-NODL-NEXT: v_ashrrev_i16_e32 v9, 8, v51069; GFX9-NODL-NEXT: v_bfe_i32 v5, v5, 0, 81070; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v21071; GFX9-NODL-NEXT: v_perm_b32 v5, v9, v5, s01072; GFX9-NODL-NEXT: v_perm_b32 v4, v8, v4, s01073; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1074; GFX9-NODL-NEXT: v_add_u16_e32 v3, v1, v31075; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v51076; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_11077; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v21078; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_11079; GFX9-NODL-NEXT: global_store_short v0, v1, s[6:7]1080; GFX9-NODL-NEXT: s_endpgm1081;1082; GFX9-DL-LABEL: idot4_acc16_vecMul:1083; GFX9-DL: ; %bb.0: ; %entry1084; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241085; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341086; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01087; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1088; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1089; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1090; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01091; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]1092; GFX9-DL-NEXT: s_mov_b32 s0, 0x50401001093; GFX9-DL-NEXT: s_waitcnt vmcnt(2)1094; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v11095; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1096; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v21097; GFX9-DL-NEXT: v_ashrrev_i16_e32 v6, 8, v11098; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 81099; GFX9-DL-NEXT: v_ashrrev_i16_e32 v7, 8, v21100; GFX9-DL-NEXT: v_bfe_i32 v2, v2, 0, 81101; GFX9-DL-NEXT: v_perm_b32 v2, v7, v2, s01102; GFX9-DL-NEXT: v_perm_b32 v1, v6, v1, s01103; GFX9-DL-NEXT: v_ashrrev_i16_e32 v8, 8, v41104; GFX9-DL-NEXT: v_bfe_i32 v4, v4, 0, 81105; GFX9-DL-NEXT: v_ashrrev_i16_e32 v9, 8, v51106; GFX9-DL-NEXT: v_bfe_i32 v5, v5, 0, 81107; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v21108; GFX9-DL-NEXT: v_perm_b32 v5, v9, v5, s01109; GFX9-DL-NEXT: v_perm_b32 v4, v8, v4, s01110; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1111; GFX9-DL-NEXT: v_add_u16_e32 v3, v1, v31112; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v51113; GFX9-DL-NEXT: v_add_u16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_11114; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v21115; GFX9-DL-NEXT: v_add_u16_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_11116; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]1117; GFX9-DL-NEXT: s_endpgm1118;1119; GFX10-DL-LABEL: idot4_acc16_vecMul:1120; GFX10-DL: ; %bb.0: ; %entry1121; GFX10-DL-NEXT: s_clause 0x11122; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241123; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341124; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01125; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1126; GFX10-DL-NEXT: s_clause 0x11127; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1128; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1129; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01130; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]1131; GFX10-DL-NEXT: s_waitcnt vmcnt(2)1132; GFX10-DL-NEXT: v_ashrrev_i16 v4, 8, v11133; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1134; GFX10-DL-NEXT: v_ashrrev_i16 v5, 8, v21135; GFX10-DL-NEXT: v_bfe_i32 v6, v2, 0, 81136; GFX10-DL-NEXT: v_bfe_i32 v7, v1, 0, 81137; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 16, v11138; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 16, v21139; GFX10-DL-NEXT: v_perm_b32 v5, v5, v6, 0x50401001140; GFX10-DL-NEXT: v_perm_b32 v4, v4, v7, 0x50401001141; GFX10-DL-NEXT: v_ashrrev_i16 v6, 8, v11142; GFX10-DL-NEXT: v_ashrrev_i16 v7, 8, v21143; GFX10-DL-NEXT: v_bfe_i32 v2, v2, 0, 81144; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 81145; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v51146; GFX10-DL-NEXT: v_perm_b32 v2, v7, v2, 0x50401001147; GFX10-DL-NEXT: v_perm_b32 v1, v6, v1, 0x50401001148; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v41149; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1150; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v31151; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v21152; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v51153; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v11154; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v11155; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v31156; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]1157; GFX10-DL-NEXT: s_endpgm1158;1159; GFX11-DL-TRUE16-LABEL: idot4_acc16_vecMul:1160; GFX11-DL-TRUE16: ; %bb.0: ; %entry1161; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x241162; GFX11-DL-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v01163; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x341164; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1165; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01166; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1167; GFX11-DL-TRUE16-NEXT: s_clause 0x11168; GFX11-DL-TRUE16-NEXT: global_load_b32 v1, v0, s[0:1]1169; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3]1170; GFX11-DL-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[4:5]1171; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(2)1172; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v1, 0, 81173; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)1174; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 81175; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v4.h, 8, v1.l1176; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h1177; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v7.h, 8, v2.l1178; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h1179; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l1180; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l1181; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v6, v1, 0, 81182; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v1.h, 8, v1.h1183; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 81184; GFX11-DL-TRUE16-NEXT: v_ashrrev_i16 v2.h, 8, v2.h1185; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v4, v4, v71186; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l1187; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)1188; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l1189; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)1190; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v4.l, v0.l1191; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)1192; GFX11-DL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v21193; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v4.h1194; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1195; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l1196; GFX11-DL-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h1197; GFX11-DL-TRUE16-NEXT: global_store_b16 v3, v0, s[4:5]1198; GFX11-DL-TRUE16-NEXT: s_endpgm1199;1200; GFX11-DL-FAKE16-LABEL: idot4_acc16_vecMul:1201; GFX11-DL-FAKE16: ; %bb.0: ; %entry1202; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x241203; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v01204; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x341205; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 01206; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1207; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01208; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1209; GFX11-DL-FAKE16-NEXT: s_clause 0x11210; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]1211; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]1212; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]1213; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)1214; GFX11-DL-FAKE16-NEXT: v_ashrrev_i16 v4, 8, v11215; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)1216; GFX11-DL-FAKE16-NEXT: v_ashrrev_i16 v5, 8, v01217; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v0, 0, 81218; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v7, v1, 0, 81219; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v11220; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v01221; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)1222; GFX11-DL-FAKE16-NEXT: v_perm_b32 v5, v5, v6, 0x50401001223; GFX11-DL-FAKE16-NEXT: v_perm_b32 v4, v4, v7, 0x50401001224; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)1225; GFX11-DL-FAKE16-NEXT: v_ashrrev_i16 v6, 8, v11226; GFX11-DL-FAKE16-NEXT: v_ashrrev_i16 v7, 8, v01227; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 81228; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 81229; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v4, v4, v51230; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)1231; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v7, v0, 0x50401001232; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v6, v1, 0x50401001233; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)1234; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v41235; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)1236; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v3, v4, v31237; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v1, v01238; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)1239; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v1, v3, v51240; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v01241; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1242; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v1, v01243; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v0, v31244; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]1245; GFX11-DL-FAKE16-NEXT: s_endpgm1246 ptr addrspace(1) %src2,1247 ptr addrspace(1) nocapture %dst) {1248entry:1249 %idx = call i32 @llvm.amdgcn.workitem.id.x()1250 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1251 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11252 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1253 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21254 1255 %cvec1 = sext <4 x i8> %vec1 to <4 x i16>1256 %cvec2 = sext <4 x i8> %vec2 to <4 x i16>1257 1258 %mul = mul <4 x i16> %cvec1, %cvec21259 %mul0 = extractelement <4 x i16> %mul, i64 01260 %mul1 = extractelement <4 x i16> %mul, i64 11261 %mul2 = extractelement <4 x i16> %mul, i64 21262 %mul3 = extractelement <4 x i16> %mul, i64 31263 1264 %acc = load i16, ptr addrspace(1) %dst, align 41265 %add1 = add i16 %mul0, %acc1266 %add2 = add i16 %add1, %mul11267 %add3 = add i16 %add2, %mul21268 %add4 = add i16 %add3, %mul31269 1270 store i16 %add4, ptr addrspace(1) %dst, align 41271 ret void1272}1273 1274define amdgpu_kernel void @idot4_acc32_2ele(ptr addrspace(1) %src1,1275; GFX7-LABEL: idot4_acc32_2ele:1276; GFX7: ; %bb.0: ; %entry1277; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91278; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1279; GFX7-NEXT: s_mov_b32 s7, 0xf0001280; GFX7-NEXT: s_mov_b32 s10, 01281; GFX7-NEXT: s_mov_b32 s11, s71282; GFX7-NEXT: s_waitcnt lgkmcnt(0)1283; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1284; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01285; GFX7-NEXT: v_mov_b32_e32 v1, 01286; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641287; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1288; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641289; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01290; GFX7-NEXT: s_mov_b32 s6, -11291; GFX7-NEXT: s_waitcnt vmcnt(1)1292; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 81293; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 81294; GFX7-NEXT: s_waitcnt vmcnt(0)1295; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 81296; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 81297; GFX7-NEXT: s_waitcnt lgkmcnt(0)1298; GFX7-NEXT: v_mad_i32_i24 v1, v1, v3, s01299; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v11300; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01301; GFX7-NEXT: s_endpgm1302;1303; GFX8-LABEL: idot4_acc32_2ele:1304; GFX8: ; %bb.0: ; %entry1305; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241306; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341307; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01308; GFX8-NEXT: s_waitcnt lgkmcnt(0)1309; GFX8-NEXT: v_mov_b32_e32 v1, s11310; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21311; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1312; GFX8-NEXT: flat_load_dword v3, v[0:1]1313; GFX8-NEXT: v_mov_b32_e32 v1, s31314; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21315; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1316; GFX8-NEXT: flat_load_dword v0, v[0:1]1317; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01318; GFX8-NEXT: s_waitcnt vmcnt(1)1319; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 81320; GFX8-NEXT: v_bfe_i32 v3, v3, 8, 81321; GFX8-NEXT: s_waitcnt vmcnt(0)1322; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 81323; GFX8-NEXT: v_bfe_i32 v0, v0, 8, 81324; GFX8-NEXT: s_waitcnt lgkmcnt(0)1325; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s01326; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v11327; GFX8-NEXT: v_mov_b32_e32 v0, s41328; GFX8-NEXT: v_mov_b32_e32 v1, s51329; GFX8-NEXT: flat_store_dword v[0:1], v21330; GFX8-NEXT: s_endpgm1331;1332; GFX9-NODL-LABEL: idot4_acc32_2ele:1333; GFX9-NODL: ; %bb.0: ; %entry1334; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241335; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341336; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01337; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1338; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1339; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1340; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01341; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01342; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1343; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_01344; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11345; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1346; GFX9-NODL-NEXT: v_add3_u32 v1, v3, s0, v11347; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1348; GFX9-NODL-NEXT: s_endpgm1349;1350; GFX9-DL-LABEL: idot4_acc32_2ele:1351; GFX9-DL: ; %bb.0: ; %entry1352; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241353; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341354; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01355; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1356; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]1357; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]1358; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01359; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c01001360; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01361; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1362; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s11363; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1364; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s11365; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1366; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s01367; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1368; GFX9-DL-NEXT: s_endpgm1369;1370; GFX10-DL-LABEL: idot4_acc32_2ele:1371; GFX10-DL: ; %bb.0: ; %entry1372; GFX10-DL-NEXT: s_clause 0x11373; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241374; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341375; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01376; GFX10-DL-NEXT: v_mov_b32_e32 v3, 01377; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1378; GFX10-DL-NEXT: s_clause 0x11379; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]1380; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1381; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1382; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01383; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1384; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0c01001385; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1386; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0c01001387; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1388; GFX10-DL-NEXT: v_mov_b32_e32 v2, s01389; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v01390; GFX10-DL-NEXT: global_store_dword v3, v2, s[6:7]1391; GFX10-DL-NEXT: s_endpgm1392;1393; GFX11-DL-LABEL: idot4_acc32_2ele:1394; GFX11-DL: ; %bb.0: ; %entry1395; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241396; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01397; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341398; GFX11-DL-NEXT: v_mov_b32_e32 v2, 01399; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)1400; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01401; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1402; GFX11-DL-NEXT: s_clause 0x11403; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]1404; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]1405; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x01406; GFX11-DL-NEXT: s_waitcnt vmcnt(1)1407; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c01001408; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1409; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c01001410; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1411; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1412; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]1413; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]1414; GFX11-DL-NEXT: s_endpgm1415 ptr addrspace(1) %src2,1416 ptr addrspace(1) nocapture %dst) {1417entry:1418 %idx = call i32 @llvm.amdgcn.workitem.id.x()1419 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1420 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11421 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1422 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21423 1424 %v1e0 = extractelement <4 x i8> %vec1, i64 01425 %cv1e0 = sext i8 %v1e0 to i321426 %v2e0 = extractelement <4 x i8> %vec2, i64 01427 %cv2e0 = sext i8 %v2e0 to i321428 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01429 1430 %v1e1 = extractelement <4 x i8> %vec1, i64 11431 %cv1e1 = sext i8 %v1e1 to i321432 %v2e1 = extractelement <4 x i8> %vec2, i64 11433 %cv2e1 = sext i8 %v2e1 to i321434 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11435 1436 %acc = load i32, ptr addrspace(1) %dst, align 41437 %add1 = add i32 %mul1, %acc1438 %add2 = add i32 %add1, %mul21439 store i32 %add2, ptr addrspace(1) %dst, align 41440 ret void1441}1442 1443 1444define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,1445; GFX7-LABEL: idot4_acc32_3ele:1446; GFX7: ; %bb.0: ; %entry1447; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91448; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1449; GFX7-NEXT: s_mov_b32 s7, 0xf0001450; GFX7-NEXT: s_mov_b32 s10, 01451; GFX7-NEXT: s_mov_b32 s11, s71452; GFX7-NEXT: s_waitcnt lgkmcnt(0)1453; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1454; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01455; GFX7-NEXT: v_mov_b32_e32 v1, 01456; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641457; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1458; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641459; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01460; GFX7-NEXT: s_mov_b32 s6, -11461; GFX7-NEXT: s_waitcnt vmcnt(1)1462; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 81463; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 81464; GFX7-NEXT: s_waitcnt vmcnt(0)1465; GFX7-NEXT: v_bfe_i32 v4, v0, 0, 81466; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 81467; GFX7-NEXT: s_waitcnt lgkmcnt(0)1468; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s01469; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 81470; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 81471; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v11472; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v11473; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01474; GFX7-NEXT: s_endpgm1475;1476; GFX8-LABEL: idot4_acc32_3ele:1477; GFX8: ; %bb.0: ; %entry1478; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241479; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341480; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01481; GFX8-NEXT: s_waitcnt lgkmcnt(0)1482; GFX8-NEXT: v_mov_b32_e32 v1, s11483; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21484; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1485; GFX8-NEXT: flat_load_dword v3, v[0:1]1486; GFX8-NEXT: v_mov_b32_e32 v1, s31487; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21488; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1489; GFX8-NEXT: flat_load_dword v0, v[0:1]1490; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01491; GFX8-NEXT: s_waitcnt vmcnt(1)1492; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 81493; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 81494; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 81495; GFX8-NEXT: s_waitcnt vmcnt(0)1496; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 81497; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 81498; GFX8-NEXT: s_waitcnt lgkmcnt(0)1499; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s01500; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 81501; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v11502; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v11503; GFX8-NEXT: v_mov_b32_e32 v0, s41504; GFX8-NEXT: v_mov_b32_e32 v1, s51505; GFX8-NEXT: flat_store_dword v[0:1], v21506; GFX8-NEXT: s_endpgm1507;1508; GFX9-NODL-LABEL: idot4_acc32_3ele:1509; GFX9-NODL: ; %bb.0: ; %entry1510; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241511; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341512; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01513; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1514; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1515; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1516; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01517; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01518; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1519; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 81520; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1521; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 81522; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11523; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_21524; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1525; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v3, v4, s01526; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v11527; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1528; GFX9-NODL-NEXT: s_endpgm1529;1530; GFX9-DL-LABEL: idot4_acc32_3ele:1531; GFX9-DL: ; %bb.0: ; %entry1532; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241533; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341534; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01535; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1536; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]1537; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]1538; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01539; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0201001540; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01541; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1542; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s11543; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1544; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s11545; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1546; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s01547; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1548; GFX9-DL-NEXT: s_endpgm1549;1550; GFX10-DL-LABEL: idot4_acc32_3ele:1551; GFX10-DL: ; %bb.0: ; %entry1552; GFX10-DL-NEXT: s_clause 0x11553; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241554; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341555; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01556; GFX10-DL-NEXT: v_mov_b32_e32 v3, 01557; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1558; GFX10-DL-NEXT: s_clause 0x11559; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]1560; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1561; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1562; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01563; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1564; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0201001565; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1566; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0201001567; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1568; GFX10-DL-NEXT: v_mov_b32_e32 v2, s01569; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v01570; GFX10-DL-NEXT: global_store_dword v3, v2, s[6:7]1571; GFX10-DL-NEXT: s_endpgm1572;1573; GFX11-DL-LABEL: idot4_acc32_3ele:1574; GFX11-DL: ; %bb.0: ; %entry1575; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241576; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01577; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341578; GFX11-DL-NEXT: v_mov_b32_e32 v2, 01579; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)1580; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01581; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1582; GFX11-DL-NEXT: s_clause 0x11583; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]1584; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]1585; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x01586; GFX11-DL-NEXT: s_waitcnt vmcnt(1)1587; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0201001588; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1589; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201001590; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1591; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1592; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]1593; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]1594; GFX11-DL-NEXT: s_endpgm1595 ptr addrspace(1) %src2,1596 ptr addrspace(1) nocapture %dst) {1597entry:1598 %idx = call i32 @llvm.amdgcn.workitem.id.x()1599 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1600 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11601 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1602 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21603 1604 %v1e0 = extractelement <4 x i8> %vec1, i64 01605 %cv1e0 = sext i8 %v1e0 to i321606 %v2e0 = extractelement <4 x i8> %vec2, i64 01607 %cv2e0 = sext i8 %v2e0 to i321608 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01609 1610 %v1e1 = extractelement <4 x i8> %vec1, i64 11611 %cv1e1 = sext i8 %v1e1 to i321612 %v2e1 = extractelement <4 x i8> %vec2, i64 11613 %cv2e1 = sext i8 %v2e1 to i321614 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11615 1616 %v1e2 = extractelement <4 x i8> %vec1, i64 21617 %cv1e2 = sext i8 %v1e2 to i321618 %v2e2 = extractelement <4 x i8> %vec2, i64 21619 %cv2e2 = sext i8 %v2e2 to i321620 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e21621 1622 %acc = load i32, ptr addrspace(1) %dst, align 41623 %add1 = add i32 %mul1, %acc1624 %add2 = add i32 %add1, %mul21625 %add3 = add i32 %add2, %mul31626 store i32 %add3, ptr addrspace(1) %dst, align 41627 ret void1628}1629 1630 1631define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,1632; GFX7-LABEL: idot4_acc32_3ele_permuted:1633; GFX7: ; %bb.0: ; %entry1634; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91635; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1636; GFX7-NEXT: s_mov_b32 s7, 0xf0001637; GFX7-NEXT: s_mov_b32 s10, 01638; GFX7-NEXT: s_mov_b32 s11, s71639; GFX7-NEXT: s_waitcnt lgkmcnt(0)1640; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1641; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01642; GFX7-NEXT: v_mov_b32_e32 v1, 01643; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641644; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1645; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641646; GFX7-NEXT: s_load_dword s0, s[4:5], 0x01647; GFX7-NEXT: s_mov_b32 s6, -11648; GFX7-NEXT: s_waitcnt vmcnt(1)1649; GFX7-NEXT: v_ashrrev_i32_e32 v1, 24, v21650; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 81651; GFX7-NEXT: s_waitcnt vmcnt(0)1652; GFX7-NEXT: v_ashrrev_i32_e32 v4, 24, v01653; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 81654; GFX7-NEXT: s_waitcnt lgkmcnt(0)1655; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s01656; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 81657; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 81658; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v11659; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v11660; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01661; GFX7-NEXT: s_endpgm1662;1663; GFX8-LABEL: idot4_acc32_3ele_permuted:1664; GFX8: ; %bb.0: ; %entry1665; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241666; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341667; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01668; GFX8-NEXT: s_waitcnt lgkmcnt(0)1669; GFX8-NEXT: v_mov_b32_e32 v1, s11670; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21671; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1672; GFX8-NEXT: flat_load_dword v3, v[0:1]1673; GFX8-NEXT: v_mov_b32_e32 v1, s31674; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21675; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1676; GFX8-NEXT: flat_load_dword v0, v[0:1]1677; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01678; GFX8-NEXT: s_waitcnt vmcnt(1)1679; GFX8-NEXT: v_ashrrev_i32_e32 v1, 24, v31680; GFX8-NEXT: v_bfe_i32 v4, v3, 0, 81681; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 81682; GFX8-NEXT: s_waitcnt vmcnt(0)1683; GFX8-NEXT: v_ashrrev_i32_e32 v2, 24, v01684; GFX8-NEXT: v_bfe_i32 v5, v0, 0, 81685; GFX8-NEXT: s_waitcnt lgkmcnt(0)1686; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s01687; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 81688; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v11689; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v11690; GFX8-NEXT: v_mov_b32_e32 v0, s41691; GFX8-NEXT: v_mov_b32_e32 v1, s51692; GFX8-NEXT: flat_store_dword v[0:1], v21693; GFX8-NEXT: s_endpgm1694;1695; GFX9-NODL-LABEL: idot4_acc32_3ele_permuted:1696; GFX9-NODL: ; %bb.0: ; %entry1697; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241698; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341699; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01700; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1701; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1702; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1703; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x01704; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01705; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1706; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v3, 24, v11707; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1708; GFX9-NODL-NEXT: v_ashrrev_i32_e32 v4, 24, v21709; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_01710; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_21711; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1712; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v3, v4, s01713; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v11714; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1715; GFX9-NODL-NEXT: s_endpgm1716;1717; GFX9-DL-LABEL: idot4_acc32_3ele_permuted:1718; GFX9-DL: ; %bb.0: ; %entry1719; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241720; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341721; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01722; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1723; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]1724; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]1725; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01726; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0200031727; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01728; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1729; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s11730; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1731; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s11732; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1733; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s01734; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1735; GFX9-DL-NEXT: s_endpgm1736;1737; GFX10-DL-LABEL: idot4_acc32_3ele_permuted:1738; GFX10-DL: ; %bb.0: ; %entry1739; GFX10-DL-NEXT: s_clause 0x11740; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241741; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341742; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01743; GFX10-DL-NEXT: v_mov_b32_e32 v3, 01744; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1745; GFX10-DL-NEXT: s_clause 0x11746; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]1747; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1748; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1749; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01750; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1751; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0200031752; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1753; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0200031754; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1755; GFX10-DL-NEXT: v_mov_b32_e32 v2, s01756; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v01757; GFX10-DL-NEXT: global_store_dword v3, v2, s[6:7]1758; GFX10-DL-NEXT: s_endpgm1759;1760; GFX11-DL-LABEL: idot4_acc32_3ele_permuted:1761; GFX11-DL: ; %bb.0: ; %entry1762; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241763; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01764; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341765; GFX11-DL-NEXT: v_mov_b32_e32 v2, 01766; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)1767; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01768; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1769; GFX11-DL-NEXT: s_clause 0x11770; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]1771; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]1772; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x01773; GFX11-DL-NEXT: s_waitcnt vmcnt(1)1774; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0200031775; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1776; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0200031777; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1778; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)1779; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]1780; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]1781; GFX11-DL-NEXT: s_endpgm1782 ptr addrspace(1) %src2,1783 ptr addrspace(1) nocapture %dst) {1784entry:1785 %idx = call i32 @llvm.amdgcn.workitem.id.x()1786 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1787 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11788 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1789 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21790 1791 %v1e0 = extractelement <4 x i8> %vec1, i64 31792 %cv1e0 = sext i8 %v1e0 to i321793 %v2e0 = extractelement <4 x i8> %vec2, i64 31794 %cv2e0 = sext i8 %v2e0 to i321795 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01796 1797 %v1e1 = extractelement <4 x i8> %vec1, i64 01798 %cv1e1 = sext i8 %v1e1 to i321799 %v2e1 = extractelement <4 x i8> %vec2, i64 01800 %cv2e1 = sext i8 %v2e1 to i321801 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11802 1803 %v1e2 = extractelement <4 x i8> %vec1, i64 21804 %cv1e2 = sext i8 %v1e2 to i321805 %v2e2 = extractelement <4 x i8> %vec2, i64 21806 %cv2e2 = sext i8 %v2e2 to i321807 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e21808 1809 %acc = load i32, ptr addrspace(1) %dst, align 41810 %add1 = add i32 %mul1, %acc1811 %add2 = add i32 %add1, %mul21812 %add3 = add i32 %add2, %mul31813 store i32 %add3, ptr addrspace(1) %dst, align 41814 ret void1815}1816 1817define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,1818; GFX7-LABEL: idot4_acc32_opt:1819; GFX7: ; %bb.0: ; %entry1820; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91821; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1822; GFX7-NEXT: s_mov_b32 s7, 0xf0001823; GFX7-NEXT: s_mov_b32 s10, 01824; GFX7-NEXT: s_mov_b32 s11, s71825; GFX7-NEXT: s_waitcnt lgkmcnt(0)1826; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]1827; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01828; GFX7-NEXT: v_mov_b32_e32 v1, 01829; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr641830; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]1831; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr641832; GFX7-NEXT: s_mov_b32 s6, -11833; GFX7-NEXT: s_waitcnt vmcnt(1)1834; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 81835; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 81836; GFX7-NEXT: s_waitcnt vmcnt(0)1837; GFX7-NEXT: v_bfe_i32 v6, v0, 8, 81838; GFX7-NEXT: v_bfe_i32 v5, v0, 0, 81839; GFX7-NEXT: v_mul_i32_i24_e32 v3, v3, v61840; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 81841; GFX7-NEXT: v_bfe_i32 v7, v0, 16, 81842; GFX7-NEXT: v_mad_i32_i24 v1, v1, v5, v31843; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v21844; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v01845; GFX7-NEXT: v_mad_i32_i24 v1, v4, v7, v11846; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v11847; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 01848; GFX7-NEXT: s_endpgm1849;1850; GFX8-LABEL: idot4_acc32_opt:1851; GFX8: ; %bb.0: ; %entry1852; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241853; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341854; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01855; GFX8-NEXT: s_waitcnt lgkmcnt(0)1856; GFX8-NEXT: v_mov_b32_e32 v1, s11857; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21858; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1859; GFX8-NEXT: flat_load_dword v3, v[0:1]1860; GFX8-NEXT: v_mov_b32_e32 v1, s31861; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21862; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1863; GFX8-NEXT: flat_load_dword v2, v[0:1]1864; GFX8-NEXT: v_mov_b32_e32 v0, s41865; GFX8-NEXT: v_mov_b32_e32 v1, s51866; GFX8-NEXT: s_waitcnt vmcnt(1)1867; GFX8-NEXT: v_bfe_i32 v4, v3, 0, 81868; GFX8-NEXT: v_bfe_i32 v7, v3, 16, 81869; GFX8-NEXT: s_waitcnt vmcnt(0)1870; GFX8-NEXT: v_bfe_i32 v5, v2, 0, 81871; GFX8-NEXT: v_mul_i32_i24_sdwa v6, sext(v3), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11872; GFX8-NEXT: v_bfe_i32 v8, v2, 16, 81873; GFX8-NEXT: v_mad_i32_i24 v4, v4, v5, v61874; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v31875; GFX8-NEXT: v_ashrrev_i32_e32 v2, 24, v21876; GFX8-NEXT: v_mad_i32_i24 v4, v7, v8, v41877; GFX8-NEXT: v_mad_i32_i24 v2, v3, v2, v41878; GFX8-NEXT: flat_store_dword v[0:1], v21879; GFX8-NEXT: s_endpgm1880;1881; GFX9-NODL-LABEL: idot4_acc32_opt:1882; GFX9-NODL: ; %bb.0: ; %entry1883; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241884; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341885; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v01886; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)1887; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]1888; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]1889; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 01890; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)1891; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 81892; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)1893; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 81894; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_11895; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v6, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_21896; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_31897; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v3, v4, v51898; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v11899; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]1900; GFX9-NODL-NEXT: s_endpgm1901;1902; GFX9-DL-LABEL: idot4_acc32_opt:1903; GFX9-DL: ; %bb.0: ; %entry1904; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241905; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341906; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01907; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1908; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1909; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1910; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01911; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1912; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, 01913; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1914; GFX9-DL-NEXT: s_endpgm1915;1916; GFX10-DL-LABEL: idot4_acc32_opt:1917; GFX10-DL: ; %bb.0: ; %entry1918; GFX10-DL-NEXT: s_clause 0x11919; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241920; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341921; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01922; GFX10-DL-NEXT: v_mov_b32_e32 v3, 01923; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1924; GFX10-DL-NEXT: s_clause 0x11925; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1926; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1927; GFX10-DL-NEXT: v_mov_b32_e32 v0, 01928; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1929; GFX10-DL-NEXT: v_dot4c_i32_i8 v0, v1, v21930; GFX10-DL-NEXT: global_store_dword v3, v0, s[6:7]1931; GFX10-DL-NEXT: s_endpgm1932;1933; GFX11-DL-LABEL: idot4_acc32_opt:1934; GFX11-DL: ; %bb.0: ; %entry1935; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x241936; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v01937; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x341938; GFX11-DL-NEXT: v_mov_b32_e32 v2, 01939; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)1940; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01941; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)1942; GFX11-DL-NEXT: s_clause 0x11943; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]1944; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]1945; GFX11-DL-NEXT: s_waitcnt vmcnt(0)1946; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v1, v0, 0 neg_lo:[1,1,0]1947; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]1948; GFX11-DL-NEXT: s_endpgm1949 ptr addrspace(1) %src2,1950 ptr addrspace(1) nocapture %dst) {1951entry:1952 %idx = call i32 @llvm.amdgcn.workitem.id.x()1953 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx1954 %vec1 = load <4 x i8>, ptr addrspace(1) %gep11955 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx1956 %vec2 = load <4 x i8>, ptr addrspace(1) %gep21957 1958 %v1e0 = extractelement <4 x i8> %vec1, i64 01959 %cv1e0 = sext i8 %v1e0 to i321960 %v2e0 = extractelement <4 x i8> %vec2, i64 01961 %cv2e0 = sext i8 %v2e0 to i321962 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e01963 1964 %v1e1 = extractelement <4 x i8> %vec1, i64 11965 %cv1e1 = sext i8 %v1e1 to i321966 %v2e1 = extractelement <4 x i8> %vec2, i64 11967 %cv2e1 = sext i8 %v2e1 to i321968 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e11969 1970 %v1e2 = extractelement <4 x i8> %vec1, i64 21971 %cv1e2 = sext i8 %v1e2 to i321972 %v2e2 = extractelement <4 x i8> %vec2, i64 21973 %cv2e2 = sext i8 %v2e2 to i321974 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e21975 1976 %v1e3 = extractelement <4 x i8> %vec1, i64 31977 %cv1e3 = sext i8 %v1e3 to i321978 %v2e3 = extractelement <4 x i8> %vec2, i64 31979 %cv2e3 = sext i8 %v2e3 to i321980 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e31981 1982 %add2 = add i32 %mul1, %mul21983 %add3 = add i32 %add2, %mul31984 %add4 = add i32 %add3, %mul41985 store i32 %add4, ptr addrspace(1) %dst, align 41986 ret void1987}1988 1989define amdgpu_kernel void @idot4_acc32_3src(ptr addrspace(1) %src1,1990; GFX7-LABEL: idot4_acc32_3src:1991; GFX7: ; %bb.0: ; %entry1992; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x91993; GFX7-NEXT: s_mov_b32 s11, 0xf0001994; GFX7-NEXT: s_mov_b32 s14, 01995; GFX7-NEXT: s_mov_b32 s15, s111996; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01997; GFX7-NEXT: s_waitcnt lgkmcnt(0)1998; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]1999; GFX7-NEXT: v_mov_b32_e32 v1, 02000; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr642001; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]2002; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr642003; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]2004; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr642005; GFX7-NEXT: s_load_dword s0, s[6:7], 0x02006; GFX7-NEXT: s_mov_b32 s10, -12007; GFX7-NEXT: s_mov_b32 s8, s62008; GFX7-NEXT: s_mov_b32 s9, s72009; GFX7-NEXT: s_waitcnt vmcnt(2)2010; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 82011; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 82012; GFX7-NEXT: s_waitcnt vmcnt(1)2013; GFX7-NEXT: v_bfe_i32 v3, v3, 8, 82014; GFX7-NEXT: s_waitcnt lgkmcnt(0)2015; GFX7-NEXT: v_mad_i32_i24 v1, v1, v1, s02016; GFX7-NEXT: v_bfe_i32 v5, v2, 16, 82017; GFX7-NEXT: s_waitcnt vmcnt(0)2018; GFX7-NEXT: v_bfe_i32 v6, v0, 16, 82019; GFX7-NEXT: v_mad_i32_i24 v1, v4, v3, v12020; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v22021; GFX7-NEXT: v_ashrrev_i32_e32 v0, 24, v02022; GFX7-NEXT: v_mad_i32_i24 v1, v5, v6, v12023; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v12024; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 02025; GFX7-NEXT: s_endpgm2026;2027; GFX8-LABEL: idot4_acc32_3src:2028; GFX8: ; %bb.0: ; %entry2029; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242030; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02031; GFX8-NEXT: s_waitcnt lgkmcnt(0)2032; GFX8-NEXT: v_mov_b32_e32 v1, s12033; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22034; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2035; GFX8-NEXT: flat_load_dword v3, v[0:1]2036; GFX8-NEXT: v_mov_b32_e32 v1, s32037; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22038; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2039; GFX8-NEXT: flat_load_dword v4, v[0:1]2040; GFX8-NEXT: v_mov_b32_e32 v1, s52041; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v22042; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2043; GFX8-NEXT: flat_load_dword v0, v[0:1]2044; GFX8-NEXT: s_load_dword s0, s[6:7], 0x02045; GFX8-NEXT: s_waitcnt vmcnt(2)2046; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 82047; GFX8-NEXT: v_bfe_i32 v2, v3, 8, 82048; GFX8-NEXT: s_waitcnt lgkmcnt(0)2049; GFX8-NEXT: v_mad_i32_i24 v1, v1, v1, s02050; GFX8-NEXT: v_bfe_i32 v5, v3, 16, 82051; GFX8-NEXT: v_ashrrev_i32_e32 v3, 24, v32052; GFX8-NEXT: s_waitcnt vmcnt(1)2053; GFX8-NEXT: v_bfe_i32 v4, v4, 8, 82054; GFX8-NEXT: v_mad_i32_i24 v1, v2, v4, v12055; GFX8-NEXT: s_waitcnt vmcnt(0)2056; GFX8-NEXT: v_bfe_i32 v6, v0, 16, 82057; GFX8-NEXT: v_ashrrev_i32_e32 v0, 24, v02058; GFX8-NEXT: v_mad_i32_i24 v1, v5, v6, v12059; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v12060; GFX8-NEXT: v_mov_b32_e32 v0, s62061; GFX8-NEXT: v_mov_b32_e32 v1, s72062; GFX8-NEXT: flat_store_dword v[0:1], v22063; GFX8-NEXT: s_endpgm2064;2065; GFX9-NODL-LABEL: idot4_acc32_3src:2066; GFX9-NODL: ; %bb.0: ; %entry2067; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242068; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02069; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2070; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]2071; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]2072; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]2073; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x02074; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02075; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2076; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_02077; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2078; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12079; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2080; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22081; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_32082; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2083; GFX9-NODL-NEXT: v_add3_u32 v2, v4, s0, v22084; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v12085; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]2086; GFX9-NODL-NEXT: s_endpgm2087;2088; GFX9-DL-LABEL: idot4_acc32_3src:2089; GFX9-DL: ; %bb.0: ; %entry2090; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242091; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02092; GFX9-DL-NEXT: s_mov_b32 s0, 0x706010c2093; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c0c002094; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2095; GFX9-DL-NEXT: global_load_dword v1, v0, s[10:11]2096; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]2097; GFX9-DL-NEXT: global_load_dword v3, v0, s[8:9]2098; GFX9-DL-NEXT: s_load_dword s1, s[14:15], 0x02099; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02100; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2101; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s02102; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2103; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s22104; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v22105; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2106; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v3, v1, s12107; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]2108; GFX9-DL-NEXT: s_endpgm2109;2110; GFX10-DL-LABEL: idot4_acc32_3src:2111; GFX10-DL: ; %bb.0: ; %entry2112; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242113; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02114; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2115; GFX10-DL-NEXT: s_clause 0x22116; GFX10-DL-NEXT: global_load_dword v1, v0, s[10:11]2117; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]2118; GFX10-DL-NEXT: global_load_dword v3, v0, s[8:9]2119; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x02120; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2121; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0x706010c2122; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2123; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c002124; GFX10-DL-NEXT: v_mov_b32_e32 v2, 02125; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v12126; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2127; GFX10-DL-NEXT: v_mov_b32_e32 v1, s02128; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v3, v02129; GFX10-DL-NEXT: global_store_dword v2, v1, s[14:15]2130; GFX10-DL-NEXT: s_endpgm2131;2132; GFX11-DL-LABEL: idot4_acc32_3src:2133; GFX11-DL: ; %bb.0: ; %entry2134; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x242135; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02136; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2137; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02138; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2139; GFX11-DL-NEXT: s_clause 0x22140; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]2141; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]2142; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]2143; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x02144; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2145; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0x706010c2146; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2147; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c002148; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)2149; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v22150; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02151; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2152; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]2153; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]2154; GFX11-DL-NEXT: s_endpgm2155 ptr addrspace(1) %src2,2156 ptr addrspace(1) %src3,2157 ptr addrspace(1) nocapture %dst) {2158entry:2159 %idx = call i32 @llvm.amdgcn.workitem.id.x()2160 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2161 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12162 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2163 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22164 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx2165 %vec3 = load <4 x i8>, ptr addrspace(1) %gep32166 2167 %v1e0 = extractelement <4 x i8> %vec1, i64 02168 %cv1e0 = sext i8 %v1e0 to i322169 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e02170 2171 %v1e1 = extractelement <4 x i8> %vec1, i64 12172 %cv1e1 = sext i8 %v1e1 to i322173 %v2e1 = extractelement <4 x i8> %vec2, i64 12174 %cv2e1 = sext i8 %v2e1 to i322175 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12176 2177 %v1e2 = extractelement <4 x i8> %vec1, i64 22178 %cv1e2 = sext i8 %v1e2 to i322179 %v3e2 = extractelement <4 x i8> %vec3, i64 22180 %cv3e2 = sext i8 %v3e2 to i322181 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e22182 2183 %v1e3 = extractelement <4 x i8> %vec1, i64 32184 %cv1e3 = sext i8 %v1e3 to i322185 %v3e3 = extractelement <4 x i8> %vec3, i64 32186 %cv3e3 = sext i8 %v3e3 to i322187 %mul4 = mul nuw nsw i32 %cv1e3, %cv3e32188 2189 %acc = load i32, ptr addrspace(1) %dst, align 42190 %add1 = add i32 %mul1, %acc2191 %add2 = add i32 %add1, %mul22192 %add3 = add i32 %add2, %mul32193 %add4 = add i32 %add3, %mul42194 store i32 %add4, ptr addrspace(1) %dst, align 42195 ret void2196}2197 2198define amdgpu_kernel void @idot4_acc32_3src_3ele(ptr addrspace(1) %src1,2199; GFX7-LABEL: idot4_acc32_3src_3ele:2200; GFX7: ; %bb.0: ; %entry2201; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92202; GFX7-NEXT: s_mov_b32 s11, 0xf0002203; GFX7-NEXT: s_mov_b32 s14, 02204; GFX7-NEXT: s_mov_b32 s15, s112205; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02206; GFX7-NEXT: s_waitcnt lgkmcnt(0)2207; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]2208; GFX7-NEXT: v_mov_b32_e32 v1, 02209; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr642210; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]2211; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr642212; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]2213; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr642214; GFX7-NEXT: s_load_dword s0, s[6:7], 0x02215; GFX7-NEXT: s_mov_b32 s10, -12216; GFX7-NEXT: s_mov_b32 s8, s62217; GFX7-NEXT: s_mov_b32 s9, s72218; GFX7-NEXT: s_waitcnt vmcnt(2)2219; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 82220; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 82221; GFX7-NEXT: s_waitcnt vmcnt(1)2222; GFX7-NEXT: v_bfe_i32 v3, v3, 8, 82223; GFX7-NEXT: s_waitcnt lgkmcnt(0)2224; GFX7-NEXT: v_mad_i32_i24 v1, v1, v1, s02225; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 82226; GFX7-NEXT: s_waitcnt vmcnt(0)2227; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 82228; GFX7-NEXT: v_mad_i32_i24 v1, v4, v3, v12229; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v12230; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 02231; GFX7-NEXT: s_endpgm2232;2233; GFX8-LABEL: idot4_acc32_3src_3ele:2234; GFX8: ; %bb.0: ; %entry2235; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242236; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02237; GFX8-NEXT: s_waitcnt lgkmcnt(0)2238; GFX8-NEXT: v_mov_b32_e32 v1, s12239; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22240; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2241; GFX8-NEXT: flat_load_dword v3, v[0:1]2242; GFX8-NEXT: v_mov_b32_e32 v1, s32243; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22244; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2245; GFX8-NEXT: flat_load_dword v4, v[0:1]2246; GFX8-NEXT: v_mov_b32_e32 v1, s52247; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v22248; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2249; GFX8-NEXT: flat_load_dword v0, v[0:1]2250; GFX8-NEXT: s_load_dword s0, s[6:7], 0x02251; GFX8-NEXT: s_waitcnt vmcnt(2)2252; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 82253; GFX8-NEXT: v_bfe_i32 v2, v3, 8, 82254; GFX8-NEXT: s_waitcnt lgkmcnt(0)2255; GFX8-NEXT: v_mad_i32_i24 v1, v1, v1, s02256; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 82257; GFX8-NEXT: s_waitcnt vmcnt(1)2258; GFX8-NEXT: v_bfe_i32 v4, v4, 8, 82259; GFX8-NEXT: v_mad_i32_i24 v1, v2, v4, v12260; GFX8-NEXT: s_waitcnt vmcnt(0)2261; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 82262; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v12263; GFX8-NEXT: v_mov_b32_e32 v0, s62264; GFX8-NEXT: v_mov_b32_e32 v1, s72265; GFX8-NEXT: flat_store_dword v[0:1], v22266; GFX8-NEXT: s_endpgm2267;2268; GFX9-NODL-LABEL: idot4_acc32_3src_3ele:2269; GFX9-NODL: ; %bb.0: ; %entry2270; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242271; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02272; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2273; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]2274; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]2275; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]2276; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x02277; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02278; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2279; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 82280; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2281; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12282; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2283; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22284; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2285; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v4, v4, s02286; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v12287; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]2288; GFX9-NODL-NEXT: s_endpgm2289;2290; GFX9-DL-LABEL: idot4_acc32_3src_3ele:2291; GFX9-DL: ; %bb.0: ; %entry2292; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242293; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02294; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c2295; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c002296; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0201002297; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2298; GFX9-DL-NEXT: global_load_dword v1, v0, s[10:11]2299; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]2300; GFX9-DL-NEXT: global_load_dword v3, v0, s[8:9]2301; GFX9-DL-NEXT: s_load_dword s3, s[14:15], 0x02302; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02303; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2304; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s02305; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2306; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s12307; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v22308; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s22309; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2310; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s32311; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]2312; GFX9-DL-NEXT: s_endpgm2313;2314; GFX10-DL-LABEL: idot4_acc32_3src_3ele:2315; GFX10-DL: ; %bb.0: ; %entry2316; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242317; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02318; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2319; GFX10-DL-NEXT: s_clause 0x22320; GFX10-DL-NEXT: global_load_dword v1, v0, s[10:11]2321; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]2322; GFX10-DL-NEXT: global_load_dword v3, v0, s[8:9]2323; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x02324; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2325; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc06010c2326; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2327; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c002328; GFX10-DL-NEXT: v_perm_b32 v2, v3, v3, 0xc0201002329; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02330; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v12331; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2332; GFX10-DL-NEXT: v_mov_b32_e32 v1, s02333; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v2, v02334; GFX10-DL-NEXT: global_store_dword v3, v1, s[14:15]2335; GFX10-DL-NEXT: s_endpgm2336;2337; GFX11-DL-LABEL: idot4_acc32_3src_3ele:2338; GFX11-DL: ; %bb.0: ; %entry2339; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x242340; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02341; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2342; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02343; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2344; GFX11-DL-NEXT: s_clause 0x22345; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]2346; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]2347; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]2348; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x02349; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2350; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc06010c2351; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2352; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c002353; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201002354; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)2355; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v22356; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02357; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2358; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]2359; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]2360; GFX11-DL-NEXT: s_endpgm2361 ptr addrspace(1) %src2,2362 ptr addrspace(1) %src3,2363 ptr addrspace(1) nocapture %dst) {2364entry:2365 %idx = call i32 @llvm.amdgcn.workitem.id.x()2366 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2367 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12368 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2369 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22370 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx2371 %vec3 = load <4 x i8>, ptr addrspace(1) %gep32372 2373 %v1e0 = extractelement <4 x i8> %vec1, i64 02374 %cv1e0 = sext i8 %v1e0 to i322375 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e02376 2377 %v1e1 = extractelement <4 x i8> %vec1, i64 12378 %cv1e1 = sext i8 %v1e1 to i322379 %v2e1 = extractelement <4 x i8> %vec2, i64 12380 %cv2e1 = sext i8 %v2e1 to i322381 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12382 2383 %v1e2 = extractelement <4 x i8> %vec1, i64 22384 %cv1e2 = sext i8 %v1e2 to i322385 %v3e2 = extractelement <4 x i8> %vec3, i64 22386 %cv3e2 = sext i8 %v3e2 to i322387 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e22388 2389 %acc = load i32, ptr addrspace(1) %dst, align 42390 %add1 = add i32 %mul1, %acc2391 %add2 = add i32 %add1, %mul22392 %add3 = add i32 %add2, %mul32393 store i32 %add3, ptr addrspace(1) %dst, align 42394 ret void2395}2396 2397define amdgpu_kernel void @idot4_bad_source(ptr addrspace(1) %src1,2398; GFX7-LABEL: idot4_bad_source:2399; GFX7: ; %bb.0: ; %entry2400; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92401; GFX7-NEXT: s_load_dword s12, s[4:5], 0xf2402; GFX7-NEXT: s_mov_b32 s7, 0xf0002403; GFX7-NEXT: s_mov_b32 s10, 02404; GFX7-NEXT: s_mov_b32 s11, s72405; GFX7-NEXT: s_waitcnt lgkmcnt(0)2406; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2407; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02408; GFX7-NEXT: v_mov_b32_e32 v1, 02409; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642410; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2411; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642412; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x112413; GFX7-NEXT: s_sext_i32_i16 s1, s122414; GFX7-NEXT: s_mov_b32 s6, -12415; GFX7-NEXT: s_waitcnt lgkmcnt(0)2416; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02417; GFX7-NEXT: s_waitcnt lgkmcnt(0)2418; GFX7-NEXT: v_mov_b32_e32 v1, s02419; GFX7-NEXT: s_waitcnt vmcnt(1)2420; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 82421; GFX7-NEXT: v_bfe_i32 v4, v2, 8, 82422; GFX7-NEXT: s_waitcnt vmcnt(0)2423; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 82424; GFX7-NEXT: v_mad_i32_i24 v1, v3, s1, v12425; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 82426; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 82427; GFX7-NEXT: v_mad_i32_i24 v1, v4, v5, v12428; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v12429; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02430; GFX7-NEXT: s_endpgm2431;2432; GFX8-LABEL: idot4_bad_source:2433; GFX8: ; %bb.0: ; %entry2434; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242435; GFX8-NEXT: s_load_dword s6, s[4:5], 0x3c2436; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02437; GFX8-NEXT: s_waitcnt lgkmcnt(0)2438; GFX8-NEXT: v_mov_b32_e32 v1, s12439; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22440; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2441; GFX8-NEXT: flat_load_dword v3, v[0:1]2442; GFX8-NEXT: v_mov_b32_e32 v1, s32443; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22444; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2445; GFX8-NEXT: flat_load_dword v0, v[0:1]2446; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x442447; GFX8-NEXT: s_sext_i32_i16 s3, s62448; GFX8-NEXT: s_waitcnt lgkmcnt(0)2449; GFX8-NEXT: s_load_dword s2, s[0:1], 0x02450; GFX8-NEXT: s_waitcnt lgkmcnt(0)2451; GFX8-NEXT: v_mov_b32_e32 v1, s22452; GFX8-NEXT: s_waitcnt vmcnt(1)2453; GFX8-NEXT: v_bfe_i32 v2, v3, 0, 82454; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 82455; GFX8-NEXT: v_mad_i32_i24 v1, v2, s3, v12456; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 82457; GFX8-NEXT: s_waitcnt vmcnt(0)2458; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 82459; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 82460; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v12461; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v12462; GFX8-NEXT: v_mov_b32_e32 v0, s02463; GFX8-NEXT: v_mov_b32_e32 v1, s12464; GFX8-NEXT: flat_store_dword v[0:1], v22465; GFX8-NEXT: s_endpgm2466;2467; GFX9-NODL-LABEL: idot4_bad_source:2468; GFX9-NODL: ; %bb.0: ; %entry2469; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242470; GFX9-NODL-NEXT: s_load_dword s6, s[4:5], 0x3c2471; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02472; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2473; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2474; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2475; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x442476; GFX9-NODL-NEXT: s_sext_i32_i16 s3, s62477; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02478; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2479; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x02480; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2481; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 82482; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2483; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12484; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22485; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2486; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s22487; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v3, s3, v22488; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v4, v12489; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]2490; GFX9-NODL-NEXT: s_endpgm2491;2492; GFX9-DL-LABEL: idot4_bad_source:2493; GFX9-DL: ; %bb.0: ; %entry2494; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242495; GFX9-DL-NEXT: s_load_dword s6, s[4:5], 0x3c2496; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02497; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2498; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2499; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2500; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x442501; GFX9-DL-NEXT: s_mov_b32 s3, 0xc0c02012502; GFX9-DL-NEXT: s_sext_i32_i16 s4, s62503; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02504; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2505; GFX9-DL-NEXT: s_load_dword s2, s[0:1], 0x02506; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2507; GFX9-DL-NEXT: v_mov_b32_e32 v3, s22508; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2509; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 82510; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2511; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s32512; GFX9-DL-NEXT: v_mad_i32_i24 v3, v4, s4, v32513; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s32514; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v2, v32515; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]2516; GFX9-DL-NEXT: s_endpgm2517;2518; GFX10-DL-LABEL: idot4_bad_source:2519; GFX10-DL: ; %bb.0: ; %entry2520; GFX10-DL-NEXT: s_clause 0x12521; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242522; GFX10-DL-NEXT: s_load_dword s6, s[4:5], 0x3c2523; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02524; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02525; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2526; GFX10-DL-NEXT: s_clause 0x12527; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2528; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2529; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2530; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x442531; GFX10-DL-NEXT: s_sext_i32_i16 s3, s62532; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2533; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x02534; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2535; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 82536; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2537; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c02012538; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c02012539; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2540; GFX10-DL-NEXT: v_mad_i32_i24 v0, v0, s3, s22541; GFX10-DL-NEXT: v_dot4c_i32_i8 v0, v1, v22542; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]2543; GFX10-DL-NEXT: s_endpgm2544;2545; GFX11-DL-LABEL: idot4_bad_source:2546; GFX11-DL: ; %bb.0: ; %entry2547; GFX11-DL-NEXT: s_clause 0x12548; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x242549; GFX11-DL-NEXT: s_load_b32 s6, s[4:5], 0x3c2550; GFX11-DL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x3ff, v02551; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2552; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02553; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2554; GFX11-DL-NEXT: s_clause 0x12555; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]2556; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]2557; GFX11-DL-NEXT: s_load_b64 s[0:1], s[4:5], 0x442558; GFX11-DL-NEXT: s_sext_i32_i16 s3, s62559; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2560; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x02561; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2562; GFX11-DL-NEXT: v_bfe_i32 v2, v1, 0, 82563; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2564; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c02012565; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c02012566; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2567; GFX11-DL-NEXT: v_mad_i32_i24 v2, v2, s3, s22568; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2569; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v1, v0, v2 neg_lo:[1,1,0]2570; GFX11-DL-NEXT: global_store_b32 v3, v0, s[0:1]2571; GFX11-DL-NEXT: s_endpgm2572 ptr addrspace(1) %src2,2573 ptr addrspace(1) %src3,2574 i16 %badsource,2575 ptr addrspace(1) nocapture %dst) {2576entry:2577 %idx = call i32 @llvm.amdgcn.workitem.id.x()2578 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2579 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12580 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2581 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22582 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx2583 %vec3 = load <4 x i8>, ptr addrspace(1) %gep32584 2585 %v1e0 = extractelement <4 x i8> %vec1, i64 02586 %cv1e0 = sext i8 %v1e0 to i322587 %v2e0 = extractelement <4 x i8> %vec2, i64 02588 %other = sext i16 %badsource to i322589 %mul1 = mul nuw nsw i32 %cv1e0, %other2590 2591 %v1e1 = extractelement <4 x i8> %vec1, i64 12592 %cv1e1 = sext i8 %v1e1 to i322593 %v2e1 = extractelement <4 x i8> %vec2, i64 12594 %cv2e1 = sext i8 %v2e1 to i322595 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12596 2597 %v2e2 = extractelement <4 x i8> %vec2, i64 22598 %cv2e2 = sext i8 %v2e2 to i322599 %v1e2 = extractelement <4 x i8> %vec1, i64 22600 %cv1e2 = sext i8 %v1e2 to i322601 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e22602 2603 2604 %acc = load i32, ptr addrspace(1) %dst, align 42605 %mad1 = add i32 %mul1, %acc2606 %mad2 = add i32 %mad1, %mul22607 %mad3 = add i32 %mad2, %mul32608 2609 store i32 %mad3, ptr addrspace(1) %dst, align 42610 ret void2611}2612 2613 2614define amdgpu_kernel void @idot4_commutative(ptr addrspace(1) %src1,2615; GFX7-LABEL: idot4_commutative:2616; GFX7: ; %bb.0: ; %entry2617; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92618; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf2619; GFX7-NEXT: s_mov_b32 s7, 0xf0002620; GFX7-NEXT: s_mov_b32 s10, 02621; GFX7-NEXT: s_mov_b32 s11, s72622; GFX7-NEXT: s_waitcnt lgkmcnt(0)2623; GFX7-NEXT: s_mov_b64 s[8:9], s[0:1]2624; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02625; GFX7-NEXT: v_mov_b32_e32 v1, 02626; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr642627; GFX7-NEXT: s_mov_b64 s[8:9], s[2:3]2628; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr642629; GFX7-NEXT: s_load_dword s0, s[4:5], 0x02630; GFX7-NEXT: s_mov_b32 s6, -12631; GFX7-NEXT: s_waitcnt vmcnt(1)2632; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 82633; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 82634; GFX7-NEXT: s_waitcnt vmcnt(0)2635; GFX7-NEXT: v_bfe_i32 v4, v0, 0, 82636; GFX7-NEXT: v_bfe_i32 v5, v0, 8, 82637; GFX7-NEXT: s_waitcnt lgkmcnt(0)2638; GFX7-NEXT: v_mad_i32_i24 v1, v1, v4, s02639; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 82640; GFX7-NEXT: v_bfe_i32 v2, v2, 16, 82641; GFX7-NEXT: v_mad_i32_i24 v1, v3, v5, v12642; GFX7-NEXT: v_mad_i32_i24 v0, v2, v0, v12643; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 02644; GFX7-NEXT: s_endpgm2645;2646; GFX8-LABEL: idot4_commutative:2647; GFX8: ; %bb.0: ; %entry2648; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242649; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3c2650; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02651; GFX8-NEXT: s_waitcnt lgkmcnt(0)2652; GFX8-NEXT: v_mov_b32_e32 v1, s12653; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22654; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2655; GFX8-NEXT: flat_load_dword v3, v[0:1]2656; GFX8-NEXT: v_mov_b32_e32 v1, s32657; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22658; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2659; GFX8-NEXT: flat_load_dword v0, v[0:1]2660; GFX8-NEXT: s_load_dword s0, s[4:5], 0x02661; GFX8-NEXT: s_waitcnt vmcnt(1)2662; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 82663; GFX8-NEXT: v_bfe_i32 v4, v3, 8, 82664; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 82665; GFX8-NEXT: s_waitcnt vmcnt(0)2666; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 82667; GFX8-NEXT: v_bfe_i32 v5, v0, 8, 82668; GFX8-NEXT: s_waitcnt lgkmcnt(0)2669; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s02670; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 82671; GFX8-NEXT: v_mad_i32_i24 v1, v4, v5, v12672; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v12673; GFX8-NEXT: v_mov_b32_e32 v0, s42674; GFX8-NEXT: v_mov_b32_e32 v1, s52675; GFX8-NEXT: flat_store_dword v[0:1], v22676; GFX8-NEXT: s_endpgm2677;2678; GFX9-NODL-LABEL: idot4_commutative:2679; GFX9-NODL: ; %bb.0: ; %entry2680; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242681; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2682; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02683; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2684; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]2685; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]2686; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x02687; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02688; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2689; GFX9-NODL-NEXT: v_bfe_i32 v3, v1, 0, 82690; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2691; GFX9-NODL-NEXT: v_bfe_i32 v4, v2, 0, 82692; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v5, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_12693; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22694; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2695; GFX9-NODL-NEXT: v_mad_i32_i24 v2, v3, v4, s02696; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v12697; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]2698; GFX9-NODL-NEXT: s_endpgm2699;2700; GFX9-DL-LABEL: idot4_commutative:2701; GFX9-DL: ; %bb.0: ; %entry2702; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242703; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2704; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02705; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2706; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]2707; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]2708; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02709; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0201002710; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02711; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2712; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s12713; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2714; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s12715; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2716; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s02717; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2718; GFX9-DL-NEXT: s_endpgm2719;2720; GFX10-DL-LABEL: idot4_commutative:2721; GFX10-DL: ; %bb.0: ; %entry2722; GFX10-DL-NEXT: s_clause 0x12723; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242724; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2725; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02726; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02727; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2728; GFX10-DL-NEXT: s_clause 0x12729; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]2730; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]2731; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2732; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02733; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2734; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0201002735; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2736; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0201002737; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2738; GFX10-DL-NEXT: v_mov_b32_e32 v2, s02739; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v02740; GFX10-DL-NEXT: global_store_dword v3, v2, s[6:7]2741; GFX10-DL-NEXT: s_endpgm2742;2743; GFX11-DL-LABEL: idot4_commutative:2744; GFX11-DL: ; %bb.0: ; %entry2745; GFX11-DL-NEXT: s_load_b128 s[0:3], s[4:5], 0x242746; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02747; GFX11-DL-NEXT: s_load_b64 s[4:5], s[4:5], 0x3c2748; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02749; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)2750; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02751; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2752; GFX11-DL-NEXT: s_clause 0x12753; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]2754; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]2755; GFX11-DL-NEXT: s_load_b32 s0, s[4:5], 0x02756; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2757; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0201002758; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2759; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201002760; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2761; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2762; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]2763; GFX11-DL-NEXT: global_store_b32 v2, v0, s[4:5]2764; GFX11-DL-NEXT: s_endpgm2765 ptr addrspace(1) %src2,2766 ptr addrspace(1) %src3,2767 ptr addrspace(1) nocapture %dst) {2768entry:2769 %idx = call i32 @llvm.amdgcn.workitem.id.x()2770 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2771 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12772 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2773 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22774 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx2775 %vec3 = load <4 x i8>, ptr addrspace(1) %gep32776 2777 %v1e0 = extractelement <4 x i8> %vec1, i64 02778 %cv1e0 = sext i8 %v1e0 to i322779 %v2e0 = extractelement <4 x i8> %vec2, i64 02780 %cv2e0 = sext i8 %v2e0 to i322781 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e02782 2783 %v1e1 = extractelement <4 x i8> %vec1, i64 12784 %cv1e1 = sext i8 %v1e1 to i322785 %v2e1 = extractelement <4 x i8> %vec2, i64 12786 %cv2e1 = sext i8 %v2e1 to i322787 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12788 2789 %v2e2 = extractelement <4 x i8> %vec2, i64 22790 %cv2e2 = sext i8 %v2e2 to i322791 %v1e2 = extractelement <4 x i8> %vec1, i64 22792 %cv1e2 = sext i8 %v1e2 to i322793 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e22794 2795 2796 %acc = load i32, ptr addrspace(1) %dst, align 42797 %mad1 = add i32 %mul1, %acc2798 %mad2 = add i32 %mad1, %mul22799 %mad3 = add i32 %mad2, %mul32800 2801 store i32 %mad3, ptr addrspace(1) %dst, align 42802 ret void2803}2804 2805define amdgpu_kernel void @idot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,2806; GFX7-LABEL: idot4_acc32_3src_3ele_src0:2807; GFX7: ; %bb.0: ; %entry2808; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92809; GFX7-NEXT: s_mov_b32 s11, 0xf0002810; GFX7-NEXT: s_mov_b32 s14, 02811; GFX7-NEXT: s_mov_b32 s15, s112812; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02813; GFX7-NEXT: s_waitcnt lgkmcnt(0)2814; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]2815; GFX7-NEXT: v_mov_b32_e32 v1, 02816; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr642817; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]2818; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr642819; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]2820; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr642821; GFX7-NEXT: s_load_dword s0, s[6:7], 0x02822; GFX7-NEXT: s_mov_b32 s10, -12823; GFX7-NEXT: s_mov_b32 s8, s62824; GFX7-NEXT: s_mov_b32 s9, s72825; GFX7-NEXT: s_waitcnt vmcnt(2)2826; GFX7-NEXT: v_bfe_i32 v1, v2, 8, 82827; GFX7-NEXT: s_waitcnt vmcnt(1)2828; GFX7-NEXT: v_bfe_i32 v2, v3, 8, 82829; GFX7-NEXT: s_waitcnt lgkmcnt(0)2830; GFX7-NEXT: v_mad_i32_i24 v4, v2, v2, s02831; GFX7-NEXT: v_bfe_i32 v3, v3, 16, 82832; GFX7-NEXT: s_waitcnt vmcnt(0)2833; GFX7-NEXT: v_bfe_i32 v0, v0, 16, 82834; GFX7-NEXT: v_mad_i32_i24 v1, v1, v2, v42835; GFX7-NEXT: v_mad_i32_i24 v0, v3, v0, v12836; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 02837; GFX7-NEXT: s_endpgm2838;2839; GFX8-LABEL: idot4_acc32_3src_3ele_src0:2840; GFX8: ; %bb.0: ; %entry2841; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242842; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02843; GFX8-NEXT: s_waitcnt lgkmcnt(0)2844; GFX8-NEXT: v_mov_b32_e32 v1, s12845; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22846; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2847; GFX8-NEXT: flat_load_dword v3, v[0:1]2848; GFX8-NEXT: v_mov_b32_e32 v1, s32849; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22850; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2851; GFX8-NEXT: flat_load_dword v4, v[0:1]2852; GFX8-NEXT: v_mov_b32_e32 v1, s52853; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v22854; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2855; GFX8-NEXT: flat_load_dword v0, v[0:1]2856; GFX8-NEXT: s_load_dword s0, s[6:7], 0x02857; GFX8-NEXT: s_waitcnt vmcnt(2)2858; GFX8-NEXT: v_bfe_i32 v2, v3, 8, 82859; GFX8-NEXT: s_waitcnt vmcnt(1)2860; GFX8-NEXT: v_bfe_i32 v1, v4, 8, 82861; GFX8-NEXT: v_bfe_i32 v3, v4, 16, 82862; GFX8-NEXT: s_waitcnt lgkmcnt(0)2863; GFX8-NEXT: v_mad_i32_i24 v4, v1, v1, s02864; GFX8-NEXT: v_mad_i32_i24 v1, v2, v1, v42865; GFX8-NEXT: s_waitcnt vmcnt(0)2866; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 82867; GFX8-NEXT: v_mad_i32_i24 v2, v3, v0, v12868; GFX8-NEXT: v_mov_b32_e32 v0, s62869; GFX8-NEXT: v_mov_b32_e32 v1, s72870; GFX8-NEXT: flat_store_dword v[0:1], v22871; GFX8-NEXT: s_endpgm2872;2873; GFX9-NODL-LABEL: idot4_acc32_3src_3ele_src0:2874; GFX9-NODL: ; %bb.0: ; %entry2875; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242876; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v02877; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2878; GFX9-NODL-NEXT: global_load_dword v1, v0, s[10:11]2879; GFX9-NODL-NEXT: global_load_dword v2, v0, s[12:13]2880; GFX9-NODL-NEXT: global_load_dword v3, v0, s[8:9]2881; GFX9-NODL-NEXT: s_load_dword s0, s[14:15], 0x02882; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 02883; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)2884; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 8, 82885; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)2886; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_22887; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)2888; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v3), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD2889; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)2890; GFX9-NODL-NEXT: v_mad_i32_i24 v3, v4, v4, s02891; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v12892; GFX9-NODL-NEXT: global_store_dword v0, v1, s[14:15]2893; GFX9-NODL-NEXT: s_endpgm2894;2895; GFX9-DL-LABEL: idot4_acc32_3src_3ele_src0:2896; GFX9-DL: ; %bb.0: ; %entry2897; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242898; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02899; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c2900; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c012901; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0201012902; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2903; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]2904; GFX9-DL-NEXT: global_load_dword v2, v0, s[12:13]2905; GFX9-DL-NEXT: global_load_dword v3, v0, s[10:11]2906; GFX9-DL-NEXT: s_load_dword s3, s[14:15], 0x02907; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02908; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2909; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s02910; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2911; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s12912; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v22913; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s22914; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2915; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v2, v1, s32916; GFX9-DL-NEXT: global_store_dword v0, v1, s[14:15]2917; GFX9-DL-NEXT: s_endpgm2918;2919; GFX10-DL-LABEL: idot4_acc32_3src_3ele_src0:2920; GFX10-DL: ; %bb.0: ; %entry2921; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242922; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02923; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2924; GFX10-DL-NEXT: s_clause 0x22925; GFX10-DL-NEXT: global_load_dword v1, v0, s[8:9]2926; GFX10-DL-NEXT: global_load_dword v2, v0, s[12:13]2927; GFX10-DL-NEXT: global_load_dword v3, v0, s[10:11]2928; GFX10-DL-NEXT: s_load_dword s0, s[14:15], 0x02929; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2930; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc06010c2931; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2932; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c012933; GFX10-DL-NEXT: v_perm_b32 v2, v3, v3, 0xc0201012934; GFX10-DL-NEXT: v_mov_b32_e32 v3, 02935; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v12936; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2937; GFX10-DL-NEXT: v_mov_b32_e32 v1, s02938; GFX10-DL-NEXT: v_dot4c_i32_i8 v1, v2, v02939; GFX10-DL-NEXT: global_store_dword v3, v1, s[14:15]2940; GFX10-DL-NEXT: s_endpgm2941;2942; GFX11-DL-LABEL: idot4_acc32_3src_3ele_src0:2943; GFX11-DL: ; %bb.0: ; %entry2944; GFX11-DL-NEXT: s_load_b256 s[0:7], s[4:5], 0x242945; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v02946; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)2947; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02948; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2949; GFX11-DL-NEXT: s_clause 0x22950; GFX11-DL-NEXT: global_load_b32 v1, v0, s[0:1]2951; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]2952; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]2953; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x02954; GFX11-DL-NEXT: s_waitcnt vmcnt(1)2955; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc06010c2956; GFX11-DL-NEXT: s_waitcnt vmcnt(0)2957; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c012958; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0201012959; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)2960; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v22961; GFX11-DL-NEXT: v_mov_b32_e32 v2, 02962; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)2963; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v1, s0 neg_lo:[1,1,0]2964; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]2965; GFX11-DL-NEXT: s_endpgm2966 ptr addrspace(1) %src2,2967 ptr addrspace(1) %src3,2968 ptr addrspace(1) nocapture %dst) {2969entry:2970 %idx = call i32 @llvm.amdgcn.workitem.id.x()2971 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx2972 %vec1 = load <4 x i8>, ptr addrspace(1) %gep12973 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx2974 %vec2 = load <4 x i8>, ptr addrspace(1) %gep22975 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx2976 %vec3 = load <4 x i8>, ptr addrspace(1) %gep32977 2978 %v2e0 = extractelement <4 x i8> %vec2, i64 12979 %cv2e0 = sext i8 %v2e0 to i322980 %mul1 = mul nuw nsw i32 %cv2e0, %cv2e02981 2982 %v1e1 = extractelement <4 x i8> %vec1, i64 12983 %cv1e1 = sext i8 %v1e1 to i322984 %v2e1 = extractelement <4 x i8> %vec2, i64 12985 %cv2e1 = sext i8 %v2e1 to i322986 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e12987 2988 %v3e2 = extractelement <4 x i8> %vec3, i64 22989 %cv3e2 = sext i8 %v3e2 to i322990 %v2e2 = extractelement <4 x i8> %vec2, i64 22991 %cv2e2 = sext i8 %v2e2 to i322992 %mul3 = mul nuw nsw i32 %cv2e2, %cv3e22993 2994 2995 %acc = load i32, ptr addrspace(1) %dst, align 42996 %mad1 = add i32 %mul1, %acc2997 %mad2 = add i32 %mad1, %mul22998 %mad3 = add i32 %mad2, %mul32999 3000 store i32 %mad3, ptr addrspace(1) %dst, align 43001 ret void3002}3003 3004define amdgpu_kernel void @idot4_4src(ptr addrspace(1) %src1,3005; GFX7-LABEL: idot4_4src:3006; GFX7: ; %bb.0: ; %entry3007; GFX7-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x93008; GFX7-NEXT: s_mov_b32 s3, 0xf0003009; GFX7-NEXT: s_mov_b32 s18, 03010; GFX7-NEXT: s_mov_b32 s19, s33011; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03012; GFX7-NEXT: s_waitcnt lgkmcnt(0)3013; GFX7-NEXT: s_mov_b64 s[16:17], s[8:9]3014; GFX7-NEXT: v_mov_b32_e32 v1, 03015; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[16:19], 0 addr643016; GFX7-NEXT: s_mov_b64 s[16:17], s[10:11]3017; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[16:19], 0 addr643018; GFX7-NEXT: s_mov_b64 s[16:17], s[12:13]3019; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[16:19], 0 addr643020; GFX7-NEXT: s_mov_b64 s[16:17], s[14:15]3021; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[16:19], 0 addr643022; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x113023; GFX7-NEXT: s_mov_b32 s2, -13024; GFX7-NEXT: s_waitcnt lgkmcnt(0)3025; GFX7-NEXT: s_load_dword s4, s[0:1], 0x03026; GFX7-NEXT: s_waitcnt vmcnt(3)3027; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 83028; GFX7-NEXT: v_bfe_i32 v2, v2, 8, 83029; GFX7-NEXT: s_waitcnt vmcnt(2)3030; GFX7-NEXT: v_bfe_i32 v5, v3, 0, 83031; GFX7-NEXT: v_bfe_i32 v3, v3, 8, 83032; GFX7-NEXT: s_waitcnt lgkmcnt(0)3033; GFX7-NEXT: v_mad_i32_i24 v1, v1, v2, s43034; GFX7-NEXT: s_waitcnt vmcnt(1)3035; GFX7-NEXT: v_bfe_i32 v2, v4, 0, 83036; GFX7-NEXT: v_bfe_i32 v4, v4, 8, 83037; GFX7-NEXT: v_mad_i32_i24 v1, v5, v3, v13038; GFX7-NEXT: s_waitcnt vmcnt(0)3039; GFX7-NEXT: v_bfe_i32 v3, v0, 0, 83040; GFX7-NEXT: v_bfe_i32 v0, v0, 8, 83041; GFX7-NEXT: v_mad_i32_i24 v1, v2, v4, v13042; GFX7-NEXT: v_mad_i32_i24 v0, v3, v0, v13043; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 03044; GFX7-NEXT: s_endpgm3045;3046; GFX8-LABEL: idot4_4src:3047; GFX8: ; %bb.0: ; %entry3048; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243049; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03050; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x443051; GFX8-NEXT: s_waitcnt lgkmcnt(0)3052; GFX8-NEXT: v_mov_b32_e32 v1, s93053; GFX8-NEXT: v_add_u32_e32 v0, vcc, s8, v23054; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3055; GFX8-NEXT: flat_load_dword v3, v[0:1]3056; GFX8-NEXT: v_mov_b32_e32 v1, s113057; GFX8-NEXT: v_add_u32_e32 v0, vcc, s10, v23058; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3059; GFX8-NEXT: flat_load_dword v4, v[0:1]3060; GFX8-NEXT: v_mov_b32_e32 v1, s133061; GFX8-NEXT: v_add_u32_e32 v0, vcc, s12, v23062; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3063; GFX8-NEXT: flat_load_dword v5, v[0:1]3064; GFX8-NEXT: v_mov_b32_e32 v1, s153065; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v23066; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3067; GFX8-NEXT: flat_load_dword v0, v[0:1]3068; GFX8-NEXT: s_load_dword s2, s[0:1], 0x03069; GFX8-NEXT: s_waitcnt vmcnt(3)3070; GFX8-NEXT: v_bfe_i32 v1, v3, 0, 83071; GFX8-NEXT: v_bfe_i32 v2, v3, 8, 83072; GFX8-NEXT: s_waitcnt lgkmcnt(0)3073; GFX8-NEXT: v_mad_i32_i24 v1, v1, v2, s23074; GFX8-NEXT: s_waitcnt vmcnt(2)3075; GFX8-NEXT: v_bfe_i32 v3, v4, 0, 83076; GFX8-NEXT: v_bfe_i32 v4, v4, 8, 83077; GFX8-NEXT: v_mad_i32_i24 v1, v3, v4, v13078; GFX8-NEXT: s_waitcnt vmcnt(1)3079; GFX8-NEXT: v_bfe_i32 v6, v5, 0, 83080; GFX8-NEXT: v_bfe_i32 v5, v5, 8, 83081; GFX8-NEXT: v_mad_i32_i24 v1, v6, v5, v13082; GFX8-NEXT: s_waitcnt vmcnt(0)3083; GFX8-NEXT: v_bfe_i32 v7, v0, 0, 83084; GFX8-NEXT: v_bfe_i32 v0, v0, 8, 83085; GFX8-NEXT: v_mad_i32_i24 v2, v7, v0, v13086; GFX8-NEXT: v_mov_b32_e32 v0, s03087; GFX8-NEXT: v_mov_b32_e32 v1, s13088; GFX8-NEXT: flat_store_dword v[0:1], v23089; GFX8-NEXT: s_endpgm3090;3091; GFX9-NODL-LABEL: idot4_4src:3092; GFX9-NODL: ; %bb.0: ; %entry3093; GFX9-NODL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243094; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03095; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x443096; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3097; GFX9-NODL-NEXT: global_load_dword v1, v0, s[8:9]3098; GFX9-NODL-NEXT: global_load_dword v2, v0, s[10:11]3099; GFX9-NODL-NEXT: global_load_dword v3, v0, s[12:13]3100; GFX9-NODL-NEXT: global_load_dword v4, v0, s[14:15]3101; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03102; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x03103; GFX9-NODL-NEXT: s_waitcnt vmcnt(3)3104; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_13105; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)3106; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v2, sext(v2), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_13107; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3108; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v3, sext(v3), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_13109; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3110; GFX9-NODL-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_13111; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3112; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s2, v23113; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v43114; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]3115; GFX9-NODL-NEXT: s_endpgm3116;3117; GFX9-DL-LABEL: idot4_4src:3118; GFX9-DL: ; %bb.0: ; %entry3119; GFX9-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243120; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03121; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x443122; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c05013123; GFX9-DL-NEXT: s_mov_b32 s3, 0x5010c0c3124; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3125; GFX9-DL-NEXT: global_load_dword v1, v0, s[8:9]3126; GFX9-DL-NEXT: global_load_dword v2, v0, s[10:11]3127; GFX9-DL-NEXT: global_load_dword v3, v0, s[12:13]3128; GFX9-DL-NEXT: global_load_dword v4, v0, s[14:15]3129; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c04003130; GFX9-DL-NEXT: s_load_dword s6, s[0:1], 0x03131; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c3132; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03133; GFX9-DL-NEXT: s_waitcnt vmcnt(2)3134; GFX9-DL-NEXT: v_perm_b32 v5, v2, v1, s23135; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s43136; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3137; GFX9-DL-NEXT: v_perm_b32 v6, v4, v3, s33138; GFX9-DL-NEXT: v_perm_b32 v2, v4, v3, s53139; GFX9-DL-NEXT: v_or_b32_e32 v3, v6, v53140; GFX9-DL-NEXT: v_or_b32_e32 v1, v2, v13141; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3142; GFX9-DL-NEXT: v_dot4_i32_i8 v1, v1, v3, s63143; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]3144; GFX9-DL-NEXT: s_endpgm3145;3146; GFX10-DL-LABEL: idot4_4src:3147; GFX10-DL: ; %bb.0: ; %entry3148; GFX10-DL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243149; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03150; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x443151; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3152; GFX10-DL-NEXT: s_clause 0x33153; GFX10-DL-NEXT: global_load_dword v1, v0, s[8:9]3154; GFX10-DL-NEXT: global_load_dword v2, v0, s[10:11]3155; GFX10-DL-NEXT: global_load_dword v3, v0, s[12:13]3156; GFX10-DL-NEXT: global_load_dword v4, v0, s[14:15]3157; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x03158; GFX10-DL-NEXT: s_waitcnt vmcnt(2)3159; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc0c05013160; GFX10-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c04003161; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3162; GFX10-DL-NEXT: v_perm_b32 v5, v4, v3, 0x5010c0c3163; GFX10-DL-NEXT: v_perm_b32 v2, v4, v3, 0x4000c0c3164; GFX10-DL-NEXT: v_mov_b32_e32 v3, 03165; GFX10-DL-NEXT: v_or_b32_e32 v0, v5, v03166; GFX10-DL-NEXT: v_or_b32_e32 v1, v2, v13167; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3168; GFX10-DL-NEXT: v_mov_b32_e32 v2, s23169; GFX10-DL-NEXT: v_dot4c_i32_i8 v2, v1, v03170; GFX10-DL-NEXT: global_store_dword v3, v2, s[0:1]3171; GFX10-DL-NEXT: s_endpgm3172;3173; GFX11-DL-LABEL: idot4_4src:3174; GFX11-DL: ; %bb.0: ; %entry3175; GFX11-DL-NEXT: s_load_b256 s[8:15], s[4:5], 0x243176; GFX11-DL-NEXT: v_and_b32_e32 v0, 0x3ff, v03177; GFX11-DL-NEXT: s_load_b64 s[0:1], s[4:5], 0x443178; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)3179; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03180; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3181; GFX11-DL-NEXT: s_clause 0x33182; GFX11-DL-NEXT: global_load_b32 v1, v0, s[8:9]3183; GFX11-DL-NEXT: global_load_b32 v2, v0, s[10:11]3184; GFX11-DL-NEXT: global_load_b32 v3, v0, s[12:13]3185; GFX11-DL-NEXT: global_load_b32 v0, v0, s[14:15]3186; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x03187; GFX11-DL-NEXT: s_waitcnt vmcnt(2)3188; GFX11-DL-NEXT: v_perm_b32 v4, v2, v1, 0xc0c05013189; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c04003190; GFX11-DL-NEXT: s_waitcnt vmcnt(0)3191; GFX11-DL-NEXT: v_perm_b32 v5, v0, v3, 0x5010c0c3192; GFX11-DL-NEXT: v_perm_b32 v0, v0, v3, 0x4000c0c3193; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)3194; GFX11-DL-NEXT: v_or_b32_e32 v2, v5, v43195; GFX11-DL-NEXT: v_or_b32_e32 v0, v0, v13196; GFX11-DL-NEXT: v_mov_b32_e32 v1, 03197; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)3198; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)3199; GFX11-DL-NEXT: v_dot4_i32_iu8 v0, v0, v2, s2 neg_lo:[1,1,0]3200; GFX11-DL-NEXT: global_store_b32 v1, v0, s[0:1]3201; GFX11-DL-NEXT: s_endpgm3202 ptr addrspace(1) %src2,3203 ptr addrspace(1) %src3,3204 ptr addrspace(1) %src4,3205 ptr addrspace(1) nocapture %dst) {3206entry:3207 %idx = call i32 @llvm.amdgcn.workitem.id.x()3208 3209 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3210 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13211 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3212 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23213 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx3214 %vec3 = load <4 x i8>, ptr addrspace(1) %gep33215 %gep4 = getelementptr <4 x i8>, ptr addrspace(1) %src4, i32 %idx3216 %vec4 = load <4 x i8>, ptr addrspace(1) %gep43217 3218 3219 %v1e0 = extractelement <4 x i8> %vec1, i64 03220 %cv1e0 = sext i8 %v1e0 to i323221 %v1e1 = extractelement <4 x i8> %vec1, i64 13222 %cv1e1 = sext i8 %v1e1 to i323223 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e13224 3225 %v2e0 = extractelement <4 x i8> %vec2, i64 03226 %cv2e0 = sext i8 %v2e0 to i323227 %v2e1 = extractelement <4 x i8> %vec2, i64 13228 %cv2e1 = sext i8 %v2e1 to i323229 %mul2 = mul nuw nsw i32 %cv2e0, %cv2e13230 3231 %v3e0 = extractelement <4 x i8> %vec3, i64 03232 %cv3e0 = sext i8 %v3e0 to i323233 %v3e1 = extractelement <4 x i8> %vec3, i64 13234 %cv3e1 = sext i8 %v3e1 to i323235 %mul3 = mul nuw nsw i32 %cv3e0, %cv3e13236 3237 %v4e0 = extractelement <4 x i8> %vec4, i64 03238 %cv4e0 = sext i8 %v4e0 to i323239 %v4e1 = extractelement <4 x i8> %vec4, i64 13240 %cv4e1 = sext i8 %v4e1 to i323241 %mul4 = mul nuw nsw i32 %cv4e0, %cv4e13242 3243 3244 %acc = load i32, ptr addrspace(1) %dst, align 43245 %mad1 = add i32 %mul1, %acc3246 %mad2 = add i32 %mad1, %mul23247 %mad3 = add i32 %mad2, %mul33248 %mad4 = add i32 %mad3, %mul43249 3250 store i32 %mad4, ptr addrspace(1) %dst, align 43251 ret void3252}3253 3254define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,3255; GFX7-LABEL: idot4_nonstandard_signed:3256; GFX7: ; %bb.0: ; %entry3257; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x93258; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd3259; GFX7-NEXT: s_mov_b32 s3, 0xf0003260; GFX7-NEXT: s_mov_b32 s6, 03261; GFX7-NEXT: s_mov_b32 s7, s33262; GFX7-NEXT: s_waitcnt lgkmcnt(0)3263; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]3264; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03265; GFX7-NEXT: v_mov_b32_e32 v1, 03266; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr643267; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]3268; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr643269; GFX7-NEXT: s_mov_b32 s2, -13270; GFX7-NEXT: s_waitcnt vmcnt(1)3271; GFX7-NEXT: v_bfe_i32 v1, v2, 0, 83272; GFX7-NEXT: v_bfe_i32 v3, v2, 8, 83273; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v13274; GFX7-NEXT: s_waitcnt vmcnt(0)3275; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v03276; GFX7-NEXT: v_bfe_i32 v4, v2, 16, 83277; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 83278; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v33279; GFX7-NEXT: v_mul_u32_u24_e32 v1, v1, v53280; GFX7-NEXT: v_ashrrev_i32_e32 v2, 24, v23281; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 83282; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v43283; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v13284; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v03285; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v13286; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v23287; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v13288; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 163289; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 03290; GFX7-NEXT: s_endpgm3291;3292; GFX8-LABEL: idot4_nonstandard_signed:3293; GFX8: ; %bb.0: ; %entry3294; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243295; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343296; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03297; GFX8-NEXT: v_mov_b32_e32 v4, 0xff3298; GFX8-NEXT: s_waitcnt lgkmcnt(0)3299; GFX8-NEXT: v_mov_b32_e32 v1, s13300; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23301; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3302; GFX8-NEXT: flat_load_dword v3, v[0:1]3303; GFX8-NEXT: v_mov_b32_e32 v1, s33304; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23305; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3306; GFX8-NEXT: flat_load_dword v2, v[0:1]3307; GFX8-NEXT: v_mov_b32_e32 v0, s43308; GFX8-NEXT: v_mov_b32_e32 v1, s53309; GFX8-NEXT: s_waitcnt vmcnt(1)3310; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v33311; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v33312; GFX8-NEXT: v_bfe_i32 v7, v7, 0, 83313; GFX8-NEXT: v_bfe_i32 v5, v5, 0, 83314; GFX8-NEXT: s_waitcnt vmcnt(0)3315; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v23316; GFX8-NEXT: v_mul_lo_u16_sdwa v6, sext(v3), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_03317; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v83318; GFX8-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD3319; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v33320; GFX8-NEXT: v_mad_u16 v6, v8, v7, v63321; GFX8-NEXT: v_bfe_i32 v3, v3, 0, 83322; GFX8-NEXT: v_mad_u16 v4, v4, v5, v63323; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v23324; GFX8-NEXT: v_mad_u16 v2, v3, v2, v43325; GFX8-NEXT: v_bfe_i32 v2, v2, 0, 163326; GFX8-NEXT: flat_store_dword v[0:1], v23327; GFX8-NEXT: s_endpgm3328;3329; GFX9-NODL-LABEL: idot4_nonstandard_signed:3330; GFX9-NODL: ; %bb.0: ; %entry3331; GFX9-NODL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243332; GFX9-NODL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343333; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v03334; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)3335; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]3336; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]3337; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff3338; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 03339; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)3340; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v13341; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)3342; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v23343; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 16, v13344; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v4, sext(v1), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_03345; GFX9-NODL-NEXT: v_bfe_i32 v5, v5, 0, 83346; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v63347; GFX9-NODL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD3348; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v13349; GFX9-NODL-NEXT: v_bfe_i32 v3, v3, 0, 83350; GFX9-NODL-NEXT: v_mad_legacy_u16 v4, v6, v5, v43351; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 83352; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v3, v43353; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v23354; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v33355; GFX9-NODL-NEXT: v_bfe_i32 v1, v1, 0, 163356; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]3357; GFX9-NODL-NEXT: s_endpgm3358;3359; GFX9-DL-LABEL: idot4_nonstandard_signed:3360; GFX9-DL: ; %bb.0: ; %entry3361; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243362; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343363; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03364; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3365; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]3366; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]3367; GFX9-DL-NEXT: s_movk_i32 s0, 0xff3368; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03369; GFX9-DL-NEXT: s_waitcnt vmcnt(1)3370; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v13371; GFX9-DL-NEXT: s_waitcnt vmcnt(0)3372; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v23373; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v13374; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, sext(v1), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_03375; GFX9-DL-NEXT: v_bfe_i32 v5, v5, 0, 83376; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v63377; GFX9-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD3378; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v13379; GFX9-DL-NEXT: v_bfe_i32 v3, v3, 0, 83380; GFX9-DL-NEXT: v_mad_legacy_u16 v4, v6, v5, v43381; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 83382; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v7, v3, v43383; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v23384; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v33385; GFX9-DL-NEXT: v_bfe_i32 v1, v1, 0, 163386; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]3387; GFX9-DL-NEXT: s_endpgm3388;3389; GFX10-DL-LABEL: idot4_nonstandard_signed:3390; GFX10-DL: ; %bb.0: ; %entry3391; GFX10-DL-NEXT: s_clause 0x13392; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243393; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343394; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03395; GFX10-DL-NEXT: v_mov_b32_e32 v6, 0xff3396; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3397; GFX10-DL-NEXT: s_clause 0x13398; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]3399; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]3400; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3401; GFX10-DL-NEXT: v_bfe_i32 v0, v1, 0, 83402; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3403; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xff, v23404; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v13405; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v23406; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v13407; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v13408; GFX10-DL-NEXT: v_mul_lo_u16 v0, v0, v33409; GFX10-DL-NEXT: v_bfe_i32 v3, v4, 0, 83410; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v53411; GFX10-DL-NEXT: v_and_b32_sdwa v5, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD3412; GFX10-DL-NEXT: v_bfe_i32 v6, v7, 0, 83413; GFX10-DL-NEXT: v_bfe_i32 v1, v1, 0, 83414; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 24, v23415; GFX10-DL-NEXT: v_mad_u16 v0, v4, v3, v03416; GFX10-DL-NEXT: v_mad_u16 v0, v5, v6, v03417; GFX10-DL-NEXT: v_mad_u16 v0, v1, v2, v03418; GFX10-DL-NEXT: v_mov_b32_e32 v1, 03419; GFX10-DL-NEXT: v_bfe_i32 v0, v0, 0, 163420; GFX10-DL-NEXT: global_store_dword v1, v0, s[6:7]3421; GFX10-DL-NEXT: s_endpgm3422;3423; GFX11-DL-TRUE16-LABEL: idot4_nonstandard_signed:3424; GFX11-DL-TRUE16: ; %bb.0: ; %entry3425; GFX11-DL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x243426; GFX11-DL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v03427; GFX11-DL-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x343428; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3429; GFX11-DL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v03430; GFX11-DL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3431; GFX11-DL-TRUE16-NEXT: s_clause 0x13432; GFX11-DL-TRUE16-NEXT: global_load_b32 v2, v0, s[0:1]3433; GFX11-DL-TRUE16-NEXT: global_load_b32 v3, v0, s[2:3]3434; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(1)3435; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v23436; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v1, v2, 0, 83437; GFX11-DL-TRUE16-NEXT: s_waitcnt vmcnt(0)3438; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l3439; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33440; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h3441; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 83442; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.h3443; GFX11-DL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l3444; GFX11-DL-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l3445; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v5, v6, 0, 83446; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l3447; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v23448; GFX11-DL-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v33449; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)3450; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l3451; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l3452; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)3453; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v4, v4, 0, 83454; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.h, v2.l, v0.l3455; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3456; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l3457; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v1.l, v3.l, v0.l3458; GFX11-DL-TRUE16-NEXT: v_mov_b32_e32 v1, 03459; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3460; GFX11-DL-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 163461; GFX11-DL-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]3462; GFX11-DL-TRUE16-NEXT: s_endpgm3463;3464; GFX11-DL-FAKE16-LABEL: idot4_nonstandard_signed:3465; GFX11-DL-FAKE16: ; %bb.0: ; %entry3466; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x243467; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v03468; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x343469; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3470; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v03471; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)3472; GFX11-DL-FAKE16-NEXT: s_clause 0x13473; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]3474; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]3475; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)3476; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 83477; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)3478; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v03479; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v13480; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v03481; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v13482; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v03483; GFX11-DL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v33484; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v3, v4, 0, 83485; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v53486; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v13487; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v5, v6, 0, 83488; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v73489; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v03490; GFX11-DL-FAKE16-NEXT: v_mad_u16 v2, v4, v3, v23491; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 83492; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3493; GFX11-DL-FAKE16-NEXT: v_mad_u16 v2, v6, v5, v23494; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v23495; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v1, 03496; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3497; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 163498; GFX11-DL-FAKE16-NEXT: global_store_b32 v1, v0, s[4:5]3499; GFX11-DL-FAKE16-NEXT: s_endpgm3500 ptr addrspace(1) %src2,3501 ptr addrspace(1) nocapture %dst) {3502entry:3503 %idx = call i32 @llvm.amdgcn.workitem.id.x()3504 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx3505 %vec1 = load <4 x i8>, ptr addrspace(1) %gep13506 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx3507 %vec2 = load <4 x i8>, ptr addrspace(1) %gep23508 %v1e0 = extractelement <4 x i8> %vec1, i64 03509 %v1e0e = sext i8 %v1e0 to i163510 %v2e0 = extractelement <4 x i8> %vec2, i64 03511 %v2e0e = zext i8 %v2e0 to i163512 %mul0 = mul nsw i16 %v1e0e, %v2e0e3513 %add0 = add i16 %mul0, 03514 3515 %v1e1 = extractelement <4 x i8> %vec1, i64 13516 %v1e1e = sext i8 %v1e1 to i163517 %v2e1 = extractelement <4 x i8> %vec2, i64 13518 %v2e1e = zext i8 %v2e1 to i163519 %mul1 = mul nsw i16 %v2e1e, %v1e1e3520 %add1 = add i16 %mul1, %add03521 %v1e2 = extractelement <4 x i8> %vec1, i64 23522 %v1e2e = sext i8 %v1e2 to i163523 %v2e2 = extractelement <4 x i8> %vec2, i64 23524 %v2e2e = zext i8 %v2e2 to i163525 %mul2 = mul nsw i16 %v2e2e, %v1e2e3526 %add2 = add i16 %mul2, %add13527 %v1e3 = extractelement <4 x i8> %vec1, i64 33528 %v1e3e = sext i8 %v1e3 to i163529 %v2e3 = extractelement <4 x i8> %vec2, i64 33530 %v2e3e = zext i8 %v2e3 to i163531 %mul3 = mul nsw i16 %v1e3e, %v2e3e3532 %add3 = add i16 %mul3, %add23533 %res = sext i16 %add3 to i323534 store i32 %res, ptr addrspace(1) %dst, align 43535 ret void3536}3537 3538 3539declare i32 @llvm.amdgcn.workitem.id.x()3540