3354 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck -check-prefixes=GFX9-DL %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 < %s | FileCheck -check-prefixes=GFX10-DL %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 < %s | FileCheck -check-prefixes=GFX10-DL %s8 9define amdgpu_kernel void @udot8_acc32(ptr addrspace(1) %src1,10; GFX7-LABEL: udot8_acc32:11; GFX7: ; %bb.0: ; %entry12; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD013; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD114; GFX7-NEXT: s_mov_b32 s14, -115; GFX7-NEXT: s_mov_b32 s15, 0xe8f00016; GFX7-NEXT: s_add_u32 s12, s12, s1117; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x918; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd19; GFX7-NEXT: s_mov_b32 s3, 0xf00020; GFX7-NEXT: s_mov_b32 s6, 021; GFX7-NEXT: s_mov_b32 s7, s322; GFX7-NEXT: s_waitcnt lgkmcnt(0)23; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]24; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v025; GFX7-NEXT: v_mov_b32_e32 v1, 026; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr6427; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]28; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr6429; GFX7-NEXT: s_load_dword s4, s[0:1], 0x030; GFX7-NEXT: s_mov_b32 s2, -131; GFX7-NEXT: s_addc_u32 s13, s13, 032; GFX7-NEXT: s_waitcnt vmcnt(1)33; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v234; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 435; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 436; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 437; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 438; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 439; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 440; GFX7-NEXT: v_and_b32_e32 v2, 15, v241; GFX7-NEXT: s_waitcnt vmcnt(0)42; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v043; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 444; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 445; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 446; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 447; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 448; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 449; GFX7-NEXT: v_and_b32_e32 v0, 15, v050; GFX7-NEXT: s_waitcnt lgkmcnt(0)51; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s452; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v053; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v054; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v055; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v056; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v057; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v058; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v059; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 060; GFX7-NEXT: s_endpgm61;62; GFX8-LABEL: udot8_acc32:63; GFX8: ; %bb.0: ; %entry64; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2465; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3466; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v067; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD068; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD169; GFX8-NEXT: s_waitcnt lgkmcnt(0)70; GFX8-NEXT: v_mov_b32_e32 v1, s171; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v272; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc73; GFX8-NEXT: flat_load_dword v3, v[0:1]74; GFX8-NEXT: v_mov_b32_e32 v1, s375; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v276; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc77; GFX8-NEXT: flat_load_dword v0, v[0:1]78; GFX8-NEXT: s_load_dword s0, s[4:5], 0x079; GFX8-NEXT: s_mov_b32 s14, -180; GFX8-NEXT: s_mov_b32 s15, 0xe8000081; GFX8-NEXT: s_add_u32 s12, s12, s1182; GFX8-NEXT: s_addc_u32 s13, s13, 083; GFX8-NEXT: s_waitcnt vmcnt(1)84; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v385; GFX8-NEXT: v_bfe_u32 v2, v3, 24, 486; GFX8-NEXT: v_bfe_u32 v4, v3, 20, 487; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 488; GFX8-NEXT: v_bfe_u32 v6, v3, 12, 489; GFX8-NEXT: v_bfe_u32 v7, v3, 8, 490; GFX8-NEXT: v_bfe_u32 v8, v3, 4, 491; GFX8-NEXT: v_and_b32_e32 v3, 15, v392; GFX8-NEXT: s_waitcnt vmcnt(0)93; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v094; GFX8-NEXT: v_bfe_u32 v10, v0, 24, 495; GFX8-NEXT: v_bfe_u32 v11, v0, 20, 496; GFX8-NEXT: v_bfe_u32 v12, v0, 16, 497; GFX8-NEXT: v_bfe_u32 v13, v0, 12, 498; GFX8-NEXT: v_bfe_u32 v14, v0, 8, 499; GFX8-NEXT: v_bfe_u32 v15, v0, 4, 4100; GFX8-NEXT: v_and_b32_e32 v0, 15, v0101; GFX8-NEXT: s_waitcnt lgkmcnt(0)102; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s0103; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v0104; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v0105; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v0106; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v0107; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v0108; GFX8-NEXT: v_mad_u32_u24 v0, v2, v10, v0109; GFX8-NEXT: v_mad_u32_u24 v2, v1, v9, v0110; GFX8-NEXT: v_mov_b32_e32 v0, s4111; GFX8-NEXT: v_mov_b32_e32 v1, s5112; GFX8-NEXT: flat_store_dword v[0:1], v2113; GFX8-NEXT: s_endpgm114;115; GFX9-LABEL: udot8_acc32:116; GFX9: ; %bb.0: ; %entry117; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24118; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34119; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0120; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0121; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1122; GFX9-NEXT: s_waitcnt lgkmcnt(0)123; GFX9-NEXT: global_load_dword v1, v0, s[0:1]124; GFX9-NEXT: global_load_dword v2, v0, s[2:3]125; GFX9-NEXT: s_load_dword s0, s[6:7], 0x0126; GFX9-NEXT: s_mov_b32 s14, -1127; GFX9-NEXT: s_mov_b32 s15, 0xe00000128; GFX9-NEXT: s_add_u32 s12, s12, s11129; GFX9-NEXT: v_mov_b32_e32 v0, 0130; GFX9-NEXT: s_addc_u32 s13, s13, 0131; GFX9-NEXT: s_waitcnt vmcnt(1)132; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v1133; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 4134; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 4135; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 4136; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 4137; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 4138; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 4139; GFX9-NEXT: v_and_b32_e32 v1, 15, v1140; GFX9-NEXT: s_waitcnt vmcnt(0)141; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v2142; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 4143; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 4144; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 4145; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 4146; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 4147; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 4148; GFX9-NEXT: v_and_b32_e32 v2, 15, v2149; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v2150; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v16151; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v15152; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v14153; GFX9-NEXT: s_waitcnt lgkmcnt(0)154; GFX9-NEXT: v_add3_u32 v1, v1, s0, v2155; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v13156; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v12157; GFX9-NEXT: v_add3_u32 v1, v1, v8, v7158; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v11159; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v10160; GFX9-NEXT: v_add3_u32 v1, v1, v6, v5161; GFX9-NEXT: v_add3_u32 v1, v1, v4, v3162; GFX9-NEXT: global_store_dword v0, v1, s[6:7]163; GFX9-NEXT: s_endpgm164;165; GFX9-DL-LABEL: udot8_acc32:166; GFX9-DL: ; %bb.0: ; %entry167; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24168; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34169; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0170; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0171; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1172; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)173; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]174; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]175; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0176; GFX9-DL-NEXT: s_mov_b32 s14, -1177; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000178; GFX9-DL-NEXT: s_add_u32 s12, s12, s11179; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0180; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0181; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)182; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s0183; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]184; GFX9-DL-NEXT: s_endpgm185;186; GFX10-DL-LABEL: udot8_acc32:187; GFX10-DL: ; %bb.0: ; %entry188; GFX10-DL-NEXT: s_clause 0x1189; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24190; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34191; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0192; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0193; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1194; GFX10-DL-NEXT: s_mov_b32 s14, -1195; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000196; GFX10-DL-NEXT: s_add_u32 s12, s12, s11197; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0198; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)199; GFX10-DL-NEXT: s_clause 0x1200; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]201; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]202; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)203; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0204; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0205; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)206; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s0207; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]208; GFX10-DL-NEXT: s_endpgm209 ptr addrspace(1) %src2,210 ptr addrspace(1) nocapture %dst) {211entry:212 %idx = call i32 @llvm.amdgcn.workitem.id.x()213 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx214 %vec1 = load <8 x i4>, ptr addrspace(1) %gep1215 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx216 %vec2 = load <8 x i4>, ptr addrspace(1) %gep2217 218 %v1e0 = extractelement <8 x i4> %vec1, i64 0219 %cv1e0 = zext i4 %v1e0 to i32220 %v2e0 = extractelement <8 x i4> %vec2, i64 0221 %cv2e0 = zext i4 %v2e0 to i32222 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0223 224 %v1e1 = extractelement <8 x i4> %vec1, i64 1225 %cv1e1 = zext i4 %v1e1 to i32226 %v2e1 = extractelement <8 x i4> %vec2, i64 1227 %cv2e1 = zext i4 %v2e1 to i32228 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1229 230 %v1e2 = extractelement <8 x i4> %vec1, i64 2231 %cv1e2 = zext i4 %v1e2 to i32232 %v2e2 = extractelement <8 x i4> %vec2, i64 2233 %cv2e2 = zext i4 %v2e2 to i32234 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2235 236 %v1e3 = extractelement <8 x i4> %vec1, i64 3237 %cv1e3 = zext i4 %v1e3 to i32238 %v2e3 = extractelement <8 x i4> %vec2, i64 3239 %cv2e3 = zext i4 %v2e3 to i32240 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3241 242 %v1e4 = extractelement <8 x i4> %vec1, i64 4243 %cv1e4 = zext i4 %v1e4 to i32244 %v2e4 = extractelement <8 x i4> %vec2, i64 4245 %cv2e4 = zext i4 %v2e4 to i32246 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4247 248 %v1e5 = extractelement <8 x i4> %vec1, i64 5249 %cv1e5 = zext i4 %v1e5 to i32250 %v2e5 = extractelement <8 x i4> %vec2, i64 5251 %cv2e5 = zext i4 %v2e5 to i32252 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5253 254 %v1e6 = extractelement <8 x i4> %vec1, i64 6255 %cv1e6 = zext i4 %v1e6 to i32256 %v2e6 = extractelement <8 x i4> %vec2, i64 6257 %cv2e6 = zext i4 %v2e6 to i32258 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6259 260 %v1e7 = extractelement <8 x i4> %vec1, i64 7261 %cv1e7 = zext i4 %v1e7 to i32262 %v2e7 = extractelement <8 x i4> %vec2, i64 7263 %cv2e7 = zext i4 %v2e7 to i32264 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7265 266 %acc = load i32, ptr addrspace(1) %dst, align 4267 %add1 = add i32 %mul0, %acc268 %add2 = add i32 %add1, %mul1269 %add3 = add i32 %add2, %mul2270 %add4 = add i32 %add3, %mul3271 %add5 = add i32 %add4, %mul4272 %add6 = add i32 %add5, %mul5273 %add7 = add i32 %add6, %mul6274 %add8 = add i32 %add7, %mul7275 276 store i32 %add8, ptr addrspace(1) %dst, align 4277 ret void278}279 280; TODO: Remove the unnecessary instruction(that is zero-extending the281; 2nd MAD) to have the pattern-recognizer to kick in.282define amdgpu_kernel void @udot8_acc16(ptr addrspace(1) %src1,283; GFX7-LABEL: udot8_acc16:284; GFX7: ; %bb.0: ; %entry285; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0286; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1287; GFX7-NEXT: s_mov_b32 s14, -1288; GFX7-NEXT: s_mov_b32 s15, 0xe8f000289; GFX7-NEXT: s_add_u32 s12, s12, s11290; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9291; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd292; GFX7-NEXT: s_mov_b32 s3, 0xf000293; GFX7-NEXT: s_mov_b32 s6, 0294; GFX7-NEXT: s_mov_b32 s7, s3295; GFX7-NEXT: s_waitcnt lgkmcnt(0)296; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]297; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0298; GFX7-NEXT: v_mov_b32_e32 v1, 0299; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64300; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]301; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64302; GFX7-NEXT: s_mov_b32 s2, -1303; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0304; GFX7-NEXT: s_addc_u32 s13, s13, 0305; GFX7-NEXT: s_waitcnt vmcnt(2)306; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v2307; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 4308; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 4309; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 4310; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 4311; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 4312; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 4313; GFX7-NEXT: v_and_b32_e32 v2, 15, v2314; GFX7-NEXT: s_waitcnt vmcnt(1)315; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v0316; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 4317; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 4318; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 4319; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 4320; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 4321; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 4322; GFX7-NEXT: v_and_b32_e32 v0, 15, v0323; GFX7-NEXT: s_waitcnt vmcnt(0)324; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1325; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v0326; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0327; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0328; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0329; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0330; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0331; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0332; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0333; GFX7-NEXT: s_endpgm334;335; GFX8-LABEL: udot8_acc16:336; GFX8: ; %bb.0: ; %entry337; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24338; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34339; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0340; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0341; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1342; GFX8-NEXT: s_waitcnt lgkmcnt(0)343; GFX8-NEXT: v_mov_b32_e32 v1, s1344; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2345; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc346; GFX8-NEXT: flat_load_dword v3, v[0:1]347; GFX8-NEXT: v_mov_b32_e32 v1, s3348; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2349; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc350; GFX8-NEXT: flat_load_dword v2, v[0:1]351; GFX8-NEXT: v_mov_b32_e32 v0, s4352; GFX8-NEXT: v_mov_b32_e32 v1, s5353; GFX8-NEXT: flat_load_ushort v4, v[0:1]354; GFX8-NEXT: s_mov_b32 s14, -1355; GFX8-NEXT: s_mov_b32 s15, 0xe80000356; GFX8-NEXT: s_add_u32 s12, s12, s11357; GFX8-NEXT: s_addc_u32 s13, s13, 0358; GFX8-NEXT: s_waitcnt vmcnt(2)359; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v3360; GFX8-NEXT: v_bfe_u32 v6, v3, 24, 4361; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 4362; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 4363; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 4364; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 4365; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 4366; GFX8-NEXT: v_and_b32_e32 v3, 15, v3367; GFX8-NEXT: s_waitcnt vmcnt(1)368; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v2369; GFX8-NEXT: v_bfe_u32 v13, v2, 24, 4370; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 4371; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 4372; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 4373; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 4374; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 4375; GFX8-NEXT: v_and_b32_e32 v2, 15, v2376; GFX8-NEXT: s_waitcnt vmcnt(0)377; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4378; GFX8-NEXT: v_mad_u16 v2, v11, v18, v2379; GFX8-NEXT: v_mad_u16 v2, v10, v17, v2380; GFX8-NEXT: v_mad_u16 v2, v9, v16, v2381; GFX8-NEXT: v_mad_u16 v2, v8, v15, v2382; GFX8-NEXT: v_mad_u16 v2, v7, v14, v2383; GFX8-NEXT: v_mad_u16 v2, v6, v13, v2384; GFX8-NEXT: v_mad_u16 v2, v5, v12, v2385; GFX8-NEXT: flat_store_short v[0:1], v2386; GFX8-NEXT: s_endpgm387;388; GFX9-LABEL: udot8_acc16:389; GFX9: ; %bb.0: ; %entry390; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24391; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34392; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0393; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0394; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1395; GFX9-NEXT: s_waitcnt lgkmcnt(0)396; GFX9-NEXT: global_load_dword v1, v0, s[0:1]397; GFX9-NEXT: global_load_dword v2, v0, s[2:3]398; GFX9-NEXT: v_mov_b32_e32 v0, 0399; GFX9-NEXT: global_load_ushort v3, v0, s[6:7]400; GFX9-NEXT: s_mov_b32 s14, -1401; GFX9-NEXT: s_mov_b32 s15, 0xe00000402; GFX9-NEXT: s_add_u32 s12, s12, s11403; GFX9-NEXT: s_addc_u32 s13, s13, 0404; GFX9-NEXT: s_waitcnt vmcnt(2)405; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v1406; GFX9-NEXT: v_bfe_u32 v5, v1, 24, 4407; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 4408; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 4409; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 4410; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 4411; GFX9-NEXT: v_bfe_u32 v10, v1, 4, 4412; GFX9-NEXT: v_and_b32_e32 v1, 15, v1413; GFX9-NEXT: s_waitcnt vmcnt(1)414; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2415; GFX9-NEXT: v_bfe_u32 v12, v2, 24, 4416; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 4417; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 4418; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 4419; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 4420; GFX9-NEXT: v_bfe_u32 v17, v2, 4, 4421; GFX9-NEXT: v_and_b32_e32 v2, 15, v2422; GFX9-NEXT: s_waitcnt vmcnt(0)423; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3424; GFX9-NEXT: v_mad_legacy_u16 v1, v10, v17, v1425; GFX9-NEXT: v_mad_legacy_u16 v1, v9, v16, v1426; GFX9-NEXT: v_mad_legacy_u16 v1, v8, v15, v1427; GFX9-NEXT: v_mad_legacy_u16 v1, v7, v14, v1428; GFX9-NEXT: v_mad_legacy_u16 v1, v6, v13, v1429; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v12, v1430; GFX9-NEXT: v_mad_legacy_u16 v1, v4, v11, v1431; GFX9-NEXT: global_store_short v0, v1, s[6:7]432; GFX9-NEXT: s_endpgm433;434; GFX9-DL-LABEL: udot8_acc16:435; GFX9-DL: ; %bb.0: ; %entry436; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24437; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34438; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0439; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0440; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1441; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)442; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]443; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]444; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0445; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]446; GFX9-DL-NEXT: s_mov_b32 s14, -1447; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000448; GFX9-DL-NEXT: s_add_u32 s12, s12, s11449; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0450; GFX9-DL-NEXT: s_waitcnt vmcnt(2)451; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1452; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4453; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4454; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4455; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4456; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4457; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4458; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1459; GFX9-DL-NEXT: s_waitcnt vmcnt(1)460; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2461; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4462; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4463; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4464; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4465; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4466; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4467; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2468; GFX9-DL-NEXT: s_waitcnt vmcnt(0)469; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3470; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1471; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1472; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1473; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1474; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1475; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1476; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1477; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]478; GFX9-DL-NEXT: s_endpgm479;480; GFX10-DL-LABEL: udot8_acc16:481; GFX10-DL: ; %bb.0: ; %entry482; GFX10-DL-NEXT: s_clause 0x1483; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24484; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34485; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0486; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0487; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0488; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1489; GFX10-DL-NEXT: s_mov_b32 s14, -1490; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000491; GFX10-DL-NEXT: s_add_u32 s12, s12, s11492; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0493; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)494; GFX10-DL-NEXT: s_clause 0x1495; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]496; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]497; GFX10-DL-NEXT: global_load_ushort v4, v1, s[6:7]498; GFX10-DL-NEXT: s_waitcnt vmcnt(2)499; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2500; GFX10-DL-NEXT: s_waitcnt vmcnt(1)501; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3502; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4503; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4504; GFX10-DL-NEXT: s_waitcnt vmcnt(0)505; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4506; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4507; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4508; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0509; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4510; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4511; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0512; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4513; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4514; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0515; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4516; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4517; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0518; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 24, 4519; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4520; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2521; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3522; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0523; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0524; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0525; GFX10-DL-NEXT: global_store_short v1, v0, s[6:7]526; GFX10-DL-NEXT: s_endpgm527 ptr addrspace(1) %src2,528 ptr addrspace(1) nocapture %dst) {529entry:530 %idx = call i32 @llvm.amdgcn.workitem.id.x()531 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx532 %vec1 = load <8 x i4>, ptr addrspace(1) %gep1533 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx534 %vec2 = load <8 x i4>, ptr addrspace(1) %gep2535 536 %v1e0 = extractelement <8 x i4> %vec1, i64 0537 %cv1e0 = zext i4 %v1e0 to i16538 %v2e0 = extractelement <8 x i4> %vec2, i64 0539 %cv2e0 = zext i4 %v2e0 to i16540 %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0541 542 %v1e1 = extractelement <8 x i4> %vec1, i64 1543 %cv1e1 = zext i4 %v1e1 to i16544 %v2e1 = extractelement <8 x i4> %vec2, i64 1545 %cv2e1 = zext i4 %v2e1 to i16546 %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1547 548 %v1e2 = extractelement <8 x i4> %vec1, i64 2549 %cv1e2 = zext i4 %v1e2 to i16550 %v2e2 = extractelement <8 x i4> %vec2, i64 2551 %cv2e2 = zext i4 %v2e2 to i16552 %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2553 554 %v1e3 = extractelement <8 x i4> %vec1, i64 3555 %cv1e3 = zext i4 %v1e3 to i16556 %v2e3 = extractelement <8 x i4> %vec2, i64 3557 %cv2e3 = zext i4 %v2e3 to i16558 %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3559 560 %v1e4 = extractelement <8 x i4> %vec1, i64 4561 %cv1e4 = zext i4 %v1e4 to i16562 %v2e4 = extractelement <8 x i4> %vec2, i64 4563 %cv2e4 = zext i4 %v2e4 to i16564 %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4565 566 %v1e5 = extractelement <8 x i4> %vec1, i64 5567 %cv1e5 = zext i4 %v1e5 to i16568 %v2e5 = extractelement <8 x i4> %vec2, i64 5569 %cv2e5 = zext i4 %v2e5 to i16570 %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5571 572 %v1e6 = extractelement <8 x i4> %vec1, i64 6573 %cv1e6 = zext i4 %v1e6 to i16574 %v2e6 = extractelement <8 x i4> %vec2, i64 6575 %cv2e6 = zext i4 %v2e6 to i16576 %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6577 578 %v1e7 = extractelement <8 x i4> %vec1, i64 7579 %cv1e7 = zext i4 %v1e7 to i16580 %v2e7 = extractelement <8 x i4> %vec2, i64 7581 %cv2e7 = zext i4 %v2e7 to i16582 %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7583 584 %acc = load i16, ptr addrspace(1) %dst, align 4585 %add1 = add i16 %mul0, %acc586 %add2 = add i16 %add1, %mul1587 %add3 = add i16 %add2, %mul2588 %add4 = add i16 %add3, %mul3589 %add5 = add i16 %add4, %mul4590 %add6 = add i16 %add5, %mul5591 %add7 = add i16 %add6, %mul6592 %add8 = add i16 %add7, %mul7593 594 store i16 %add8, ptr addrspace(1) %dst, align 4595 ret void596}597 598; TODO: Remove the unnecessary instruction(that is zero-extending the599; 2nd MAD) to have the pattern-recognizer to kick in.600define amdgpu_kernel void @udot8_acc8(ptr addrspace(1) %src1,601; GFX7-LABEL: udot8_acc8:602; GFX7: ; %bb.0: ; %entry603; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0604; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1605; GFX7-NEXT: s_mov_b32 s14, -1606; GFX7-NEXT: s_mov_b32 s15, 0xe8f000607; GFX7-NEXT: s_add_u32 s12, s12, s11608; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9609; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd610; GFX7-NEXT: s_mov_b32 s3, 0xf000611; GFX7-NEXT: s_mov_b32 s6, 0612; GFX7-NEXT: s_mov_b32 s7, s3613; GFX7-NEXT: s_waitcnt lgkmcnt(0)614; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]615; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0616; GFX7-NEXT: v_mov_b32_e32 v1, 0617; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64618; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]619; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64620; GFX7-NEXT: s_mov_b32 s2, -1621; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0622; GFX7-NEXT: s_addc_u32 s13, s13, 0623; GFX7-NEXT: s_waitcnt vmcnt(2)624; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v2625; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 4626; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 4627; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 4628; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 4629; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 4630; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 4631; GFX7-NEXT: v_and_b32_e32 v2, 15, v2632; GFX7-NEXT: s_waitcnt vmcnt(1)633; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v0634; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 4635; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 4636; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 4637; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 4638; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 4639; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 4640; GFX7-NEXT: v_and_b32_e32 v0, 15, v0641; GFX7-NEXT: s_waitcnt vmcnt(0)642; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1643; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v0644; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0645; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0646; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0647; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0648; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0649; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0650; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0651; GFX7-NEXT: s_endpgm652;653; GFX8-LABEL: udot8_acc8:654; GFX8: ; %bb.0: ; %entry655; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24656; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34657; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0658; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0659; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1660; GFX8-NEXT: s_waitcnt lgkmcnt(0)661; GFX8-NEXT: v_mov_b32_e32 v1, s1662; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2663; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc664; GFX8-NEXT: flat_load_dword v3, v[0:1]665; GFX8-NEXT: v_mov_b32_e32 v1, s3666; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2667; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc668; GFX8-NEXT: flat_load_dword v2, v[0:1]669; GFX8-NEXT: v_mov_b32_e32 v0, s4670; GFX8-NEXT: v_mov_b32_e32 v1, s5671; GFX8-NEXT: flat_load_ubyte v4, v[0:1]672; GFX8-NEXT: s_mov_b32 s14, -1673; GFX8-NEXT: s_mov_b32 s15, 0xe80000674; GFX8-NEXT: s_add_u32 s12, s12, s11675; GFX8-NEXT: s_addc_u32 s13, s13, 0676; GFX8-NEXT: s_waitcnt vmcnt(2)677; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v3678; GFX8-NEXT: v_bfe_u32 v6, v3, 24, 4679; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 4680; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 4681; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 4682; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 4683; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 4684; GFX8-NEXT: v_and_b32_e32 v3, 15, v3685; GFX8-NEXT: s_waitcnt vmcnt(1)686; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v2687; GFX8-NEXT: v_bfe_u32 v13, v2, 24, 4688; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 4689; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 4690; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 4691; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 4692; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 4693; GFX8-NEXT: v_and_b32_e32 v2, 15, v2694; GFX8-NEXT: s_waitcnt vmcnt(0)695; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4696; GFX8-NEXT: v_mad_u16 v2, v11, v18, v2697; GFX8-NEXT: v_mad_u16 v2, v10, v17, v2698; GFX8-NEXT: v_mad_u16 v2, v9, v16, v2699; GFX8-NEXT: v_mad_u16 v2, v8, v15, v2700; GFX8-NEXT: v_mad_u16 v2, v7, v14, v2701; GFX8-NEXT: v_mad_u16 v2, v6, v13, v2702; GFX8-NEXT: v_mad_u16 v2, v5, v12, v2703; GFX8-NEXT: flat_store_byte v[0:1], v2704; GFX8-NEXT: s_endpgm705;706; GFX9-LABEL: udot8_acc8:707; GFX9: ; %bb.0: ; %entry708; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24709; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34710; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0711; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0712; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1713; GFX9-NEXT: s_waitcnt lgkmcnt(0)714; GFX9-NEXT: global_load_dword v1, v0, s[0:1]715; GFX9-NEXT: global_load_dword v2, v0, s[2:3]716; GFX9-NEXT: v_mov_b32_e32 v0, 0717; GFX9-NEXT: global_load_ubyte v3, v0, s[6:7]718; GFX9-NEXT: s_mov_b32 s14, -1719; GFX9-NEXT: s_mov_b32 s15, 0xe00000720; GFX9-NEXT: s_add_u32 s12, s12, s11721; GFX9-NEXT: s_addc_u32 s13, s13, 0722; GFX9-NEXT: s_waitcnt vmcnt(2)723; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v1724; GFX9-NEXT: v_bfe_u32 v5, v1, 24, 4725; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 4726; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 4727; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 4728; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 4729; GFX9-NEXT: v_bfe_u32 v10, v1, 4, 4730; GFX9-NEXT: v_and_b32_e32 v1, 15, v1731; GFX9-NEXT: s_waitcnt vmcnt(1)732; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v2733; GFX9-NEXT: v_bfe_u32 v12, v2, 24, 4734; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 4735; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 4736; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 4737; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 4738; GFX9-NEXT: v_bfe_u32 v17, v2, 4, 4739; GFX9-NEXT: v_and_b32_e32 v2, 15, v2740; GFX9-NEXT: s_waitcnt vmcnt(0)741; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v3742; GFX9-NEXT: v_mad_legacy_u16 v1, v10, v17, v1743; GFX9-NEXT: v_mad_legacy_u16 v1, v9, v16, v1744; GFX9-NEXT: v_mad_legacy_u16 v1, v8, v15, v1745; GFX9-NEXT: v_mad_legacy_u16 v1, v7, v14, v1746; GFX9-NEXT: v_mad_legacy_u16 v1, v6, v13, v1747; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v12, v1748; GFX9-NEXT: v_mad_legacy_u16 v1, v4, v11, v1749; GFX9-NEXT: global_store_byte v0, v1, s[6:7]750; GFX9-NEXT: s_endpgm751;752; GFX9-DL-LABEL: udot8_acc8:753; GFX9-DL: ; %bb.0: ; %entry754; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24755; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34756; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0757; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0758; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1759; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)760; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]761; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]762; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0763; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]764; GFX9-DL-NEXT: s_mov_b32 s14, -1765; GFX9-DL-NEXT: s_mov_b32 s15, 0xe00000766; GFX9-DL-NEXT: s_add_u32 s12, s12, s11767; GFX9-DL-NEXT: s_addc_u32 s13, s13, 0768; GFX9-DL-NEXT: s_waitcnt vmcnt(2)769; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v1770; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 4771; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 4772; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 4773; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 4774; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 4775; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 4776; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v1777; GFX9-DL-NEXT: s_waitcnt vmcnt(1)778; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v2779; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 4780; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 4781; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 4782; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 4783; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 4784; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 4785; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2786; GFX9-DL-NEXT: s_waitcnt vmcnt(0)787; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3788; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v1789; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v1790; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v1791; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v1792; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v1793; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v1794; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v1795; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]796; GFX9-DL-NEXT: s_endpgm797;798; GFX10-DL-LABEL: udot8_acc8:799; GFX10-DL: ; %bb.0: ; %entry800; GFX10-DL-NEXT: s_clause 0x1801; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24802; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34803; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0804; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0805; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0806; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1807; GFX10-DL-NEXT: s_mov_b32 s14, -1808; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c16000809; GFX10-DL-NEXT: s_add_u32 s12, s12, s11810; GFX10-DL-NEXT: s_addc_u32 s13, s13, 0811; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)812; GFX10-DL-NEXT: s_clause 0x1813; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]814; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]815; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]816; GFX10-DL-NEXT: s_waitcnt vmcnt(2)817; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v2818; GFX10-DL-NEXT: s_waitcnt vmcnt(1)819; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v3820; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 4821; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 4822; GFX10-DL-NEXT: s_waitcnt vmcnt(0)823; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v4824; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 4825; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 4826; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0827; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 4828; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 4829; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0830; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 4831; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 4832; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0833; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 4834; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 4835; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0836; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 24, 4837; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 24, 4838; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v2839; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v3840; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v0841; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v0842; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v0843; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]844; GFX10-DL-NEXT: s_endpgm845 ptr addrspace(1) %src2,846 ptr addrspace(1) nocapture %dst) {847entry:848 %idx = call i32 @llvm.amdgcn.workitem.id.x()849 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx850 %vec1 = load <8 x i4>, ptr addrspace(1) %gep1851 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx852 %vec2 = load <8 x i4>, ptr addrspace(1) %gep2853 854 %v1e0 = extractelement <8 x i4> %vec1, i64 0855 %cv1e0 = zext i4 %v1e0 to i8856 %v2e0 = extractelement <8 x i4> %vec2, i64 0857 %cv2e0 = zext i4 %v2e0 to i8858 %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0859 860 %v1e1 = extractelement <8 x i4> %vec1, i64 1861 %cv1e1 = zext i4 %v1e1 to i8862 %v2e1 = extractelement <8 x i4> %vec2, i64 1863 %cv2e1 = zext i4 %v2e1 to i8864 %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1865 866 %v1e2 = extractelement <8 x i4> %vec1, i64 2867 %cv1e2 = zext i4 %v1e2 to i8868 %v2e2 = extractelement <8 x i4> %vec2, i64 2869 %cv2e2 = zext i4 %v2e2 to i8870 %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2871 872 %v1e3 = extractelement <8 x i4> %vec1, i64 3873 %cv1e3 = zext i4 %v1e3 to i8874 %v2e3 = extractelement <8 x i4> %vec2, i64 3875 %cv2e3 = zext i4 %v2e3 to i8876 %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3877 878 %v1e4 = extractelement <8 x i4> %vec1, i64 4879 %cv1e4 = zext i4 %v1e4 to i8880 %v2e4 = extractelement <8 x i4> %vec2, i64 4881 %cv2e4 = zext i4 %v2e4 to i8882 %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4883 884 %v1e5 = extractelement <8 x i4> %vec1, i64 5885 %cv1e5 = zext i4 %v1e5 to i8886 %v2e5 = extractelement <8 x i4> %vec2, i64 5887 %cv2e5 = zext i4 %v2e5 to i8888 %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5889 890 %v1e6 = extractelement <8 x i4> %vec1, i64 6891 %cv1e6 = zext i4 %v1e6 to i8892 %v2e6 = extractelement <8 x i4> %vec2, i64 6893 %cv2e6 = zext i4 %v2e6 to i8894 %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6895 896 %v1e7 = extractelement <8 x i4> %vec1, i64 7897 %cv1e7 = zext i4 %v1e7 to i8898 %v2e7 = extractelement <8 x i4> %vec2, i64 7899 %cv2e7 = zext i4 %v2e7 to i8900 %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7901 902 %acc = load i8, ptr addrspace(1) %dst, align 4903 %add1 = add i8 %mul0, %acc904 %add2 = add i8 %add1, %mul1905 %add3 = add i8 %add2, %mul2906 %add4 = add i8 %add3, %mul3907 %add5 = add i8 %add4, %mul4908 %add6 = add i8 %add5, %mul5909 %add7 = add i8 %add6, %mul6910 %add8 = add i8 %add7, %mul7911 912 store i8 %add8, ptr addrspace(1) %dst, align 4913 ret void914}915 916; TODO: Remove the two unnecessary instructions(and+add after 2nd MAD)917; to have the pattern-recognizer to kick in.918define amdgpu_kernel void @udot8_acc4(ptr addrspace(1) %src1,919; GFX7-LABEL: udot8_acc4:920; GFX7: ; %bb.0: ; %entry921; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0922; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1923; GFX7-NEXT: s_mov_b32 s14, -1924; GFX7-NEXT: s_mov_b32 s15, 0xe8f000925; GFX7-NEXT: s_add_u32 s12, s12, s11926; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9927; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd928; GFX7-NEXT: s_mov_b32 s3, 0xf000929; GFX7-NEXT: s_mov_b32 s6, 0930; GFX7-NEXT: s_mov_b32 s7, s3931; GFX7-NEXT: s_waitcnt lgkmcnt(0)932; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]933; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0934; GFX7-NEXT: v_mov_b32_e32 v1, 0935; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64936; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]937; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64938; GFX7-NEXT: s_mov_b32 s2, -1939; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0940; GFX7-NEXT: s_addc_u32 s13, s13, 0941; GFX7-NEXT: s_waitcnt vmcnt(2)942; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v2943; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 4944; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 4945; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 4946; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 4947; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 4948; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 4949; GFX7-NEXT: v_and_b32_e32 v2, 15, v2950; GFX7-NEXT: s_waitcnt vmcnt(1)951; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v0952; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 4953; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 4954; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 4955; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 4956; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 4957; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 4958; GFX7-NEXT: v_and_b32_e32 v0, 15, v0959; GFX7-NEXT: s_waitcnt vmcnt(0)960; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1961; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v0962; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v0963; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v0964; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v0965; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v0966; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v0967; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v0968; GFX7-NEXT: v_and_b32_e32 v0, 15, v0969; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0970; GFX7-NEXT: s_endpgm971;972; GFX8-LABEL: udot8_acc4:973; GFX8: ; %bb.0: ; %entry974; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24975; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34976; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0977; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0978; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1979; GFX8-NEXT: s_waitcnt lgkmcnt(0)980; GFX8-NEXT: v_mov_b32_e32 v1, s1981; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2982; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc983; GFX8-NEXT: flat_load_dword v3, v[0:1]984; GFX8-NEXT: v_mov_b32_e32 v1, s3985; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2986; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc987; GFX8-NEXT: flat_load_dword v2, v[0:1]988; GFX8-NEXT: v_mov_b32_e32 v0, s4989; GFX8-NEXT: v_mov_b32_e32 v1, s5990; GFX8-NEXT: flat_load_ubyte v4, v[0:1]991; GFX8-NEXT: s_mov_b32 s14, -1992; GFX8-NEXT: s_mov_b32 s15, 0xe80000993; GFX8-NEXT: s_add_u32 s12, s12, s11994; GFX8-NEXT: s_addc_u32 s13, s13, 0995; GFX8-NEXT: s_waitcnt vmcnt(2)996; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v3997; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v3998; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 4999; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 41000; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 41001; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 41002; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 41003; GFX8-NEXT: v_and_b32_e32 v3, 15, v31004; GFX8-NEXT: s_waitcnt vmcnt(1)1005; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v21006; GFX8-NEXT: v_lshrrev_b32_e32 v13, 24, v21007; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 41008; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 41009; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 41010; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 41011; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 41012; GFX8-NEXT: v_and_b32_e32 v2, 15, v21013; GFX8-NEXT: s_waitcnt vmcnt(0)1014; GFX8-NEXT: v_mad_u16 v2, v3, v2, v41015; GFX8-NEXT: v_mad_u16 v2, v11, v18, v21016; GFX8-NEXT: v_mad_u16 v2, v10, v17, v21017; GFX8-NEXT: v_mad_u16 v2, v9, v16, v21018; GFX8-NEXT: v_mad_u16 v2, v8, v15, v21019; GFX8-NEXT: v_mad_u16 v2, v7, v14, v21020; GFX8-NEXT: v_mad_u16 v2, v6, v13, v21021; GFX8-NEXT: v_mad_u16 v2, v5, v12, v21022; GFX8-NEXT: v_and_b32_e32 v2, 15, v21023; GFX8-NEXT: flat_store_byte v[0:1], v21024; GFX8-NEXT: s_endpgm1025;1026; GFX9-LABEL: udot8_acc4:1027; GFX9: ; %bb.0: ; %entry1028; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241029; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341030; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01031; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01032; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11033; GFX9-NEXT: s_waitcnt lgkmcnt(0)1034; GFX9-NEXT: global_load_dword v1, v0, s[0:1]1035; GFX9-NEXT: global_load_dword v2, v0, s[2:3]1036; GFX9-NEXT: v_mov_b32_e32 v0, 01037; GFX9-NEXT: global_load_ubyte v3, v0, s[6:7]1038; GFX9-NEXT: s_mov_b32 s14, -11039; GFX9-NEXT: s_mov_b32 s15, 0xe000001040; GFX9-NEXT: s_add_u32 s12, s12, s111041; GFX9-NEXT: s_addc_u32 s13, s13, 01042; GFX9-NEXT: s_waitcnt vmcnt(2)1043; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v11044; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v11045; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 41046; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 41047; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 41048; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 41049; GFX9-NEXT: v_bfe_u32 v10, v1, 4, 41050; GFX9-NEXT: v_and_b32_e32 v1, 15, v11051; GFX9-NEXT: s_waitcnt vmcnt(1)1052; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v21053; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v21054; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 41055; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 41056; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 41057; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 41058; GFX9-NEXT: v_bfe_u32 v17, v2, 4, 41059; GFX9-NEXT: v_and_b32_e32 v2, 15, v21060; GFX9-NEXT: s_waitcnt vmcnt(0)1061; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v31062; GFX9-NEXT: v_mad_legacy_u16 v1, v10, v17, v11063; GFX9-NEXT: v_mad_legacy_u16 v1, v9, v16, v11064; GFX9-NEXT: v_mad_legacy_u16 v1, v8, v15, v11065; GFX9-NEXT: v_mad_legacy_u16 v1, v7, v14, v11066; GFX9-NEXT: v_mad_legacy_u16 v1, v6, v13, v11067; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v12, v11068; GFX9-NEXT: v_mad_legacy_u16 v1, v4, v11, v11069; GFX9-NEXT: v_and_b32_e32 v1, 15, v11070; GFX9-NEXT: global_store_byte v0, v1, s[6:7]1071; GFX9-NEXT: s_endpgm1072;1073; GFX9-DL-LABEL: udot8_acc4:1074; GFX9-DL: ; %bb.0: ; %entry1075; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241076; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341077; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01078; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01079; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11080; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1081; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1082; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1083; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01084; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]1085; GFX9-DL-NEXT: s_mov_b32 s14, -11086; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000001087; GFX9-DL-NEXT: s_add_u32 s12, s12, s111088; GFX9-DL-NEXT: s_addc_u32 s13, s13, 01089; GFX9-DL-NEXT: s_waitcnt vmcnt(2)1090; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v11091; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v11092; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 41093; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 41094; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 41095; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 41096; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 41097; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v11098; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1099; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v21100; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 24, v21101; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 41102; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 41103; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 41104; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 41105; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 41106; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v21107; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1108; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v31109; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v11110; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v11111; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v11112; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v11113; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v11114; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v11115; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v11116; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v11117; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]1118; GFX9-DL-NEXT: s_endpgm1119;1120; GFX10-DL-LABEL: udot8_acc4:1121; GFX10-DL: ; %bb.0: ; %entry1122; GFX10-DL-NEXT: s_clause 0x11123; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241124; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341125; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01126; GFX10-DL-NEXT: v_mov_b32_e32 v1, 01127; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01128; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11129; GFX10-DL-NEXT: s_mov_b32 s14, -11130; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160001131; GFX10-DL-NEXT: s_add_u32 s12, s12, s111132; GFX10-DL-NEXT: s_addc_u32 s13, s13, 01133; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1134; GFX10-DL-NEXT: s_clause 0x11135; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1136; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]1137; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]1138; GFX10-DL-NEXT: s_waitcnt vmcnt(2)1139; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v21140; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1141; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v31142; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 41143; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 41144; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1145; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v41146; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 41147; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 41148; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01149; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 41150; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 41151; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01152; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 41153; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 41154; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01155; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 41156; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 41157; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01158; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v21159; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v31160; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v21161; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v31162; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01163; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01164; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v01165; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v01166; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]1167; GFX10-DL-NEXT: s_endpgm1168 ptr addrspace(1) %src2,1169 ptr addrspace(1) nocapture %dst) {1170entry:1171 %idx = call i32 @llvm.amdgcn.workitem.id.x()1172 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx1173 %vec1 = load <8 x i4>, ptr addrspace(1) %gep11174 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx1175 %vec2 = load <8 x i4>, ptr addrspace(1) %gep21176 1177 %v1e0 = extractelement <8 x i4> %vec1, i64 01178 %v2e0 = extractelement <8 x i4> %vec2, i64 01179 %mul0 = mul nuw nsw i4 %v1e0, %v2e01180 1181 %v1e1 = extractelement <8 x i4> %vec1, i64 11182 %v2e1 = extractelement <8 x i4> %vec2, i64 11183 %mul1 = mul nuw nsw i4 %v1e1, %v2e11184 1185 %v1e2 = extractelement <8 x i4> %vec1, i64 21186 %v2e2 = extractelement <8 x i4> %vec2, i64 21187 %mul2 = mul nuw nsw i4 %v1e2, %v2e21188 1189 %v1e3 = extractelement <8 x i4> %vec1, i64 31190 %v2e3 = extractelement <8 x i4> %vec2, i64 31191 %mul3 = mul nuw nsw i4 %v1e3, %v2e31192 1193 %v1e4 = extractelement <8 x i4> %vec1, i64 41194 %v2e4 = extractelement <8 x i4> %vec2, i64 41195 %mul4 = mul nuw nsw i4 %v1e4, %v2e41196 1197 %v1e5 = extractelement <8 x i4> %vec1, i64 51198 %v2e5 = extractelement <8 x i4> %vec2, i64 51199 %mul5 = mul nuw nsw i4 %v1e5, %v2e51200 1201 %v1e6 = extractelement <8 x i4> %vec1, i64 61202 %v2e6 = extractelement <8 x i4> %vec2, i64 61203 %mul6 = mul nuw nsw i4 %v1e6, %v2e61204 1205 %v1e7 = extractelement <8 x i4> %vec1, i64 71206 %v2e7 = extractelement <8 x i4> %vec2, i64 71207 %mul7 = mul nuw nsw i4 %v1e7, %v2e71208 1209 %acc = load i4, ptr addrspace(1) %dst, align 41210 %add1 = add i4 %mul0, %acc1211 %add2 = add i4 %add1, %mul11212 %add3 = add i4 %add2, %mul21213 %add4 = add i4 %add3, %mul31214 %add5 = add i4 %add4, %mul41215 %add6 = add i4 %add5, %mul51216 %add7 = add i4 %add6, %mul61217 %add8 = add i4 %add7, %mul71218 1219 store i4 %add8, ptr addrspace(1) %dst, align 41220 ret void1221}1222 1223; TODO: Currently, permutation of udot8 is turned off due to a huge increase1224; in the compile time.1225define amdgpu_kernel void @udot8_CommutationInsideMAD(ptr addrspace(1) %src1,1226; GFX7-LABEL: udot8_CommutationInsideMAD:1227; GFX7: ; %bb.0: ; %entry1228; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01229; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11230; GFX7-NEXT: s_mov_b32 s14, -11231; GFX7-NEXT: s_mov_b32 s15, 0xe8f0001232; GFX7-NEXT: s_add_u32 s12, s12, s111233; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91234; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1235; GFX7-NEXT: s_mov_b32 s3, 0xf0001236; GFX7-NEXT: s_mov_b32 s6, 01237; GFX7-NEXT: s_mov_b32 s7, s31238; GFX7-NEXT: s_waitcnt lgkmcnt(0)1239; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1240; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01241; GFX7-NEXT: v_mov_b32_e32 v1, 01242; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641243; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1244; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641245; GFX7-NEXT: s_mov_b32 s2, -11246; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 01247; GFX7-NEXT: s_addc_u32 s13, s13, 01248; GFX7-NEXT: s_waitcnt vmcnt(2)1249; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v21250; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 41251; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 41252; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 41253; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 41254; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 41255; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 41256; GFX7-NEXT: v_and_b32_e32 v2, 15, v21257; GFX7-NEXT: s_waitcnt vmcnt(1)1258; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v01259; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 41260; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 41261; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 41262; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 41263; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 41264; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 41265; GFX7-NEXT: v_and_b32_e32 v0, 15, v01266; GFX7-NEXT: s_waitcnt vmcnt(0)1267; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v11268; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v01269; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v01270; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v01271; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v01272; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v01273; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v01274; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v01275; GFX7-NEXT: v_and_b32_e32 v0, 15, v01276; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 01277; GFX7-NEXT: s_endpgm1278;1279; GFX8-LABEL: udot8_CommutationInsideMAD:1280; GFX8: ; %bb.0: ; %entry1281; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241282; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341283; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01284; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01285; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11286; GFX8-NEXT: s_waitcnt lgkmcnt(0)1287; GFX8-NEXT: v_mov_b32_e32 v1, s11288; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21289; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1290; GFX8-NEXT: flat_load_dword v3, v[0:1]1291; GFX8-NEXT: v_mov_b32_e32 v1, s31292; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21293; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1294; GFX8-NEXT: flat_load_dword v2, v[0:1]1295; GFX8-NEXT: v_mov_b32_e32 v0, s41296; GFX8-NEXT: v_mov_b32_e32 v1, s51297; GFX8-NEXT: flat_load_ubyte v4, v[0:1]1298; GFX8-NEXT: s_mov_b32 s14, -11299; GFX8-NEXT: s_mov_b32 s15, 0xe800001300; GFX8-NEXT: s_add_u32 s12, s12, s111301; GFX8-NEXT: s_addc_u32 s13, s13, 01302; GFX8-NEXT: s_waitcnt vmcnt(2)1303; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v31304; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v31305; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 41306; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 41307; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 41308; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 41309; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 41310; GFX8-NEXT: v_and_b32_e32 v3, 15, v31311; GFX8-NEXT: s_waitcnt vmcnt(1)1312; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v21313; GFX8-NEXT: v_lshrrev_b32_e32 v13, 24, v21314; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 41315; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 41316; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 41317; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 41318; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 41319; GFX8-NEXT: v_and_b32_e32 v2, 15, v21320; GFX8-NEXT: s_waitcnt vmcnt(0)1321; GFX8-NEXT: v_mad_u16 v2, v3, v2, v41322; GFX8-NEXT: v_mad_u16 v2, v11, v18, v21323; GFX8-NEXT: v_mad_u16 v2, v10, v17, v21324; GFX8-NEXT: v_mad_u16 v2, v9, v16, v21325; GFX8-NEXT: v_mad_u16 v2, v8, v15, v21326; GFX8-NEXT: v_mad_u16 v2, v7, v14, v21327; GFX8-NEXT: v_mad_u16 v2, v6, v13, v21328; GFX8-NEXT: v_mad_u16 v2, v5, v12, v21329; GFX8-NEXT: v_and_b32_e32 v2, 15, v21330; GFX8-NEXT: flat_store_byte v[0:1], v21331; GFX8-NEXT: s_endpgm1332;1333; GFX9-LABEL: udot8_CommutationInsideMAD:1334; GFX9: ; %bb.0: ; %entry1335; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241336; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341337; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01338; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01339; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11340; GFX9-NEXT: s_waitcnt lgkmcnt(0)1341; GFX9-NEXT: global_load_dword v1, v0, s[0:1]1342; GFX9-NEXT: global_load_dword v2, v0, s[2:3]1343; GFX9-NEXT: v_mov_b32_e32 v0, 01344; GFX9-NEXT: global_load_ubyte v3, v0, s[6:7]1345; GFX9-NEXT: s_mov_b32 s14, -11346; GFX9-NEXT: s_mov_b32 s15, 0xe000001347; GFX9-NEXT: s_add_u32 s12, s12, s111348; GFX9-NEXT: s_addc_u32 s13, s13, 01349; GFX9-NEXT: s_waitcnt vmcnt(2)1350; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v11351; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v11352; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 41353; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 41354; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 41355; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 41356; GFX9-NEXT: v_bfe_u32 v10, v1, 4, 41357; GFX9-NEXT: v_and_b32_e32 v1, 15, v11358; GFX9-NEXT: s_waitcnt vmcnt(1)1359; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v21360; GFX9-NEXT: v_lshrrev_b32_e32 v12, 24, v21361; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 41362; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 41363; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 41364; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 41365; GFX9-NEXT: v_bfe_u32 v17, v2, 4, 41366; GFX9-NEXT: v_and_b32_e32 v2, 15, v21367; GFX9-NEXT: s_waitcnt vmcnt(0)1368; GFX9-NEXT: v_mad_legacy_u16 v1, v1, v2, v31369; GFX9-NEXT: v_mad_legacy_u16 v1, v10, v17, v11370; GFX9-NEXT: v_mad_legacy_u16 v1, v9, v16, v11371; GFX9-NEXT: v_mad_legacy_u16 v1, v8, v15, v11372; GFX9-NEXT: v_mad_legacy_u16 v1, v7, v14, v11373; GFX9-NEXT: v_mad_legacy_u16 v1, v6, v13, v11374; GFX9-NEXT: v_mad_legacy_u16 v1, v5, v12, v11375; GFX9-NEXT: v_mad_legacy_u16 v1, v4, v11, v11376; GFX9-NEXT: v_and_b32_e32 v1, 15, v11377; GFX9-NEXT: global_store_byte v0, v1, s[6:7]1378; GFX9-NEXT: s_endpgm1379;1380; GFX9-DL-LABEL: udot8_CommutationInsideMAD:1381; GFX9-DL: ; %bb.0: ; %entry1382; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241383; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341384; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01385; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01386; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11387; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1388; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1389; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1390; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01391; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]1392; GFX9-DL-NEXT: s_mov_b32 s14, -11393; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000001394; GFX9-DL-NEXT: s_add_u32 s12, s12, s111395; GFX9-DL-NEXT: s_addc_u32 s13, s13, 01396; GFX9-DL-NEXT: s_waitcnt vmcnt(2)1397; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v11398; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v11399; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 41400; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 41401; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 41402; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 41403; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 4, 41404; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v11405; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1406; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v21407; GFX9-DL-NEXT: v_lshrrev_b32_e32 v12, 24, v21408; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 41409; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 41410; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 41411; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 41412; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 4, 41413; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v21414; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1415; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v31416; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v10, v17, v11417; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v9, v16, v11418; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v8, v15, v11419; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v7, v14, v11420; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v6, v13, v11421; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v5, v12, v11422; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v11, v11423; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v11424; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]1425; GFX9-DL-NEXT: s_endpgm1426;1427; GFX10-DL-LABEL: udot8_CommutationInsideMAD:1428; GFX10-DL: ; %bb.0: ; %entry1429; GFX10-DL-NEXT: s_clause 0x11430; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241431; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341432; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01433; GFX10-DL-NEXT: v_mov_b32_e32 v1, 01434; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01435; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11436; GFX10-DL-NEXT: s_mov_b32 s14, -11437; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160001438; GFX10-DL-NEXT: s_add_u32 s12, s12, s111439; GFX10-DL-NEXT: s_addc_u32 s13, s13, 01440; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1441; GFX10-DL-NEXT: s_clause 0x11442; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]1443; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]1444; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[6:7]1445; GFX10-DL-NEXT: s_waitcnt vmcnt(2)1446; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v21447; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1448; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v31449; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 41450; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 4, 41451; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1452; GFX10-DL-NEXT: v_mad_u16 v0, v0, v5, v41453; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 8, 41454; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 8, 41455; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01456; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 12, 41457; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 12, 41458; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01459; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 41460; GFX10-DL-NEXT: v_bfe_u32 v5, v3, 16, 41461; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01462; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 20, 41463; GFX10-DL-NEXT: v_bfe_u32 v7, v3, 20, 41464; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01465; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v21466; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v31467; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v21468; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v31469; GFX10-DL-NEXT: v_mad_u16 v0, v6, v7, v01470; GFX10-DL-NEXT: v_mad_u16 v0, v4, v5, v01471; GFX10-DL-NEXT: v_mad_u16 v0, v2, v3, v01472; GFX10-DL-NEXT: v_and_b32_e32 v0, 15, v01473; GFX10-DL-NEXT: global_store_byte v1, v0, s[6:7]1474; GFX10-DL-NEXT: s_endpgm1475 ptr addrspace(1) %src2,1476 ptr addrspace(1) nocapture %dst) {1477entry:1478 %idx = call i32 @llvm.amdgcn.workitem.id.x()1479 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx1480 %vec1 = load <8 x i4>, ptr addrspace(1) %gep11481 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx1482 %vec2 = load <8 x i4>, ptr addrspace(1) %gep21483 1484 %v1e0 = extractelement <8 x i4> %vec1, i64 01485 %v2e0 = extractelement <8 x i4> %vec2, i64 01486 %mul0 = mul nuw nsw i4 %v1e0, %v2e01487 1488 %v1e1 = extractelement <8 x i4> %vec1, i64 11489 %v2e1 = extractelement <8 x i4> %vec2, i64 11490 %mul1 = mul nuw nsw i4 %v1e1, %v2e11491 1492 %v1e2 = extractelement <8 x i4> %vec1, i64 21493 %v2e2 = extractelement <8 x i4> %vec2, i64 21494 %mul2 = mul nuw nsw i4 %v1e2, %v2e21495 1496 %v1e3 = extractelement <8 x i4> %vec1, i64 31497 %v2e3 = extractelement <8 x i4> %vec2, i64 31498 %mul3 = mul nuw nsw i4 %v1e3, %v2e31499 1500 %v1e4 = extractelement <8 x i4> %vec1, i64 41501 %v2e4 = extractelement <8 x i4> %vec2, i64 41502 %mul4 = mul nuw nsw i4 %v1e4, %v2e41503 1504 %v1e5 = extractelement <8 x i4> %vec1, i64 51505 %v2e5 = extractelement <8 x i4> %vec2, i64 51506 %mul5 = mul nuw nsw i4 %v1e5, %v2e51507 1508 %v1e6 = extractelement <8 x i4> %vec1, i64 61509 %v2e6 = extractelement <8 x i4> %vec2, i64 61510 %mul6 = mul nuw nsw i4 %v1e6, %v2e61511 1512 %v1e7 = extractelement <8 x i4> %vec1, i64 71513 %v2e7 = extractelement <8 x i4> %vec2, i64 71514 %mul7 = mul nuw nsw i4 %v1e7, %v2e71515 1516 %acc = load i4, ptr addrspace(1) %dst, align 41517 %add1 = add i4 %mul0, %acc1518 %add2 = add i4 %mul1, %add11519 %add3 = add i4 %mul2, %add21520 %add4 = add i4 %mul3, %add31521 %add5 = add i4 %mul4, %add41522 %add6 = add i4 %mul5, %add51523 %add7 = add i4 %mul6, %add61524 %add8 = add i4 %mul7, %add71525 1526 store i4 %add8, ptr addrspace(1) %dst, align 41527 ret void1528}1529 1530define amdgpu_kernel void @udot8_multiuses_mul1(ptr addrspace(1) %src1,1531; GFX7-LABEL: udot8_multiuses_mul1:1532; GFX7: ; %bb.0: ; %entry1533; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01534; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11535; GFX7-NEXT: s_mov_b32 s14, -11536; GFX7-NEXT: s_mov_b32 s15, 0xe8f0001537; GFX7-NEXT: s_add_u32 s12, s12, s111538; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91539; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1540; GFX7-NEXT: s_mov_b32 s3, 0xf0001541; GFX7-NEXT: s_mov_b32 s6, 01542; GFX7-NEXT: s_mov_b32 s7, s31543; GFX7-NEXT: s_waitcnt lgkmcnt(0)1544; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1545; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01546; GFX7-NEXT: v_mov_b32_e32 v1, 01547; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641548; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1549; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641550; GFX7-NEXT: s_load_dword s4, s[0:1], 0x01551; GFX7-NEXT: s_mov_b32 s2, -11552; GFX7-NEXT: s_addc_u32 s13, s13, 01553; GFX7-NEXT: s_waitcnt vmcnt(1)1554; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v21555; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 41556; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 41557; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 41558; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 41559; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 41560; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 41561; GFX7-NEXT: v_and_b32_e32 v2, 15, v21562; GFX7-NEXT: s_waitcnt vmcnt(0)1563; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v01564; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 41565; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 41566; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 41567; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 41568; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 41569; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 41570; GFX7-NEXT: v_and_b32_e32 v0, 15, v01571; GFX7-NEXT: s_waitcnt lgkmcnt(0)1572; GFX7-NEXT: v_mad_u32_u24 v16, v2, v0, s41573; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v161574; GFX7-NEXT: v_mad_u32_u24 v2, v8, v15, v161575; GFX7-NEXT: v_mad_u32_u24 v2, v7, v14, v21576; GFX7-NEXT: v_mad_u32_u24 v2, v6, v13, v21577; GFX7-NEXT: v_mad_u32_u24 v2, v5, v12, v21578; GFX7-NEXT: v_mad_u32_u24 v2, v4, v11, v21579; GFX7-NEXT: v_mad_u32_u24 v2, v3, v10, v21580; GFX7-NEXT: v_mad_u32_u24 v1, v1, v9, v21581; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v11582; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 01583; GFX7-NEXT: s_endpgm1584;1585; GFX8-LABEL: udot8_multiuses_mul1:1586; GFX8: ; %bb.0: ; %entry1587; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241588; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341589; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01590; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01591; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11592; GFX8-NEXT: s_waitcnt lgkmcnt(0)1593; GFX8-NEXT: v_mov_b32_e32 v1, s11594; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21595; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1596; GFX8-NEXT: flat_load_dword v3, v[0:1]1597; GFX8-NEXT: v_mov_b32_e32 v1, s31598; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21599; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1600; GFX8-NEXT: flat_load_dword v0, v[0:1]1601; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01602; GFX8-NEXT: s_mov_b32 s14, -11603; GFX8-NEXT: s_mov_b32 s15, 0xe800001604; GFX8-NEXT: s_add_u32 s12, s12, s111605; GFX8-NEXT: s_addc_u32 s13, s13, 01606; GFX8-NEXT: s_waitcnt vmcnt(1)1607; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v31608; GFX8-NEXT: v_bfe_u32 v2, v3, 24, 41609; GFX8-NEXT: v_bfe_u32 v4, v3, 20, 41610; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 41611; GFX8-NEXT: v_bfe_u32 v6, v3, 12, 41612; GFX8-NEXT: v_bfe_u32 v7, v3, 8, 41613; GFX8-NEXT: v_bfe_u32 v8, v3, 4, 41614; GFX8-NEXT: v_and_b32_e32 v3, 15, v31615; GFX8-NEXT: s_waitcnt vmcnt(0)1616; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v01617; GFX8-NEXT: v_bfe_u32 v10, v0, 24, 41618; GFX8-NEXT: v_bfe_u32 v11, v0, 20, 41619; GFX8-NEXT: v_bfe_u32 v12, v0, 16, 41620; GFX8-NEXT: v_bfe_u32 v13, v0, 12, 41621; GFX8-NEXT: v_bfe_u32 v14, v0, 8, 41622; GFX8-NEXT: v_bfe_u32 v15, v0, 4, 41623; GFX8-NEXT: v_and_b32_e32 v0, 15, v01624; GFX8-NEXT: s_waitcnt lgkmcnt(0)1625; GFX8-NEXT: v_mad_u32_u24 v16, v3, v0, s01626; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, v161627; GFX8-NEXT: v_mad_u32_u24 v3, v8, v15, v161628; GFX8-NEXT: v_mad_u32_u24 v3, v7, v14, v31629; GFX8-NEXT: v_mad_u32_u24 v3, v6, v13, v31630; GFX8-NEXT: v_mad_u32_u24 v3, v5, v12, v31631; GFX8-NEXT: v_mad_u32_u24 v3, v4, v11, v31632; GFX8-NEXT: v_mad_u32_u24 v2, v2, v10, v31633; GFX8-NEXT: v_mad_u32_u24 v1, v1, v9, v21634; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v11635; GFX8-NEXT: v_mov_b32_e32 v0, s41636; GFX8-NEXT: v_mov_b32_e32 v1, s51637; GFX8-NEXT: flat_store_dword v[0:1], v21638; GFX8-NEXT: s_endpgm1639;1640; GFX9-LABEL: udot8_multiuses_mul1:1641; GFX9: ; %bb.0: ; %entry1642; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241643; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341644; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01645; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01646; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11647; GFX9-NEXT: s_waitcnt lgkmcnt(0)1648; GFX9-NEXT: global_load_dword v1, v0, s[0:1]1649; GFX9-NEXT: global_load_dword v2, v0, s[2:3]1650; GFX9-NEXT: s_load_dword s0, s[6:7], 0x01651; GFX9-NEXT: s_mov_b32 s14, -11652; GFX9-NEXT: s_mov_b32 s15, 0xe000001653; GFX9-NEXT: s_add_u32 s12, s12, s111654; GFX9-NEXT: v_mov_b32_e32 v0, 01655; GFX9-NEXT: s_addc_u32 s13, s13, 01656; GFX9-NEXT: s_waitcnt vmcnt(1)1657; GFX9-NEXT: v_bfe_u32 v3, v1, 4, 41658; GFX9-NEXT: v_lshrrev_b32_e32 v4, 28, v11659; GFX9-NEXT: v_bfe_u32 v5, v1, 24, 41660; GFX9-NEXT: v_bfe_u32 v6, v1, 20, 41661; GFX9-NEXT: v_bfe_u32 v7, v1, 16, 41662; GFX9-NEXT: v_bfe_u32 v8, v1, 12, 41663; GFX9-NEXT: v_bfe_u32 v9, v1, 8, 41664; GFX9-NEXT: v_and_b32_e32 v1, 15, v11665; GFX9-NEXT: s_waitcnt vmcnt(0)1666; GFX9-NEXT: v_bfe_u32 v10, v2, 4, 41667; GFX9-NEXT: v_lshrrev_b32_e32 v11, 28, v21668; GFX9-NEXT: v_bfe_u32 v12, v2, 24, 41669; GFX9-NEXT: v_bfe_u32 v13, v2, 20, 41670; GFX9-NEXT: v_bfe_u32 v14, v2, 16, 41671; GFX9-NEXT: v_bfe_u32 v15, v2, 12, 41672; GFX9-NEXT: v_bfe_u32 v16, v2, 8, 41673; GFX9-NEXT: v_and_b32_e32 v2, 15, v21674; GFX9-NEXT: v_mul_u32_u24_e32 v17, v1, v21675; GFX9-NEXT: s_waitcnt lgkmcnt(0)1676; GFX9-NEXT: v_mad_u32_u24 v1, v1, v2, s01677; GFX9-NEXT: v_mul_u32_u24_e32 v9, v9, v161678; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v151679; GFX9-NEXT: v_mad_u32_u24 v2, v3, v10, v11680; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v141681; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v131682; GFX9-NEXT: v_add3_u32 v2, v2, v9, v81683; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v121684; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v111685; GFX9-NEXT: v_add3_u32 v2, v2, v7, v61686; GFX9-NEXT: v_add3_u32 v2, v2, v5, v41687; GFX9-NEXT: v_add3_u32 v1, v17, v1, v21688; GFX9-NEXT: global_store_dword v0, v1, s[6:7]1689; GFX9-NEXT: s_endpgm1690;1691; GFX9-DL-LABEL: udot8_multiuses_mul1:1692; GFX9-DL: ; %bb.0: ; %entry1693; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241694; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341695; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01696; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01697; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11698; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1699; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]1700; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]1701; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x01702; GFX9-DL-NEXT: s_mov_b32 s14, -11703; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000001704; GFX9-DL-NEXT: s_add_u32 s12, s12, s111705; GFX9-DL-NEXT: v_mov_b32_e32 v0, 01706; GFX9-DL-NEXT: s_addc_u32 s13, s13, 01707; GFX9-DL-NEXT: s_waitcnt vmcnt(1)1708; GFX9-DL-NEXT: v_bfe_u32 v3, v1, 4, 41709; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 28, v11710; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 24, 41711; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 20, 41712; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 16, 41713; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 12, 41714; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 8, 41715; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v11716; GFX9-DL-NEXT: s_waitcnt vmcnt(0)1717; GFX9-DL-NEXT: v_bfe_u32 v10, v2, 4, 41718; GFX9-DL-NEXT: v_lshrrev_b32_e32 v11, 28, v21719; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 24, 41720; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 20, 41721; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 16, 41722; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 12, 41723; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 8, 41724; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v21725; GFX9-DL-NEXT: v_mul_u32_u24_e32 v17, v1, v21726; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)1727; GFX9-DL-NEXT: v_mad_u32_u24 v1, v1, v2, s01728; GFX9-DL-NEXT: v_mul_u32_u24_e32 v9, v9, v161729; GFX9-DL-NEXT: v_mul_u32_u24_e32 v8, v8, v151730; GFX9-DL-NEXT: v_mad_u32_u24 v2, v3, v10, v11731; GFX9-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v141732; GFX9-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v131733; GFX9-DL-NEXT: v_add3_u32 v2, v2, v9, v81734; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v121735; GFX9-DL-NEXT: v_mul_u32_u24_e32 v4, v4, v111736; GFX9-DL-NEXT: v_add3_u32 v2, v2, v7, v61737; GFX9-DL-NEXT: v_add3_u32 v2, v2, v5, v41738; GFX9-DL-NEXT: v_add3_u32 v1, v17, v1, v21739; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]1740; GFX9-DL-NEXT: s_endpgm1741;1742; GFX10-DL-LABEL: udot8_multiuses_mul1:1743; GFX10-DL: ; %bb.0: ; %entry1744; GFX10-DL-NEXT: s_clause 0x11745; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241746; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341747; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v01748; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01749; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11750; GFX10-DL-NEXT: s_mov_b32 s14, -11751; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160001752; GFX10-DL-NEXT: s_add_u32 s12, s12, s111753; GFX10-DL-NEXT: s_addc_u32 s13, s13, 01754; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1755; GFX10-DL-NEXT: s_clause 0x11756; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]1757; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]1758; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1759; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x01760; GFX10-DL-NEXT: s_waitcnt vmcnt(1)1761; GFX10-DL-NEXT: v_and_b32_e32 v8, 15, v11762; GFX10-DL-NEXT: s_waitcnt vmcnt(0)1763; GFX10-DL-NEXT: v_and_b32_e32 v9, 15, v21764; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 41765; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 28, v11766; GFX10-DL-NEXT: v_bfe_u32 v4, v1, 24, 41767; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 20, 41768; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 16, 41769; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 12, 41770; GFX10-DL-NEXT: v_bfe_u32 v1, v1, 8, 41771; GFX10-DL-NEXT: v_bfe_u32 v10, v2, 4, 41772; GFX10-DL-NEXT: v_bfe_u32 v11, v2, 8, 41773; GFX10-DL-NEXT: v_bfe_u32 v12, v2, 12, 41774; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)1775; GFX10-DL-NEXT: v_mad_u32_u24 v13, v8, v9, s01776; GFX10-DL-NEXT: v_bfe_u32 v14, v2, 20, 41777; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 16, 41778; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v1, v111779; GFX10-DL-NEXT: v_mul_u32_u24_e32 v7, v7, v121780; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v10, v131781; GFX10-DL-NEXT: v_lshrrev_b32_e32 v10, 28, v21782; GFX10-DL-NEXT: v_bfe_u32 v2, v2, 24, 41783; GFX10-DL-NEXT: v_mul_u32_u24_e32 v6, v6, v151784; GFX10-DL-NEXT: v_mul_u32_u24_e32 v5, v5, v141785; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v71786; GFX10-DL-NEXT: v_mul_u32_u24_e32 v1, v4, v21787; GFX10-DL-NEXT: v_mul_u32_u24_e32 v2, v3, v101788; GFX10-DL-NEXT: v_mul_u32_u24_e32 v3, v8, v91789; GFX10-DL-NEXT: v_add3_u32 v0, v0, v6, v51790; GFX10-DL-NEXT: v_add3_u32 v0, v0, v1, v21791; GFX10-DL-NEXT: v_mov_b32_e32 v1, 01792; GFX10-DL-NEXT: v_add3_u32 v0, v3, v13, v01793; GFX10-DL-NEXT: global_store_dword v1, v0, s[6:7]1794; GFX10-DL-NEXT: s_endpgm1795 ptr addrspace(1) %src2,1796 ptr addrspace(1) nocapture %dst) {1797entry:1798 %idx = call i32 @llvm.amdgcn.workitem.id.x()1799 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx1800 %vec1 = load <8 x i4>, ptr addrspace(1) %gep11801 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx1802 %vec2 = load <8 x i4>, ptr addrspace(1) %gep21803 1804 %v1e0 = extractelement <8 x i4> %vec1, i64 01805 %cv1e0 = zext i4 %v1e0 to i321806 %v2e0 = extractelement <8 x i4> %vec2, i64 01807 %cv2e0 = zext i4 %v2e0 to i321808 %mul0 = mul nuw nsw i32 %cv1e0, %cv2e01809 1810 %v1e1 = extractelement <8 x i4> %vec1, i64 11811 %cv1e1 = zext i4 %v1e1 to i321812 %v2e1 = extractelement <8 x i4> %vec2, i64 11813 %cv2e1 = zext i4 %v2e1 to i321814 %mul1 = mul nuw nsw i32 %cv1e1, %cv2e11815 1816 %v1e2 = extractelement <8 x i4> %vec1, i64 21817 %cv1e2 = zext i4 %v1e2 to i321818 %v2e2 = extractelement <8 x i4> %vec2, i64 21819 %cv2e2 = zext i4 %v2e2 to i321820 %mul2 = mul nuw nsw i32 %cv1e2, %cv2e21821 1822 %v1e3 = extractelement <8 x i4> %vec1, i64 31823 %cv1e3 = zext i4 %v1e3 to i321824 %v2e3 = extractelement <8 x i4> %vec2, i64 31825 %cv2e3 = zext i4 %v2e3 to i321826 %mul3 = mul nuw nsw i32 %cv1e3, %cv2e31827 1828 %v1e4 = extractelement <8 x i4> %vec1, i64 41829 %cv1e4 = zext i4 %v1e4 to i321830 %v2e4 = extractelement <8 x i4> %vec2, i64 41831 %cv2e4 = zext i4 %v2e4 to i321832 %mul4 = mul nuw nsw i32 %cv1e4, %cv2e41833 1834 %v1e5 = extractelement <8 x i4> %vec1, i64 51835 %cv1e5 = zext i4 %v1e5 to i321836 %v2e5 = extractelement <8 x i4> %vec2, i64 51837 %cv2e5 = zext i4 %v2e5 to i321838 %mul5 = mul nuw nsw i32 %cv1e5, %cv2e51839 1840 %v1e6 = extractelement <8 x i4> %vec1, i64 61841 %cv1e6 = zext i4 %v1e6 to i321842 %v2e6 = extractelement <8 x i4> %vec2, i64 61843 %cv2e6 = zext i4 %v2e6 to i321844 %mul6 = mul nuw nsw i32 %cv1e6, %cv2e61845 1846 %v1e7 = extractelement <8 x i4> %vec1, i64 71847 %cv1e7 = zext i4 %v1e7 to i321848 %v2e7 = extractelement <8 x i4> %vec2, i64 71849 %cv2e7 = zext i4 %v2e7 to i321850 %mul7 = mul nuw nsw i32 %cv1e7, %cv2e71851 1852 %acc = load i32, ptr addrspace(1) %dst, align 41853 %add1 = add i32 %mul0, %acc1854 %add = add i32 %mul0, %add11855 %add2 = add i32 %add1, %mul11856 %add3 = add i32 %add2, %mul21857 %add4 = add i32 %add3, %mul31858 %add5 = add i32 %add4, %mul41859 %add6 = add i32 %add5, %mul51860 %add7 = add i32 %add6, %mul61861 %add8 = add i32 %add7, %mul71862 1863 %res = add i32 %add, %add81864 store i32 %res, ptr addrspace(1) %dst, align 41865 ret void1866}1867 1868define amdgpu_kernel void @udot8_acc32_vecMul(ptr addrspace(1) %src1,1869; GFX7-LABEL: udot8_acc32_vecMul:1870; GFX7: ; %bb.0: ; %entry1871; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01872; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11873; GFX7-NEXT: s_mov_b32 s14, -11874; GFX7-NEXT: s_mov_b32 s15, 0xe8f0001875; GFX7-NEXT: s_add_u32 s12, s12, s111876; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x91877; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1878; GFX7-NEXT: s_mov_b32 s3, 0xf0001879; GFX7-NEXT: s_mov_b32 s6, 01880; GFX7-NEXT: s_mov_b32 s7, s31881; GFX7-NEXT: s_waitcnt lgkmcnt(0)1882; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]1883; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v01884; GFX7-NEXT: v_mov_b32_e32 v1, 01885; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr641886; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]1887; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr641888; GFX7-NEXT: s_load_dword s4, s[0:1], 0x01889; GFX7-NEXT: s_mov_b32 s2, -11890; GFX7-NEXT: s_addc_u32 s13, s13, 01891; GFX7-NEXT: s_waitcnt vmcnt(1)1892; GFX7-NEXT: v_lshrrev_b32_e32 v1, 28, v21893; GFX7-NEXT: v_bfe_u32 v3, v2, 24, 41894; GFX7-NEXT: v_bfe_u32 v4, v2, 20, 41895; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 41896; GFX7-NEXT: v_bfe_u32 v6, v2, 12, 41897; GFX7-NEXT: v_bfe_u32 v7, v2, 8, 41898; GFX7-NEXT: v_bfe_u32 v8, v2, 4, 41899; GFX7-NEXT: v_and_b32_e32 v2, 15, v21900; GFX7-NEXT: s_waitcnt vmcnt(0)1901; GFX7-NEXT: v_lshrrev_b32_e32 v9, 28, v01902; GFX7-NEXT: v_bfe_u32 v10, v0, 24, 41903; GFX7-NEXT: v_bfe_u32 v11, v0, 20, 41904; GFX7-NEXT: v_bfe_u32 v12, v0, 16, 41905; GFX7-NEXT: v_bfe_u32 v13, v0, 12, 41906; GFX7-NEXT: v_bfe_u32 v14, v0, 8, 41907; GFX7-NEXT: v_bfe_u32 v15, v0, 4, 41908; GFX7-NEXT: v_and_b32_e32 v0, 15, v01909; GFX7-NEXT: s_waitcnt lgkmcnt(0)1910; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s41911; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v01912; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v01913; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v01914; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v01915; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v01916; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v01917; GFX7-NEXT: v_mad_u32_u24 v0, v1, v9, v01918; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 01919; GFX7-NEXT: s_endpgm1920;1921; GFX8-LABEL: udot8_acc32_vecMul:1922; GFX8: ; %bb.0: ; %entry1923; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241924; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341925; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v01926; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01927; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11928; GFX8-NEXT: s_waitcnt lgkmcnt(0)1929; GFX8-NEXT: v_mov_b32_e32 v1, s11930; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v21931; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1932; GFX8-NEXT: flat_load_dword v3, v[0:1]1933; GFX8-NEXT: v_mov_b32_e32 v1, s31934; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v21935; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1936; GFX8-NEXT: flat_load_dword v0, v[0:1]1937; GFX8-NEXT: s_load_dword s0, s[4:5], 0x01938; GFX8-NEXT: s_mov_b32 s14, -11939; GFX8-NEXT: s_mov_b32 s15, 0xe800001940; GFX8-NEXT: s_add_u32 s12, s12, s111941; GFX8-NEXT: s_addc_u32 s13, s13, 01942; GFX8-NEXT: s_waitcnt vmcnt(1)1943; GFX8-NEXT: v_lshrrev_b32_e32 v1, 28, v31944; GFX8-NEXT: v_bfe_u32 v2, v3, 24, 41945; GFX8-NEXT: v_bfe_u32 v4, v3, 20, 41946; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 41947; GFX8-NEXT: v_bfe_u32 v6, v3, 12, 41948; GFX8-NEXT: v_bfe_u32 v7, v3, 8, 41949; GFX8-NEXT: v_bfe_u32 v8, v3, 4, 41950; GFX8-NEXT: v_and_b32_e32 v3, 15, v31951; GFX8-NEXT: s_waitcnt vmcnt(0)1952; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v01953; GFX8-NEXT: v_bfe_u32 v10, v0, 24, 41954; GFX8-NEXT: v_bfe_u32 v11, v0, 20, 41955; GFX8-NEXT: v_bfe_u32 v12, v0, 16, 41956; GFX8-NEXT: v_bfe_u32 v13, v0, 12, 41957; GFX8-NEXT: v_bfe_u32 v14, v0, 8, 41958; GFX8-NEXT: v_bfe_u32 v15, v0, 4, 41959; GFX8-NEXT: v_and_b32_e32 v0, 15, v01960; GFX8-NEXT: s_waitcnt lgkmcnt(0)1961; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s01962; GFX8-NEXT: v_mad_u32_u24 v0, v8, v15, v01963; GFX8-NEXT: v_mad_u32_u24 v0, v7, v14, v01964; GFX8-NEXT: v_mad_u32_u24 v0, v6, v13, v01965; GFX8-NEXT: v_mad_u32_u24 v0, v5, v12, v01966; GFX8-NEXT: v_mad_u32_u24 v0, v4, v11, v01967; GFX8-NEXT: v_mad_u32_u24 v0, v2, v10, v01968; GFX8-NEXT: v_mad_u32_u24 v2, v1, v9, v01969; GFX8-NEXT: v_mov_b32_e32 v0, s41970; GFX8-NEXT: v_mov_b32_e32 v1, s51971; GFX8-NEXT: flat_store_dword v[0:1], v21972; GFX8-NEXT: s_endpgm1973;1974; GFX9-LABEL: udot8_acc32_vecMul:1975; GFX9: ; %bb.0: ; %entry1976; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241977; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341978; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01979; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD01980; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD11981; GFX9-NEXT: s_waitcnt lgkmcnt(0)1982; GFX9-NEXT: global_load_dword v1, v0, s[0:1]1983; GFX9-NEXT: global_load_dword v2, v0, s[2:3]1984; GFX9-NEXT: s_load_dword s0, s[6:7], 0x01985; GFX9-NEXT: s_mov_b32 s14, -11986; GFX9-NEXT: s_mov_b32 s15, 0xe000001987; GFX9-NEXT: s_add_u32 s12, s12, s111988; GFX9-NEXT: v_mov_b32_e32 v0, 01989; GFX9-NEXT: s_addc_u32 s13, s13, 01990; GFX9-NEXT: s_waitcnt vmcnt(1)1991; GFX9-NEXT: v_lshrrev_b32_e32 v3, 28, v11992; GFX9-NEXT: v_bfe_u32 v4, v1, 24, 41993; GFX9-NEXT: v_bfe_u32 v5, v1, 20, 41994; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 41995; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 41996; GFX9-NEXT: v_bfe_u32 v8, v1, 8, 41997; GFX9-NEXT: v_bfe_u32 v9, v1, 4, 41998; GFX9-NEXT: v_and_b32_e32 v1, 15, v11999; GFX9-NEXT: s_waitcnt vmcnt(0)2000; GFX9-NEXT: v_lshrrev_b32_e32 v10, 28, v22001; GFX9-NEXT: v_bfe_u32 v11, v2, 24, 42002; GFX9-NEXT: v_bfe_u32 v12, v2, 20, 42003; GFX9-NEXT: v_bfe_u32 v13, v2, 16, 42004; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 42005; GFX9-NEXT: v_bfe_u32 v15, v2, 8, 42006; GFX9-NEXT: v_bfe_u32 v16, v2, 4, 42007; GFX9-NEXT: v_and_b32_e32 v2, 15, v22008; GFX9-NEXT: v_mul_u32_u24_e32 v1, v1, v22009; GFX9-NEXT: v_mul_u32_u24_e32 v2, v9, v162010; GFX9-NEXT: v_mul_u32_u24_e32 v8, v8, v152011; GFX9-NEXT: v_mul_u32_u24_e32 v7, v7, v142012; GFX9-NEXT: s_waitcnt lgkmcnt(0)2013; GFX9-NEXT: v_add3_u32 v1, v1, s0, v22014; GFX9-NEXT: v_mul_u32_u24_e32 v6, v6, v132015; GFX9-NEXT: v_mul_u32_u24_e32 v5, v5, v122016; GFX9-NEXT: v_add3_u32 v1, v1, v8, v72017; GFX9-NEXT: v_mul_u32_u24_e32 v4, v4, v112018; GFX9-NEXT: v_mul_u32_u24_e32 v3, v3, v102019; GFX9-NEXT: v_add3_u32 v1, v1, v6, v52020; GFX9-NEXT: v_add3_u32 v1, v1, v4, v32021; GFX9-NEXT: global_store_dword v0, v1, s[6:7]2022; GFX9-NEXT: s_endpgm2023;2024; GFX9-DL-LABEL: udot8_acc32_vecMul:2025; GFX9-DL: ; %bb.0: ; %entry2026; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242027; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342028; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02029; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02030; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12031; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2032; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2033; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2034; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x02035; GFX9-DL-NEXT: s_mov_b32 s14, -12036; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000002037; GFX9-DL-NEXT: s_add_u32 s12, s12, s112038; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02039; GFX9-DL-NEXT: s_addc_u32 s13, s13, 02040; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2041; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s02042; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]2043; GFX9-DL-NEXT: s_endpgm2044;2045; GFX10-DL-LABEL: udot8_acc32_vecMul:2046; GFX10-DL: ; %bb.0: ; %entry2047; GFX10-DL-NEXT: s_clause 0x12048; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242049; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342050; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02051; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02052; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12053; GFX10-DL-NEXT: s_mov_b32 s14, -12054; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160002055; GFX10-DL-NEXT: s_add_u32 s12, s12, s112056; GFX10-DL-NEXT: s_addc_u32 s13, s13, 02057; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2058; GFX10-DL-NEXT: s_clause 0x12059; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2060; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2061; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2062; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x02063; GFX10-DL-NEXT: v_mov_b32_e32 v0, 02064; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2065; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v1, v2, s02066; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]2067; GFX10-DL-NEXT: s_endpgm2068 ptr addrspace(1) %src2,2069 ptr addrspace(1) nocapture %dst) {2070entry:2071 %idx = call i32 @llvm.amdgcn.workitem.id.x()2072 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx2073 %vec1 = load <8 x i4>, ptr addrspace(1) %gep12074 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx2075 %vec2 = load <8 x i4>, ptr addrspace(1) %gep22076 2077 %cvec1 = zext <8 x i4> %vec1 to <8 x i32>2078 %cvec2 = zext <8 x i4> %vec2 to <8 x i32>2079 2080 %mul = mul <8 x i32> %cvec1, %cvec22081 %mul0 = extractelement <8 x i32> %mul, i64 02082 %mul1 = extractelement <8 x i32> %mul, i64 12083 %mul2 = extractelement <8 x i32> %mul, i64 22084 %mul3 = extractelement <8 x i32> %mul, i64 32085 %mul4 = extractelement <8 x i32> %mul, i64 42086 %mul5 = extractelement <8 x i32> %mul, i64 52087 %mul6 = extractelement <8 x i32> %mul, i64 62088 %mul7 = extractelement <8 x i32> %mul, i64 72089 2090 %acc = load i32, ptr addrspace(1) %dst, align 42091 %add1 = add i32 %mul0, %acc2092 %add2 = add i32 %add1, %mul12093 %add3 = add i32 %add2, %mul22094 %add4 = add i32 %add3, %mul32095 %add5 = add i32 %add4, %mul42096 %add6 = add i32 %add5, %mul52097 %add7 = add i32 %add6, %mul62098 %add8 = add i32 %add7, %mul72099 2100 store i32 %add8, ptr addrspace(1) %dst, align 42101 ret void2102}2103 2104; TODO: Clean up the code(by default pk_mad_I16 should be generated), then2105; support the pattern.2106define amdgpu_kernel void @udot8_acc16_vecMul(ptr addrspace(1) %src1,2107; GFX7-LABEL: udot8_acc16_vecMul:2108; GFX7: ; %bb.0: ; %entry2109; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02110; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12111; GFX7-NEXT: s_mov_b32 s14, -12112; GFX7-NEXT: s_mov_b32 s15, 0xe8f0002113; GFX7-NEXT: s_add_u32 s12, s12, s112114; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x92115; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2116; GFX7-NEXT: s_mov_b32 s3, 0xf0002117; GFX7-NEXT: s_mov_b32 s6, 02118; GFX7-NEXT: s_mov_b32 s7, s32119; GFX7-NEXT: s_waitcnt lgkmcnt(0)2120; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]2121; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02122; GFX7-NEXT: v_mov_b32_e32 v1, 02123; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr642124; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]2125; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr642126; GFX7-NEXT: s_mov_b32 s2, -12127; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 02128; GFX7-NEXT: s_addc_u32 s13, s13, 02129; GFX7-NEXT: s_waitcnt vmcnt(2)2130; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v22131; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 42132; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 42133; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 42134; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 42135; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 42136; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 42137; GFX7-NEXT: v_and_b32_e32 v2, 15, v22138; GFX7-NEXT: s_waitcnt vmcnt(1)2139; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v02140; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 42141; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 42142; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 42143; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 42144; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 42145; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 42146; GFX7-NEXT: v_and_b32_e32 v0, 15, v02147; GFX7-NEXT: s_waitcnt vmcnt(0)2148; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12149; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v02150; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v02151; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v02152; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v02153; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v02154; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v02155; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v02156; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 02157; GFX7-NEXT: s_endpgm2158;2159; GFX8-LABEL: udot8_acc16_vecMul:2160; GFX8: ; %bb.0: ; %entry2161; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242162; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342163; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02164; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02165; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12166; GFX8-NEXT: s_waitcnt lgkmcnt(0)2167; GFX8-NEXT: v_mov_b32_e32 v1, s12168; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22169; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2170; GFX8-NEXT: flat_load_dword v3, v[0:1]2171; GFX8-NEXT: v_mov_b32_e32 v1, s32172; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22173; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2174; GFX8-NEXT: flat_load_dword v2, v[0:1]2175; GFX8-NEXT: v_mov_b32_e32 v0, s42176; GFX8-NEXT: v_mov_b32_e32 v1, s52177; GFX8-NEXT: flat_load_ushort v4, v[0:1]2178; GFX8-NEXT: s_mov_b32 s14, -12179; GFX8-NEXT: s_mov_b32 s15, 0xe800002180; GFX8-NEXT: s_add_u32 s12, s12, s112181; GFX8-NEXT: s_addc_u32 s13, s13, 02182; GFX8-NEXT: s_waitcnt vmcnt(2)2183; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v32184; GFX8-NEXT: v_bfe_u32 v6, v3, 24, 42185; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 42186; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 42187; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 42188; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 42189; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 42190; GFX8-NEXT: v_and_b32_e32 v3, 15, v32191; GFX8-NEXT: s_waitcnt vmcnt(1)2192; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v22193; GFX8-NEXT: v_bfe_u32 v13, v2, 24, 42194; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 42195; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 42196; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 42197; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 42198; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 42199; GFX8-NEXT: v_and_b32_e32 v2, 15, v22200; GFX8-NEXT: s_waitcnt vmcnt(0)2201; GFX8-NEXT: v_mad_u16 v2, v3, v2, v42202; GFX8-NEXT: v_mad_u16 v2, v11, v18, v22203; GFX8-NEXT: v_mad_u16 v2, v10, v17, v22204; GFX8-NEXT: v_mad_u16 v2, v9, v16, v22205; GFX8-NEXT: v_mad_u16 v2, v8, v15, v22206; GFX8-NEXT: v_mad_u16 v2, v7, v14, v22207; GFX8-NEXT: v_mad_u16 v2, v6, v13, v22208; GFX8-NEXT: v_mad_u16 v2, v5, v12, v22209; GFX8-NEXT: flat_store_short v[0:1], v22210; GFX8-NEXT: s_endpgm2211;2212; GFX9-LABEL: udot8_acc16_vecMul:2213; GFX9: ; %bb.0: ; %entry2214; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242215; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342216; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v02217; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02218; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12219; GFX9-NEXT: s_waitcnt lgkmcnt(0)2220; GFX9-NEXT: global_load_dword v1, v0, s[0:1]2221; GFX9-NEXT: global_load_dword v2, v0, s[2:3]2222; GFX9-NEXT: v_mov_b32_e32 v0, 02223; GFX9-NEXT: global_load_ushort v3, v0, s[6:7]2224; GFX9-NEXT: s_mov_b32 s0, 0x50401002225; GFX9-NEXT: s_mov_b32 s14, -12226; GFX9-NEXT: s_mov_b32 s15, 0xe000002227; GFX9-NEXT: s_add_u32 s12, s12, s112228; GFX9-NEXT: s_addc_u32 s13, s13, 02229; GFX9-NEXT: s_waitcnt vmcnt(2)2230; GFX9-NEXT: v_and_b32_e32 v4, 15, v12231; GFX9-NEXT: v_bfe_u32 v5, v1, 4, 42232; GFX9-NEXT: v_bfe_u32 v6, v1, 8, 42233; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 42234; GFX9-NEXT: s_waitcnt vmcnt(1)2235; GFX9-NEXT: v_and_b32_e32 v11, 15, v22236; GFX9-NEXT: v_bfe_u32 v12, v2, 4, 42237; GFX9-NEXT: v_perm_b32 v6, v7, v6, s02238; GFX9-NEXT: v_perm_b32 v7, v12, v11, s02239; GFX9-NEXT: v_perm_b32 v4, v5, v4, s02240; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 42241; GFX9-NEXT: v_bfe_u32 v9, v1, 20, 42242; GFX9-NEXT: v_bfe_u32 v13, v2, 8, 42243; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 42244; GFX9-NEXT: v_pk_mul_lo_u16 v4, v4, v72245; GFX9-NEXT: v_perm_b32 v8, v9, v8, s02246; GFX9-NEXT: v_perm_b32 v9, v14, v13, s02247; GFX9-NEXT: s_waitcnt vmcnt(0)2248; GFX9-NEXT: v_add_u16_e32 v3, v4, v32249; GFX9-NEXT: v_bfe_u32 v10, v1, 24, 42250; GFX9-NEXT: v_lshrrev_b32_e32 v1, 28, v12251; GFX9-NEXT: v_bfe_u32 v15, v2, 16, 42252; GFX9-NEXT: v_bfe_u32 v16, v2, 20, 42253; GFX9-NEXT: v_bfe_u32 v17, v2, 24, 42254; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v22255; GFX9-NEXT: v_pk_mul_lo_u16 v5, v6, v92256; GFX9-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12257; GFX9-NEXT: v_perm_b32 v2, v2, v17, s02258; GFX9-NEXT: v_perm_b32 v1, v1, v10, s02259; GFX9-NEXT: v_perm_b32 v10, v16, v15, s02260; GFX9-NEXT: v_add_u16_e32 v3, v3, v52261; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v22262; GFX9-NEXT: v_pk_mul_lo_u16 v2, v8, v102263; GFX9-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12264; GFX9-NEXT: v_add_u16_e32 v3, v3, v22265; GFX9-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12266; GFX9-NEXT: v_add_u16_e32 v2, v2, v12267; GFX9-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12268; GFX9-NEXT: global_store_short v0, v1, s[6:7]2269; GFX9-NEXT: s_endpgm2270;2271; GFX9-DL-LABEL: udot8_acc16_vecMul:2272; GFX9-DL: ; %bb.0: ; %entry2273; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242274; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342275; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02276; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02277; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12278; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2279; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2280; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2281; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02282; GFX9-DL-NEXT: global_load_ushort v3, v0, s[6:7]2283; GFX9-DL-NEXT: s_mov_b32 s0, 0x50401002284; GFX9-DL-NEXT: s_mov_b32 s14, -12285; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000002286; GFX9-DL-NEXT: s_add_u32 s12, s12, s112287; GFX9-DL-NEXT: s_addc_u32 s13, s13, 02288; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2289; GFX9-DL-NEXT: v_and_b32_e32 v4, 15, v12290; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 4, 42291; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 8, 42292; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 12, 42293; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2294; GFX9-DL-NEXT: v_and_b32_e32 v11, 15, v22295; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 4, 42296; GFX9-DL-NEXT: v_perm_b32 v6, v7, v6, s02297; GFX9-DL-NEXT: v_perm_b32 v7, v12, v11, s02298; GFX9-DL-NEXT: v_perm_b32 v4, v5, v4, s02299; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 16, 42300; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 20, 42301; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 8, 42302; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 12, 42303; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v72304; GFX9-DL-NEXT: v_perm_b32 v8, v9, v8, s02305; GFX9-DL-NEXT: v_perm_b32 v9, v14, v13, s02306; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2307; GFX9-DL-NEXT: v_add_u16_e32 v3, v4, v32308; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 24, 42309; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v12310; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 16, 42311; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 20, 42312; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 24, 42313; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v22314; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v6, v92315; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12316; GFX9-DL-NEXT: v_perm_b32 v2, v2, v17, s02317; GFX9-DL-NEXT: v_perm_b32 v1, v1, v10, s02318; GFX9-DL-NEXT: v_perm_b32 v10, v16, v15, s02319; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v52320; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v22321; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v8, v102322; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12323; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v22324; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12325; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v12326; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12327; GFX9-DL-NEXT: global_store_short v0, v1, s[6:7]2328; GFX9-DL-NEXT: s_endpgm2329;2330; GFX10-DL-LABEL: udot8_acc16_vecMul:2331; GFX10-DL: ; %bb.0: ; %entry2332; GFX10-DL-NEXT: s_clause 0x12333; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242334; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342335; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02336; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02337; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12338; GFX10-DL-NEXT: s_mov_b32 s14, -12339; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160002340; GFX10-DL-NEXT: s_add_u32 s12, s12, s112341; GFX10-DL-NEXT: s_addc_u32 s13, s13, 02342; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2343; GFX10-DL-NEXT: s_clause 0x12344; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2345; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2346; GFX10-DL-NEXT: v_mov_b32_e32 v0, 02347; GFX10-DL-NEXT: global_load_ushort v3, v0, s[6:7]2348; GFX10-DL-NEXT: s_waitcnt vmcnt(2)2349; GFX10-DL-NEXT: v_and_b32_e32 v4, 15, v12350; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2351; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v22352; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 42353; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 4, 42354; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 12, 42355; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 42356; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 20, 42357; GFX10-DL-NEXT: v_perm_b32 v5, v6, v5, 0x50401002358; GFX10-DL-NEXT: v_perm_b32 v4, v7, v4, 0x50401002359; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 8, 42360; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 8, 42361; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v52362; GFX10-DL-NEXT: v_perm_b32 v6, v9, v6, 0x50401002363; GFX10-DL-NEXT: v_perm_b32 v7, v8, v7, 0x50401002364; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 16, 42365; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 20, 42366; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v42367; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2368; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v32369; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 42370; GFX10-DL-NEXT: v_pk_mul_lo_u16 v6, v6, v72371; GFX10-DL-NEXT: v_perm_b32 v5, v10, v5, 0x50401002372; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 24, 42373; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v82374; GFX10-DL-NEXT: v_perm_b32 v4, v9, v4, 0x50401002375; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v62376; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v12377; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v62378; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 24, 42379; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v22380; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v42381; GFX10-DL-NEXT: v_perm_b32 v1, v1, v7, 0x50401002382; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v82383; GFX10-DL-NEXT: v_perm_b32 v2, v2, v6, 0x50401002384; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v42385; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v42386; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v22387; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v52388; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v12389; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v12390; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v32391; GFX10-DL-NEXT: global_store_short v0, v1, s[6:7]2392; GFX10-DL-NEXT: s_endpgm2393 ptr addrspace(1) %src2,2394 ptr addrspace(1) nocapture %dst) {2395entry:2396 %idx = call i32 @llvm.amdgcn.workitem.id.x()2397 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx2398 %vec1 = load <8 x i4>, ptr addrspace(1) %gep12399 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx2400 %vec2 = load <8 x i4>, ptr addrspace(1) %gep22401 2402 %cvec1 = zext <8 x i4> %vec1 to <8 x i16>2403 %cvec2 = zext <8 x i4> %vec2 to <8 x i16>2404 2405 %mul = mul <8 x i16> %cvec1, %cvec22406 %mul0 = extractelement <8 x i16> %mul, i64 02407 %mul1 = extractelement <8 x i16> %mul, i64 12408 %mul2 = extractelement <8 x i16> %mul, i64 22409 %mul3 = extractelement <8 x i16> %mul, i64 32410 %mul4 = extractelement <8 x i16> %mul, i64 42411 %mul5 = extractelement <8 x i16> %mul, i64 52412 %mul6 = extractelement <8 x i16> %mul, i64 62413 %mul7 = extractelement <8 x i16> %mul, i64 72414 2415 %acc = load i16, ptr addrspace(1) %dst, align 42416 %add1 = add i16 %mul0, %acc2417 %add2 = add i16 %add1, %mul12418 %add3 = add i16 %add2, %mul22419 %add4 = add i16 %add3, %mul32420 %add5 = add i16 %add4, %mul42421 %add6 = add i16 %add5, %mul52422 %add7 = add i16 %add6, %mul62423 %add8 = add i16 %add7, %mul72424 2425 store i16 %add8, ptr addrspace(1) %dst, align 42426 ret void2427}2428 2429; TODO: Cleanup the code to generate MAD; pattern should be recognized then.2430define amdgpu_kernel void @udot8_acc8_vecMul(ptr addrspace(1) %src1,2431; GFX7-LABEL: udot8_acc8_vecMul:2432; GFX7: ; %bb.0: ; %entry2433; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02434; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12435; GFX7-NEXT: s_mov_b32 s14, -12436; GFX7-NEXT: s_mov_b32 s15, 0xe8f0002437; GFX7-NEXT: s_add_u32 s12, s12, s112438; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x92439; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2440; GFX7-NEXT: s_mov_b32 s3, 0xf0002441; GFX7-NEXT: s_mov_b32 s6, 02442; GFX7-NEXT: s_mov_b32 s7, s32443; GFX7-NEXT: s_waitcnt lgkmcnt(0)2444; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]2445; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02446; GFX7-NEXT: v_mov_b32_e32 v1, 02447; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr642448; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]2449; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr642450; GFX7-NEXT: s_mov_b32 s2, -12451; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 02452; GFX7-NEXT: s_addc_u32 s13, s13, 02453; GFX7-NEXT: s_waitcnt vmcnt(2)2454; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v22455; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 42456; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 42457; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 42458; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 42459; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 42460; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 42461; GFX7-NEXT: v_and_b32_e32 v2, 15, v22462; GFX7-NEXT: s_waitcnt vmcnt(1)2463; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v02464; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 42465; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 42466; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 42467; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 42468; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 42469; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 42470; GFX7-NEXT: v_and_b32_e32 v0, 15, v02471; GFX7-NEXT: s_waitcnt vmcnt(0)2472; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12473; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v02474; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v02475; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v02476; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v02477; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v02478; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v02479; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v02480; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 02481; GFX7-NEXT: s_endpgm2482;2483; GFX8-LABEL: udot8_acc8_vecMul:2484; GFX8: ; %bb.0: ; %entry2485; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242486; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342487; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02488; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02489; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12490; GFX8-NEXT: s_waitcnt lgkmcnt(0)2491; GFX8-NEXT: v_mov_b32_e32 v1, s12492; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22493; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2494; GFX8-NEXT: flat_load_dword v3, v[0:1]2495; GFX8-NEXT: v_mov_b32_e32 v1, s32496; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22497; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2498; GFX8-NEXT: flat_load_dword v2, v[0:1]2499; GFX8-NEXT: v_mov_b32_e32 v0, s42500; GFX8-NEXT: v_mov_b32_e32 v1, s52501; GFX8-NEXT: flat_load_ubyte v4, v[0:1]2502; GFX8-NEXT: s_mov_b32 s14, -12503; GFX8-NEXT: s_mov_b32 s15, 0xe800002504; GFX8-NEXT: s_add_u32 s12, s12, s112505; GFX8-NEXT: s_addc_u32 s13, s13, 02506; GFX8-NEXT: s_waitcnt vmcnt(2)2507; GFX8-NEXT: v_lshrrev_b32_e32 v9, 28, v32508; GFX8-NEXT: v_bfe_u32 v10, v3, 24, 42509; GFX8-NEXT: v_bfe_u32 v11, v3, 20, 42510; GFX8-NEXT: v_bfe_u32 v7, v3, 12, 42511; GFX8-NEXT: v_bfe_u32 v8, v3, 8, 42512; GFX8-NEXT: v_bfe_u32 v12, v3, 16, 42513; GFX8-NEXT: s_waitcnt vmcnt(1)2514; GFX8-NEXT: v_lshrrev_b32_e32 v16, 28, v22515; GFX8-NEXT: v_bfe_u32 v17, v2, 24, 42516; GFX8-NEXT: v_bfe_u32 v18, v2, 20, 42517; GFX8-NEXT: v_bfe_u32 v14, v2, 12, 42518; GFX8-NEXT: v_bfe_u32 v15, v2, 8, 42519; GFX8-NEXT: v_bfe_u32 v19, v2, 16, 42520; GFX8-NEXT: v_mul_lo_u16_sdwa v11, v11, v18 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2521; GFX8-NEXT: v_mul_lo_u16_e32 v18, v10, v172522; GFX8-NEXT: v_mul_lo_u16_sdwa v9, v9, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2523; GFX8-NEXT: v_bfe_u32 v5, v3, 4, 42524; GFX8-NEXT: v_and_b32_e32 v6, 15, v32525; GFX8-NEXT: v_bfe_u32 v3, v2, 4, 42526; GFX8-NEXT: v_and_b32_e32 v13, 15, v22527; GFX8-NEXT: v_mul_lo_u16_e32 v2, v12, v192528; GFX8-NEXT: v_mul_lo_u16_e32 v8, v8, v152529; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v7, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2530; GFX8-NEXT: v_or_b32_e32 v9, v18, v92531; GFX8-NEXT: v_mul_lo_u16_sdwa v5, v5, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2532; GFX8-NEXT: v_or_b32_e32 v3, v2, v112533; GFX8-NEXT: v_or_b32_e32 v7, v8, v72534; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v92535; GFX8-NEXT: v_mul_lo_u16_e32 v6, v6, v132536; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v72537; GFX8-NEXT: v_or_b32_sdwa v3, v3, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2538; GFX8-NEXT: v_or_b32_e32 v6, v6, v52539; GFX8-NEXT: v_or_b32_e32 v5, v5, v22540; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v32541; GFX8-NEXT: v_lshrrev_b64 v[2:3], 24, v[2:3]2542; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v52543; GFX8-NEXT: s_waitcnt vmcnt(0)2544; GFX8-NEXT: v_add_u16_e32 v3, v6, v42545; GFX8-NEXT: v_add_u16_e32 v3, v3, v52546; GFX8-NEXT: v_add_u16_e32 v3, v3, v72547; GFX8-NEXT: v_add_u16_e32 v2, v3, v22548; GFX8-NEXT: v_mad_u16 v2, v12, v19, v22549; GFX8-NEXT: v_add_u16_e32 v2, v2, v82550; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v92551; GFX8-NEXT: v_mad_u16 v2, v10, v17, v22552; GFX8-NEXT: v_add_u16_e32 v2, v2, v92553; GFX8-NEXT: flat_store_byte v[0:1], v22554; GFX8-NEXT: s_endpgm2555;2556; GFX9-LABEL: udot8_acc8_vecMul:2557; GFX9: ; %bb.0: ; %entry2558; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242559; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342560; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v02561; GFX9-NEXT: v_mov_b32_e32 v3, 02562; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02563; GFX9-NEXT: s_waitcnt lgkmcnt(0)2564; GFX9-NEXT: global_load_dword v1, v0, s[0:1]2565; GFX9-NEXT: global_load_dword v2, v0, s[2:3]2566; GFX9-NEXT: global_load_ubyte v4, v3, s[6:7]2567; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12568; GFX9-NEXT: s_mov_b32 s14, -12569; GFX9-NEXT: s_mov_b32 s15, 0xe000002570; GFX9-NEXT: s_add_u32 s12, s12, s112571; GFX9-NEXT: s_addc_u32 s13, s13, 02572; GFX9-NEXT: s_waitcnt vmcnt(2)2573; GFX9-NEXT: v_bfe_u32 v0, v1, 4, 42574; GFX9-NEXT: v_and_b32_e32 v5, 15, v12575; GFX9-NEXT: v_bfe_u32 v6, v1, 12, 42576; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 42577; GFX9-NEXT: v_lshrrev_b32_e32 v8, 28, v12578; GFX9-NEXT: v_bfe_u32 v9, v1, 24, 42579; GFX9-NEXT: v_bfe_u32 v10, v1, 20, 42580; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 42581; GFX9-NEXT: s_waitcnt vmcnt(1)2582; GFX9-NEXT: v_bfe_u32 v1, v2, 4, 42583; GFX9-NEXT: v_and_b32_e32 v12, 15, v22584; GFX9-NEXT: v_bfe_u32 v13, v2, 12, 42585; GFX9-NEXT: v_bfe_u32 v14, v2, 8, 42586; GFX9-NEXT: v_lshrrev_b32_e32 v15, 28, v22587; GFX9-NEXT: v_bfe_u32 v16, v2, 24, 42588; GFX9-NEXT: v_bfe_u32 v17, v2, 20, 42589; GFX9-NEXT: v_bfe_u32 v2, v2, 16, 42590; GFX9-NEXT: v_mul_lo_u16_e32 v18, v11, v22591; GFX9-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2592; GFX9-NEXT: v_mul_lo_u16_e32 v17, v9, v162593; GFX9-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2594; GFX9-NEXT: v_mul_lo_u16_e32 v7, v7, v142595; GFX9-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2596; GFX9-NEXT: v_mul_lo_u16_e32 v5, v5, v122597; GFX9-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2598; GFX9-NEXT: v_or_b32_e32 v0, v18, v102599; GFX9-NEXT: v_or_b32_sdwa v1, v17, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2600; GFX9-NEXT: v_or_b32_e32 v6, v7, v62601; GFX9-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2602; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v62603; GFX9-NEXT: v_or_b32_e32 v5, v5, v122604; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v82605; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v12606; GFX9-NEXT: v_or_b32_e32 v10, v12, v02607; GFX9-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]2608; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v102609; GFX9-NEXT: s_waitcnt vmcnt(0)2610; GFX9-NEXT: v_add_u16_e32 v4, v5, v42611; GFX9-NEXT: v_add_u16_e32 v1, v4, v12612; GFX9-NEXT: v_add_u16_e32 v1, v1, v62613; GFX9-NEXT: v_add_u16_e32 v0, v1, v02614; GFX9-NEXT: v_mad_legacy_u16 v0, v11, v2, v02615; GFX9-NEXT: v_add_u16_e32 v0, v0, v82616; GFX9-NEXT: v_mad_legacy_u16 v0, v9, v16, v02617; GFX9-NEXT: v_add_u16_e32 v0, v0, v72618; GFX9-NEXT: global_store_byte v3, v0, s[6:7]2619; GFX9-NEXT: s_endpgm2620;2621; GFX9-DL-LABEL: udot8_acc8_vecMul:2622; GFX9-DL: ; %bb.0: ; %entry2623; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242624; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342625; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02626; GFX9-DL-NEXT: v_mov_b32_e32 v3, 02627; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02628; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2629; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2630; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2631; GFX9-DL-NEXT: global_load_ubyte v4, v3, s[6:7]2632; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12633; GFX9-DL-NEXT: s_mov_b32 s14, -12634; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000002635; GFX9-DL-NEXT: s_add_u32 s12, s12, s112636; GFX9-DL-NEXT: s_addc_u32 s13, s13, 02637; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2638; GFX9-DL-NEXT: v_bfe_u32 v0, v1, 4, 42639; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v12640; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 12, 42641; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 8, 42642; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 28, v12643; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 24, 42644; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 20, 42645; GFX9-DL-NEXT: v_bfe_u32 v11, v1, 16, 42646; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2647; GFX9-DL-NEXT: v_bfe_u32 v1, v2, 4, 42648; GFX9-DL-NEXT: v_and_b32_e32 v12, 15, v22649; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 12, 42650; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 8, 42651; GFX9-DL-NEXT: v_lshrrev_b32_e32 v15, 28, v22652; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 24, 42653; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 20, 42654; GFX9-DL-NEXT: v_bfe_u32 v2, v2, 16, 42655; GFX9-DL-NEXT: v_mul_lo_u16_e32 v18, v11, v22656; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, v10, v17 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2657; GFX9-DL-NEXT: v_mul_lo_u16_e32 v17, v9, v162658; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, v8, v15 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2659; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v7, v142660; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v6, v13 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2661; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, v5, v122662; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v12, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2663; GFX9-DL-NEXT: v_or_b32_e32 v0, v18, v102664; GFX9-DL-NEXT: v_or_b32_sdwa v1, v17, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2665; GFX9-DL-NEXT: v_or_b32_e32 v6, v7, v62666; GFX9-DL-NEXT: v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2667; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v62668; GFX9-DL-NEXT: v_or_b32_e32 v5, v5, v122669; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v82670; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v12671; GFX9-DL-NEXT: v_or_b32_e32 v10, v12, v02672; GFX9-DL-NEXT: v_lshrrev_b64 v[0:1], 24, v[0:1]2673; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v102674; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2675; GFX9-DL-NEXT: v_add_u16_e32 v4, v5, v42676; GFX9-DL-NEXT: v_add_u16_e32 v1, v4, v12677; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v62678; GFX9-DL-NEXT: v_add_u16_e32 v0, v1, v02679; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v11, v2, v02680; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v82681; GFX9-DL-NEXT: v_mad_legacy_u16 v0, v9, v16, v02682; GFX9-DL-NEXT: v_add_u16_e32 v0, v0, v72683; GFX9-DL-NEXT: global_store_byte v3, v0, s[6:7]2684; GFX9-DL-NEXT: s_endpgm2685;2686; GFX10-DL-LABEL: udot8_acc8_vecMul:2687; GFX10-DL: ; %bb.0: ; %entry2688; GFX10-DL-NEXT: s_clause 0x12689; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242690; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342691; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02692; GFX10-DL-NEXT: v_mov_b32_e32 v4, 02693; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02694; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12695; GFX10-DL-NEXT: s_mov_b32 s14, -12696; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160002697; GFX10-DL-NEXT: s_add_u32 s12, s12, s112698; GFX10-DL-NEXT: s_addc_u32 s13, s13, 02699; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)2700; GFX10-DL-NEXT: s_clause 0x12701; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]2702; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]2703; GFX10-DL-NEXT: global_load_ubyte v3, v4, s[6:7]2704; GFX10-DL-NEXT: s_waitcnt vmcnt(2)2705; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 12, 42706; GFX10-DL-NEXT: s_waitcnt vmcnt(1)2707; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 12, 42708; GFX10-DL-NEXT: v_bfe_u32 v0, v1, 4, 42709; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v12710; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 8, 42711; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 28, v12712; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 24, 42713; GFX10-DL-NEXT: v_bfe_u32 v11, v1, 20, 42714; GFX10-DL-NEXT: v_bfe_u32 v12, v1, 16, 42715; GFX10-DL-NEXT: v_bfe_u32 v1, v2, 8, 42716; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v92717; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 4, 42718; GFX10-DL-NEXT: v_lshrrev_b32_e32 v14, 28, v22719; GFX10-DL-NEXT: v_bfe_u32 v15, v2, 20, 42720; GFX10-DL-NEXT: v_mul_lo_u16 v1, v7, v12721; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v62722; GFX10-DL-NEXT: v_and_b32_e32 v13, 15, v22723; GFX10-DL-NEXT: v_mul_lo_u16 v0, v0, v92724; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 16, 42725; GFX10-DL-NEXT: v_bfe_u32 v16, v2, 24, 42726; GFX10-DL-NEXT: v_or_b32_e32 v6, v1, v62727; GFX10-DL-NEXT: v_mul_lo_u16 v2, v11, v152728; GFX10-DL-NEXT: v_mul_lo_u16 v8, v8, v142729; GFX10-DL-NEXT: v_lshlrev_b16 v9, 8, v02730; GFX10-DL-NEXT: v_mul_lo_u16 v5, v5, v132731; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 16, v62732; GFX10-DL-NEXT: v_mul_lo_u16 v1, v12, v72733; GFX10-DL-NEXT: v_mul_lo_u16 v11, v10, v162734; GFX10-DL-NEXT: v_lshlrev_b16 v2, 8, v22735; GFX10-DL-NEXT: v_lshlrev_b16 v8, 8, v82736; GFX10-DL-NEXT: v_or_b32_sdwa v13, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2737; GFX10-DL-NEXT: v_or_b32_e32 v5, v5, v92738; GFX10-DL-NEXT: v_or_b32_e32 v1, v1, v22739; GFX10-DL-NEXT: v_or_b32_sdwa v2, v11, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD2740; GFX10-DL-NEXT: v_lshrrev_b32_e32 v9, 8, v132741; GFX10-DL-NEXT: s_waitcnt vmcnt(0)2742; GFX10-DL-NEXT: v_add_nc_u16 v3, v5, v32743; GFX10-DL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2744; GFX10-DL-NEXT: v_add_nc_u16 v5, v3, v92745; GFX10-DL-NEXT: v_lshrrev_b64 v[2:3], 24, v[0:1]2746; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v12747; GFX10-DL-NEXT: v_add_nc_u16 v0, v5, v62748; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v22749; GFX10-DL-NEXT: v_mad_u16 v0, v12, v7, v02750; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v12751; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v82752; GFX10-DL-NEXT: v_mad_u16 v0, v10, v16, v02753; GFX10-DL-NEXT: v_add_nc_u16 v0, v0, v12754; GFX10-DL-NEXT: global_store_byte v4, v0, s[6:7]2755; GFX10-DL-NEXT: s_endpgm2756 ptr addrspace(1) %src2,2757 ptr addrspace(1) nocapture %dst) {2758entry:2759 %idx = call i32 @llvm.amdgcn.workitem.id.x()2760 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx2761 %vec1 = load <8 x i4>, ptr addrspace(1) %gep12762 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx2763 %vec2 = load <8 x i4>, ptr addrspace(1) %gep22764 2765 %cvec1 = zext <8 x i4> %vec1 to <8 x i8>2766 %cvec2 = zext <8 x i4> %vec2 to <8 x i8>2767 2768 %mul = mul <8 x i8> %cvec1, %cvec22769 %mul0 = extractelement <8 x i8> %mul, i64 02770 %mul1 = extractelement <8 x i8> %mul, i64 12771 %mul2 = extractelement <8 x i8> %mul, i64 22772 %mul3 = extractelement <8 x i8> %mul, i64 32773 %mul4 = extractelement <8 x i8> %mul, i64 42774 %mul5 = extractelement <8 x i8> %mul, i64 52775 %mul6 = extractelement <8 x i8> %mul, i64 62776 %mul7 = extractelement <8 x i8> %mul, i64 72777 2778 %acc = load i8, ptr addrspace(1) %dst, align 42779 %add1 = add i8 %mul0, %acc2780 %add2 = add i8 %add1, %mul12781 %add3 = add i8 %add2, %mul22782 %add4 = add i8 %add3, %mul32783 %add5 = add i8 %add4, %mul42784 %add6 = add i8 %add5, %mul52785 %add7 = add i8 %add6, %mul62786 %add8 = add i8 %add7, %mul72787 2788 store i8 %add8, ptr addrspace(1) %dst, align 42789 ret void2790}2791 2792; TODO: Once the adictional "and+add" are removed, the pattern will be recognized.2793define amdgpu_kernel void @udot8_acc4_vecMul(ptr addrspace(1) %src1,2794; GFX7-LABEL: udot8_acc4_vecMul:2795; GFX7: ; %bb.0: ; %entry2796; GFX7-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02797; GFX7-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12798; GFX7-NEXT: s_mov_b32 s14, -12799; GFX7-NEXT: s_mov_b32 s15, 0xe8f0002800; GFX7-NEXT: s_add_u32 s12, s12, s112801; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x92802; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2803; GFX7-NEXT: s_mov_b32 s3, 0xf0002804; GFX7-NEXT: s_mov_b32 s6, 02805; GFX7-NEXT: s_mov_b32 s7, s32806; GFX7-NEXT: s_waitcnt lgkmcnt(0)2807; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]2808; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v02809; GFX7-NEXT: v_mov_b32_e32 v1, 02810; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr642811; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]2812; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr642813; GFX7-NEXT: s_mov_b32 s2, -12814; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 02815; GFX7-NEXT: s_addc_u32 s13, s13, 02816; GFX7-NEXT: s_waitcnt vmcnt(2)2817; GFX7-NEXT: v_lshrrev_b32_e32 v3, 28, v22818; GFX7-NEXT: v_bfe_u32 v4, v2, 24, 42819; GFX7-NEXT: v_bfe_u32 v5, v2, 20, 42820; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 42821; GFX7-NEXT: v_bfe_u32 v7, v2, 12, 42822; GFX7-NEXT: v_bfe_u32 v8, v2, 8, 42823; GFX7-NEXT: v_bfe_u32 v9, v2, 4, 42824; GFX7-NEXT: v_and_b32_e32 v2, 15, v22825; GFX7-NEXT: s_waitcnt vmcnt(1)2826; GFX7-NEXT: v_lshrrev_b32_e32 v10, 28, v02827; GFX7-NEXT: v_bfe_u32 v11, v0, 24, 42828; GFX7-NEXT: v_bfe_u32 v12, v0, 20, 42829; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 42830; GFX7-NEXT: v_bfe_u32 v14, v0, 12, 42831; GFX7-NEXT: v_bfe_u32 v15, v0, 8, 42832; GFX7-NEXT: v_bfe_u32 v16, v0, 4, 42833; GFX7-NEXT: v_and_b32_e32 v0, 15, v02834; GFX7-NEXT: s_waitcnt vmcnt(0)2835; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v12836; GFX7-NEXT: v_mad_u32_u24 v0, v9, v16, v02837; GFX7-NEXT: v_mad_u32_u24 v0, v8, v15, v02838; GFX7-NEXT: v_mad_u32_u24 v0, v7, v14, v02839; GFX7-NEXT: v_mad_u32_u24 v0, v6, v13, v02840; GFX7-NEXT: v_mad_u32_u24 v0, v5, v12, v02841; GFX7-NEXT: v_mad_u32_u24 v0, v4, v11, v02842; GFX7-NEXT: v_mad_u32_u24 v0, v3, v10, v02843; GFX7-NEXT: v_and_b32_e32 v0, 15, v02844; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 02845; GFX7-NEXT: s_endpgm2846;2847; GFX8-LABEL: udot8_acc4_vecMul:2848; GFX8: ; %bb.0: ; %entry2849; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242850; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342851; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v02852; GFX8-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02853; GFX8-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12854; GFX8-NEXT: s_waitcnt lgkmcnt(0)2855; GFX8-NEXT: v_mov_b32_e32 v1, s12856; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v22857; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2858; GFX8-NEXT: flat_load_dword v3, v[0:1]2859; GFX8-NEXT: v_mov_b32_e32 v1, s32860; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v22861; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2862; GFX8-NEXT: flat_load_dword v2, v[0:1]2863; GFX8-NEXT: v_mov_b32_e32 v0, s42864; GFX8-NEXT: v_mov_b32_e32 v1, s52865; GFX8-NEXT: flat_load_ubyte v4, v[0:1]2866; GFX8-NEXT: s_mov_b32 s14, -12867; GFX8-NEXT: s_mov_b32 s15, 0xe800002868; GFX8-NEXT: s_add_u32 s12, s12, s112869; GFX8-NEXT: s_addc_u32 s13, s13, 02870; GFX8-NEXT: s_waitcnt vmcnt(2)2871; GFX8-NEXT: v_lshrrev_b32_e32 v5, 28, v32872; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v32873; GFX8-NEXT: v_bfe_u32 v7, v3, 20, 42874; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 42875; GFX8-NEXT: v_bfe_u32 v9, v3, 12, 42876; GFX8-NEXT: v_bfe_u32 v10, v3, 8, 42877; GFX8-NEXT: v_bfe_u32 v11, v3, 4, 42878; GFX8-NEXT: v_and_b32_e32 v3, 15, v32879; GFX8-NEXT: s_waitcnt vmcnt(1)2880; GFX8-NEXT: v_lshrrev_b32_e32 v12, 28, v22881; GFX8-NEXT: v_lshrrev_b32_e32 v13, 24, v22882; GFX8-NEXT: v_bfe_u32 v14, v2, 20, 42883; GFX8-NEXT: v_bfe_u32 v15, v2, 16, 42884; GFX8-NEXT: v_bfe_u32 v16, v2, 12, 42885; GFX8-NEXT: v_bfe_u32 v17, v2, 8, 42886; GFX8-NEXT: v_bfe_u32 v18, v2, 4, 42887; GFX8-NEXT: v_and_b32_e32 v2, 15, v22888; GFX8-NEXT: s_waitcnt vmcnt(0)2889; GFX8-NEXT: v_mad_u16 v2, v3, v2, v42890; GFX8-NEXT: v_mad_u16 v2, v11, v18, v22891; GFX8-NEXT: v_mad_u16 v2, v10, v17, v22892; GFX8-NEXT: v_mad_u16 v2, v9, v16, v22893; GFX8-NEXT: v_mad_u16 v2, v8, v15, v22894; GFX8-NEXT: v_mad_u16 v2, v7, v14, v22895; GFX8-NEXT: v_mad_u16 v2, v6, v13, v22896; GFX8-NEXT: v_mad_u16 v2, v5, v12, v22897; GFX8-NEXT: v_and_b32_e32 v2, 15, v22898; GFX8-NEXT: flat_store_byte v[0:1], v22899; GFX8-NEXT: s_endpgm2900;2901; GFX9-LABEL: udot8_acc4_vecMul:2902; GFX9: ; %bb.0: ; %entry2903; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242904; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342905; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v02906; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02907; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12908; GFX9-NEXT: s_waitcnt lgkmcnt(0)2909; GFX9-NEXT: global_load_dword v1, v0, s[0:1]2910; GFX9-NEXT: global_load_dword v2, v0, s[2:3]2911; GFX9-NEXT: v_mov_b32_e32 v0, 02912; GFX9-NEXT: global_load_ubyte v3, v0, s[6:7]2913; GFX9-NEXT: s_mov_b32 s0, 0x50401002914; GFX9-NEXT: s_mov_b32 s14, -12915; GFX9-NEXT: s_mov_b32 s15, 0xe000002916; GFX9-NEXT: s_add_u32 s12, s12, s112917; GFX9-NEXT: s_addc_u32 s13, s13, 02918; GFX9-NEXT: s_waitcnt vmcnt(2)2919; GFX9-NEXT: v_and_b32_e32 v4, 15, v12920; GFX9-NEXT: v_bfe_u32 v5, v1, 4, 42921; GFX9-NEXT: v_bfe_u32 v6, v1, 8, 42922; GFX9-NEXT: v_bfe_u32 v7, v1, 12, 42923; GFX9-NEXT: s_waitcnt vmcnt(1)2924; GFX9-NEXT: v_and_b32_e32 v11, 15, v22925; GFX9-NEXT: v_bfe_u32 v12, v2, 4, 42926; GFX9-NEXT: v_perm_b32 v6, v7, v6, s02927; GFX9-NEXT: v_perm_b32 v7, v12, v11, s02928; GFX9-NEXT: v_perm_b32 v4, v5, v4, s02929; GFX9-NEXT: v_bfe_u32 v8, v1, 16, 42930; GFX9-NEXT: v_bfe_u32 v9, v1, 20, 42931; GFX9-NEXT: v_bfe_u32 v13, v2, 8, 42932; GFX9-NEXT: v_bfe_u32 v14, v2, 12, 42933; GFX9-NEXT: v_pk_mul_lo_u16 v4, v4, v72934; GFX9-NEXT: v_perm_b32 v8, v9, v8, s02935; GFX9-NEXT: v_perm_b32 v9, v14, v13, s02936; GFX9-NEXT: s_waitcnt vmcnt(0)2937; GFX9-NEXT: v_add_u16_e32 v3, v4, v32938; GFX9-NEXT: v_bfe_u32 v10, v1, 24, 42939; GFX9-NEXT: v_lshrrev_b32_e32 v1, 28, v12940; GFX9-NEXT: v_bfe_u32 v15, v2, 16, 42941; GFX9-NEXT: v_bfe_u32 v16, v2, 20, 42942; GFX9-NEXT: v_bfe_u32 v17, v2, 24, 42943; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v22944; GFX9-NEXT: v_pk_mul_lo_u16 v5, v6, v92945; GFX9-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12946; GFX9-NEXT: v_perm_b32 v2, v2, v17, s02947; GFX9-NEXT: v_perm_b32 v1, v1, v10, s02948; GFX9-NEXT: v_perm_b32 v10, v16, v15, s02949; GFX9-NEXT: v_add_u16_e32 v3, v3, v52950; GFX9-NEXT: v_pk_mul_lo_u16 v1, v1, v22951; GFX9-NEXT: v_pk_mul_lo_u16 v2, v8, v102952; GFX9-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12953; GFX9-NEXT: v_add_u16_e32 v3, v3, v22954; GFX9-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12955; GFX9-NEXT: v_add_u16_e32 v2, v2, v12956; GFX9-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_12957; GFX9-NEXT: v_and_b32_e32 v1, 15, v12958; GFX9-NEXT: global_store_byte v0, v1, s[6:7]2959; GFX9-NEXT: s_endpgm2960;2961; GFX9-DL-LABEL: udot8_acc4_vecMul:2962; GFX9-DL: ; %bb.0: ; %entry2963; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242964; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342965; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v02966; GFX9-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD02967; GFX9-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD12968; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)2969; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]2970; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]2971; GFX9-DL-NEXT: v_mov_b32_e32 v0, 02972; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[6:7]2973; GFX9-DL-NEXT: s_mov_b32 s0, 0x50401002974; GFX9-DL-NEXT: s_mov_b32 s14, -12975; GFX9-DL-NEXT: s_mov_b32 s15, 0xe000002976; GFX9-DL-NEXT: s_add_u32 s12, s12, s112977; GFX9-DL-NEXT: s_addc_u32 s13, s13, 02978; GFX9-DL-NEXT: s_waitcnt vmcnt(2)2979; GFX9-DL-NEXT: v_and_b32_e32 v4, 15, v12980; GFX9-DL-NEXT: v_bfe_u32 v5, v1, 4, 42981; GFX9-DL-NEXT: v_bfe_u32 v6, v1, 8, 42982; GFX9-DL-NEXT: v_bfe_u32 v7, v1, 12, 42983; GFX9-DL-NEXT: s_waitcnt vmcnt(1)2984; GFX9-DL-NEXT: v_and_b32_e32 v11, 15, v22985; GFX9-DL-NEXT: v_bfe_u32 v12, v2, 4, 42986; GFX9-DL-NEXT: v_perm_b32 v6, v7, v6, s02987; GFX9-DL-NEXT: v_perm_b32 v7, v12, v11, s02988; GFX9-DL-NEXT: v_perm_b32 v4, v5, v4, s02989; GFX9-DL-NEXT: v_bfe_u32 v8, v1, 16, 42990; GFX9-DL-NEXT: v_bfe_u32 v9, v1, 20, 42991; GFX9-DL-NEXT: v_bfe_u32 v13, v2, 8, 42992; GFX9-DL-NEXT: v_bfe_u32 v14, v2, 12, 42993; GFX9-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v72994; GFX9-DL-NEXT: v_perm_b32 v8, v9, v8, s02995; GFX9-DL-NEXT: v_perm_b32 v9, v14, v13, s02996; GFX9-DL-NEXT: s_waitcnt vmcnt(0)2997; GFX9-DL-NEXT: v_add_u16_e32 v3, v4, v32998; GFX9-DL-NEXT: v_bfe_u32 v10, v1, 24, 42999; GFX9-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v13000; GFX9-DL-NEXT: v_bfe_u32 v15, v2, 16, 43001; GFX9-DL-NEXT: v_bfe_u32 v16, v2, 20, 43002; GFX9-DL-NEXT: v_bfe_u32 v17, v2, 24, 43003; GFX9-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v23004; GFX9-DL-NEXT: v_pk_mul_lo_u16 v5, v6, v93005; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13006; GFX9-DL-NEXT: v_perm_b32 v2, v2, v17, s03007; GFX9-DL-NEXT: v_perm_b32 v1, v1, v10, s03008; GFX9-DL-NEXT: v_perm_b32 v10, v16, v15, s03009; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v53010; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v23011; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v8, v103012; GFX9-DL-NEXT: v_add_u16_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13013; GFX9-DL-NEXT: v_add_u16_e32 v3, v3, v23014; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13015; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v13016; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13017; GFX9-DL-NEXT: v_and_b32_e32 v1, 15, v13018; GFX9-DL-NEXT: global_store_byte v0, v1, s[6:7]3019; GFX9-DL-NEXT: s_endpgm3020;3021; GFX10-DL-LABEL: udot8_acc4_vecMul:3022; GFX10-DL: ; %bb.0: ; %entry3023; GFX10-DL-NEXT: s_clause 0x13024; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243025; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343026; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03027; GFX10-DL-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD03028; GFX10-DL-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD13029; GFX10-DL-NEXT: s_mov_b32 s14, -13030; GFX10-DL-NEXT: s_mov_b32 s15, 0x31c160003031; GFX10-DL-NEXT: s_add_u32 s12, s12, s113032; GFX10-DL-NEXT: s_addc_u32 s13, s13, 03033; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3034; GFX10-DL-NEXT: s_clause 0x13035; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]3036; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]3037; GFX10-DL-NEXT: v_mov_b32_e32 v0, 03038; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[6:7]3039; GFX10-DL-NEXT: s_waitcnt vmcnt(2)3040; GFX10-DL-NEXT: v_and_b32_e32 v4, 15, v13041; GFX10-DL-NEXT: s_waitcnt vmcnt(1)3042; GFX10-DL-NEXT: v_and_b32_e32 v5, 15, v23043; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 4, 43044; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 4, 43045; GFX10-DL-NEXT: v_bfe_u32 v8, v2, 12, 43046; GFX10-DL-NEXT: v_bfe_u32 v9, v1, 12, 43047; GFX10-DL-NEXT: v_bfe_u32 v10, v1, 20, 43048; GFX10-DL-NEXT: v_perm_b32 v5, v6, v5, 0x50401003049; GFX10-DL-NEXT: v_perm_b32 v4, v7, v4, 0x50401003050; GFX10-DL-NEXT: v_bfe_u32 v6, v1, 8, 43051; GFX10-DL-NEXT: v_bfe_u32 v7, v2, 8, 43052; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v53053; GFX10-DL-NEXT: v_perm_b32 v6, v9, v6, 0x50401003054; GFX10-DL-NEXT: v_perm_b32 v7, v8, v7, 0x50401003055; GFX10-DL-NEXT: v_bfe_u32 v5, v1, 16, 43056; GFX10-DL-NEXT: v_bfe_u32 v9, v2, 20, 43057; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v43058; GFX10-DL-NEXT: s_waitcnt vmcnt(0)3059; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v33060; GFX10-DL-NEXT: v_bfe_u32 v4, v2, 16, 43061; GFX10-DL-NEXT: v_pk_mul_lo_u16 v6, v6, v73062; GFX10-DL-NEXT: v_perm_b32 v5, v10, v5, 0x50401003063; GFX10-DL-NEXT: v_bfe_u32 v7, v1, 24, 43064; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v83065; GFX10-DL-NEXT: v_perm_b32 v4, v9, v4, 0x50401003066; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v63067; GFX10-DL-NEXT: v_lshrrev_b32_e32 v1, 28, v13068; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v63069; GFX10-DL-NEXT: v_bfe_u32 v6, v2, 24, 43070; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 28, v23071; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v5, v43072; GFX10-DL-NEXT: v_perm_b32 v1, v1, v7, 0x50401003073; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v83074; GFX10-DL-NEXT: v_perm_b32 v2, v2, v6, 0x50401003075; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v43076; GFX10-DL-NEXT: v_add_nc_u16 v3, v3, v43077; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v23078; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v53079; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v13080; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v13081; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v33082; GFX10-DL-NEXT: v_and_b32_e32 v1, 15, v13083; GFX10-DL-NEXT: global_store_byte v0, v1, s[6:7]3084; GFX10-DL-NEXT: s_endpgm3085 ptr addrspace(1) %src2,3086 ptr addrspace(1) nocapture %dst) {3087entry:3088 %idx = call i32 @llvm.amdgcn.workitem.id.x()3089 %gep1 = getelementptr <8 x i4>, ptr addrspace(1) %src1, i32 %idx3090 %vec1 = load <8 x i4>, ptr addrspace(1) %gep13091 %gep2 = getelementptr <8 x i4>, ptr addrspace(1) %src2, i32 %idx3092 %vec2 = load <8 x i4>, ptr addrspace(1) %gep23093 3094 %mul = mul <8 x i4> %vec1, %vec23095 %mul0 = extractelement <8 x i4> %mul, i64 03096 %mul1 = extractelement <8 x i4> %mul, i64 13097 %mul2 = extractelement <8 x i4> %mul, i64 23098 %mul3 = extractelement <8 x i4> %mul, i64 33099 %mul4 = extractelement <8 x i4> %mul, i64 43100 %mul5 = extractelement <8 x i4> %mul, i64 53101 %mul6 = extractelement <8 x i4> %mul, i64 63102 %mul7 = extractelement <8 x i4> %mul, i64 73103 3104 %acc = load i4, ptr addrspace(1) %dst, align 43105 %add1 = add i4 %mul0, %acc3106 %add2 = add i4 %add1, %mul13107 %add3 = add i4 %add2, %mul23108 %add4 = add i4 %add3, %mul33109 %add5 = add i4 %add4, %mul43110 %add6 = add i4 %add5, %mul53111 %add7 = add i4 %add6, %mul63112 %add8 = add i4 %add7, %mul73113 3114 store i4 %add8, ptr addrspace(1) %dst, align 43115 ret void3116}3117 3118define amdgpu_kernel void @udot8_variant1(ptr addrspace(1) %v1addr,3119; GFX7-LABEL: udot8_variant1:3120; GFX7: ; %bb.0: ; %entry3121; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x93122; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd3123; GFX7-NEXT: s_mov_b32 s3, 0xf0003124; GFX7-NEXT: s_mov_b32 s6, 03125; GFX7-NEXT: s_mov_b32 s7, s33126; GFX7-NEXT: s_waitcnt lgkmcnt(0)3127; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]3128; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v03129; GFX7-NEXT: v_mov_b32_e32 v1, 03130; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr643131; GFX7-NEXT: s_mov_b64 s[4:5], s[10:11]3132; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr643133; GFX7-NEXT: s_load_dword s4, s[0:1], 0x03134; GFX7-NEXT: s_mov_b32 s2, -13135; GFX7-NEXT: s_waitcnt vmcnt(1)3136; GFX7-NEXT: v_and_b32_e32 v1, 15, v23137; GFX7-NEXT: v_bfe_u32 v3, v2, 4, 43138; GFX7-NEXT: s_waitcnt vmcnt(0)3139; GFX7-NEXT: v_and_b32_e32 v9, 15, v03140; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 43141; GFX7-NEXT: v_bfe_u32 v5, v2, 12, 43142; GFX7-NEXT: v_bfe_u32 v6, v2, 16, 43143; GFX7-NEXT: v_bfe_u32 v7, v2, 20, 43144; GFX7-NEXT: v_bfe_u32 v8, v2, 24, 43145; GFX7-NEXT: v_lshrrev_b32_e32 v2, 28, v23146; GFX7-NEXT: v_bfe_u32 v10, v0, 4, 43147; GFX7-NEXT: v_bfe_u32 v11, v0, 8, 43148; GFX7-NEXT: v_bfe_u32 v12, v0, 12, 43149; GFX7-NEXT: v_bfe_u32 v13, v0, 16, 43150; GFX7-NEXT: v_bfe_u32 v14, v0, 20, 43151; GFX7-NEXT: v_bfe_u32 v15, v0, 24, 43152; GFX7-NEXT: v_lshrrev_b32_e32 v0, 28, v03153; GFX7-NEXT: s_waitcnt lgkmcnt(0)3154; GFX7-NEXT: v_mad_u32_u24 v1, v9, v1, s43155; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v13156; GFX7-NEXT: v_mad_u32_u24 v0, v10, v3, v03157; GFX7-NEXT: v_mad_u32_u24 v0, v11, v4, v03158; GFX7-NEXT: v_mad_u32_u24 v0, v12, v5, v03159; GFX7-NEXT: v_mad_u32_u24 v0, v13, v6, v03160; GFX7-NEXT: v_mad_u32_u24 v0, v14, v7, v03161; GFX7-NEXT: v_mad_u32_u24 v0, v15, v8, v03162; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 03163; GFX7-NEXT: s_endpgm3164;3165; GFX8-LABEL: udot8_variant1:3166; GFX8: ; %bb.0: ; %entry3167; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243168; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343169; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v03170; GFX8-NEXT: s_waitcnt lgkmcnt(0)3171; GFX8-NEXT: v_mov_b32_e32 v1, s13172; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v23173; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3174; GFX8-NEXT: flat_load_dword v3, v[0:1]3175; GFX8-NEXT: v_mov_b32_e32 v1, s33176; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v23177; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3178; GFX8-NEXT: flat_load_dword v0, v[0:1]3179; GFX8-NEXT: s_load_dword s0, s[4:5], 0x03180; GFX8-NEXT: s_waitcnt vmcnt(1)3181; GFX8-NEXT: v_and_b32_e32 v1, 15, v33182; GFX8-NEXT: v_bfe_u32 v4, v3, 4, 43183; GFX8-NEXT: v_bfe_u32 v6, v3, 8, 43184; GFX8-NEXT: v_bfe_u32 v8, v3, 12, 43185; GFX8-NEXT: v_bfe_u32 v10, v3, 16, 43186; GFX8-NEXT: v_bfe_u32 v12, v3, 20, 43187; GFX8-NEXT: s_waitcnt vmcnt(0)3188; GFX8-NEXT: v_and_b32_e32 v2, 15, v03189; GFX8-NEXT: v_bfe_u32 v5, v0, 4, 43190; GFX8-NEXT: v_bfe_u32 v7, v0, 8, 43191; GFX8-NEXT: v_bfe_u32 v9, v0, 12, 43192; GFX8-NEXT: v_bfe_u32 v11, v0, 16, 43193; GFX8-NEXT: v_bfe_u32 v13, v0, 20, 43194; GFX8-NEXT: v_bfe_u32 v14, v3, 24, 43195; GFX8-NEXT: v_bfe_u32 v15, v0, 24, 43196; GFX8-NEXT: v_lshrrev_b32_e32 v3, 28, v33197; GFX8-NEXT: v_lshrrev_b32_e32 v0, 28, v03198; GFX8-NEXT: s_waitcnt lgkmcnt(0)3199; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, s03200; GFX8-NEXT: v_mad_u32_u24 v0, v0, v3, v13201; GFX8-NEXT: v_mad_u32_u24 v0, v5, v4, v03202; GFX8-NEXT: v_mad_u32_u24 v0, v7, v6, v03203; GFX8-NEXT: v_mad_u32_u24 v0, v9, v8, v03204; GFX8-NEXT: v_mad_u32_u24 v0, v11, v10, v03205; GFX8-NEXT: v_mad_u32_u24 v0, v13, v12, v03206; GFX8-NEXT: v_mad_u32_u24 v2, v15, v14, v03207; GFX8-NEXT: v_mov_b32_e32 v0, s43208; GFX8-NEXT: v_mov_b32_e32 v1, s53209; GFX8-NEXT: flat_store_dword v[0:1], v23210; GFX8-NEXT: s_endpgm3211;3212; GFX9-LABEL: udot8_variant1:3213; GFX9: ; %bb.0: ; %entry3214; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243215; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343216; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v03217; GFX9-NEXT: s_waitcnt lgkmcnt(0)3218; GFX9-NEXT: global_load_dword v1, v0, s[0:1]3219; GFX9-NEXT: global_load_dword v2, v0, s[2:3]3220; GFX9-NEXT: s_load_dword s0, s[6:7], 0x03221; GFX9-NEXT: v_mov_b32_e32 v0, 03222; GFX9-NEXT: s_waitcnt vmcnt(1)3223; GFX9-NEXT: v_and_b32_e32 v3, 15, v13224; GFX9-NEXT: s_waitcnt vmcnt(0)3225; GFX9-NEXT: v_and_b32_e32 v4, 15, v23226; GFX9-NEXT: v_bfe_u32 v5, v1, 4, 43227; GFX9-NEXT: v_bfe_u32 v6, v2, 4, 43228; GFX9-NEXT: v_bfe_u32 v7, v1, 8, 43229; GFX9-NEXT: v_bfe_u32 v8, v2, 8, 43230; GFX9-NEXT: v_bfe_u32 v9, v1, 12, 43231; GFX9-NEXT: v_bfe_u32 v10, v2, 12, 43232; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 43233; GFX9-NEXT: v_bfe_u32 v12, v2, 16, 43234; GFX9-NEXT: v_bfe_u32 v13, v1, 20, 43235; GFX9-NEXT: v_bfe_u32 v14, v2, 20, 43236; GFX9-NEXT: v_bfe_u32 v15, v1, 24, 43237; GFX9-NEXT: v_bfe_u32 v16, v2, 24, 43238; GFX9-NEXT: v_lshrrev_b32_e32 v1, 28, v13239; GFX9-NEXT: v_lshrrev_b32_e32 v2, 28, v23240; GFX9-NEXT: v_mul_u32_u24_e32 v3, v4, v33241; GFX9-NEXT: v_mul_u32_u24_e32 v1, v2, v13242; GFX9-NEXT: v_mul_u32_u24_e32 v4, v6, v53243; GFX9-NEXT: v_mul_u32_u24_e32 v5, v8, v73244; GFX9-NEXT: s_waitcnt lgkmcnt(0)3245; GFX9-NEXT: v_add3_u32 v1, v3, s0, v13246; GFX9-NEXT: v_mul_u32_u24_e32 v6, v10, v93247; GFX9-NEXT: v_mul_u32_u24_e32 v7, v12, v113248; GFX9-NEXT: v_add3_u32 v1, v1, v4, v53249; GFX9-NEXT: v_mul_u32_u24_e32 v8, v14, v133250; GFX9-NEXT: v_mul_u32_u24_e32 v9, v16, v153251; GFX9-NEXT: v_add3_u32 v1, v1, v6, v73252; GFX9-NEXT: v_add3_u32 v1, v1, v8, v93253; GFX9-NEXT: global_store_dword v0, v1, s[6:7]3254; GFX9-NEXT: s_endpgm3255;3256; GFX9-DL-LABEL: udot8_variant1:3257; GFX9-DL: ; %bb.0: ; %entry3258; GFX9-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243259; GFX9-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343260; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03261; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)3262; GFX9-DL-NEXT: global_load_dword v1, v0, s[0:1]3263; GFX9-DL-NEXT: global_load_dword v2, v0, s[2:3]3264; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x03265; GFX9-DL-NEXT: v_mov_b32_e32 v0, 03266; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3267; GFX9-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s03268; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]3269; GFX9-DL-NEXT: s_endpgm3270;3271; GFX10-DL-LABEL: udot8_variant1:3272; GFX10-DL: ; %bb.0: ; %entry3273; GFX10-DL-NEXT: s_clause 0x13274; GFX10-DL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243275; GFX10-DL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343276; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v03277; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)3278; GFX10-DL-NEXT: s_clause 0x13279; GFX10-DL-NEXT: global_load_dword v1, v0, s[0:1]3280; GFX10-DL-NEXT: global_load_dword v2, v0, s[2:3]3281; GFX10-DL-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)3282; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x03283; GFX10-DL-NEXT: v_mov_b32_e32 v0, 03284; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3285; GFX10-DL-NEXT: v_dot8_u32_u4 v1, v2, v1, s03286; GFX10-DL-NEXT: global_store_dword v0, v1, s[6:7]3287; GFX10-DL-NEXT: s_endpgm3288 ptr addrspace(1) %v2addr,3289 ptr addrspace(1) %dst) {3290entry:3291 %idx = call i32 @llvm.amdgcn.workitem.id.x()3292 %gep1 = getelementptr i32, ptr addrspace(1) %v1addr, i32 %idx3293 %v1 = load i32, ptr addrspace(1) %gep1, align 43294 %gep2 = getelementptr i32, ptr addrspace(1) %v2addr, i32 %idx3295 %v2 = load i32, ptr addrspace(1) %gep2, align 43296 %and = and i32 %v1, 153297 %and1 = and i32 %v2, 153298 %mul1 = mul nuw nsw i32 %and1, %and3299 3300 %shr = lshr i32 %v1, 43301 %and2 = and i32 %shr, 153302 %shr3 = lshr i32 %v2, 43303 %and4 = and i32 %shr3, 153304 %mul2 = mul nuw nsw i32 %and4, %and23305 3306 %shr6 = lshr i32 %v1, 83307 %and7 = and i32 %shr6, 153308 %shr8 = lshr i32 %v2, 83309 %and9 = and i32 %shr8, 153310 %mul3 = mul nuw nsw i32 %and9, %and73311 3312 %shr12 = lshr i32 %v1, 123313 %and13 = and i32 %shr12, 153314 %shr14 = lshr i32 %v2, 123315 %and15 = and i32 %shr14, 153316 %mul4 = mul nuw nsw i32 %and15, %and133317 3318 %shr18 = lshr i32 %v1, 163319 %and19 = and i32 %shr18, 153320 %shr20 = lshr i32 %v2, 163321 %and21 = and i32 %shr20, 153322 %mul5 = mul nuw nsw i32 %and21, %and193323 3324 %shr24 = lshr i32 %v1, 203325 %and25 = and i32 %shr24, 153326 %shr26 = lshr i32 %v2, 203327 %and27 = and i32 %shr26, 153328 %mul6 = mul nuw nsw i32 %and27, %and253329 3330 %shr30 = lshr i32 %v1, 243331 %and31 = and i32 %shr30, 153332 %shr32 = lshr i32 %v2, 243333 %and33 = and i32 %shr32, 153334 %mul7 = mul nuw nsw i32 %and33, %and313335 3336 %shr36 = lshr i32 %v1, 283337 %shr37 = lshr i32 %v2, 283338 %mul8 = mul nuw nsw i32 %shr37, %shr363339 %acc = load i32, ptr addrspace(1) %dst, align 43340 3341 %add1 = add i32 %mul1, %acc3342 %add2 = add i32 %add1, %mul83343 %add3 = add i32 %add2, %mul23344 %add4 = add i32 %add3, %mul33345 %add5 = add i32 %add4, %mul43346 %add6 = add i32 %add5, %mul53347 %add7 = add i32 %add6, %mul63348 %add8 = add i32 %add7, %mul73349 store i32 %add8, ptr addrspace(1) %dst, align 43350 ret void3351}3352 3353declare i32 @llvm.amdgcn.workitem.id.x()3354