1503 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple aarch64 -mattr=+neon < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NODOT3; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-DOT4; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-DOT-I8MM5 6define <4 x i32> @udot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {7; CHECK-NODOT-LABEL: udot:8; CHECK-NODOT: // %bb.0:9; CHECK-NODOT-NEXT: umull v3.8h, v2.8b, v1.8b10; CHECK-NODOT-NEXT: umull2 v1.8h, v2.16b, v1.16b11; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v3.4h12; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v3.8h13; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h14; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v1.8h15; CHECK-NODOT-NEXT: ret16;17; CHECK-DOT-LABEL: udot:18; CHECK-DOT: // %bb.0:19; CHECK-DOT-NEXT: udot v0.4s, v2.16b, v1.16b20; CHECK-DOT-NEXT: ret21;22; CHECK-DOT-I8MM-LABEL: udot:23; CHECK-DOT-I8MM: // %bb.0:24; CHECK-DOT-I8MM-NEXT: udot v0.4s, v2.16b, v1.16b25; CHECK-DOT-I8MM-NEXT: ret26 %u.wide = zext <16 x i8> %u to <16 x i32>27 %s.wide = zext <16 x i8> %s to <16 x i32>28 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide29 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mult)30 ret <4 x i32> %partial.reduce31}32 33define <4 x i32> @udot_in_loop(ptr %p1, ptr %p2){34; CHECK-NODOT-LABEL: udot_in_loop:35; CHECK-NODOT: // %bb.0: // %entry36; CHECK-NODOT-NEXT: movi v1.2d, #000000000000000037; CHECK-NODOT-NEXT: mov x8, xzr38; CHECK-NODOT-NEXT: .LBB1_1: // %vector.body39; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=140; CHECK-NODOT-NEXT: ldr q2, [x0, x8]41; CHECK-NODOT-NEXT: ldr q3, [x1, x8]42; CHECK-NODOT-NEXT: mov v0.16b, v1.16b43; CHECK-NODOT-NEXT: add x8, x8, #1644; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b45; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b46; CHECK-NODOT-NEXT: cmp x8, #1647; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v4.4h48; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v4.8h49; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v2.4h50; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v2.8h51; CHECK-NODOT-NEXT: b.ne .LBB1_152; CHECK-NODOT-NEXT: // %bb.2: // %end53; CHECK-NODOT-NEXT: ret54;55; CHECK-DOT-LABEL: udot_in_loop:56; CHECK-DOT: // %bb.0: // %entry57; CHECK-DOT-NEXT: movi v1.2d, #000000000000000058; CHECK-DOT-NEXT: mov x8, xzr59; CHECK-DOT-NEXT: .LBB1_1: // %vector.body60; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=161; CHECK-DOT-NEXT: ldr q2, [x0, x8]62; CHECK-DOT-NEXT: ldr q3, [x1, x8]63; CHECK-DOT-NEXT: mov v0.16b, v1.16b64; CHECK-DOT-NEXT: add x8, x8, #1665; CHECK-DOT-NEXT: udot v1.4s, v2.16b, v3.16b66; CHECK-DOT-NEXT: cmp x8, #1667; CHECK-DOT-NEXT: b.ne .LBB1_168; CHECK-DOT-NEXT: // %bb.2: // %end69; CHECK-DOT-NEXT: ret70;71; CHECK-DOT-I8MM-LABEL: udot_in_loop:72; CHECK-DOT-I8MM: // %bb.0: // %entry73; CHECK-DOT-I8MM-NEXT: movi v1.2d, #000000000000000074; CHECK-DOT-I8MM-NEXT: mov x8, xzr75; CHECK-DOT-I8MM-NEXT: .LBB1_1: // %vector.body76; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=177; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]78; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]79; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b80; CHECK-DOT-I8MM-NEXT: add x8, x8, #1681; CHECK-DOT-I8MM-NEXT: udot v1.4s, v2.16b, v3.16b82; CHECK-DOT-I8MM-NEXT: cmp x8, #1683; CHECK-DOT-I8MM-NEXT: b.ne .LBB1_184; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end85; CHECK-DOT-I8MM-NEXT: ret86entry:87 br label %vector.body88 89vector.body:90 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]91 %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %partial.reduce, %vector.body ]92 %gep1 = getelementptr i8, ptr %p1, i64 %index93 %load1 = load <16 x i8>, ptr %gep1, align 1694 %load1.wide = zext <16 x i8> %load1 to <16 x i32>95 %gep2 = getelementptr i8, ptr %p2, i64 %index96 %load2 = load <16 x i8>, ptr %gep2, align 1697 %load2.wide = zext <16 x i8> %load2 to <16 x i32>98 %mul = mul nuw nsw <16 x i32> %load1.wide, %load2.wide99 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mul)100 %index.next = add nuw i64 %index, 16101 %cmp = icmp eq i64 %index.next, 16102 br i1 %cmp, label %end, label %vector.body103 104end:105 ret <4 x i32> %acc106}107 108define <2 x i32> @udot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) {109; CHECK-NODOT-LABEL: udot_narrow:110; CHECK-NODOT: // %bb.0:111; CHECK-NODOT-NEXT: umull v1.8h, v2.8b, v1.8b112; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0113; CHECK-NODOT-NEXT: ushll v2.4s, v1.4h, #0114; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h115; CHECK-NODOT-NEXT: ushll2 v3.4s, v1.8h, #0116; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8117; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #8118; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s119; CHECK-NODOT-NEXT: ext v2.16b, v3.16b, v3.16b, #8120; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h121; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s122; CHECK-NODOT-NEXT: ret123;124; CHECK-DOT-LABEL: udot_narrow:125; CHECK-DOT: // %bb.0:126; CHECK-DOT-NEXT: udot v0.2s, v2.8b, v1.8b127; CHECK-DOT-NEXT: ret128;129; CHECK-DOT-I8MM-LABEL: udot_narrow:130; CHECK-DOT-I8MM: // %bb.0:131; CHECK-DOT-I8MM-NEXT: udot v0.2s, v2.8b, v1.8b132; CHECK-DOT-I8MM-NEXT: ret133 %u.wide = zext <8 x i8> %u to <8 x i32>134 %s.wide = zext <8 x i8> %s to <8 x i32>135 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide136 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<2 x i32> %acc, <8 x i32> %mult)137 ret <2 x i32> %partial.reduce138}139 140define <4 x i32> @sdot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {141; CHECK-NODOT-LABEL: sdot:142; CHECK-NODOT: // %bb.0:143; CHECK-NODOT-NEXT: smull v3.8h, v2.8b, v1.8b144; CHECK-NODOT-NEXT: smull2 v1.8h, v2.16b, v1.16b145; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v3.4h146; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v3.8h147; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h148; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v1.8h149; CHECK-NODOT-NEXT: ret150;151; CHECK-DOT-LABEL: sdot:152; CHECK-DOT: // %bb.0:153; CHECK-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b154; CHECK-DOT-NEXT: ret155;156; CHECK-DOT-I8MM-LABEL: sdot:157; CHECK-DOT-I8MM: // %bb.0:158; CHECK-DOT-I8MM-NEXT: sdot v0.4s, v2.16b, v1.16b159; CHECK-DOT-I8MM-NEXT: ret160 %u.wide = sext <16 x i8> %u to <16 x i32>161 %s.wide = sext <16 x i8> %s to <16 x i32>162 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide163 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mult)164 ret <4 x i32> %partial.reduce165}166 167define <2 x i32> @sdot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) {168; CHECK-NODOT-LABEL: sdot_narrow:169; CHECK-NODOT: // %bb.0:170; CHECK-NODOT-NEXT: smull v1.8h, v2.8b, v1.8b171; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0172; CHECK-NODOT-NEXT: sshll v2.4s, v1.4h, #0173; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h174; CHECK-NODOT-NEXT: sshll2 v3.4s, v1.8h, #0175; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8176; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #8177; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s178; CHECK-NODOT-NEXT: ext v2.16b, v3.16b, v3.16b, #8179; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h180; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s181; CHECK-NODOT-NEXT: ret182;183; CHECK-DOT-LABEL: sdot_narrow:184; CHECK-DOT: // %bb.0:185; CHECK-DOT-NEXT: sdot v0.2s, v2.8b, v1.8b186; CHECK-DOT-NEXT: ret187;188; CHECK-DOT-I8MM-LABEL: sdot_narrow:189; CHECK-DOT-I8MM: // %bb.0:190; CHECK-DOT-I8MM-NEXT: sdot v0.2s, v2.8b, v1.8b191; CHECK-DOT-I8MM-NEXT: ret192 %u.wide = sext <8 x i8> %u to <8 x i32>193 %s.wide = sext <8 x i8> %s to <8 x i32>194 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide195 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<2 x i32> %acc, <8 x i32> %mult)196 ret <2 x i32> %partial.reduce197}198 199define <4 x i32> @usdot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {200; CHECK-NODOT-LABEL: usdot:201; CHECK-NODOT: // %bb.0:202; CHECK-NODOT-NEXT: ushll v3.8h, v1.8b, #0203; CHECK-NODOT-NEXT: sshll v4.8h, v2.8b, #0204; CHECK-NODOT-NEXT: ushll2 v1.8h, v1.16b, #0205; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #0206; CHECK-NODOT-NEXT: smlal v0.4s, v4.4h, v3.4h207; CHECK-NODOT-NEXT: smlal2 v0.4s, v4.8h, v3.8h208; CHECK-NODOT-NEXT: smlal v0.4s, v2.4h, v1.4h209; CHECK-NODOT-NEXT: smlal2 v0.4s, v2.8h, v1.8h210; CHECK-NODOT-NEXT: ret211;212; CHECK-DOT-LABEL: usdot:213; CHECK-DOT: // %bb.0:214; CHECK-DOT-NEXT: ushll v3.8h, v1.8b, #0215; CHECK-DOT-NEXT: sshll v4.8h, v2.8b, #0216; CHECK-DOT-NEXT: ushll2 v1.8h, v1.16b, #0217; CHECK-DOT-NEXT: sshll2 v2.8h, v2.16b, #0218; CHECK-DOT-NEXT: smlal v0.4s, v4.4h, v3.4h219; CHECK-DOT-NEXT: smlal2 v0.4s, v4.8h, v3.8h220; CHECK-DOT-NEXT: smlal v0.4s, v2.4h, v1.4h221; CHECK-DOT-NEXT: smlal2 v0.4s, v2.8h, v1.8h222; CHECK-DOT-NEXT: ret223;224; CHECK-DOT-I8MM-LABEL: usdot:225; CHECK-DOT-I8MM: // %bb.0:226; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v1.16b, v2.16b227; CHECK-DOT-I8MM-NEXT: ret228 %u.wide = zext <16 x i8> %u to <16 x i32>229 %s.wide = sext <16 x i8> %s to <16 x i32>230 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide231 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mult)232 ret <4 x i32> %partial.reduce233}234 235define <4 x i32> @usdot_in_loop(ptr %p1, ptr %p2){236; CHECK-NODOT-LABEL: usdot_in_loop:237; CHECK-NODOT: // %bb.0: // %entry238; CHECK-NODOT-NEXT: movi v1.2d, #0000000000000000239; CHECK-NODOT-NEXT: mov x8, xzr240; CHECK-NODOT-NEXT: .LBB6_1: // %vector.body241; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1242; CHECK-NODOT-NEXT: ldr q2, [x0, x8]243; CHECK-NODOT-NEXT: ldr q3, [x1, x8]244; CHECK-NODOT-NEXT: mov v0.16b, v1.16b245; CHECK-NODOT-NEXT: add x8, x8, #16246; CHECK-NODOT-NEXT: sshll v4.8h, v2.8b, #0247; CHECK-NODOT-NEXT: ushll v5.8h, v3.8b, #0248; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #0249; CHECK-NODOT-NEXT: ushll2 v3.8h, v3.16b, #0250; CHECK-NODOT-NEXT: cmp x8, #16251; CHECK-NODOT-NEXT: smlal v1.4s, v4.4h, v5.4h252; CHECK-NODOT-NEXT: smlal2 v1.4s, v4.8h, v5.8h253; CHECK-NODOT-NEXT: smlal v1.4s, v2.4h, v3.4h254; CHECK-NODOT-NEXT: smlal2 v1.4s, v2.8h, v3.8h255; CHECK-NODOT-NEXT: b.ne .LBB6_1256; CHECK-NODOT-NEXT: // %bb.2: // %end257; CHECK-NODOT-NEXT: ret258;259; CHECK-DOT-LABEL: usdot_in_loop:260; CHECK-DOT: // %bb.0: // %entry261; CHECK-DOT-NEXT: movi v1.2d, #0000000000000000262; CHECK-DOT-NEXT: mov x8, xzr263; CHECK-DOT-NEXT: .LBB6_1: // %vector.body264; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=1265; CHECK-DOT-NEXT: ldr q2, [x0, x8]266; CHECK-DOT-NEXT: ldr q3, [x1, x8]267; CHECK-DOT-NEXT: mov v0.16b, v1.16b268; CHECK-DOT-NEXT: add x8, x8, #16269; CHECK-DOT-NEXT: sshll v4.8h, v2.8b, #0270; CHECK-DOT-NEXT: ushll v5.8h, v3.8b, #0271; CHECK-DOT-NEXT: sshll2 v2.8h, v2.16b, #0272; CHECK-DOT-NEXT: ushll2 v3.8h, v3.16b, #0273; CHECK-DOT-NEXT: cmp x8, #16274; CHECK-DOT-NEXT: smlal v1.4s, v4.4h, v5.4h275; CHECK-DOT-NEXT: smlal2 v1.4s, v4.8h, v5.8h276; CHECK-DOT-NEXT: smlal v1.4s, v2.4h, v3.4h277; CHECK-DOT-NEXT: smlal2 v1.4s, v2.8h, v3.8h278; CHECK-DOT-NEXT: b.ne .LBB6_1279; CHECK-DOT-NEXT: // %bb.2: // %end280; CHECK-DOT-NEXT: ret281;282; CHECK-DOT-I8MM-LABEL: usdot_in_loop:283; CHECK-DOT-I8MM: // %bb.0: // %entry284; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000285; CHECK-DOT-I8MM-NEXT: mov x8, xzr286; CHECK-DOT-I8MM-NEXT: .LBB6_1: // %vector.body287; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1288; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]289; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]290; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b291; CHECK-DOT-I8MM-NEXT: add x8, x8, #16292; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v3.16b, v2.16b293; CHECK-DOT-I8MM-NEXT: cmp x8, #16294; CHECK-DOT-I8MM-NEXT: b.ne .LBB6_1295; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end296; CHECK-DOT-I8MM-NEXT: ret297entry:298 br label %vector.body299 300vector.body:301 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]302 %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %partial.reduce, %vector.body ]303 %gep1 = getelementptr i8, ptr %p1, i64 %index304 %load1 = load <16 x i8>, ptr %gep1, align 16305 %load1.wide = sext <16 x i8> %load1 to <16 x i32>306 %gep2 = getelementptr i8, ptr %p2, i64 %index307 %load2 = load <16 x i8>, ptr %gep2, align 16308 %load2.wide = zext <16 x i8> %load2 to <16 x i32>309 %mul = mul nuw nsw <16 x i32> %load1.wide, %load2.wide310 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mul)311 %index.next = add nuw i64 %index, 16312 %cmp = icmp eq i64 %index.next, 16313 br i1 %cmp, label %end, label %vector.body314 315end:316 ret <4 x i32> %acc317}318 319define <2 x i32> @usdot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{320; CHECK-NODOT-LABEL: usdot_narrow:321; CHECK-NODOT: // %bb.0:322; CHECK-NODOT-NEXT: ushll v1.8h, v1.8b, #0323; CHECK-NODOT-NEXT: sshll v2.8h, v2.8b, #0324; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0325; CHECK-NODOT-NEXT: smull v3.4s, v2.4h, v1.4h326; CHECK-NODOT-NEXT: smlal v0.4s, v2.4h, v1.4h327; CHECK-NODOT-NEXT: ext v4.16b, v1.16b, v1.16b, #8328; CHECK-NODOT-NEXT: ext v5.16b, v2.16b, v2.16b, #8329; CHECK-NODOT-NEXT: smull2 v1.4s, v2.8h, v1.8h330; CHECK-NODOT-NEXT: ext v3.16b, v3.16b, v3.16b, #8331; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8332; CHECK-NODOT-NEXT: add v0.2s, v3.2s, v0.2s333; CHECK-NODOT-NEXT: smlal v0.4s, v5.4h, v4.4h334; CHECK-NODOT-NEXT: add v0.2s, v1.2s, v0.2s335; CHECK-NODOT-NEXT: ret336;337; CHECK-DOT-LABEL: usdot_narrow:338; CHECK-DOT: // %bb.0:339; CHECK-DOT-NEXT: ushll v1.8h, v1.8b, #0340; CHECK-DOT-NEXT: sshll v2.8h, v2.8b, #0341; CHECK-DOT-NEXT: // kill: def $d0 killed $d0 def $q0342; CHECK-DOT-NEXT: smull v3.4s, v2.4h, v1.4h343; CHECK-DOT-NEXT: smlal v0.4s, v2.4h, v1.4h344; CHECK-DOT-NEXT: ext v4.16b, v1.16b, v1.16b, #8345; CHECK-DOT-NEXT: ext v5.16b, v2.16b, v2.16b, #8346; CHECK-DOT-NEXT: smull2 v1.4s, v2.8h, v1.8h347; CHECK-DOT-NEXT: ext v3.16b, v3.16b, v3.16b, #8348; CHECK-DOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8349; CHECK-DOT-NEXT: add v0.2s, v3.2s, v0.2s350; CHECK-DOT-NEXT: smlal v0.4s, v5.4h, v4.4h351; CHECK-DOT-NEXT: add v0.2s, v1.2s, v0.2s352; CHECK-DOT-NEXT: ret353;354; CHECK-DOT-I8MM-LABEL: usdot_narrow:355; CHECK-DOT-I8MM: // %bb.0:356; CHECK-DOT-I8MM-NEXT: usdot v0.2s, v1.8b, v2.8b357; CHECK-DOT-I8MM-NEXT: ret358 %u.wide = zext <8 x i8> %u to <8 x i32>359 %s.wide = sext <8 x i8> %s to <8 x i32>360 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide361 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<2 x i32> %acc, <8 x i32> %mult)362 ret <2 x i32> %partial.reduce363}364 365define <4 x i32> @sudot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) #0{366; CHECK-NODOT-LABEL: sudot:367; CHECK-NODOT: // %bb.0:368; CHECK-NODOT-NEXT: sshll v3.8h, v1.8b, #0369; CHECK-NODOT-NEXT: ushll v4.8h, v2.8b, #0370; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0371; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0372; CHECK-NODOT-NEXT: smlal v0.4s, v4.4h, v3.4h373; CHECK-NODOT-NEXT: smlal2 v0.4s, v4.8h, v3.8h374; CHECK-NODOT-NEXT: smlal v0.4s, v2.4h, v1.4h375; CHECK-NODOT-NEXT: smlal2 v0.4s, v2.8h, v1.8h376; CHECK-NODOT-NEXT: ret377;378; CHECK-DOT-LABEL: sudot:379; CHECK-DOT: // %bb.0:380; CHECK-DOT-NEXT: sshll v3.8h, v1.8b, #0381; CHECK-DOT-NEXT: ushll v4.8h, v2.8b, #0382; CHECK-DOT-NEXT: sshll2 v1.8h, v1.16b, #0383; CHECK-DOT-NEXT: ushll2 v2.8h, v2.16b, #0384; CHECK-DOT-NEXT: smlal v0.4s, v4.4h, v3.4h385; CHECK-DOT-NEXT: smlal2 v0.4s, v4.8h, v3.8h386; CHECK-DOT-NEXT: smlal v0.4s, v2.4h, v1.4h387; CHECK-DOT-NEXT: smlal2 v0.4s, v2.8h, v1.8h388; CHECK-DOT-NEXT: ret389;390; CHECK-DOT-I8MM-LABEL: sudot:391; CHECK-DOT-I8MM: // %bb.0:392; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v2.16b, v1.16b393; CHECK-DOT-I8MM-NEXT: ret394 %s.wide = sext <16 x i8> %u to <16 x i32>395 %u.wide = zext <16 x i8> %s to <16 x i32>396 %mult = mul nuw nsw <16 x i32> %u.wide, %s.wide397 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mult)398 ret <4 x i32> %partial.reduce399}400 401define <4 x i32> @sudot_in_loop(ptr %p1, ptr %p2){402; CHECK-NODOT-LABEL: sudot_in_loop:403; CHECK-NODOT: // %bb.0: // %entry404; CHECK-NODOT-NEXT: movi v1.2d, #0000000000000000405; CHECK-NODOT-NEXT: mov x8, xzr406; CHECK-NODOT-NEXT: .LBB9_1: // %vector.body407; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1408; CHECK-NODOT-NEXT: ldr q2, [x0, x8]409; CHECK-NODOT-NEXT: ldr q3, [x1, x8]410; CHECK-NODOT-NEXT: mov v0.16b, v1.16b411; CHECK-NODOT-NEXT: add x8, x8, #16412; CHECK-NODOT-NEXT: ushll v4.8h, v2.8b, #0413; CHECK-NODOT-NEXT: sshll v5.8h, v3.8b, #0414; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0415; CHECK-NODOT-NEXT: sshll2 v3.8h, v3.16b, #0416; CHECK-NODOT-NEXT: cmp x8, #16417; CHECK-NODOT-NEXT: smlal v1.4s, v4.4h, v5.4h418; CHECK-NODOT-NEXT: smlal2 v1.4s, v4.8h, v5.8h419; CHECK-NODOT-NEXT: smlal v1.4s, v2.4h, v3.4h420; CHECK-NODOT-NEXT: smlal2 v1.4s, v2.8h, v3.8h421; CHECK-NODOT-NEXT: b.ne .LBB9_1422; CHECK-NODOT-NEXT: // %bb.2: // %end423; CHECK-NODOT-NEXT: ret424;425; CHECK-DOT-LABEL: sudot_in_loop:426; CHECK-DOT: // %bb.0: // %entry427; CHECK-DOT-NEXT: movi v1.2d, #0000000000000000428; CHECK-DOT-NEXT: mov x8, xzr429; CHECK-DOT-NEXT: .LBB9_1: // %vector.body430; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=1431; CHECK-DOT-NEXT: ldr q2, [x0, x8]432; CHECK-DOT-NEXT: ldr q3, [x1, x8]433; CHECK-DOT-NEXT: mov v0.16b, v1.16b434; CHECK-DOT-NEXT: add x8, x8, #16435; CHECK-DOT-NEXT: ushll v4.8h, v2.8b, #0436; CHECK-DOT-NEXT: sshll v5.8h, v3.8b, #0437; CHECK-DOT-NEXT: ushll2 v2.8h, v2.16b, #0438; CHECK-DOT-NEXT: sshll2 v3.8h, v3.16b, #0439; CHECK-DOT-NEXT: cmp x8, #16440; CHECK-DOT-NEXT: smlal v1.4s, v4.4h, v5.4h441; CHECK-DOT-NEXT: smlal2 v1.4s, v4.8h, v5.8h442; CHECK-DOT-NEXT: smlal v1.4s, v2.4h, v3.4h443; CHECK-DOT-NEXT: smlal2 v1.4s, v2.8h, v3.8h444; CHECK-DOT-NEXT: b.ne .LBB9_1445; CHECK-DOT-NEXT: // %bb.2: // %end446; CHECK-DOT-NEXT: ret447;448; CHECK-DOT-I8MM-LABEL: sudot_in_loop:449; CHECK-DOT-I8MM: // %bb.0: // %entry450; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000451; CHECK-DOT-I8MM-NEXT: mov x8, xzr452; CHECK-DOT-I8MM-NEXT: .LBB9_1: // %vector.body453; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1454; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]455; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]456; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b457; CHECK-DOT-I8MM-NEXT: add x8, x8, #16458; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v2.16b, v3.16b459; CHECK-DOT-I8MM-NEXT: cmp x8, #16460; CHECK-DOT-I8MM-NEXT: b.ne .LBB9_1461; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end462; CHECK-DOT-I8MM-NEXT: ret463entry:464 br label %vector.body465 466vector.body:467 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]468 %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %partial.reduce, %vector.body ]469 %gep1 = getelementptr i8, ptr %p1, i64 %index470 %load1 = load <16 x i8>, ptr %gep1, align 16471 %load1.wide = zext <16 x i8> %load1 to <16 x i32>472 %gep2 = getelementptr i8, ptr %p2, i64 %index473 %load2 = load <16 x i8>, ptr %gep2, align 16474 %load2.wide = sext <16 x i8> %load2 to <16 x i32>475 %mul = mul nuw nsw <16 x i32> %load1.wide, %load2.wide476 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %mul)477 %index.next = add nuw i64 %index, 16478 %cmp = icmp eq i64 %index.next, 16479 br i1 %cmp, label %end, label %vector.body480 481end:482 ret <4 x i32> %acc483}484 485define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{486; CHECK-NODOT-LABEL: sudot_narrow:487; CHECK-NODOT: // %bb.0:488; CHECK-NODOT-NEXT: sshll v1.8h, v1.8b, #0489; CHECK-NODOT-NEXT: ushll v2.8h, v2.8b, #0490; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0491; CHECK-NODOT-NEXT: smull v3.4s, v2.4h, v1.4h492; CHECK-NODOT-NEXT: smlal v0.4s, v2.4h, v1.4h493; CHECK-NODOT-NEXT: ext v4.16b, v1.16b, v1.16b, #8494; CHECK-NODOT-NEXT: ext v5.16b, v2.16b, v2.16b, #8495; CHECK-NODOT-NEXT: smull2 v1.4s, v2.8h, v1.8h496; CHECK-NODOT-NEXT: ext v3.16b, v3.16b, v3.16b, #8497; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8498; CHECK-NODOT-NEXT: add v0.2s, v3.2s, v0.2s499; CHECK-NODOT-NEXT: smlal v0.4s, v5.4h, v4.4h500; CHECK-NODOT-NEXT: add v0.2s, v1.2s, v0.2s501; CHECK-NODOT-NEXT: ret502;503; CHECK-DOT-LABEL: sudot_narrow:504; CHECK-DOT: // %bb.0:505; CHECK-DOT-NEXT: sshll v1.8h, v1.8b, #0506; CHECK-DOT-NEXT: ushll v2.8h, v2.8b, #0507; CHECK-DOT-NEXT: // kill: def $d0 killed $d0 def $q0508; CHECK-DOT-NEXT: smull v3.4s, v2.4h, v1.4h509; CHECK-DOT-NEXT: smlal v0.4s, v2.4h, v1.4h510; CHECK-DOT-NEXT: ext v4.16b, v1.16b, v1.16b, #8511; CHECK-DOT-NEXT: ext v5.16b, v2.16b, v2.16b, #8512; CHECK-DOT-NEXT: smull2 v1.4s, v2.8h, v1.8h513; CHECK-DOT-NEXT: ext v3.16b, v3.16b, v3.16b, #8514; CHECK-DOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8515; CHECK-DOT-NEXT: add v0.2s, v3.2s, v0.2s516; CHECK-DOT-NEXT: smlal v0.4s, v5.4h, v4.4h517; CHECK-DOT-NEXT: add v0.2s, v1.2s, v0.2s518; CHECK-DOT-NEXT: ret519;520; CHECK-DOT-I8MM-LABEL: sudot_narrow:521; CHECK-DOT-I8MM: // %bb.0:522; CHECK-DOT-I8MM-NEXT: usdot v0.2s, v2.8b, v1.8b523; CHECK-DOT-I8MM-NEXT: ret524 %u.wide = sext <8 x i8> %u to <8 x i32>525 %s.wide = zext <8 x i8> %s to <8 x i32>526 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide527 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<2 x i32> %acc, <8 x i32> %mult)528 ret <2 x i32> %partial.reduce529}530 531define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {532; CHECK-NODOT-LABEL: udot_8to64:533; CHECK-NODOT: // %bb.0: // %entry534; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b535; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b536; CHECK-NODOT-NEXT: ushll v3.4s, v4.4h, #0537; CHECK-NODOT-NEXT: ushll v5.4s, v2.4h, #0538; CHECK-NODOT-NEXT: ushll2 v4.4s, v4.8h, #0539; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0540; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v5.2s541; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s542; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v5.4s543; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s544; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v2.2s545; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s546; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s547; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s548; CHECK-NODOT-NEXT: ret549;550; CHECK-DOT-LABEL: udot_8to64:551; CHECK-DOT: // %bb.0: // %entry552; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000553; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b554; CHECK-DOT-NEXT: uaddw v0.2d, v0.2d, v4.2s555; CHECK-DOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s556; CHECK-DOT-NEXT: ret557;558; CHECK-DOT-I8MM-LABEL: udot_8to64:559; CHECK-DOT-I8MM: // %bb.0: // %entry560; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000561; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b562; CHECK-DOT-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s563; CHECK-DOT-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s564; CHECK-DOT-I8MM-NEXT: ret565entry:566 %a.wide = zext <16 x i8> %a to <16 x i64>567 %b.wide = zext <16 x i8> %b to <16 x i64>568 %mult = mul nuw nsw <16 x i64> %a.wide, %b.wide569 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(570 <4 x i64> %acc, <16 x i64> %mult)571 ret <4 x i64> %partial.reduce572}573 574define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){575; CHECK-NODOT-LABEL: sdot_8to64:576; CHECK-NODOT: // %bb.0: // %entry577; CHECK-NODOT-NEXT: smull v4.8h, v2.8b, v3.8b578; CHECK-NODOT-NEXT: smull2 v2.8h, v2.16b, v3.16b579; CHECK-NODOT-NEXT: sshll v3.4s, v4.4h, #0580; CHECK-NODOT-NEXT: sshll v5.4s, v2.4h, #0581; CHECK-NODOT-NEXT: sshll2 v4.4s, v4.8h, #0582; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0583; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v5.2s584; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s585; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v5.4s586; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s587; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v2.2s588; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s589; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s590; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s591; CHECK-NODOT-NEXT: ret592;593; CHECK-DOT-LABEL: sdot_8to64:594; CHECK-DOT: // %bb.0: // %entry595; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000596; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b597; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s598; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s599; CHECK-DOT-NEXT: ret600;601; CHECK-DOT-I8MM-LABEL: sdot_8to64:602; CHECK-DOT-I8MM: // %bb.0: // %entry603; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000604; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b605; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s606; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s607; CHECK-DOT-I8MM-NEXT: ret608entry:609 %a.wide = sext <16 x i8> %a to <16 x i64>610 %b.wide = sext <16 x i8> %b to <16 x i64>611 %mult = mul nuw nsw <16 x i64> %a.wide, %b.wide612 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(613 <4 x i64> %acc, <16 x i64> %mult)614 ret <4 x i64> %partial.reduce615}616 617define <4 x i64> @usdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){618; CHECK-NODOT-LABEL: usdot_8to64:619; CHECK-NODOT: // %bb.0: // %entry620; CHECK-NODOT-NEXT: ushll v4.8h, v2.8b, #0621; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0622; CHECK-NODOT-NEXT: sshll v5.8h, v3.8b, #0623; CHECK-NODOT-NEXT: sshll2 v3.8h, v3.16b, #0624; CHECK-NODOT-NEXT: ushll v6.4s, v4.4h, #0625; CHECK-NODOT-NEXT: ushll v7.4s, v2.4h, #0626; CHECK-NODOT-NEXT: sshll v16.4s, v5.4h, #0627; CHECK-NODOT-NEXT: sshll v17.4s, v3.4h, #0628; CHECK-NODOT-NEXT: ushll2 v4.4s, v4.8h, #0629; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0630; CHECK-NODOT-NEXT: sshll2 v5.4s, v5.8h, #0631; CHECK-NODOT-NEXT: sshll2 v3.4s, v3.8h, #0632; CHECK-NODOT-NEXT: smlal v0.2d, v6.2s, v16.2s633; CHECK-NODOT-NEXT: smlal v1.2d, v7.2s, v17.2s634; CHECK-NODOT-NEXT: smlal2 v0.2d, v6.4s, v16.4s635; CHECK-NODOT-NEXT: smlal2 v1.2d, v7.4s, v17.4s636; CHECK-NODOT-NEXT: smlal v0.2d, v4.2s, v5.2s637; CHECK-NODOT-NEXT: smlal v1.2d, v2.2s, v3.2s638; CHECK-NODOT-NEXT: smlal2 v0.2d, v4.4s, v5.4s639; CHECK-NODOT-NEXT: smlal2 v1.2d, v2.4s, v3.4s640; CHECK-NODOT-NEXT: ret641;642; CHECK-DOT-LABEL: usdot_8to64:643; CHECK-DOT: // %bb.0: // %entry644; CHECK-DOT-NEXT: ushll v4.8h, v2.8b, #0645; CHECK-DOT-NEXT: ushll2 v2.8h, v2.16b, #0646; CHECK-DOT-NEXT: sshll v5.8h, v3.8b, #0647; CHECK-DOT-NEXT: sshll2 v3.8h, v3.16b, #0648; CHECK-DOT-NEXT: ushll v6.4s, v4.4h, #0649; CHECK-DOT-NEXT: ushll v7.4s, v2.4h, #0650; CHECK-DOT-NEXT: sshll v16.4s, v5.4h, #0651; CHECK-DOT-NEXT: sshll v17.4s, v3.4h, #0652; CHECK-DOT-NEXT: ushll2 v4.4s, v4.8h, #0653; CHECK-DOT-NEXT: ushll2 v2.4s, v2.8h, #0654; CHECK-DOT-NEXT: sshll2 v5.4s, v5.8h, #0655; CHECK-DOT-NEXT: sshll2 v3.4s, v3.8h, #0656; CHECK-DOT-NEXT: smlal v0.2d, v6.2s, v16.2s657; CHECK-DOT-NEXT: smlal v1.2d, v7.2s, v17.2s658; CHECK-DOT-NEXT: smlal2 v0.2d, v6.4s, v16.4s659; CHECK-DOT-NEXT: smlal2 v1.2d, v7.4s, v17.4s660; CHECK-DOT-NEXT: smlal v0.2d, v4.2s, v5.2s661; CHECK-DOT-NEXT: smlal v1.2d, v2.2s, v3.2s662; CHECK-DOT-NEXT: smlal2 v0.2d, v4.4s, v5.4s663; CHECK-DOT-NEXT: smlal2 v1.2d, v2.4s, v3.4s664; CHECK-DOT-NEXT: ret665;666; CHECK-DOT-I8MM-LABEL: usdot_8to64:667; CHECK-DOT-I8MM: // %bb.0: // %entry668; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000669; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v2.16b, v3.16b670; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s671; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s672; CHECK-DOT-I8MM-NEXT: ret673entry:674 %a.wide = zext <16 x i8> %a to <16 x i64>675 %b.wide = sext <16 x i8> %b to <16 x i64>676 %mult = mul nuw nsw <16 x i64> %a.wide, %b.wide677 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(678 <4 x i64> %acc, <16 x i64> %mult)679 ret <4 x i64> %partial.reduce680}681 682define <4 x i64> @sudot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {683; CHECK-NODOT-LABEL: sudot_8to64:684; CHECK-NODOT: // %bb.0: // %entry685; CHECK-NODOT-NEXT: sshll v4.8h, v2.8b, #0686; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #0687; CHECK-NODOT-NEXT: ushll v5.8h, v3.8b, #0688; CHECK-NODOT-NEXT: ushll2 v3.8h, v3.16b, #0689; CHECK-NODOT-NEXT: sshll v6.4s, v4.4h, #0690; CHECK-NODOT-NEXT: sshll v7.4s, v2.4h, #0691; CHECK-NODOT-NEXT: ushll v16.4s, v5.4h, #0692; CHECK-NODOT-NEXT: ushll v17.4s, v3.4h, #0693; CHECK-NODOT-NEXT: sshll2 v4.4s, v4.8h, #0694; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0695; CHECK-NODOT-NEXT: ushll2 v5.4s, v5.8h, #0696; CHECK-NODOT-NEXT: ushll2 v3.4s, v3.8h, #0697; CHECK-NODOT-NEXT: smlal v0.2d, v6.2s, v16.2s698; CHECK-NODOT-NEXT: smlal v1.2d, v7.2s, v17.2s699; CHECK-NODOT-NEXT: smlal2 v0.2d, v6.4s, v16.4s700; CHECK-NODOT-NEXT: smlal2 v1.2d, v7.4s, v17.4s701; CHECK-NODOT-NEXT: smlal v0.2d, v4.2s, v5.2s702; CHECK-NODOT-NEXT: smlal v1.2d, v2.2s, v3.2s703; CHECK-NODOT-NEXT: smlal2 v0.2d, v4.4s, v5.4s704; CHECK-NODOT-NEXT: smlal2 v1.2d, v2.4s, v3.4s705; CHECK-NODOT-NEXT: ret706;707; CHECK-DOT-LABEL: sudot_8to64:708; CHECK-DOT: // %bb.0: // %entry709; CHECK-DOT-NEXT: sshll v4.8h, v2.8b, #0710; CHECK-DOT-NEXT: sshll2 v2.8h, v2.16b, #0711; CHECK-DOT-NEXT: ushll v5.8h, v3.8b, #0712; CHECK-DOT-NEXT: ushll2 v3.8h, v3.16b, #0713; CHECK-DOT-NEXT: sshll v6.4s, v4.4h, #0714; CHECK-DOT-NEXT: sshll v7.4s, v2.4h, #0715; CHECK-DOT-NEXT: ushll v16.4s, v5.4h, #0716; CHECK-DOT-NEXT: ushll v17.4s, v3.4h, #0717; CHECK-DOT-NEXT: sshll2 v4.4s, v4.8h, #0718; CHECK-DOT-NEXT: sshll2 v2.4s, v2.8h, #0719; CHECK-DOT-NEXT: ushll2 v5.4s, v5.8h, #0720; CHECK-DOT-NEXT: ushll2 v3.4s, v3.8h, #0721; CHECK-DOT-NEXT: smlal v0.2d, v6.2s, v16.2s722; CHECK-DOT-NEXT: smlal v1.2d, v7.2s, v17.2s723; CHECK-DOT-NEXT: smlal2 v0.2d, v6.4s, v16.4s724; CHECK-DOT-NEXT: smlal2 v1.2d, v7.4s, v17.4s725; CHECK-DOT-NEXT: smlal v0.2d, v4.2s, v5.2s726; CHECK-DOT-NEXT: smlal v1.2d, v2.2s, v3.2s727; CHECK-DOT-NEXT: smlal2 v0.2d, v4.4s, v5.4s728; CHECK-DOT-NEXT: smlal2 v1.2d, v2.4s, v3.4s729; CHECK-DOT-NEXT: ret730;731; CHECK-DOT-I8MM-LABEL: sudot_8to64:732; CHECK-DOT-I8MM: // %bb.0: // %entry733; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000734; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v3.16b, v2.16b735; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s736; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s737; CHECK-DOT-I8MM-NEXT: ret738entry:739 %a.wide = sext <16 x i8> %a to <16 x i64>740 %b.wide = zext <16 x i8> %b to <16 x i64>741 %mult = mul nuw nsw <16 x i64> %a.wide, %b.wide742 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(743 <4 x i64> %acc, <16 x i64> %mult)744 ret <4 x i64> %partial.reduce745}746 747define <4 x i32> @udot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){748; CHECK-NODOT-LABEL: udot_no_bin_op:749; CHECK-NODOT: // %bb.0:750; CHECK-NODOT-NEXT: ushll v2.8h, v1.8b, #0751; CHECK-NODOT-NEXT: ushll2 v1.8h, v1.16b, #0752; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v2.4h753; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v2.8h754; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h755; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v1.8h756; CHECK-NODOT-NEXT: ret757;758; CHECK-DOT-LABEL: udot_no_bin_op:759; CHECK-DOT: // %bb.0:760; CHECK-DOT-NEXT: movi v2.16b, #1761; CHECK-DOT-NEXT: udot v0.4s, v1.16b, v2.16b762; CHECK-DOT-NEXT: ret763;764; CHECK-DOT-I8MM-LABEL: udot_no_bin_op:765; CHECK-DOT-I8MM: // %bb.0:766; CHECK-DOT-I8MM-NEXT: movi v2.16b, #1767; CHECK-DOT-I8MM-NEXT: udot v0.4s, v1.16b, v2.16b768; CHECK-DOT-I8MM-NEXT: ret769 %a.wide = zext <16 x i8> %a to <16 x i32>770 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %a.wide)771 ret <4 x i32> %partial.reduce772}773 774define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){775; CHECK-NODOT-LABEL: udot_no_bin_op_in_loop:776; CHECK-NODOT: // %bb.0: // %entry777; CHECK-NODOT-NEXT: movi v1.2d, #0000000000000000778; CHECK-NODOT-NEXT: mov x8, xzr779; CHECK-NODOT-NEXT: .LBB16_1: // %vector.body780; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1781; CHECK-NODOT-NEXT: ldr q2, [x0, x8]782; CHECK-NODOT-NEXT: mov v0.16b, v1.16b783; CHECK-NODOT-NEXT: add x8, x8, #16784; CHECK-NODOT-NEXT: cmp x8, #16785; CHECK-NODOT-NEXT: ushll v3.8h, v2.8b, #0786; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0787; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v3.4h788; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v3.8h789; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v2.4h790; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v2.8h791; CHECK-NODOT-NEXT: b.ne .LBB16_1792; CHECK-NODOT-NEXT: // %bb.2: // %end793; CHECK-NODOT-NEXT: ret794;795; CHECK-DOT-LABEL: udot_no_bin_op_in_loop:796; CHECK-DOT: // %bb.0: // %entry797; CHECK-DOT-NEXT: movi v1.2d, #0000000000000000798; CHECK-DOT-NEXT: movi v2.16b, #1799; CHECK-DOT-NEXT: mov x8, xzr800; CHECK-DOT-NEXT: .LBB16_1: // %vector.body801; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=1802; CHECK-DOT-NEXT: ldr q3, [x0, x8]803; CHECK-DOT-NEXT: mov v0.16b, v1.16b804; CHECK-DOT-NEXT: add x8, x8, #16805; CHECK-DOT-NEXT: cmp x8, #16806; CHECK-DOT-NEXT: udot v1.4s, v3.16b, v2.16b807; CHECK-DOT-NEXT: b.ne .LBB16_1808; CHECK-DOT-NEXT: // %bb.2: // %end809; CHECK-DOT-NEXT: ret810;811; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_in_loop:812; CHECK-DOT-I8MM: // %bb.0: // %entry813; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000814; CHECK-DOT-I8MM-NEXT: movi v2.16b, #1815; CHECK-DOT-I8MM-NEXT: mov x8, xzr816; CHECK-DOT-I8MM-NEXT: .LBB16_1: // %vector.body817; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1818; CHECK-DOT-I8MM-NEXT: ldr q3, [x0, x8]819; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b820; CHECK-DOT-I8MM-NEXT: add x8, x8, #16821; CHECK-DOT-I8MM-NEXT: cmp x8, #16822; CHECK-DOT-I8MM-NEXT: udot v1.4s, v3.16b, v2.16b823; CHECK-DOT-I8MM-NEXT: b.ne .LBB16_1824; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end825; CHECK-DOT-I8MM-NEXT: ret826entry:827 br label %vector.body828 829vector.body:830 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]831 %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %partial.reduce, %vector.body ]832 %gep = getelementptr i8, ptr %p, i64 %index833 %load = load <16 x i8>, ptr %gep, align 16834 %load.wide = zext <16 x i8> %load to <16 x i32>835 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %load.wide)836 %index.next = add nuw i64 %index, 16837 %cmp = icmp eq i64 %index.next, 16838 br i1 %cmp, label %end, label %vector.body839 840end:841 ret <4 x i32> %acc842}843 844define <4 x i32> @sdot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){845; CHECK-NODOT-LABEL: sdot_no_bin_op:846; CHECK-NODOT: // %bb.0:847; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0848; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0849; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v2.4h850; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v2.8h851; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h852; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v1.8h853; CHECK-NODOT-NEXT: ret854;855; CHECK-DOT-LABEL: sdot_no_bin_op:856; CHECK-DOT: // %bb.0:857; CHECK-DOT-NEXT: movi v2.16b, #1858; CHECK-DOT-NEXT: sdot v0.4s, v1.16b, v2.16b859; CHECK-DOT-NEXT: ret860;861; CHECK-DOT-I8MM-LABEL: sdot_no_bin_op:862; CHECK-DOT-I8MM: // %bb.0:863; CHECK-DOT-I8MM-NEXT: movi v2.16b, #1864; CHECK-DOT-I8MM-NEXT: sdot v0.4s, v1.16b, v2.16b865; CHECK-DOT-I8MM-NEXT: ret866 %a.wide = sext <16 x i8> %a to <16 x i32>867 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <16 x i32> %a.wide)868 ret <4 x i32> %partial.reduce869}870 871define <2 x i32> @udot_no_bin_op_narrow(<2 x i32> %acc, <8 x i8> %a){872; CHECK-NODOT-LABEL: udot_no_bin_op_narrow:873; CHECK-NODOT: // %bb.0:874; CHECK-NODOT-NEXT: ushll v1.8h, v1.8b, #0875; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0876; CHECK-NODOT-NEXT: ushll v2.4s, v1.4h, #0877; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h878; CHECK-NODOT-NEXT: ushll2 v3.4s, v1.8h, #0879; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8880; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #8881; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s882; CHECK-NODOT-NEXT: ext v2.16b, v3.16b, v3.16b, #8883; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h884; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s885; CHECK-NODOT-NEXT: ret886;887; CHECK-DOT-LABEL: udot_no_bin_op_narrow:888; CHECK-DOT: // %bb.0:889; CHECK-DOT-NEXT: movi v2.8b, #1890; CHECK-DOT-NEXT: udot v0.2s, v1.8b, v2.8b891; CHECK-DOT-NEXT: ret892;893; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_narrow:894; CHECK-DOT-I8MM: // %bb.0:895; CHECK-DOT-I8MM-NEXT: movi v2.8b, #1896; CHECK-DOT-I8MM-NEXT: udot v0.2s, v1.8b, v2.8b897; CHECK-DOT-I8MM-NEXT: ret898 %a.wide = zext <8 x i8> %a to <8 x i32>899 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> %acc, <8 x i32> %a.wide)900 ret <2 x i32> %partial.reduce901}902 903define <2 x i32> @sdot_no_bin_op_narrow(<2 x i32> %acc, <8 x i8> %a){904; CHECK-NODOT-LABEL: sdot_no_bin_op_narrow:905; CHECK-NODOT: // %bb.0:906; CHECK-NODOT-NEXT: sshll v1.8h, v1.8b, #0907; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q0908; CHECK-NODOT-NEXT: sshll v2.4s, v1.4h, #0909; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h910; CHECK-NODOT-NEXT: sshll2 v3.4s, v1.8h, #0911; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #8912; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #8913; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s914; CHECK-NODOT-NEXT: ext v2.16b, v3.16b, v3.16b, #8915; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h916; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s917; CHECK-NODOT-NEXT: ret918;919; CHECK-DOT-LABEL: sdot_no_bin_op_narrow:920; CHECK-DOT: // %bb.0:921; CHECK-DOT-NEXT: movi v2.8b, #1922; CHECK-DOT-NEXT: sdot v0.2s, v1.8b, v2.8b923; CHECK-DOT-NEXT: ret924;925; CHECK-DOT-I8MM-LABEL: sdot_no_bin_op_narrow:926; CHECK-DOT-I8MM: // %bb.0:927; CHECK-DOT-I8MM-NEXT: movi v2.8b, #1928; CHECK-DOT-I8MM-NEXT: sdot v0.2s, v1.8b, v2.8b929; CHECK-DOT-I8MM-NEXT: ret930 %a.wide = sext <8 x i8> %a to <8 x i32>931 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> %acc, <8 x i32> %a.wide)932 ret <2 x i32> %partial.reduce933}934 935define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){936; CHECK-NODOT-LABEL: udot_no_bin_op_8to64:937; CHECK-NODOT: // %bb.0:938; CHECK-NODOT-NEXT: ushll v3.8h, v2.8b, #0939; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0940; CHECK-NODOT-NEXT: ushll v4.4s, v3.4h, #0941; CHECK-NODOT-NEXT: ushll v5.4s, v2.4h, #0942; CHECK-NODOT-NEXT: ushll2 v3.4s, v3.8h, #0943; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0944; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v5.2s945; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s946; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v5.4s947; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s948; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v2.2s949; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s950; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s951; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s952; CHECK-NODOT-NEXT: ret953;954; CHECK-DOT-LABEL: udot_no_bin_op_8to64:955; CHECK-DOT: // %bb.0:956; CHECK-DOT-NEXT: movi v3.16b, #1957; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000958; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b959; CHECK-DOT-NEXT: uaddw v0.2d, v0.2d, v4.2s960; CHECK-DOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s961; CHECK-DOT-NEXT: ret962;963; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_8to64:964; CHECK-DOT-I8MM: // %bb.0:965; CHECK-DOT-I8MM-NEXT: movi v3.16b, #1966; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000967; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b968; CHECK-DOT-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s969; CHECK-DOT-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s970; CHECK-DOT-I8MM-NEXT: ret971 %a.wide = zext <16 x i8> %a to <16 x i64>972 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)973 ret <4 x i64> %partial.reduce974}975 976define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){977; CHECK-NODOT-LABEL: sdot_no_bin_op_8to64:978; CHECK-NODOT: // %bb.0:979; CHECK-NODOT-NEXT: sshll v3.8h, v2.8b, #0980; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #0981; CHECK-NODOT-NEXT: sshll v4.4s, v3.4h, #0982; CHECK-NODOT-NEXT: sshll v5.4s, v2.4h, #0983; CHECK-NODOT-NEXT: sshll2 v3.4s, v3.8h, #0984; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0985; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v5.2s986; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s987; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v5.4s988; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s989; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v2.2s990; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s991; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s992; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s993; CHECK-NODOT-NEXT: ret994;995; CHECK-DOT-LABEL: sdot_no_bin_op_8to64:996; CHECK-DOT: // %bb.0:997; CHECK-DOT-NEXT: movi v3.16b, #1998; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000999; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b1000; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s1001; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s1002; CHECK-DOT-NEXT: ret1003;1004; CHECK-DOT-I8MM-LABEL: sdot_no_bin_op_8to64:1005; CHECK-DOT-I8MM: // %bb.0:1006; CHECK-DOT-I8MM-NEXT: movi v3.16b, #11007; CHECK-DOT-I8MM-NEXT: movi v4.2d, #00000000000000001008; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b1009; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s1010; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s1011; CHECK-DOT-I8MM-NEXT: ret1012 %a.wide = sext <16 x i8> %a to <16 x i64>1013 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)1014 ret <4 x i64> %partial.reduce1015}1016 1017define <4 x i32> @not_udot(<4 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{1018; CHECK-COMMON-LABEL: not_udot:1019; CHECK-COMMON: // %bb.0:1020; CHECK-COMMON-NEXT: umull v1.8h, v2.8b, v1.8b1021; CHECK-COMMON-NEXT: uaddw v0.4s, v0.4s, v1.4h1022; CHECK-COMMON-NEXT: uaddw2 v0.4s, v0.4s, v1.8h1023; CHECK-COMMON-NEXT: ret1024 %u.wide = zext <8 x i8> %u to <8 x i32>1025 %s.wide = zext <8 x i8> %s to <8 x i32>1026 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide1027 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc, <8 x i32> %mult)1028 ret <4 x i32> %partial.reduce1029}1030 1031define <2 x i32> @not_udot_narrow(<2 x i32> %acc, <4 x i8> %u, <4 x i8> %s) {1032; CHECK-COMMON-LABEL: not_udot_narrow:1033; CHECK-COMMON: // %bb.0:1034; CHECK-COMMON-NEXT: bic v1.4h, #255, lsl #81035; CHECK-COMMON-NEXT: bic v2.4h, #255, lsl #81036; CHECK-COMMON-NEXT: // kill: def $d0 killed $d0 def $q01037; CHECK-COMMON-NEXT: umull v3.4s, v2.4h, v1.4h1038; CHECK-COMMON-NEXT: umlal v0.4s, v2.4h, v1.4h1039; CHECK-COMMON-NEXT: ext v1.16b, v3.16b, v3.16b, #81040; CHECK-COMMON-NEXT: add v0.2s, v1.2s, v0.2s1041; CHECK-COMMON-NEXT: ret1042 %u.wide = zext <4 x i8> %u to <4 x i32>1043 %s.wide = zext <4 x i8> %s to <4 x i32>1044 %mult = mul nuw nsw <4 x i32> %s.wide, %u.wide1045 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<2 x i32> %acc, <4 x i32> %mult)1046 ret <2 x i32> %partial.reduce1047}1048 1049define <2 x i64> @udot_different_types(<2 x i64> %acc, <8 x i16> %a, <8 x i8> %b){1050; CHECK-COMMON-LABEL: udot_different_types:1051; CHECK-COMMON: // %bb.0: // %entry1052; CHECK-COMMON-NEXT: ushll v2.8h, v2.8b, #01053; CHECK-COMMON-NEXT: ushll v3.4s, v1.4h, #01054; CHECK-COMMON-NEXT: ushll2 v1.4s, v1.8h, #01055; CHECK-COMMON-NEXT: ushll v4.4s, v2.4h, #01056; CHECK-COMMON-NEXT: ushll2 v2.4s, v2.8h, #01057; CHECK-COMMON-NEXT: umlal v0.2d, v3.2s, v4.2s1058; CHECK-COMMON-NEXT: umlal2 v0.2d, v3.4s, v4.4s1059; CHECK-COMMON-NEXT: umlal v0.2d, v1.2s, v2.2s1060; CHECK-COMMON-NEXT: umlal2 v0.2d, v1.4s, v2.4s1061; CHECK-COMMON-NEXT: ret1062entry:1063 %a.wide = zext <8 x i16> %a to <8 x i64>1064 %b.wide = zext <8 x i8> %b to <8 x i64>1065 %mult = mul nuw nsw <8 x i64> %a.wide, %b.wide1066 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %mult)1067 ret <2 x i64> %partial.reduce1068}1069 1070define <2 x i64> @sdot_different_types(<2 x i64> %acc, <8 x i16> %a, <8 x i8> %b){1071; CHECK-COMMON-LABEL: sdot_different_types:1072; CHECK-COMMON: // %bb.0: // %entry1073; CHECK-COMMON-NEXT: sshll v2.8h, v2.8b, #01074; CHECK-COMMON-NEXT: sshll v3.4s, v1.4h, #01075; CHECK-COMMON-NEXT: sshll2 v1.4s, v1.8h, #01076; CHECK-COMMON-NEXT: sshll v4.4s, v2.4h, #01077; CHECK-COMMON-NEXT: sshll2 v2.4s, v2.8h, #01078; CHECK-COMMON-NEXT: smlal v0.2d, v3.2s, v4.2s1079; CHECK-COMMON-NEXT: smlal2 v0.2d, v3.4s, v4.4s1080; CHECK-COMMON-NEXT: smlal v0.2d, v1.2s, v2.2s1081; CHECK-COMMON-NEXT: smlal2 v0.2d, v1.4s, v2.4s1082; CHECK-COMMON-NEXT: ret1083entry:1084 %a.wide = sext <8 x i16> %a to <8 x i64>1085 %b.wide = sext <8 x i8> %b to <8 x i64>1086 %mult = mul nuw nsw <8 x i64> %a.wide, %b.wide1087 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %mult)1088 ret <2 x i64> %partial.reduce1089}1090 1091define <2 x i64> @usdot_different_types(<2 x i64> %acc, <8 x i16> %a, <8 x i8> %b){1092; CHECK-COMMON-LABEL: usdot_different_types:1093; CHECK-COMMON: // %bb.0: // %entry1094; CHECK-COMMON-NEXT: sshll v2.8h, v2.8b, #01095; CHECK-COMMON-NEXT: ushll v3.4s, v1.4h, #01096; CHECK-COMMON-NEXT: ushll2 v1.4s, v1.8h, #01097; CHECK-COMMON-NEXT: sshll v4.4s, v2.4h, #01098; CHECK-COMMON-NEXT: sshll2 v2.4s, v2.8h, #01099; CHECK-COMMON-NEXT: smlal v0.2d, v3.2s, v4.2s1100; CHECK-COMMON-NEXT: smlal2 v0.2d, v3.4s, v4.4s1101; CHECK-COMMON-NEXT: smlal v0.2d, v1.2s, v2.2s1102; CHECK-COMMON-NEXT: smlal2 v0.2d, v1.4s, v2.4s1103; CHECK-COMMON-NEXT: ret1104entry:1105 %a.wide = zext <8 x i16> %a to <8 x i64>1106 %b.wide = sext <8 x i8> %b to <8 x i64>1107 %mult = mul nuw nsw <8 x i64> %a.wide, %b.wide1108 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %mult)1109 ret <2 x i64> %partial.reduce1110}1111 1112define <2 x i64> @sudot_different_types(<2 x i64> %acc, <8 x i16> %a, <8 x i8> %b){1113; CHECK-COMMON-LABEL: sudot_different_types:1114; CHECK-COMMON: // %bb.0: // %entry1115; CHECK-COMMON-NEXT: ushll v2.8h, v2.8b, #01116; CHECK-COMMON-NEXT: sshll v3.4s, v1.4h, #01117; CHECK-COMMON-NEXT: sshll2 v1.4s, v1.8h, #01118; CHECK-COMMON-NEXT: ushll v4.4s, v2.4h, #01119; CHECK-COMMON-NEXT: ushll2 v2.4s, v2.8h, #01120; CHECK-COMMON-NEXT: smlal v0.2d, v3.2s, v4.2s1121; CHECK-COMMON-NEXT: smlal2 v0.2d, v3.4s, v4.4s1122; CHECK-COMMON-NEXT: smlal v0.2d, v1.2s, v2.2s1123; CHECK-COMMON-NEXT: smlal2 v0.2d, v1.4s, v2.4s1124; CHECK-COMMON-NEXT: ret1125entry:1126 %a.wide = sext <8 x i16> %a to <8 x i64>1127 %b.wide = zext <8 x i8> %b to <8 x i64>1128 %mult = mul nuw nsw <8 x i64> %a.wide, %b.wide1129 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %mult)1130 ret <2 x i64> %partial.reduce1131}1132 1133define <4 x i32> @usdot_multiple_zext_users(ptr %p1, ptr %p2, ptr %p3) {1134; CHECK-NODOT-LABEL: usdot_multiple_zext_users:1135; CHECK-NODOT: // %bb.0: // %entry1136; CHECK-NODOT-NEXT: movi v0.2d, #00000000000000001137; CHECK-NODOT-NEXT: movi v1.2d, #00000000000000001138; CHECK-NODOT-NEXT: mov x8, xzr1139; CHECK-NODOT-NEXT: .LBB28_1: // %vector.body1140; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=11141; CHECK-NODOT-NEXT: ldr q2, [x0, x8]1142; CHECK-NODOT-NEXT: ldr q3, [x1, x8]1143; CHECK-NODOT-NEXT: ldr q4, [x2, x8]1144; CHECK-NODOT-NEXT: add x8, x8, #161145; CHECK-NODOT-NEXT: sshll v5.8h, v2.8b, #01146; CHECK-NODOT-NEXT: ushll v6.8h, v4.8b, #01147; CHECK-NODOT-NEXT: sshll v7.8h, v3.8b, #01148; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #01149; CHECK-NODOT-NEXT: ushll2 v4.8h, v4.16b, #01150; CHECK-NODOT-NEXT: sshll2 v3.8h, v3.16b, #01151; CHECK-NODOT-NEXT: cmp x8, #10241152; CHECK-NODOT-NEXT: smlal v0.4s, v5.4h, v6.4h1153; CHECK-NODOT-NEXT: smlal v1.4s, v7.4h, v6.4h1154; CHECK-NODOT-NEXT: smlal2 v0.4s, v5.8h, v6.8h1155; CHECK-NODOT-NEXT: smlal2 v1.4s, v7.8h, v6.8h1156; CHECK-NODOT-NEXT: smlal v0.4s, v2.4h, v4.4h1157; CHECK-NODOT-NEXT: smlal v1.4s, v3.4h, v4.4h1158; CHECK-NODOT-NEXT: smlal2 v0.4s, v2.8h, v4.8h1159; CHECK-NODOT-NEXT: smlal2 v1.4s, v3.8h, v4.8h1160; CHECK-NODOT-NEXT: b.ne .LBB28_11161; CHECK-NODOT-NEXT: // %bb.2: // %end1162; CHECK-NODOT-NEXT: add v0.4s, v1.4s, v0.4s1163; CHECK-NODOT-NEXT: ret1164;1165; CHECK-DOT-LABEL: usdot_multiple_zext_users:1166; CHECK-DOT: // %bb.0: // %entry1167; CHECK-DOT-NEXT: movi v0.2d, #00000000000000001168; CHECK-DOT-NEXT: movi v1.2d, #00000000000000001169; CHECK-DOT-NEXT: mov x8, xzr1170; CHECK-DOT-NEXT: .LBB28_1: // %vector.body1171; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=11172; CHECK-DOT-NEXT: ldr q2, [x0, x8]1173; CHECK-DOT-NEXT: ldr q3, [x1, x8]1174; CHECK-DOT-NEXT: ldr q4, [x2, x8]1175; CHECK-DOT-NEXT: add x8, x8, #161176; CHECK-DOT-NEXT: sshll v5.8h, v2.8b, #01177; CHECK-DOT-NEXT: ushll v6.8h, v4.8b, #01178; CHECK-DOT-NEXT: sshll v7.8h, v3.8b, #01179; CHECK-DOT-NEXT: sshll2 v2.8h, v2.16b, #01180; CHECK-DOT-NEXT: ushll2 v4.8h, v4.16b, #01181; CHECK-DOT-NEXT: sshll2 v3.8h, v3.16b, #01182; CHECK-DOT-NEXT: cmp x8, #10241183; CHECK-DOT-NEXT: smlal v0.4s, v5.4h, v6.4h1184; CHECK-DOT-NEXT: smlal v1.4s, v7.4h, v6.4h1185; CHECK-DOT-NEXT: smlal2 v0.4s, v5.8h, v6.8h1186; CHECK-DOT-NEXT: smlal2 v1.4s, v7.8h, v6.8h1187; CHECK-DOT-NEXT: smlal v0.4s, v2.4h, v4.4h1188; CHECK-DOT-NEXT: smlal v1.4s, v3.4h, v4.4h1189; CHECK-DOT-NEXT: smlal2 v0.4s, v2.8h, v4.8h1190; CHECK-DOT-NEXT: smlal2 v1.4s, v3.8h, v4.8h1191; CHECK-DOT-NEXT: b.ne .LBB28_11192; CHECK-DOT-NEXT: // %bb.2: // %end1193; CHECK-DOT-NEXT: add v0.4s, v1.4s, v0.4s1194; CHECK-DOT-NEXT: ret1195;1196; CHECK-DOT-I8MM-LABEL: usdot_multiple_zext_users:1197; CHECK-DOT-I8MM: // %bb.0: // %entry1198; CHECK-DOT-I8MM-NEXT: movi v0.2d, #00000000000000001199; CHECK-DOT-I8MM-NEXT: movi v1.2d, #00000000000000001200; CHECK-DOT-I8MM-NEXT: mov x8, xzr1201; CHECK-DOT-I8MM-NEXT: .LBB28_1: // %vector.body1202; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=11203; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]1204; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]1205; CHECK-DOT-I8MM-NEXT: ldr q4, [x2, x8]1206; CHECK-DOT-I8MM-NEXT: add x8, x8, #161207; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v4.16b, v2.16b1208; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v4.16b, v3.16b1209; CHECK-DOT-I8MM-NEXT: cmp x8, #10241210; CHECK-DOT-I8MM-NEXT: b.ne .LBB28_11211; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end1212; CHECK-DOT-I8MM-NEXT: add v0.4s, v1.4s, v0.4s1213; CHECK-DOT-I8MM-NEXT: ret1214entry:1215 br label %vector.body1216 1217vector.body:1218 %iv = phi i64 [ 0, %entry ], [ %iv.next, %vector.body ]1219 %acc1 = phi <4 x i32> [ zeroinitializer, %entry], [ %psum1, %vector.body]1220 %acc2 = phi <4 x i32> [ zeroinitializer, %entry], [ %psum2, %vector.body]1221 %ptr1 = getelementptr i8, ptr %p1, i64 %iv1222 %ptr2 = getelementptr i8, ptr %p2, i64 %iv1223 %ptr3 = getelementptr i8, ptr %p3, i64 %iv1224 %load1 = load <16 x i8>, ptr %ptr11225 %load2 = load <16 x i8>, ptr %ptr21226 %load3 = load <16 x i8>, ptr %ptr31227 %sext1 = sext <16 x i8> %load1 to <16 x i32>1228 %zext = zext <16 x i8> %load3 to <16 x i32>1229 %mul1 = mul <16 x i32> %sext1, %zext1230 %psum1 = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc1, <16 x i32> %mul1)1231 %sext2 = sext <16 x i8> %load2 to <16 x i32>1232 %mul2 = mul <16 x i32> %sext2, %zext1233 %psum2 = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %acc2, <16 x i32> %mul2)1234 %iv.next = add i64 %iv, 161235 %1 = icmp eq i64 %iv.next, 10241236 br i1 %1, label %end, label %vector.body1237 1238end:1239 %2 = add <4 x i32> %psum2, %psum11240 ret <4 x i32> %21241}1242 1243define <2 x i64> @udot_16to64(<2 x i64> %acc, <8 x i16> %input){1244; CHECK-COMMON-LABEL: udot_16to64:1245; CHECK-COMMON: // %bb.0: // %entry1246; CHECK-COMMON-NEXT: ushll v2.4s, v1.4h, #01247; CHECK-COMMON-NEXT: ushll2 v1.4s, v1.8h, #01248; CHECK-COMMON-NEXT: uaddw v0.2d, v0.2d, v2.2s1249; CHECK-COMMON-NEXT: uaddw2 v0.2d, v0.2d, v2.4s1250; CHECK-COMMON-NEXT: uaddw v0.2d, v0.2d, v1.2s1251; CHECK-COMMON-NEXT: uaddw2 v0.2d, v0.2d, v1.4s1252; CHECK-COMMON-NEXT: ret1253entry:1254 %input.wide = zext <8 x i16> %input to <8 x i64>1255 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %input.wide)1256 ret <2 x i64> %partial.reduce1257}1258 1259define <4 x i32> @partial_reduce_shl_sext_const_rhs6(<16 x i8> %l, <4 x i32> %part) {1260; CHECK-NODOT-LABEL: partial_reduce_shl_sext_const_rhs6:1261; CHECK-NODOT: // %bb.0:1262; CHECK-NODOT-NEXT: sshll v2.8h, v0.8b, #01263; CHECK-NODOT-NEXT: sshll2 v0.8h, v0.16b, #01264; CHECK-NODOT-NEXT: sshll v3.4s, v0.4h, #61265; CHECK-NODOT-NEXT: sshll2 v4.4s, v2.8h, #61266; CHECK-NODOT-NEXT: sshll v2.4s, v2.4h, #61267; CHECK-NODOT-NEXT: sshll2 v0.4s, v0.8h, #61268; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s1269; CHECK-NODOT-NEXT: add v2.4s, v4.4s, v3.4s1270; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s1271; CHECK-NODOT-NEXT: add v0.4s, v1.4s, v0.4s1272; CHECK-NODOT-NEXT: ret1273;1274; CHECK-DOT-LABEL: partial_reduce_shl_sext_const_rhs6:1275; CHECK-DOT: // %bb.0:1276; CHECK-DOT-NEXT: movi v2.16b, #641277; CHECK-DOT-NEXT: sdot v1.4s, v0.16b, v2.16b1278; CHECK-DOT-NEXT: mov v0.16b, v1.16b1279; CHECK-DOT-NEXT: ret1280;1281; CHECK-DOT-I8MM-LABEL: partial_reduce_shl_sext_const_rhs6:1282; CHECK-DOT-I8MM: // %bb.0:1283; CHECK-DOT-I8MM-NEXT: movi v2.16b, #641284; CHECK-DOT-I8MM-NEXT: sdot v1.4s, v0.16b, v2.16b1285; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b1286; CHECK-DOT-I8MM-NEXT: ret1287 %ext = sext <16 x i8> %l to <16 x i32>1288 %shift = shl nsw <16 x i32> %ext, splat (i32 6)1289 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1290 ret <4 x i32> %red1291}1292 1293define <4 x i32> @partial_reduce_shl_sext_const_rhs7(<16 x i8> %l, <4 x i32> %part) {1294; CHECK-COMMON-LABEL: partial_reduce_shl_sext_const_rhs7:1295; CHECK-COMMON: // %bb.0:1296; CHECK-COMMON-NEXT: sshll v2.8h, v0.8b, #01297; CHECK-COMMON-NEXT: sshll2 v0.8h, v0.16b, #01298; CHECK-COMMON-NEXT: sshll v3.4s, v0.4h, #71299; CHECK-COMMON-NEXT: sshll2 v4.4s, v2.8h, #71300; CHECK-COMMON-NEXT: sshll v2.4s, v2.4h, #71301; CHECK-COMMON-NEXT: sshll2 v0.4s, v0.8h, #71302; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1303; CHECK-COMMON-NEXT: add v2.4s, v4.4s, v3.4s1304; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1305; CHECK-COMMON-NEXT: add v0.4s, v1.4s, v0.4s1306; CHECK-COMMON-NEXT: ret1307 %ext = sext <16 x i8> %l to <16 x i32>1308 %shift = shl nsw <16 x i32> %ext, splat (i32 7)1309 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1310 ret <4 x i32> %red1311}1312 1313define <4 x i32> @partial_reduce_shl_sext_const_rhs8(<16 x i8> %l, <4 x i32> %part) {1314; CHECK-COMMON-LABEL: partial_reduce_shl_sext_const_rhs8:1315; CHECK-COMMON: // %bb.0:1316; CHECK-COMMON-NEXT: sshll v2.8h, v0.8b, #01317; CHECK-COMMON-NEXT: sshll2 v0.8h, v0.16b, #01318; CHECK-COMMON-NEXT: sshll v3.4s, v0.4h, #81319; CHECK-COMMON-NEXT: sshll2 v4.4s, v2.8h, #81320; CHECK-COMMON-NEXT: sshll v2.4s, v2.4h, #81321; CHECK-COMMON-NEXT: sshll2 v0.4s, v0.8h, #81322; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1323; CHECK-COMMON-NEXT: add v2.4s, v4.4s, v3.4s1324; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1325; CHECK-COMMON-NEXT: add v0.4s, v1.4s, v0.4s1326; CHECK-COMMON-NEXT: ret1327 %ext = sext <16 x i8> %l to <16 x i32>1328 %shift = shl nsw <16 x i32> %ext, splat (i32 8)1329 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1330 ret <4 x i32> %red1331}1332 1333define <4 x i32> @partial_reduce_shl_sext_const_rhs_9(<16 x i8> %l, <4 x i32> %part) {1334; CHECK-COMMON-LABEL: partial_reduce_shl_sext_const_rhs_9:1335; CHECK-COMMON: // %bb.0:1336; CHECK-COMMON-NEXT: ret1337 %ext = sext <16 x i8> %l to <16 x i32>1338 %shift = shl nsw <16 x i32> %ext, splat (i32 32)1339 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1340 ret <4 x i32> %red1341}1342 1343define <4 x i32> @partial_reduce_shl_sext_non_const_rhs(<16 x i8> %l, <4 x i32> %part) {1344; CHECK-COMMON-LABEL: partial_reduce_shl_sext_non_const_rhs:1345; CHECK-COMMON: // %bb.0:1346; CHECK-COMMON-NEXT: sshll v2.8h, v0.8b, #01347; CHECK-COMMON-NEXT: sshll2 v0.8h, v0.16b, #01348; CHECK-COMMON-NEXT: sshll v3.4s, v2.4h, #01349; CHECK-COMMON-NEXT: sshll2 v2.4s, v2.8h, #01350; CHECK-COMMON-NEXT: sshll v4.4s, v0.4h, #01351; CHECK-COMMON-NEXT: sshll2 v0.4s, v0.8h, #01352; CHECK-COMMON-NEXT: ushl v4.4s, v4.4s, v4.4s1353; CHECK-COMMON-NEXT: ushl v2.4s, v2.4s, v2.4s1354; CHECK-COMMON-NEXT: ushl v3.4s, v3.4s, v3.4s1355; CHECK-COMMON-NEXT: ushl v0.4s, v0.4s, v0.4s1356; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v3.4s1357; CHECK-COMMON-NEXT: add v2.4s, v2.4s, v4.4s1358; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1359; CHECK-COMMON-NEXT: add v0.4s, v1.4s, v0.4s1360; CHECK-COMMON-NEXT: ret1361 %ext = sext <16 x i8> %l to <16 x i32>1362 %shift = shl nsw <16 x i32> %ext, %ext1363 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1364 ret <4 x i32> %red1365}1366 1367define <4 x i32> @partial_reduce_shl_zext_const_rhs6(<16 x i8> %l, <4 x i32> %part) {1368; CHECK-NODOT-LABEL: partial_reduce_shl_zext_const_rhs6:1369; CHECK-NODOT: // %bb.0:1370; CHECK-NODOT-NEXT: ushll v2.8h, v0.8b, #01371; CHECK-NODOT-NEXT: ushll2 v0.8h, v0.16b, #01372; CHECK-NODOT-NEXT: ushll v3.4s, v0.4h, #61373; CHECK-NODOT-NEXT: ushll2 v4.4s, v2.8h, #61374; CHECK-NODOT-NEXT: ushll v2.4s, v2.4h, #61375; CHECK-NODOT-NEXT: ushll2 v0.4s, v0.8h, #61376; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s1377; CHECK-NODOT-NEXT: add v2.4s, v4.4s, v3.4s1378; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s1379; CHECK-NODOT-NEXT: add v0.4s, v1.4s, v0.4s1380; CHECK-NODOT-NEXT: ret1381;1382; CHECK-DOT-LABEL: partial_reduce_shl_zext_const_rhs6:1383; CHECK-DOT: // %bb.0:1384; CHECK-DOT-NEXT: movi v2.16b, #641385; CHECK-DOT-NEXT: udot v1.4s, v0.16b, v2.16b1386; CHECK-DOT-NEXT: mov v0.16b, v1.16b1387; CHECK-DOT-NEXT: ret1388;1389; CHECK-DOT-I8MM-LABEL: partial_reduce_shl_zext_const_rhs6:1390; CHECK-DOT-I8MM: // %bb.0:1391; CHECK-DOT-I8MM-NEXT: movi v2.16b, #641392; CHECK-DOT-I8MM-NEXT: udot v1.4s, v0.16b, v2.16b1393; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b1394; CHECK-DOT-I8MM-NEXT: ret1395 %ext = zext <16 x i8> %l to <16 x i32>1396 %shift = shl nsw <16 x i32> %ext, splat (i32 6)1397 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1398 ret <4 x i32> %red1399}1400 1401define <4 x i32> @partial_reduce_shl_zext_const_rhs8(<16 x i8> %l, <4 x i32> %part) {1402; CHECK-COMMON-LABEL: partial_reduce_shl_zext_const_rhs8:1403; CHECK-COMMON: // %bb.0:1404; CHECK-COMMON-NEXT: ushll v2.8h, v0.8b, #01405; CHECK-COMMON-NEXT: ushll2 v0.8h, v0.16b, #01406; CHECK-COMMON-NEXT: ushll v3.4s, v0.4h, #81407; CHECK-COMMON-NEXT: ushll2 v4.4s, v2.8h, #81408; CHECK-COMMON-NEXT: ushll v2.4s, v2.4h, #81409; CHECK-COMMON-NEXT: ushll2 v0.4s, v0.8h, #81410; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1411; CHECK-COMMON-NEXT: add v2.4s, v4.4s, v3.4s1412; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1413; CHECK-COMMON-NEXT: add v0.4s, v1.4s, v0.4s1414; CHECK-COMMON-NEXT: ret1415 %ext = zext <16 x i8> %l to <16 x i32>1416 %shift = shl nsw <16 x i32> %ext, splat (i32 8)1417 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1418 ret <4 x i32> %red1419}1420 1421define <4 x i32> @partial_reduce_shl_zext_const_rhs_9(<16 x i8> %l, <4 x i32> %part) {1422; CHECK-COMMON-LABEL: partial_reduce_shl_zext_const_rhs_9:1423; CHECK-COMMON: // %bb.0:1424; CHECK-COMMON-NEXT: ret1425 %ext = zext <16 x i8> %l to <16 x i32>1426 %shift = shl nsw <16 x i32> %ext, splat (i32 32)1427 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1428 ret <4 x i32> %red1429}1430 1431define <4 x i32> @partial_reduce_shl_zext_non_const_rhs(<16 x i8> %l, <4 x i32> %part) {1432; CHECK-COMMON-LABEL: partial_reduce_shl_zext_non_const_rhs:1433; CHECK-COMMON: // %bb.0:1434; CHECK-COMMON-NEXT: ushll v2.8h, v0.8b, #01435; CHECK-COMMON-NEXT: ushll2 v0.8h, v0.16b, #01436; CHECK-COMMON-NEXT: ushll v3.4s, v2.4h, #01437; CHECK-COMMON-NEXT: ushll2 v2.4s, v2.8h, #01438; CHECK-COMMON-NEXT: ushll v4.4s, v0.4h, #01439; CHECK-COMMON-NEXT: ushll2 v0.4s, v0.8h, #01440; CHECK-COMMON-NEXT: ushl v4.4s, v4.4s, v4.4s1441; CHECK-COMMON-NEXT: ushl v2.4s, v2.4s, v2.4s1442; CHECK-COMMON-NEXT: ushl v3.4s, v3.4s, v3.4s1443; CHECK-COMMON-NEXT: ushl v0.4s, v0.4s, v0.4s1444; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v3.4s1445; CHECK-COMMON-NEXT: add v2.4s, v2.4s, v4.4s1446; CHECK-COMMON-NEXT: add v1.4s, v1.4s, v2.4s1447; CHECK-COMMON-NEXT: add v0.4s, v1.4s, v0.4s1448; CHECK-COMMON-NEXT: ret1449 %ext = zext <16 x i8> %l to <16 x i32>1450 %shift = shl nsw <16 x i32> %ext, %ext1451 %red = tail call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> %part, <16 x i32> %shift)1452 ret <4 x i32> %red1453}1454 1455define <2 x i32> @udot_v16i8tov2i32(<2 x i32> %acc, <16 x i8> %input) {1456; CHECK-NODOT-LABEL: udot_v16i8tov2i32:1457; CHECK-NODOT: // %bb.0: // %entry1458; CHECK-NODOT-NEXT: ushll v2.8h, v1.8b, #01459; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 def $q01460; CHECK-NODOT-NEXT: ushll2 v1.8h, v1.16b, #01461; CHECK-NODOT-NEXT: ushll v3.4s, v2.4h, #01462; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v2.4h1463; CHECK-NODOT-NEXT: ushll2 v4.4s, v2.8h, #01464; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #81465; CHECK-NODOT-NEXT: ext v3.16b, v3.16b, v3.16b, #81466; CHECK-NODOT-NEXT: add v0.2s, v3.2s, v0.2s1467; CHECK-NODOT-NEXT: ext v3.16b, v4.16b, v4.16b, #81468; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v2.4h1469; CHECK-NODOT-NEXT: ushll v2.4s, v1.4h, #01470; CHECK-NODOT-NEXT: add v0.2s, v3.2s, v0.2s1471; CHECK-NODOT-NEXT: ext v2.16b, v2.16b, v2.16b, #81472; CHECK-NODOT-NEXT: ushll2 v3.4s, v1.8h, #01473; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h1474; CHECK-NODOT-NEXT: ext v1.16b, v1.16b, v1.16b, #81475; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s1476; CHECK-NODOT-NEXT: ext v2.16b, v3.16b, v3.16b, #81477; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h1478; CHECK-NODOT-NEXT: add v0.2s, v2.2s, v0.2s1479; CHECK-NODOT-NEXT: ret1480;1481; CHECK-DOT-LABEL: udot_v16i8tov2i32:1482; CHECK-DOT: // %bb.0: // %entry1483; CHECK-DOT-NEXT: movi v2.16b, #11484; CHECK-DOT-NEXT: fmov d0, d01485; CHECK-DOT-NEXT: udot v0.4s, v1.16b, v2.16b1486; CHECK-DOT-NEXT: addp v0.4s, v0.4s, v0.4s1487; CHECK-DOT-NEXT: // kill: def $d0 killed $d0 killed $q01488; CHECK-DOT-NEXT: ret1489;1490; CHECK-DOT-I8MM-LABEL: udot_v16i8tov2i32:1491; CHECK-DOT-I8MM: // %bb.0: // %entry1492; CHECK-DOT-I8MM-NEXT: movi v2.16b, #11493; CHECK-DOT-I8MM-NEXT: fmov d0, d01494; CHECK-DOT-I8MM-NEXT: udot v0.4s, v1.16b, v2.16b1495; CHECK-DOT-I8MM-NEXT: addp v0.4s, v0.4s, v0.4s1496; CHECK-DOT-I8MM-NEXT: // kill: def $d0 killed $d0 killed $q01497; CHECK-DOT-I8MM-NEXT: ret1498entry:1499 %input.wide = zext <16 x i8> %input to <16 x i32>1500 %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> %acc, <16 x i32> %input.wide)1501 ret <2 x i32> %partial.reduce1502}1503