1016 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mattr=+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=COMMON,NEON3; RUN: llc -mattr=+sve,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=COMMON,SVE4; RUN: llc -mattr=+sme,+i8mm -force-streaming < %s | FileCheck %s --check-prefix=SME5 6target triple = "aarch64"7 8;9; Two-way mla (i8 -> i16)10;11 12define <8 x i16> @two_way_i8_i16_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {13;14; COMMON-LABEL: two_way_i8_i16_vl128:15; COMMON: // %bb.0:16; COMMON-NEXT: ldr q0, [x0]17; COMMON-NEXT: ldr q1, [x1]18; COMMON-NEXT: ldr q2, [x2]19; COMMON-NEXT: umlal v0.8h, v2.8b, v1.8b20; COMMON-NEXT: umlal2 v0.8h, v2.16b, v1.16b21; COMMON-NEXT: ret22;23; SME-LABEL: two_way_i8_i16_vl128:24; SME: // %bb.0:25; SME-NEXT: ldr q0, [x0]26; SME-NEXT: ldr q1, [x1]27; SME-NEXT: ldr q2, [x2]28; SME-NEXT: umlalb z0.h, z2.b, z1.b29; SME-NEXT: umlalt z0.h, z2.b, z1.b30; SME-NEXT: // kill: def $q0 killed $q0 killed $z031; SME-NEXT: ret32 %acc = load <8 x i16>, ptr %accptr33 %u = load <16 x i8>, ptr %uptr34 %s = load <16 x i8>, ptr %sptr35 %u.wide = zext <16 x i8> %u to <16 x i16>36 %s.wide = zext <16 x i8> %s to <16 x i16>37 %mult = mul nuw nsw <16 x i16> %s.wide, %u.wide38 %partial.reduce = tail call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> %acc, <16 x i16> %mult)39 ret <8 x i16> %partial.reduce40}41 42define <16 x i16> @two_way_i8_i16_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {43;44; COMMON-LABEL: two_way_i8_i16_vl128_double_width:45; COMMON: // %bb.0:46; COMMON-NEXT: ldp q0, q1, [x0]47; COMMON-NEXT: ldp q2, q3, [x1]48; COMMON-NEXT: ldp q4, q5, [x2]49; COMMON-NEXT: umlal v0.8h, v4.8b, v2.8b50; COMMON-NEXT: umlal v1.8h, v5.8b, v3.8b51; COMMON-NEXT: umlal2 v0.8h, v4.16b, v2.16b52; COMMON-NEXT: umlal2 v1.8h, v5.16b, v3.16b53; COMMON-NEXT: ret54;55; SME-LABEL: two_way_i8_i16_vl128_double_width:56; SME: // %bb.0:57; SME-NEXT: ldp q0, q1, [x0]58; SME-NEXT: ldp q3, q2, [x1]59; SME-NEXT: ldp q5, q4, [x2]60; SME-NEXT: umlalb z0.h, z5.b, z3.b61; SME-NEXT: umlalb z1.h, z4.b, z2.b62; SME-NEXT: umlalt z0.h, z5.b, z3.b63; SME-NEXT: umlalt z1.h, z4.b, z2.b64; SME-NEXT: // kill: def $q0 killed $q0 killed $z065; SME-NEXT: // kill: def $q1 killed $q1 killed $z166; SME-NEXT: ret67 %acc = load <16 x i16>, ptr %accptr68 %u = load <32 x i8>, ptr %uptr69 %s = load <32 x i8>, ptr %sptr70 %u.wide = zext <32 x i8> %u to <32 x i16>71 %s.wide = zext <32 x i8> %s to <32 x i16>72 %mult = mul nuw nsw <32 x i16> %s.wide, %u.wide73 %partial.reduce = tail call <16 x i16> @llvm.vector.partial.reduce.add(<16 x i16> %acc, <32 x i16> %mult)74 ret <16 x i16> %partial.reduce75}76 77define <16 x i16> @two_way_i8_i16_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {78;79;80; NEON-LABEL: two_way_i8_i16_vl256:81; NEON: // %bb.0:82; NEON-NEXT: ldp q0, q1, [x0]83; NEON-NEXT: ldp q2, q3, [x1]84; NEON-NEXT: ldp q4, q5, [x2]85; NEON-NEXT: umlal v0.8h, v4.8b, v2.8b86; NEON-NEXT: umlal v1.8h, v5.8b, v3.8b87; NEON-NEXT: umlal2 v0.8h, v4.16b, v2.16b88; NEON-NEXT: umlal2 v1.8h, v5.16b, v3.16b89; NEON-NEXT: ret90;91; SVE-LABEL: two_way_i8_i16_vl256:92; SVE: // %bb.0:93; SVE-NEXT: ldr z0, [x1]94; SVE-NEXT: ldr z1, [x2]95; SVE-NEXT: ptrue p0.h96; SVE-NEXT: ldr z4, [x0]97; SVE-NEXT: uunpklo z2.h, z0.b98; SVE-NEXT: uunpklo z3.h, z1.b99; SVE-NEXT: uunpkhi z0.h, z0.b100; SVE-NEXT: uunpkhi z1.h, z1.b101; SVE-NEXT: mad z2.h, p0/m, z3.h, z4.h102; SVE-NEXT: mad z0.h, p0/m, z1.h, z2.h103; SVE-NEXT: movprfx z1, z0104; SVE-NEXT: ext z1.b, z1.b, z0.b, #16105; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0106; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1107; SVE-NEXT: ret108;109; SME-LABEL: two_way_i8_i16_vl256:110; SME: // %bb.0:111; SME-NEXT: ldr z0, [x0]112; SME-NEXT: ldr z1, [x1]113; SME-NEXT: ldr z2, [x2]114; SME-NEXT: umlalb z0.h, z2.b, z1.b115; SME-NEXT: umlalt z0.h, z2.b, z1.b116; SME-NEXT: movprfx z1, z0117; SME-NEXT: ext z1.b, z1.b, z0.b, #16118; SME-NEXT: // kill: def $q0 killed $q0 killed $z0119; SME-NEXT: // kill: def $q1 killed $q1 killed $z1120; SME-NEXT: ret121 %acc = load <16 x i16>, ptr %accptr122 %u = load <32 x i8>, ptr %uptr123 %s = load <32 x i8>, ptr %sptr124 %u.wide = zext <32 x i8> %u to <32 x i16>125 %s.wide = zext <32 x i8> %s to <32 x i16>126 %mult = mul nuw nsw <32 x i16> %s.wide, %u.wide127 %partial.reduce = tail call <16 x i16> @llvm.vector.partial.reduce.add(<16 x i16> %acc, <32 x i16> %mult)128 ret <16 x i16> %partial.reduce129}130 131;132; Two-way mla (i16 -> i32)133;134 135define <4 x i32> @two_way_i16_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {136;137; COMMON-LABEL: two_way_i16_i32_vl128:138; COMMON: // %bb.0:139; COMMON-NEXT: ldr q0, [x0]140; COMMON-NEXT: ldr q1, [x1]141; COMMON-NEXT: ldr q2, [x2]142; COMMON-NEXT: umlal v0.4s, v2.4h, v1.4h143; COMMON-NEXT: umlal2 v0.4s, v2.8h, v1.8h144; COMMON-NEXT: ret145;146; SME-LABEL: two_way_i16_i32_vl128:147; SME: // %bb.0:148; SME-NEXT: ldr q0, [x0]149; SME-NEXT: ldr q1, [x1]150; SME-NEXT: ldr q2, [x2]151; SME-NEXT: umlalb z0.s, z2.h, z1.h152; SME-NEXT: umlalt z0.s, z2.h, z1.h153; SME-NEXT: // kill: def $q0 killed $q0 killed $z0154; SME-NEXT: ret155 %acc = load <4 x i32>, ptr %accptr156 %u = load <8 x i16>, ptr %uptr157 %s = load <8 x i16>, ptr %sptr158 %u.wide = zext <8 x i16> %u to <8 x i32>159 %s.wide = zext <8 x i16> %s to <8 x i32>160 %mult = mul nuw nsw <8 x i32> %s.wide, %u.wide161 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <8 x i32> %mult)162 ret <4 x i32> %partial.reduce163}164 165define <8 x i32> @two_way_i16_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {166;167; COMMON-LABEL: two_way_i16_i32_vl128_double_width:168; COMMON: // %bb.0:169; COMMON-NEXT: ldp q0, q1, [x0]170; COMMON-NEXT: ldp q2, q3, [x1]171; COMMON-NEXT: ldp q4, q5, [x2]172; COMMON-NEXT: umlal v0.4s, v4.4h, v2.4h173; COMMON-NEXT: umlal v1.4s, v5.4h, v3.4h174; COMMON-NEXT: umlal2 v0.4s, v4.8h, v2.8h175; COMMON-NEXT: umlal2 v1.4s, v5.8h, v3.8h176; COMMON-NEXT: ret177;178; SME-LABEL: two_way_i16_i32_vl128_double_width:179; SME: // %bb.0:180; SME-NEXT: ldp q0, q1, [x0]181; SME-NEXT: ldp q3, q2, [x1]182; SME-NEXT: ldp q5, q4, [x2]183; SME-NEXT: umlalb z0.s, z5.h, z3.h184; SME-NEXT: umlalb z1.s, z4.h, z2.h185; SME-NEXT: umlalt z0.s, z5.h, z3.h186; SME-NEXT: umlalt z1.s, z4.h, z2.h187; SME-NEXT: // kill: def $q0 killed $q0 killed $z0188; SME-NEXT: // kill: def $q1 killed $q1 killed $z1189; SME-NEXT: ret190 %acc = load <8 x i32>, ptr %accptr191 %u = load <16 x i16>, ptr %uptr192 %s = load <16 x i16>, ptr %sptr193 %u.wide = zext <16 x i16> %u to <16 x i32>194 %s.wide = zext <16 x i16> %s to <16 x i32>195 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide196 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <16 x i32> %mult)197 ret <8 x i32> %partial.reduce198}199 200define <8 x i32> @two_way_i16_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {201;202;203; NEON-LABEL: two_way_i16_i32_vl256:204; NEON: // %bb.0:205; NEON-NEXT: ldp q0, q1, [x0]206; NEON-NEXT: ldp q2, q3, [x1]207; NEON-NEXT: ldp q4, q5, [x2]208; NEON-NEXT: umlal v0.4s, v4.4h, v2.4h209; NEON-NEXT: umlal v1.4s, v5.4h, v3.4h210; NEON-NEXT: umlal2 v0.4s, v4.8h, v2.8h211; NEON-NEXT: umlal2 v1.4s, v5.8h, v3.8h212; NEON-NEXT: ret213;214; SVE-LABEL: two_way_i16_i32_vl256:215; SVE: // %bb.0:216; SVE-NEXT: ldr z0, [x1]217; SVE-NEXT: ldr z1, [x2]218; SVE-NEXT: ptrue p0.s219; SVE-NEXT: ldr z4, [x0]220; SVE-NEXT: uunpklo z2.s, z0.h221; SVE-NEXT: uunpklo z3.s, z1.h222; SVE-NEXT: uunpkhi z0.s, z0.h223; SVE-NEXT: uunpkhi z1.s, z1.h224; SVE-NEXT: mad z2.s, p0/m, z3.s, z4.s225; SVE-NEXT: mad z0.s, p0/m, z1.s, z2.s226; SVE-NEXT: movprfx z1, z0227; SVE-NEXT: ext z1.b, z1.b, z0.b, #16228; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0229; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1230; SVE-NEXT: ret231;232; SME-LABEL: two_way_i16_i32_vl256:233; SME: // %bb.0:234; SME-NEXT: ldr z0, [x0]235; SME-NEXT: ldr z1, [x1]236; SME-NEXT: ldr z2, [x2]237; SME-NEXT: umlalb z0.s, z2.h, z1.h238; SME-NEXT: umlalt z0.s, z2.h, z1.h239; SME-NEXT: movprfx z1, z0240; SME-NEXT: ext z1.b, z1.b, z0.b, #16241; SME-NEXT: // kill: def $q0 killed $q0 killed $z0242; SME-NEXT: // kill: def $q1 killed $q1 killed $z1243; SME-NEXT: ret244 %acc = load <8 x i32>, ptr %accptr245 %u = load <16 x i16>, ptr %uptr246 %s = load <16 x i16>, ptr %sptr247 %u.wide = zext <16 x i16> %u to <16 x i32>248 %s.wide = zext <16 x i16> %s to <16 x i32>249 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide250 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <16 x i32> %mult)251 ret <8 x i32> %partial.reduce252}253 254;255; Two-way mla (i32 -> i64)256;257 258define <2 x i64> @two_way_i32_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {259;260; COMMON-LABEL: two_way_i32_i64_vl128:261; COMMON: // %bb.0:262; COMMON-NEXT: ldr q0, [x0]263; COMMON-NEXT: ldr q1, [x1]264; COMMON-NEXT: ldr q2, [x2]265; COMMON-NEXT: umlal v0.2d, v2.2s, v1.2s266; COMMON-NEXT: umlal2 v0.2d, v2.4s, v1.4s267; COMMON-NEXT: ret268;269; SME-LABEL: two_way_i32_i64_vl128:270; SME: // %bb.0:271; SME-NEXT: ldr q0, [x0]272; SME-NEXT: ldr q1, [x1]273; SME-NEXT: ldr q2, [x2]274; SME-NEXT: umlalb z0.d, z2.s, z1.s275; SME-NEXT: umlalt z0.d, z2.s, z1.s276; SME-NEXT: // kill: def $q0 killed $q0 killed $z0277; SME-NEXT: ret278 %acc = load <2 x i64>, ptr %accptr279 %u = load <4 x i32>, ptr %uptr280 %s = load <4 x i32>, ptr %sptr281 %u.wide = zext <4 x i32> %u to <4 x i64>282 %s.wide = zext <4 x i32> %s to <4 x i64>283 %mult = mul nuw nsw <4 x i64> %s.wide, %u.wide284 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <4 x i64> %mult)285 ret <2 x i64> %partial.reduce286}287 288define <4 x i64> @two_way_i32_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {289;290; COMMON-LABEL: two_way_i32_i64_vl128_double_width:291; COMMON: // %bb.0:292; COMMON-NEXT: ldp q0, q1, [x0]293; COMMON-NEXT: ldp q2, q3, [x1]294; COMMON-NEXT: ldp q4, q5, [x2]295; COMMON-NEXT: umlal v0.2d, v4.2s, v2.2s296; COMMON-NEXT: umlal v1.2d, v5.2s, v3.2s297; COMMON-NEXT: umlal2 v0.2d, v4.4s, v2.4s298; COMMON-NEXT: umlal2 v1.2d, v5.4s, v3.4s299; COMMON-NEXT: ret300;301; SME-LABEL: two_way_i32_i64_vl128_double_width:302; SME: // %bb.0:303; SME-NEXT: ldp q0, q1, [x0]304; SME-NEXT: ldp q3, q2, [x1]305; SME-NEXT: ldp q5, q4, [x2]306; SME-NEXT: umlalb z0.d, z5.s, z3.s307; SME-NEXT: umlalb z1.d, z4.s, z2.s308; SME-NEXT: umlalt z0.d, z5.s, z3.s309; SME-NEXT: umlalt z1.d, z4.s, z2.s310; SME-NEXT: // kill: def $q0 killed $q0 killed $z0311; SME-NEXT: // kill: def $q1 killed $q1 killed $z1312; SME-NEXT: ret313 %acc = load <4 x i64>, ptr %accptr314 %u = load <8 x i32>, ptr %uptr315 %s = load <8 x i32>, ptr %sptr316 %u.wide = zext <8 x i32> %u to <8 x i64>317 %s.wide = zext <8 x i32> %s to <8 x i64>318 %mult = mul nuw nsw <8 x i64> %s.wide, %u.wide319 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <8 x i64> %mult)320 ret <4 x i64> %partial.reduce321}322 323define <4 x i64> @two_way_i32_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {324;325;326; NEON-LABEL: two_way_i32_i64_vl256:327; NEON: // %bb.0:328; NEON-NEXT: ldp q0, q1, [x0]329; NEON-NEXT: ldp q2, q3, [x1]330; NEON-NEXT: ldp q4, q5, [x2]331; NEON-NEXT: umlal v0.2d, v4.2s, v2.2s332; NEON-NEXT: umlal v1.2d, v5.2s, v3.2s333; NEON-NEXT: umlal2 v0.2d, v4.4s, v2.4s334; NEON-NEXT: umlal2 v1.2d, v5.4s, v3.4s335; NEON-NEXT: ret336;337; SVE-LABEL: two_way_i32_i64_vl256:338; SVE: // %bb.0:339; SVE-NEXT: ldr z0, [x1]340; SVE-NEXT: ldr z1, [x2]341; SVE-NEXT: ptrue p0.d342; SVE-NEXT: ldr z4, [x0]343; SVE-NEXT: uunpklo z2.d, z0.s344; SVE-NEXT: uunpklo z3.d, z1.s345; SVE-NEXT: uunpkhi z0.d, z0.s346; SVE-NEXT: uunpkhi z1.d, z1.s347; SVE-NEXT: mad z2.d, p0/m, z3.d, z4.d348; SVE-NEXT: mad z0.d, p0/m, z1.d, z2.d349; SVE-NEXT: movprfx z1, z0350; SVE-NEXT: ext z1.b, z1.b, z0.b, #16351; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0352; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1353; SVE-NEXT: ret354;355; SME-LABEL: two_way_i32_i64_vl256:356; SME: // %bb.0:357; SME-NEXT: ldr z0, [x0]358; SME-NEXT: ldr z1, [x1]359; SME-NEXT: ldr z2, [x2]360; SME-NEXT: umlalb z0.d, z2.s, z1.s361; SME-NEXT: umlalt z0.d, z2.s, z1.s362; SME-NEXT: movprfx z1, z0363; SME-NEXT: ext z1.b, z1.b, z0.b, #16364; SME-NEXT: // kill: def $q0 killed $q0 killed $z0365; SME-NEXT: // kill: def $q1 killed $q1 killed $z1366; SME-NEXT: ret367 %acc = load <4 x i64>, ptr %accptr368 %u = load <8 x i32>, ptr %uptr369 %s = load <8 x i32>, ptr %sptr370 %u.wide = zext <8 x i32> %u to <8 x i64>371 %s.wide = zext <8 x i32> %s to <8 x i64>372 %mult = mul nuw nsw <8 x i64> %s.wide, %u.wide373 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <8 x i64> %mult)374 ret <4 x i64> %partial.reduce375}376 377 378;379; Four-way dot (i8 -> i32)380;381 382define <4 x i32> @four_way_i8_i32_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {383;384; COMMON-LABEL: four_way_i8_i32_vl128:385; COMMON: // %bb.0:386; COMMON-NEXT: ldr q0, [x0]387; COMMON-NEXT: ldr q1, [x1]388; COMMON-NEXT: ldr q2, [x2]389; COMMON-NEXT: udot v0.4s, v2.16b, v1.16b390; COMMON-NEXT: ret391;392; SME-LABEL: four_way_i8_i32_vl128:393; SME: // %bb.0:394; SME-NEXT: ldr q0, [x0]395; SME-NEXT: ldr q1, [x1]396; SME-NEXT: ldr q2, [x2]397; SME-NEXT: udot z0.s, z2.b, z1.b398; SME-NEXT: // kill: def $q0 killed $q0 killed $z0399; SME-NEXT: ret400 %acc = load <4 x i32>, ptr %accptr401 %u = load <16 x i8>, ptr %uptr402 %s = load <16 x i8>, ptr %sptr403 %u.wide = zext <16 x i8> %u to <16 x i32>404 %s.wide = zext <16 x i8> %s to <16 x i32>405 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide406 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <16 x i32> %mult)407 ret <4 x i32> %partial.reduce408}409 410define <4 x i32> @four_way_i8_i32_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {411; COMMON-LABEL: four_way_i8_i32_vl128_usdot:412; COMMON: // %bb.0:413; COMMON-NEXT: ldr q0, [x0]414; COMMON-NEXT: ldr q1, [x1]415; COMMON-NEXT: ldr q2, [x2]416; COMMON-NEXT: usdot v0.4s, v1.16b, v2.16b417; COMMON-NEXT: ret418;419; SME-LABEL: four_way_i8_i32_vl128_usdot:420; SME: // %bb.0:421; SME-NEXT: ldr q0, [x0]422; SME-NEXT: ldr q1, [x1]423; SME-NEXT: ldr q2, [x2]424; SME-NEXT: usdot z0.s, z1.b, z2.b425; SME-NEXT: // kill: def $q0 killed $q0 killed $z0426; SME-NEXT: ret427 %acc = load <4 x i32>, ptr %accptr428 %u = load <16 x i8>, ptr %uptr429 %s = load <16 x i8>, ptr %sptr430 %u.wide = zext <16 x i8> %u to <16 x i32>431 %s.wide = sext <16 x i8> %s to <16 x i32>432 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide433 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <16 x i32> %mult)434 ret <4 x i32> %partial.reduce435}436 437define <4 x i32> @four_way_i8_i32_vl128_sudot(ptr %accptr, ptr %uptr, ptr %sptr) {438; COMMON-LABEL: four_way_i8_i32_vl128_sudot:439; COMMON: // %bb.0:440; COMMON-NEXT: ldr q0, [x0]441; COMMON-NEXT: ldr q1, [x1]442; COMMON-NEXT: ldr q2, [x2]443; COMMON-NEXT: usdot v0.4s, v2.16b, v1.16b444; COMMON-NEXT: ret445;446; SME-LABEL: four_way_i8_i32_vl128_sudot:447; SME: // %bb.0:448; SME-NEXT: ldr q0, [x0]449; SME-NEXT: ldr q1, [x1]450; SME-NEXT: ldr q2, [x2]451; SME-NEXT: usdot z0.s, z2.b, z1.b452; SME-NEXT: // kill: def $q0 killed $q0 killed $z0453; SME-NEXT: ret454 %acc = load <4 x i32>, ptr %accptr455 %u = load <16 x i8>, ptr %uptr456 %s = load <16 x i8>, ptr %sptr457 %u.wide = sext <16 x i8> %u to <16 x i32>458 %s.wide = zext <16 x i8> %s to <16 x i32>459 %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide460 %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <16 x i32> %mult)461 ret <4 x i32> %partial.reduce462}463 464define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {465; NEON-LABEL: four_way_i8_i64_vl128_usdot:466; NEON: // %bb.0:467; NEON-NEXT: movi v0.2d, #0000000000000000468; NEON-NEXT: ldr q1, [x1]469; NEON-NEXT: ldr q2, [x2]470; NEON-NEXT: usdot v0.4s, v1.16b, v2.16b471; NEON-NEXT: ldr q1, [x0]472; NEON-NEXT: saddw v1.2d, v1.2d, v0.2s473; NEON-NEXT: saddw2 v0.2d, v1.2d, v0.4s474; NEON-NEXT: ret475;476; SVE-LABEL: four_way_i8_i64_vl128_usdot:477; SVE: // %bb.0:478; SVE-NEXT: movi v0.2d, #0000000000000000479; SVE-NEXT: ldr q1, [x1]480; SVE-NEXT: ldr q2, [x2]481; SVE-NEXT: usdot z0.s, z1.b, z2.b482; SVE-NEXT: ldr q2, [x0]483; SVE-NEXT: sunpklo z1.d, z0.s484; SVE-NEXT: sunpkhi z0.d, z0.s485; SVE-NEXT: add z1.d, z2.d, z1.d486; SVE-NEXT: add z0.d, z1.d, z0.d487; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0488; SVE-NEXT: ret489;490; SME-LABEL: four_way_i8_i64_vl128_usdot:491; SME: // %bb.0:492; SME-NEXT: mov z0.s, #0 // =0x0493; SME-NEXT: ldr q1, [x1]494; SME-NEXT: ldr q2, [x2]495; SME-NEXT: usdot z0.s, z1.b, z2.b496; SME-NEXT: ldr q1, [x0]497; SME-NEXT: saddwb z1.d, z1.d, z0.s498; SME-NEXT: saddwt z0.d, z1.d, z0.s499; SME-NEXT: // kill: def $q0 killed $q0 killed $z0500; SME-NEXT: ret501 %acc = load <2 x i64>, ptr %accptr502 %u = load <16 x i8>, ptr %uptr503 %s = load <16 x i8>, ptr %sptr504 %u.wide = zext <16 x i8> %u to <16 x i64>505 %s.wide = sext <16 x i8> %s to <16 x i64>506 %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide507 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <16 x i64> %mult)508 ret <2 x i64> %partial.reduce509}510 511define <2 x i64> @four_way_i16_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {512; COMMON-LABEL: four_way_i16_i64_vl128_usdot:513; COMMON: // %bb.0:514; COMMON-NEXT: ldr q1, [x1]515; COMMON-NEXT: ldr q2, [x2]516; COMMON-NEXT: ldr q0, [x0]517; COMMON-NEXT: ushll v3.4s, v1.4h, #0518; COMMON-NEXT: sshll v4.4s, v2.4h, #0519; COMMON-NEXT: ushll2 v1.4s, v1.8h, #0520; COMMON-NEXT: sshll2 v2.4s, v2.8h, #0521; COMMON-NEXT: smlal v0.2d, v4.2s, v3.2s522; COMMON-NEXT: smlal2 v0.2d, v4.4s, v3.4s523; COMMON-NEXT: smlal v0.2d, v2.2s, v1.2s524; COMMON-NEXT: smlal2 v0.2d, v2.4s, v1.4s525; COMMON-NEXT: ret526;527; SME-LABEL: four_way_i16_i64_vl128_usdot:528; SME: // %bb.0:529; SME-NEXT: ptrue p0.d, vl2530; SME-NEXT: ldr q2, [x0]531; SME-NEXT: mov x8, #2 // =0x2532; SME-NEXT: ld1h { z0.d }, p0/z, [x1]533; SME-NEXT: ld1sh { z1.d }, p0/z, [x2]534; SME-NEXT: mad z0.d, p0/m, z1.d, z2.d535; SME-NEXT: ld1h { z1.d }, p0/z, [x1, x8, lsl #1]536; SME-NEXT: ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]537; SME-NEXT: mov x8, #4 // =0x4538; SME-NEXT: mla z0.d, p0/m, z2.d, z1.d539; SME-NEXT: ld1h { z1.d }, p0/z, [x1, x8, lsl #1]540; SME-NEXT: ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]541; SME-NEXT: mov x8, #6 // =0x6542; SME-NEXT: mla z0.d, p0/m, z2.d, z1.d543; SME-NEXT: ld1h { z1.d }, p0/z, [x1, x8, lsl #1]544; SME-NEXT: ld1sh { z2.d }, p0/z, [x2, x8, lsl #1]545; SME-NEXT: mla z0.d, p0/m, z2.d, z1.d546; SME-NEXT: // kill: def $q0 killed $q0 killed $z0547; SME-NEXT: ret548 %acc = load <2 x i64>, ptr %accptr549 %u = load <8 x i16>, ptr %uptr550 %s = load <8 x i16>, ptr %sptr551 %u.wide = zext <8 x i16> %u to <8 x i64>552 %s.wide = sext <8 x i16> %s to <8 x i64>553 %mult = mul nuw nsw <8 x i64> %s.wide, %u.wide554 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %mult)555 ret <2 x i64> %partial.reduce556}557 558define <8 x i32> @four_way_i8_i32_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {559;560; COMMON-LABEL: four_way_i8_i32_vl128_double_width:561; COMMON: // %bb.0:562; COMMON-NEXT: ldp q0, q1, [x0]563; COMMON-NEXT: ldp q3, q2, [x1]564; COMMON-NEXT: ldp q5, q4, [x2]565; COMMON-NEXT: udot v0.4s, v5.16b, v3.16b566; COMMON-NEXT: udot v1.4s, v4.16b, v2.16b567; COMMON-NEXT: ret568;569; SME-LABEL: four_way_i8_i32_vl128_double_width:570; SME: // %bb.0:571; SME-NEXT: ldp q0, q1, [x0]572; SME-NEXT: ldp q3, q2, [x1]573; SME-NEXT: ldp q5, q4, [x2]574; SME-NEXT: udot z0.s, z5.b, z3.b575; SME-NEXT: udot z1.s, z4.b, z2.b576; SME-NEXT: // kill: def $q0 killed $q0 killed $z0577; SME-NEXT: // kill: def $q1 killed $q1 killed $z1578; SME-NEXT: ret579 %acc = load <8 x i32>, ptr %accptr580 %u = load <32 x i8>, ptr %uptr581 %s = load <32 x i8>, ptr %sptr582 %u.wide = zext <32 x i8> %u to <32 x i32>583 %s.wide = zext <32 x i8> %s to <32 x i32>584 %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide585 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)586 ret <8 x i32> %partial.reduce587}588 589define <8 x i32> @four_way_i8_i32_vl128_double_width_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {590;591; COMMON-LABEL: four_way_i8_i32_vl128_double_width_usdot:592; COMMON: // %bb.0:593; COMMON-NEXT: ldp q0, q1, [x0]594; COMMON-NEXT: ldp q3, q2, [x1]595; COMMON-NEXT: ldp q5, q4, [x2]596; COMMON-NEXT: usdot v0.4s, v3.16b, v5.16b597; COMMON-NEXT: usdot v1.4s, v2.16b, v4.16b598; COMMON-NEXT: ret599;600; SME-LABEL: four_way_i8_i32_vl128_double_width_usdot:601; SME: // %bb.0:602; SME-NEXT: ldp q0, q1, [x0]603; SME-NEXT: ldp q3, q2, [x1]604; SME-NEXT: ldp q5, q4, [x2]605; SME-NEXT: usdot z0.s, z3.b, z5.b606; SME-NEXT: usdot z1.s, z2.b, z4.b607; SME-NEXT: // kill: def $q0 killed $q0 killed $z0608; SME-NEXT: // kill: def $q1 killed $q1 killed $z1609; SME-NEXT: ret610 %acc = load <8 x i32>, ptr %accptr611 %u = load <32 x i8>, ptr %uptr612 %s = load <32 x i8>, ptr %sptr613 %u.wide = zext <32 x i8> %u to <32 x i32>614 %s.wide = sext <32 x i8> %s to <32 x i32>615 %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide616 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)617 ret <8 x i32> %partial.reduce618}619 620define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {621;622;623; NEON-LABEL: four_way_i8_i32_vl256:624; NEON: // %bb.0:625; NEON-NEXT: ldp q0, q1, [x0]626; NEON-NEXT: ldp q3, q2, [x1]627; NEON-NEXT: ldp q5, q4, [x2]628; NEON-NEXT: udot v0.4s, v5.16b, v3.16b629; NEON-NEXT: udot v1.4s, v4.16b, v2.16b630; NEON-NEXT: ret631;632; SVE-LABEL: four_way_i8_i32_vl256:633; SVE: // %bb.0:634; SVE-NEXT: ldr z0, [x0]635; SVE-NEXT: ldr z1, [x1]636; SVE-NEXT: ldr z2, [x2]637; SVE-NEXT: udot z0.s, z2.b, z1.b638; SVE-NEXT: movprfx z1, z0639; SVE-NEXT: ext z1.b, z1.b, z0.b, #16640; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0641; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1642; SVE-NEXT: ret643;644; SME-LABEL: four_way_i8_i32_vl256:645; SME: // %bb.0:646; SME-NEXT: ldr z0, [x0]647; SME-NEXT: ldr z1, [x1]648; SME-NEXT: ldr z2, [x2]649; SME-NEXT: udot z0.s, z2.b, z1.b650; SME-NEXT: movprfx z1, z0651; SME-NEXT: ext z1.b, z1.b, z0.b, #16652; SME-NEXT: // kill: def $q0 killed $q0 killed $z0653; SME-NEXT: // kill: def $q1 killed $q1 killed $z1654; SME-NEXT: ret655 %acc = load <8 x i32>, ptr %accptr656 %u = load <32 x i8>, ptr %uptr657 %s = load <32 x i8>, ptr %sptr658 %u.wide = zext <32 x i8> %u to <32 x i32>659 %s.wide = zext <32 x i8> %s to <32 x i32>660 %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide661 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)662 ret <8 x i32> %partial.reduce663}664 665define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {666;667;668; NEON-LABEL: four_way_i8_i32_vl256_usdot:669; NEON: // %bb.0:670; NEON-NEXT: ldp q0, q1, [x0]671; NEON-NEXT: ldp q3, q2, [x1]672; NEON-NEXT: ldp q5, q4, [x2]673; NEON-NEXT: usdot v0.4s, v3.16b, v5.16b674; NEON-NEXT: usdot v1.4s, v2.16b, v4.16b675; NEON-NEXT: ret676;677; SVE-LABEL: four_way_i8_i32_vl256_usdot:678; SVE: // %bb.0:679; SVE-NEXT: ldr z0, [x0]680; SVE-NEXT: ldr z1, [x1]681; SVE-NEXT: ldr z2, [x2]682; SVE-NEXT: usdot z0.s, z1.b, z2.b683; SVE-NEXT: movprfx z1, z0684; SVE-NEXT: ext z1.b, z1.b, z0.b, #16685; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0686; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1687; SVE-NEXT: ret688;689; SME-LABEL: four_way_i8_i32_vl256_usdot:690; SME: // %bb.0:691; SME-NEXT: ldr z0, [x0]692; SME-NEXT: ldr z1, [x1]693; SME-NEXT: ldr z2, [x2]694; SME-NEXT: usdot z0.s, z1.b, z2.b695; SME-NEXT: movprfx z1, z0696; SME-NEXT: ext z1.b, z1.b, z0.b, #16697; SME-NEXT: // kill: def $q0 killed $q0 killed $z0698; SME-NEXT: // kill: def $q1 killed $q1 killed $z1699; SME-NEXT: ret700 %acc = load <8 x i32>, ptr %accptr701 %u = load <32 x i8>, ptr %uptr702 %s = load <32 x i8>, ptr %sptr703 %u.wide = zext <32 x i8> %u to <32 x i32>704 %s.wide = sext <32 x i8> %s to <32 x i32>705 %mult = mul nuw nsw <32 x i32> %s.wide, %u.wide706 %partial.reduce = tail call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <32 x i32> %mult)707 ret <8 x i32> %partial.reduce708}709 710;711; Four-way dot (i16 -> i64)712;713 714define <2 x i64> @four_way_i16_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {715;716; COMMON-LABEL: four_way_i16_i64_vl128:717; COMMON: // %bb.0:718; COMMON-NEXT: ldr q0, [x1]719; COMMON-NEXT: ldr q1, [x2]720; COMMON-NEXT: ldr q3, [x0]721; COMMON-NEXT: umull v2.4s, v1.4h, v0.4h722; COMMON-NEXT: umull2 v0.4s, v1.8h, v0.8h723; COMMON-NEXT: uaddw v3.2d, v3.2d, v2.2s724; COMMON-NEXT: uaddw2 v1.2d, v3.2d, v2.4s725; COMMON-NEXT: uaddw v1.2d, v1.2d, v0.2s726; COMMON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s727; COMMON-NEXT: ret728;729; SME-LABEL: four_way_i16_i64_vl128:730; SME: // %bb.0:731; SME-NEXT: ldr q0, [x0]732; SME-NEXT: ldr q1, [x1]733; SME-NEXT: ldr q2, [x2]734; SME-NEXT: udot z0.d, z2.h, z1.h735; SME-NEXT: // kill: def $q0 killed $q0 killed $z0736; SME-NEXT: ret737 %acc = load <2 x i64>, ptr %accptr738 %u = load <8 x i16>, ptr %uptr739 %s = load <8 x i16>, ptr %sptr740 %u.wide = zext <8 x i16> %u to <8 x i64>741 %s.wide = zext <8 x i16> %s to <8 x i64>742 %mult = mul nuw nsw <8 x i64> %s.wide, %u.wide743 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %mult)744 ret <2 x i64> %partial.reduce745}746 747define <4 x i64> @four_way_i16_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {748;749; COMMON-LABEL: four_way_i16_i64_vl128_double_width:750; COMMON: // %bb.0:751; COMMON-NEXT: ldp q0, q1, [x1]752; COMMON-NEXT: ldp q2, q3, [x2]753; COMMON-NEXT: ldp q7, q6, [x0]754; COMMON-NEXT: umull v4.4s, v3.4h, v1.4h755; COMMON-NEXT: umull v5.4s, v2.4h, v0.4h756; COMMON-NEXT: umull2 v1.4s, v3.8h, v1.8h757; COMMON-NEXT: umull2 v0.4s, v2.8h, v0.8h758; COMMON-NEXT: uaddw v7.2d, v7.2d, v5.2s759; COMMON-NEXT: uaddw v6.2d, v6.2d, v4.2s760; COMMON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s761; COMMON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s762; COMMON-NEXT: uaddw v2.2d, v2.2d, v0.2s763; COMMON-NEXT: uaddw v3.2d, v3.2d, v1.2s764; COMMON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s765; COMMON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s766; COMMON-NEXT: ret767;768; SME-LABEL: four_way_i16_i64_vl128_double_width:769; SME: // %bb.0:770; SME-NEXT: ldp q0, q1, [x0]771; SME-NEXT: ldp q3, q2, [x1]772; SME-NEXT: ldp q5, q4, [x2]773; SME-NEXT: udot z0.d, z5.h, z3.h774; SME-NEXT: udot z1.d, z4.h, z2.h775; SME-NEXT: // kill: def $q0 killed $q0 killed $z0776; SME-NEXT: // kill: def $q1 killed $q1 killed $z1777; SME-NEXT: ret778 %acc = load <4 x i64>, ptr %accptr779 %u = load <16 x i16>, ptr %uptr780 %s = load <16 x i16>, ptr %sptr781 %u.wide = zext <16 x i16> %u to <16 x i64>782 %s.wide = zext <16 x i16> %s to <16 x i64>783 %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide784 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <16 x i64> %mult)785 ret <4 x i64> %partial.reduce786}787 788define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {789;790;791; NEON-LABEL: four_way_i16_i64_vl256:792; NEON: // %bb.0:793; NEON-NEXT: ldp q0, q1, [x1]794; NEON-NEXT: ldp q2, q3, [x2]795; NEON-NEXT: ldp q7, q6, [x0]796; NEON-NEXT: umull v4.4s, v3.4h, v1.4h797; NEON-NEXT: umull v5.4s, v2.4h, v0.4h798; NEON-NEXT: umull2 v1.4s, v3.8h, v1.8h799; NEON-NEXT: umull2 v0.4s, v2.8h, v0.8h800; NEON-NEXT: uaddw v7.2d, v7.2d, v5.2s801; NEON-NEXT: uaddw v6.2d, v6.2d, v4.2s802; NEON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s803; NEON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s804; NEON-NEXT: uaddw v2.2d, v2.2d, v0.2s805; NEON-NEXT: uaddw v3.2d, v3.2d, v1.2s806; NEON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s807; NEON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s808; NEON-NEXT: ret809;810; SVE-LABEL: four_way_i16_i64_vl256:811; SVE: // %bb.0:812; SVE-NEXT: ldr z0, [x0]813; SVE-NEXT: ldr z1, [x1]814; SVE-NEXT: ldr z2, [x2]815; SVE-NEXT: udot z0.d, z2.h, z1.h816; SVE-NEXT: movprfx z1, z0817; SVE-NEXT: ext z1.b, z1.b, z0.b, #16818; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0819; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1820; SVE-NEXT: ret821;822; SME-LABEL: four_way_i16_i64_vl256:823; SME: // %bb.0:824; SME-NEXT: ldr z0, [x0]825; SME-NEXT: ldr z1, [x1]826; SME-NEXT: ldr z2, [x2]827; SME-NEXT: udot z0.d, z2.h, z1.h828; SME-NEXT: movprfx z1, z0829; SME-NEXT: ext z1.b, z1.b, z0.b, #16830; SME-NEXT: // kill: def $q0 killed $q0 killed $z0831; SME-NEXT: // kill: def $q1 killed $q1 killed $z1832; SME-NEXT: ret833 %acc = load <4 x i64>, ptr %accptr834 %u = load <16 x i16>, ptr %uptr835 %s = load <16 x i16>, ptr %sptr836 %u.wide = zext <16 x i16> %u to <16 x i64>837 %s.wide = zext <16 x i16> %s to <16 x i64>838 %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide839 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <16 x i64> %mult)840 ret <4 x i64> %partial.reduce841}842 843 844;845; Eight-way dot, requires two steps (i8 -> i64)846;847 848define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {849;850; NEON-LABEL: eight_way_i8_i64_vl128:851; NEON: // %bb.0:852; NEON-NEXT: movi v0.2d, #0000000000000000853; NEON-NEXT: ldr q1, [x1]854; NEON-NEXT: ldr q2, [x2]855; NEON-NEXT: udot v0.4s, v2.16b, v1.16b856; NEON-NEXT: ldr q1, [x0]857; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s858; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s859; NEON-NEXT: ret860;861; SVE-LABEL: eight_way_i8_i64_vl128:862; SVE: // %bb.0:863; SVE-NEXT: movi v0.2d, #0000000000000000864; SVE-NEXT: ldr q1, [x1]865; SVE-NEXT: ldr q2, [x2]866; SVE-NEXT: udot z0.s, z2.b, z1.b867; SVE-NEXT: ldr q2, [x0]868; SVE-NEXT: uunpklo z1.d, z0.s869; SVE-NEXT: uunpkhi z0.d, z0.s870; SVE-NEXT: add z1.d, z2.d, z1.d871; SVE-NEXT: add z0.d, z1.d, z0.d872; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0873; SVE-NEXT: ret874;875; SME-LABEL: eight_way_i8_i64_vl128:876; SME: // %bb.0:877; SME-NEXT: mov z0.s, #0 // =0x0878; SME-NEXT: ldr q1, [x1]879; SME-NEXT: ldr q2, [x2]880; SME-NEXT: udot z0.s, z2.b, z1.b881; SME-NEXT: ldr q1, [x0]882; SME-NEXT: uaddwb z1.d, z1.d, z0.s883; SME-NEXT: uaddwt z0.d, z1.d, z0.s884; SME-NEXT: // kill: def $q0 killed $q0 killed $z0885; SME-NEXT: ret886 %acc = load <2 x i64>, ptr %accptr887 %u = load <16 x i8>, ptr %uptr888 %s = load <16 x i8>, ptr %sptr889 %u.wide = zext <16 x i8> %u to <16 x i64>890 %s.wide = zext <16 x i8> %s to <16 x i64>891 %mult = mul nuw nsw <16 x i64> %s.wide, %u.wide892 %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <16 x i64> %mult)893 ret <2 x i64> %partial.reduce894}895 896define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr %sptr) {897;898; NEON-LABEL: four_way_i8_i64_vl128_double_width:899; NEON: // %bb.0:900; NEON-NEXT: movi v1.2d, #0000000000000000901; NEON-NEXT: movi v0.2d, #0000000000000000902; NEON-NEXT: ldp q3, q2, [x1]903; NEON-NEXT: ldp q5, q4, [x2]904; NEON-NEXT: udot v0.4s, v5.16b, v3.16b905; NEON-NEXT: udot v1.4s, v4.16b, v2.16b906; NEON-NEXT: ldp q3, q2, [x0]907; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s908; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s909; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s910; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s911; NEON-NEXT: ret912;913; SVE-LABEL: four_way_i8_i64_vl128_double_width:914; SVE: // %bb.0:915; SVE-NEXT: movi v0.2d, #0000000000000000916; SVE-NEXT: movi v1.2d, #0000000000000000917; SVE-NEXT: ldp q3, q2, [x1]918; SVE-NEXT: ldp q5, q4, [x2]919; SVE-NEXT: udot z1.s, z5.b, z3.b920; SVE-NEXT: udot z0.s, z4.b, z2.b921; SVE-NEXT: ldp q5, q4, [x0]922; SVE-NEXT: uunpklo z2.d, z1.s923; SVE-NEXT: uunpklo z3.d, z0.s924; SVE-NEXT: uunpkhi z1.d, z1.s925; SVE-NEXT: uunpkhi z6.d, z0.s926; SVE-NEXT: add z0.d, z5.d, z2.d927; SVE-NEXT: add z2.d, z4.d, z3.d928; SVE-NEXT: add z0.d, z0.d, z1.d929; SVE-NEXT: add z1.d, z2.d, z6.d930; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0931; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1932; SVE-NEXT: ret933;934; SME-LABEL: four_way_i8_i64_vl128_double_width:935; SME: // %bb.0:936; SME-NEXT: mov z1.s, #0 // =0x0937; SME-NEXT: mov z0.s, #0 // =0x0938; SME-NEXT: ldp q3, q2, [x1]939; SME-NEXT: ldp q5, q4, [x2]940; SME-NEXT: udot z0.s, z5.b, z3.b941; SME-NEXT: udot z1.s, z4.b, z2.b942; SME-NEXT: ldp q3, q2, [x0]943; SME-NEXT: uaddwb z3.d, z3.d, z0.s944; SME-NEXT: uaddwb z2.d, z2.d, z1.s945; SME-NEXT: uaddwt z0.d, z3.d, z0.s946; SME-NEXT: uaddwt z1.d, z2.d, z1.s947; SME-NEXT: // kill: def $q0 killed $q0 killed $z0948; SME-NEXT: // kill: def $q1 killed $q1 killed $z1949; SME-NEXT: ret950 %acc = load <4 x i64>, ptr %accptr951 %u = load <32 x i8>, ptr %uptr952 %s = load <32 x i8>, ptr %sptr953 %u.wide = zext <32 x i8> %u to <32 x i64>954 %s.wide = zext <32 x i8> %s to <32 x i64>955 %mult = mul nuw nsw <32 x i64> %s.wide, %u.wide956 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <32 x i64> %mult)957 ret <4 x i64> %partial.reduce958}959 960define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {961; NEON-LABEL: four_way_i8_i64_vl256:962; NEON: // %bb.0:963; NEON-NEXT: movi v1.2d, #0000000000000000964; NEON-NEXT: movi v0.2d, #0000000000000000965; NEON-NEXT: ldp q3, q2, [x1]966; NEON-NEXT: ldp q5, q4, [x2]967; NEON-NEXT: udot v0.4s, v5.16b, v3.16b968; NEON-NEXT: udot v1.4s, v4.16b, v2.16b969; NEON-NEXT: ldp q3, q2, [x0]970; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s971; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s972; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s973; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s974; NEON-NEXT: ret975;976; SVE-LABEL: four_way_i8_i64_vl256:977; SVE: // %bb.0:978; SVE-NEXT: movi v0.2d, #0000000000000000979; SVE-NEXT: ldr z1, [x1]980; SVE-NEXT: ldr z2, [x2]981; SVE-NEXT: udot z0.s, z2.b, z1.b982; SVE-NEXT: ldr z2, [x0]983; SVE-NEXT: uunpklo z1.d, z0.s984; SVE-NEXT: uunpkhi z0.d, z0.s985; SVE-NEXT: add z1.d, z2.d, z1.d986; SVE-NEXT: add z0.d, z1.d, z0.d987; SVE-NEXT: movprfx z1, z0988; SVE-NEXT: ext z1.b, z1.b, z0.b, #16989; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0990; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1991; SVE-NEXT: ret992;993; SME-LABEL: four_way_i8_i64_vl256:994; SME: // %bb.0:995; SME-NEXT: ldr z0, [x1]996; SME-NEXT: ldr z1, [x2]997; SME-NEXT: mov z2.s, #0 // =0x0998; SME-NEXT: udot z2.s, z1.b, z0.b999; SME-NEXT: ldr z0, [x0]1000; SME-NEXT: uaddwb z0.d, z0.d, z2.s1001; SME-NEXT: uaddwt z0.d, z0.d, z2.s1002; SME-NEXT: movprfx z1, z01003; SME-NEXT: ext z1.b, z1.b, z0.b, #161004; SME-NEXT: // kill: def $q0 killed $q0 killed $z01005; SME-NEXT: // kill: def $q1 killed $q1 killed $z11006; SME-NEXT: ret1007 %acc = load <4 x i64>, ptr %accptr1008 %u = load <32 x i8>, ptr %uptr1009 %s = load <32 x i8>, ptr %sptr1010 %u.wide = zext <32 x i8> %u to <32 x i64>1011 %s.wide = zext <32 x i8> %s to <32 x i64>1012 %mult = mul nuw nsw <32 x i64> %s.wide, %u.wide1013 %partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <32 x i64> %mult)1014 ret <4 x i64> %partial.reduce1015}1016