1302 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=aarch64 -mattr=+sve2 %s -o - | FileCheck %s --check-prefix=CHECK-SVE23; RUN: llc -mtriple=aarch64 -mattr=+sve2,+i8mm %s -o - | FileCheck %s --check-prefix=CHECK-SVE2-I8MM4; RUN: llc -mtriple=aarch64 -mattr=+sve2,+sme,+i8mm -force-streaming %s -o - | FileCheck %s --check-prefix=CHECK-SME5 6define <vscale x 4 x i32> @udot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {7; CHECK-SVE2-LABEL: udot:8; CHECK-SVE2: // %bb.0: // %entry9; CHECK-SVE2-NEXT: udot z0.s, z1.b, z2.b10; CHECK-SVE2-NEXT: ret11;12; CHECK-SVE2-I8MM-LABEL: udot:13; CHECK-SVE2-I8MM: // %bb.0: // %entry14; CHECK-SVE2-I8MM-NEXT: udot z0.s, z1.b, z2.b15; CHECK-SVE2-I8MM-NEXT: ret16;17; CHECK-SME-LABEL: udot:18; CHECK-SME: // %bb.0: // %entry19; CHECK-SME-NEXT: udot z0.s, z1.b, z2.b20; CHECK-SME-NEXT: ret21entry:22 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>23 %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>24 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, %b.wide25 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)26 ret <vscale x 4 x i32> %partial.reduce27}28 29define <vscale x 2 x i64> @udot_wide(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {30; CHECK-SVE2-LABEL: udot_wide:31; CHECK-SVE2: // %bb.0: // %entry32; CHECK-SVE2-NEXT: udot z0.d, z1.h, z2.h33; CHECK-SVE2-NEXT: ret34;35; CHECK-SVE2-I8MM-LABEL: udot_wide:36; CHECK-SVE2-I8MM: // %bb.0: // %entry37; CHECK-SVE2-I8MM-NEXT: udot z0.d, z1.h, z2.h38; CHECK-SVE2-I8MM-NEXT: ret39;40; CHECK-SME-LABEL: udot_wide:41; CHECK-SME: // %bb.0: // %entry42; CHECK-SME-NEXT: udot z0.d, z1.h, z2.h43; CHECK-SME-NEXT: ret44entry:45 %a.wide = zext <vscale x 8 x i16> %a to <vscale x 8 x i64>46 %b.wide = zext <vscale x 8 x i16> %b to <vscale x 8 x i64>47 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide48 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)49 ret <vscale x 2 x i64> %partial.reduce50}51 52define <vscale x 4 x i32> @sdot(<vscale x 4 x i32> %accc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {53; CHECK-SVE2-LABEL: sdot:54; CHECK-SVE2: // %bb.0: // %entry55; CHECK-SVE2-NEXT: sdot z0.s, z1.b, z2.b56; CHECK-SVE2-NEXT: ret57;58; CHECK-SVE2-I8MM-LABEL: sdot:59; CHECK-SVE2-I8MM: // %bb.0: // %entry60; CHECK-SVE2-I8MM-NEXT: sdot z0.s, z1.b, z2.b61; CHECK-SVE2-I8MM-NEXT: ret62;63; CHECK-SME-LABEL: sdot:64; CHECK-SME: // %bb.0: // %entry65; CHECK-SME-NEXT: sdot z0.s, z1.b, z2.b66; CHECK-SME-NEXT: ret67entry:68 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>69 %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>70 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, %b.wide71 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %accc, <vscale x 16 x i32> %mult)72 ret <vscale x 4 x i32> %partial.reduce73}74 75define <vscale x 2 x i64> @sdot_wide(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {76; CHECK-SVE2-LABEL: sdot_wide:77; CHECK-SVE2: // %bb.0: // %entry78; CHECK-SVE2-NEXT: sdot z0.d, z1.h, z2.h79; CHECK-SVE2-NEXT: ret80;81; CHECK-SVE2-I8MM-LABEL: sdot_wide:82; CHECK-SVE2-I8MM: // %bb.0: // %entry83; CHECK-SVE2-I8MM-NEXT: sdot z0.d, z1.h, z2.h84; CHECK-SVE2-I8MM-NEXT: ret85;86; CHECK-SME-LABEL: sdot_wide:87; CHECK-SME: // %bb.0: // %entry88; CHECK-SME-NEXT: sdot z0.d, z1.h, z2.h89; CHECK-SME-NEXT: ret90entry:91 %a.wide = sext <vscale x 8 x i16> %a to <vscale x 8 x i64>92 %b.wide = sext <vscale x 8 x i16> %b to <vscale x 8 x i64>93 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide94 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)95 ret <vscale x 2 x i64> %partial.reduce96}97 98define <vscale x 4 x i32> @usdot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {99; CHECK-SVE2-LABEL: usdot:100; CHECK-SVE2: // %bb.0: // %entry101; CHECK-SVE2-NEXT: uunpklo z3.h, z1.b102; CHECK-SVE2-NEXT: sunpklo z4.h, z2.b103; CHECK-SVE2-NEXT: ptrue p0.s104; CHECK-SVE2-NEXT: uunpkhi z1.h, z1.b105; CHECK-SVE2-NEXT: sunpkhi z2.h, z2.b106; CHECK-SVE2-NEXT: uunpklo z5.s, z3.h107; CHECK-SVE2-NEXT: sunpklo z6.s, z4.h108; CHECK-SVE2-NEXT: uunpkhi z3.s, z3.h109; CHECK-SVE2-NEXT: sunpkhi z4.s, z4.h110; CHECK-SVE2-NEXT: mla z0.s, p0/m, z5.s, z6.s111; CHECK-SVE2-NEXT: uunpklo z5.s, z1.h112; CHECK-SVE2-NEXT: sunpklo z6.s, z2.h113; CHECK-SVE2-NEXT: uunpkhi z1.s, z1.h114; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h115; CHECK-SVE2-NEXT: mla z0.s, p0/m, z3.s, z4.s116; CHECK-SVE2-NEXT: mla z0.s, p0/m, z5.s, z6.s117; CHECK-SVE2-NEXT: mla z0.s, p0/m, z1.s, z2.s118; CHECK-SVE2-NEXT: ret119;120; CHECK-SVE2-I8MM-LABEL: usdot:121; CHECK-SVE2-I8MM: // %bb.0: // %entry122; CHECK-SVE2-I8MM-NEXT: usdot z0.s, z1.b, z2.b123; CHECK-SVE2-I8MM-NEXT: ret124;125; CHECK-SME-LABEL: usdot:126; CHECK-SME: // %bb.0: // %entry127; CHECK-SME-NEXT: usdot z0.s, z1.b, z2.b128; CHECK-SME-NEXT: ret129entry:130 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>131 %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>132 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, %b.wide133 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)134 ret <vscale x 4 x i32> %partial.reduce135}136 137define <vscale x 4 x i32> @sudot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {138; CHECK-SVE2-LABEL: sudot:139; CHECK-SVE2: // %bb.0: // %entry140; CHECK-SVE2-NEXT: sunpklo z3.h, z1.b141; CHECK-SVE2-NEXT: uunpklo z4.h, z2.b142; CHECK-SVE2-NEXT: ptrue p0.s143; CHECK-SVE2-NEXT: sunpkhi z1.h, z1.b144; CHECK-SVE2-NEXT: uunpkhi z2.h, z2.b145; CHECK-SVE2-NEXT: sunpklo z5.s, z3.h146; CHECK-SVE2-NEXT: uunpklo z6.s, z4.h147; CHECK-SVE2-NEXT: sunpkhi z3.s, z3.h148; CHECK-SVE2-NEXT: uunpkhi z4.s, z4.h149; CHECK-SVE2-NEXT: mla z0.s, p0/m, z5.s, z6.s150; CHECK-SVE2-NEXT: sunpklo z5.s, z1.h151; CHECK-SVE2-NEXT: uunpklo z6.s, z2.h152; CHECK-SVE2-NEXT: sunpkhi z1.s, z1.h153; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h154; CHECK-SVE2-NEXT: mla z0.s, p0/m, z3.s, z4.s155; CHECK-SVE2-NEXT: mla z0.s, p0/m, z5.s, z6.s156; CHECK-SVE2-NEXT: mla z0.s, p0/m, z1.s, z2.s157; CHECK-SVE2-NEXT: ret158;159; CHECK-SVE2-I8MM-LABEL: sudot:160; CHECK-SVE2-I8MM: // %bb.0: // %entry161; CHECK-SVE2-I8MM-NEXT: usdot z0.s, z2.b, z1.b162; CHECK-SVE2-I8MM-NEXT: ret163;164; CHECK-SME-LABEL: sudot:165; CHECK-SME: // %bb.0: // %entry166; CHECK-SME-NEXT: usdot z0.s, z2.b, z1.b167; CHECK-SME-NEXT: ret168entry:169 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>170 %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>171 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, %b.wide172 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)173 ret <vscale x 4 x i32> %partial.reduce174}175 176define <vscale x 4 x i64> @udot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {177; CHECK-SVE2-LABEL: udot_8to64:178; CHECK-SVE2: // %bb.0: // %entry179; CHECK-SVE2-NEXT: movi v4.2d, #0000000000000000180; CHECK-SVE2-NEXT: udot z4.s, z2.b, z3.b181; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z4.s182; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z4.s183; CHECK-SVE2-NEXT: ret184;185; CHECK-SVE2-I8MM-LABEL: udot_8to64:186; CHECK-SVE2-I8MM: // %bb.0: // %entry187; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000188; CHECK-SVE2-I8MM-NEXT: udot z4.s, z2.b, z3.b189; CHECK-SVE2-I8MM-NEXT: uaddwb z0.d, z0.d, z4.s190; CHECK-SVE2-I8MM-NEXT: uaddwt z0.d, z0.d, z4.s191; CHECK-SVE2-I8MM-NEXT: ret192;193; CHECK-SME-LABEL: udot_8to64:194; CHECK-SME: // %bb.0: // %entry195; CHECK-SME-NEXT: mov z4.s, #0 // =0x0196; CHECK-SME-NEXT: udot z4.s, z2.b, z3.b197; CHECK-SME-NEXT: uaddwb z0.d, z0.d, z4.s198; CHECK-SME-NEXT: uaddwt z0.d, z0.d, z4.s199; CHECK-SME-NEXT: ret200entry:201 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>202 %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i64>203 %mult = mul nuw nsw <vscale x 16 x i64> %a.wide, %b.wide204 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(205 <vscale x 4 x i64> %acc, <vscale x 16 x i64> %mult)206 ret <vscale x 4 x i64> %partial.reduce207}208 209define <vscale x 4 x i64> @sdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b){210; CHECK-SVE2-LABEL: sdot_8to64:211; CHECK-SVE2: // %bb.0: // %entry212; CHECK-SVE2-NEXT: movi v4.2d, #0000000000000000213; CHECK-SVE2-NEXT: sdot z4.s, z2.b, z3.b214; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z4.s215; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z4.s216; CHECK-SVE2-NEXT: ret217;218; CHECK-SVE2-I8MM-LABEL: sdot_8to64:219; CHECK-SVE2-I8MM: // %bb.0: // %entry220; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000221; CHECK-SVE2-I8MM-NEXT: sdot z4.s, z2.b, z3.b222; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s223; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s224; CHECK-SVE2-I8MM-NEXT: ret225;226; CHECK-SME-LABEL: sdot_8to64:227; CHECK-SME: // %bb.0: // %entry228; CHECK-SME-NEXT: mov z4.s, #0 // =0x0229; CHECK-SME-NEXT: sdot z4.s, z2.b, z3.b230; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s231; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s232; CHECK-SME-NEXT: ret233entry:234 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>235 %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i64>236 %mult = mul nuw nsw <vscale x 16 x i64> %a.wide, %b.wide237 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(238 <vscale x 4 x i64> %acc, <vscale x 16 x i64> %mult)239 ret <vscale x 4 x i64> %partial.reduce240}241 242define <vscale x 4 x i64> @usdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b){243; CHECK-SVE2-LABEL: usdot_8to64:244; CHECK-SVE2: // %bb.0: // %entry245; CHECK-SVE2-NEXT: uunpkhi z4.h, z2.b246; CHECK-SVE2-NEXT: uunpklo z2.h, z2.b247; CHECK-SVE2-NEXT: sunpkhi z5.h, z3.b248; CHECK-SVE2-NEXT: sunpklo z3.h, z3.b249; CHECK-SVE2-NEXT: ptrue p0.d250; CHECK-SVE2-NEXT: uunpklo z6.s, z4.h251; CHECK-SVE2-NEXT: uunpklo z7.s, z2.h252; CHECK-SVE2-NEXT: sunpklo z24.s, z5.h253; CHECK-SVE2-NEXT: sunpklo z25.s, z3.h254; CHECK-SVE2-NEXT: uunpkhi z4.s, z4.h255; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h256; CHECK-SVE2-NEXT: sunpkhi z5.s, z5.h257; CHECK-SVE2-NEXT: sunpkhi z3.s, z3.h258; CHECK-SVE2-NEXT: uunpklo z26.d, z6.s259; CHECK-SVE2-NEXT: uunpklo z27.d, z7.s260; CHECK-SVE2-NEXT: sunpklo z28.d, z24.s261; CHECK-SVE2-NEXT: sunpklo z29.d, z25.s262; CHECK-SVE2-NEXT: uunpkhi z6.d, z6.s263; CHECK-SVE2-NEXT: uunpkhi z7.d, z7.s264; CHECK-SVE2-NEXT: sunpkhi z24.d, z24.s265; CHECK-SVE2-NEXT: sunpkhi z25.d, z25.s266; CHECK-SVE2-NEXT: mla z1.d, p0/m, z26.d, z28.d267; CHECK-SVE2-NEXT: uunpklo z26.d, z4.s268; CHECK-SVE2-NEXT: sunpklo z28.d, z5.s269; CHECK-SVE2-NEXT: mla z0.d, p0/m, z27.d, z29.d270; CHECK-SVE2-NEXT: uunpklo z27.d, z2.s271; CHECK-SVE2-NEXT: sunpklo z29.d, z3.s272; CHECK-SVE2-NEXT: uunpkhi z4.d, z4.s273; CHECK-SVE2-NEXT: uunpkhi z2.d, z2.s274; CHECK-SVE2-NEXT: sunpkhi z5.d, z5.s275; CHECK-SVE2-NEXT: sunpkhi z3.d, z3.s276; CHECK-SVE2-NEXT: mla z1.d, p0/m, z6.d, z24.d277; CHECK-SVE2-NEXT: mla z0.d, p0/m, z7.d, z25.d278; CHECK-SVE2-NEXT: mla z1.d, p0/m, z26.d, z28.d279; CHECK-SVE2-NEXT: mla z0.d, p0/m, z27.d, z29.d280; CHECK-SVE2-NEXT: mla z1.d, p0/m, z4.d, z5.d281; CHECK-SVE2-NEXT: mla z0.d, p0/m, z2.d, z3.d282; CHECK-SVE2-NEXT: ret283;284; CHECK-SVE2-I8MM-LABEL: usdot_8to64:285; CHECK-SVE2-I8MM: // %bb.0: // %entry286; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000287; CHECK-SVE2-I8MM-NEXT: usdot z4.s, z2.b, z3.b288; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s289; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s290; CHECK-SVE2-I8MM-NEXT: ret291;292; CHECK-SME-LABEL: usdot_8to64:293; CHECK-SME: // %bb.0: // %entry294; CHECK-SME-NEXT: mov z4.s, #0 // =0x0295; CHECK-SME-NEXT: usdot z4.s, z2.b, z3.b296; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s297; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s298; CHECK-SME-NEXT: ret299entry:300 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>301 %b.wide = sext <vscale x 16 x i8> %b to <vscale x 16 x i64>302 %mult = mul nuw nsw <vscale x 16 x i64> %a.wide, %b.wide303 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(304 <vscale x 4 x i64> %acc, <vscale x 16 x i64> %mult)305 ret <vscale x 4 x i64> %partial.reduce306}307 308define <vscale x 4 x i64> @sudot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {309; CHECK-SVE2-LABEL: sudot_8to64:310; CHECK-SVE2: // %bb.0: // %entry311; CHECK-SVE2-NEXT: sunpkhi z4.h, z2.b312; CHECK-SVE2-NEXT: sunpklo z2.h, z2.b313; CHECK-SVE2-NEXT: uunpkhi z5.h, z3.b314; CHECK-SVE2-NEXT: uunpklo z3.h, z3.b315; CHECK-SVE2-NEXT: ptrue p0.d316; CHECK-SVE2-NEXT: sunpklo z6.s, z4.h317; CHECK-SVE2-NEXT: sunpklo z7.s, z2.h318; CHECK-SVE2-NEXT: uunpklo z24.s, z5.h319; CHECK-SVE2-NEXT: uunpklo z25.s, z3.h320; CHECK-SVE2-NEXT: sunpkhi z4.s, z4.h321; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h322; CHECK-SVE2-NEXT: uunpkhi z5.s, z5.h323; CHECK-SVE2-NEXT: uunpkhi z3.s, z3.h324; CHECK-SVE2-NEXT: sunpklo z26.d, z6.s325; CHECK-SVE2-NEXT: sunpklo z27.d, z7.s326; CHECK-SVE2-NEXT: uunpklo z28.d, z24.s327; CHECK-SVE2-NEXT: uunpklo z29.d, z25.s328; CHECK-SVE2-NEXT: sunpkhi z6.d, z6.s329; CHECK-SVE2-NEXT: sunpkhi z7.d, z7.s330; CHECK-SVE2-NEXT: uunpkhi z24.d, z24.s331; CHECK-SVE2-NEXT: uunpkhi z25.d, z25.s332; CHECK-SVE2-NEXT: mla z1.d, p0/m, z26.d, z28.d333; CHECK-SVE2-NEXT: sunpklo z26.d, z4.s334; CHECK-SVE2-NEXT: uunpklo z28.d, z5.s335; CHECK-SVE2-NEXT: mla z0.d, p0/m, z27.d, z29.d336; CHECK-SVE2-NEXT: sunpklo z27.d, z2.s337; CHECK-SVE2-NEXT: uunpklo z29.d, z3.s338; CHECK-SVE2-NEXT: sunpkhi z4.d, z4.s339; CHECK-SVE2-NEXT: sunpkhi z2.d, z2.s340; CHECK-SVE2-NEXT: uunpkhi z5.d, z5.s341; CHECK-SVE2-NEXT: uunpkhi z3.d, z3.s342; CHECK-SVE2-NEXT: mla z1.d, p0/m, z6.d, z24.d343; CHECK-SVE2-NEXT: mla z0.d, p0/m, z7.d, z25.d344; CHECK-SVE2-NEXT: mla z1.d, p0/m, z26.d, z28.d345; CHECK-SVE2-NEXT: mla z0.d, p0/m, z27.d, z29.d346; CHECK-SVE2-NEXT: mla z1.d, p0/m, z4.d, z5.d347; CHECK-SVE2-NEXT: mla z0.d, p0/m, z2.d, z3.d348; CHECK-SVE2-NEXT: ret349;350; CHECK-SVE2-I8MM-LABEL: sudot_8to64:351; CHECK-SVE2-I8MM: // %bb.0: // %entry352; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000353; CHECK-SVE2-I8MM-NEXT: usdot z4.s, z3.b, z2.b354; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s355; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s356; CHECK-SVE2-I8MM-NEXT: ret357;358; CHECK-SME-LABEL: sudot_8to64:359; CHECK-SME: // %bb.0: // %entry360; CHECK-SME-NEXT: mov z4.s, #0 // =0x0361; CHECK-SME-NEXT: usdot z4.s, z3.b, z2.b362; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s363; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s364; CHECK-SME-NEXT: ret365entry:366 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>367 %b.wide = zext <vscale x 16 x i8> %b to <vscale x 16 x i64>368 %mult = mul nuw nsw <vscale x 16 x i64> %a.wide, %b.wide369 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(370 <vscale x 4 x i64> %acc, <vscale x 16 x i64> %mult)371 ret <vscale x 4 x i64> %partial.reduce372}373 374define <vscale x 4 x i32> @udot_no_bin_op(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a){375; CHECK-SVE2-LABEL: udot_no_bin_op:376; CHECK-SVE2: // %bb.0:377; CHECK-SVE2-NEXT: mov z2.b, #1 // =0x1378; CHECK-SVE2-NEXT: udot z0.s, z1.b, z2.b379; CHECK-SVE2-NEXT: ret380;381; CHECK-SVE2-I8MM-LABEL: udot_no_bin_op:382; CHECK-SVE2-I8MM: // %bb.0:383; CHECK-SVE2-I8MM-NEXT: mov z2.b, #1 // =0x1384; CHECK-SVE2-I8MM-NEXT: udot z0.s, z1.b, z2.b385; CHECK-SVE2-I8MM-NEXT: ret386;387; CHECK-SME-LABEL: udot_no_bin_op:388; CHECK-SME: // %bb.0:389; CHECK-SME-NEXT: mov z2.b, #1 // =0x1390; CHECK-SME-NEXT: udot z0.s, z1.b, z2.b391; CHECK-SME-NEXT: ret392 %a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>393 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %a.ext)394 ret <vscale x 4 x i32> %partial.reduce395}396 397define <vscale x 4 x i32> @sdot_no_bin_op(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a){398; CHECK-SVE2-LABEL: sdot_no_bin_op:399; CHECK-SVE2: // %bb.0:400; CHECK-SVE2-NEXT: mov z2.b, #1 // =0x1401; CHECK-SVE2-NEXT: sdot z0.s, z1.b, z2.b402; CHECK-SVE2-NEXT: ret403;404; CHECK-SVE2-I8MM-LABEL: sdot_no_bin_op:405; CHECK-SVE2-I8MM: // %bb.0:406; CHECK-SVE2-I8MM-NEXT: mov z2.b, #1 // =0x1407; CHECK-SVE2-I8MM-NEXT: sdot z0.s, z1.b, z2.b408; CHECK-SVE2-I8MM-NEXT: ret409;410; CHECK-SME-LABEL: sdot_no_bin_op:411; CHECK-SME: // %bb.0:412; CHECK-SME-NEXT: mov z2.b, #1 // =0x1413; CHECK-SME-NEXT: sdot z0.s, z1.b, z2.b414; CHECK-SME-NEXT: ret415 %a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>416 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %a.ext)417 ret <vscale x 4 x i32> %partial.reduce418}419 420define <vscale x 2 x i64> @udot_no_bin_op_wide(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b){421; CHECK-SVE2-LABEL: udot_no_bin_op_wide:422; CHECK-SVE2: // %bb.0: // %entry423; CHECK-SVE2-NEXT: mov z2.h, #1 // =0x1424; CHECK-SVE2-NEXT: udot z0.d, z1.h, z2.h425; CHECK-SVE2-NEXT: ret426;427; CHECK-SVE2-I8MM-LABEL: udot_no_bin_op_wide:428; CHECK-SVE2-I8MM: // %bb.0: // %entry429; CHECK-SVE2-I8MM-NEXT: mov z2.h, #1 // =0x1430; CHECK-SVE2-I8MM-NEXT: udot z0.d, z1.h, z2.h431; CHECK-SVE2-I8MM-NEXT: ret432;433; CHECK-SME-LABEL: udot_no_bin_op_wide:434; CHECK-SME: // %bb.0: // %entry435; CHECK-SME-NEXT: mov z2.h, #1 // =0x1436; CHECK-SME-NEXT: udot z0.d, z1.h, z2.h437; CHECK-SME-NEXT: ret438entry:439 %a.wide = zext <vscale x 8 x i16> %a to <vscale x 8 x i64>440 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %a.wide)441 ret <vscale x 2 x i64> %partial.reduce442}443 444define <vscale x 2 x i64> @sdot_no_bin_op_wide(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b){445; CHECK-SVE2-LABEL: sdot_no_bin_op_wide:446; CHECK-SVE2: // %bb.0: // %entry447; CHECK-SVE2-NEXT: mov z2.h, #1 // =0x1448; CHECK-SVE2-NEXT: sdot z0.d, z1.h, z2.h449; CHECK-SVE2-NEXT: ret450;451; CHECK-SVE2-I8MM-LABEL: sdot_no_bin_op_wide:452; CHECK-SVE2-I8MM: // %bb.0: // %entry453; CHECK-SVE2-I8MM-NEXT: mov z2.h, #1 // =0x1454; CHECK-SVE2-I8MM-NEXT: sdot z0.d, z1.h, z2.h455; CHECK-SVE2-I8MM-NEXT: ret456;457; CHECK-SME-LABEL: sdot_no_bin_op_wide:458; CHECK-SME: // %bb.0: // %entry459; CHECK-SME-NEXT: mov z2.h, #1 // =0x1460; CHECK-SME-NEXT: sdot z0.d, z1.h, z2.h461; CHECK-SME-NEXT: ret462entry:463 %a.wide = sext <vscale x 8 x i16> %a to <vscale x 8 x i64>464 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %a.wide)465 ret <vscale x 2 x i64> %partial.reduce466}467 468define <vscale x 4 x i64> @udot_no_bin_op_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a){469; CHECK-SVE2-LABEL: udot_no_bin_op_8to64:470; CHECK-SVE2: // %bb.0:471; CHECK-SVE2-NEXT: movi v3.2d, #0000000000000000472; CHECK-SVE2-NEXT: mov z4.b, #1 // =0x1473; CHECK-SVE2-NEXT: udot z3.s, z2.b, z4.b474; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z3.s475; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z3.s476; CHECK-SVE2-NEXT: ret477;478; CHECK-SVE2-I8MM-LABEL: udot_no_bin_op_8to64:479; CHECK-SVE2-I8MM: // %bb.0:480; CHECK-SVE2-I8MM-NEXT: movi v3.2d, #0000000000000000481; CHECK-SVE2-I8MM-NEXT: mov z4.b, #1 // =0x1482; CHECK-SVE2-I8MM-NEXT: udot z3.s, z2.b, z4.b483; CHECK-SVE2-I8MM-NEXT: uaddwb z0.d, z0.d, z3.s484; CHECK-SVE2-I8MM-NEXT: uaddwt z0.d, z0.d, z3.s485; CHECK-SVE2-I8MM-NEXT: ret486;487; CHECK-SME-LABEL: udot_no_bin_op_8to64:488; CHECK-SME: // %bb.0:489; CHECK-SME-NEXT: mov z3.b, #1 // =0x1490; CHECK-SME-NEXT: mov z4.s, #0 // =0x0491; CHECK-SME-NEXT: udot z4.s, z2.b, z3.b492; CHECK-SME-NEXT: uaddwb z0.d, z0.d, z4.s493; CHECK-SME-NEXT: uaddwt z0.d, z0.d, z4.s494; CHECK-SME-NEXT: ret495 %a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>496 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(<vscale x 4 x i64> %acc, <vscale x 16 x i64> %a.ext)497 ret <vscale x 4 x i64> %partial.reduce498}499 500define <vscale x 4 x i64> @sdot_no_bin_op_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8> %a){501; CHECK-SVE2-LABEL: sdot_no_bin_op_8to64:502; CHECK-SVE2: // %bb.0:503; CHECK-SVE2-NEXT: movi v3.2d, #0000000000000000504; CHECK-SVE2-NEXT: mov z4.b, #1 // =0x1505; CHECK-SVE2-NEXT: sdot z3.s, z2.b, z4.b506; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z3.s507; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z3.s508; CHECK-SVE2-NEXT: ret509;510; CHECK-SVE2-I8MM-LABEL: sdot_no_bin_op_8to64:511; CHECK-SVE2-I8MM: // %bb.0:512; CHECK-SVE2-I8MM-NEXT: movi v3.2d, #0000000000000000513; CHECK-SVE2-I8MM-NEXT: mov z4.b, #1 // =0x1514; CHECK-SVE2-I8MM-NEXT: sdot z3.s, z2.b, z4.b515; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z3.s516; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z3.s517; CHECK-SVE2-I8MM-NEXT: ret518;519; CHECK-SME-LABEL: sdot_no_bin_op_8to64:520; CHECK-SME: // %bb.0:521; CHECK-SME-NEXT: mov z3.b, #1 // =0x1522; CHECK-SME-NEXT: mov z4.s, #0 // =0x0523; CHECK-SME-NEXT: sdot z4.s, z2.b, z3.b524; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s525; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s526; CHECK-SME-NEXT: ret527 %a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>528 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(<vscale x 4 x i64> %acc, <vscale x 16 x i64> %a.ext)529 ret <vscale x 4 x i64> %partial.reduce530}531 532define <vscale x 4 x i32> @not_udot(<vscale x 4 x i32> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {533; CHECK-SVE2-LABEL: not_udot:534; CHECK-SVE2: // %bb.0: // %entry535; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff536; CHECK-SVE2-NEXT: and z1.h, z1.h, #0xff537; CHECK-SVE2-NEXT: umlalb z0.s, z1.h, z2.h538; CHECK-SVE2-NEXT: umlalt z0.s, z1.h, z2.h539; CHECK-SVE2-NEXT: ret540;541; CHECK-SVE2-I8MM-LABEL: not_udot:542; CHECK-SVE2-I8MM: // %bb.0: // %entry543; CHECK-SVE2-I8MM-NEXT: and z2.h, z2.h, #0xff544; CHECK-SVE2-I8MM-NEXT: and z1.h, z1.h, #0xff545; CHECK-SVE2-I8MM-NEXT: umlalb z0.s, z1.h, z2.h546; CHECK-SVE2-I8MM-NEXT: umlalt z0.s, z1.h, z2.h547; CHECK-SVE2-I8MM-NEXT: ret548;549; CHECK-SME-LABEL: not_udot:550; CHECK-SME: // %bb.0: // %entry551; CHECK-SME-NEXT: and z2.h, z2.h, #0xff552; CHECK-SME-NEXT: and z1.h, z1.h, #0xff553; CHECK-SME-NEXT: umlalb z0.s, z1.h, z2.h554; CHECK-SME-NEXT: umlalt z0.s, z1.h, z2.h555; CHECK-SME-NEXT: ret556entry:557 %a.wide = zext <vscale x 8 x i8> %a to <vscale x 8 x i32>558 %b.wide = zext <vscale x 8 x i8> %b to <vscale x 8 x i32>559 %mult = mul nuw nsw <vscale x 8 x i32> %a.wide, %b.wide560 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %mult)561 ret <vscale x 4 x i32> %partial.reduce562}563 564define <vscale x 2 x i64> @not_udot_wide(<vscale x 2 x i64> %acc, <vscale x 4 x i16> %a, <vscale x 4 x i16> %b) {565; CHECK-SVE2-LABEL: not_udot_wide:566; CHECK-SVE2: // %bb.0: // %entry567; CHECK-SVE2-NEXT: and z2.s, z2.s, #0xffff568; CHECK-SVE2-NEXT: and z1.s, z1.s, #0xffff569; CHECK-SVE2-NEXT: umlalb z0.d, z1.s, z2.s570; CHECK-SVE2-NEXT: umlalt z0.d, z1.s, z2.s571; CHECK-SVE2-NEXT: ret572;573; CHECK-SVE2-I8MM-LABEL: not_udot_wide:574; CHECK-SVE2-I8MM: // %bb.0: // %entry575; CHECK-SVE2-I8MM-NEXT: and z2.s, z2.s, #0xffff576; CHECK-SVE2-I8MM-NEXT: and z1.s, z1.s, #0xffff577; CHECK-SVE2-I8MM-NEXT: umlalb z0.d, z1.s, z2.s578; CHECK-SVE2-I8MM-NEXT: umlalt z0.d, z1.s, z2.s579; CHECK-SVE2-I8MM-NEXT: ret580;581; CHECK-SME-LABEL: not_udot_wide:582; CHECK-SME: // %bb.0: // %entry583; CHECK-SME-NEXT: and z2.s, z2.s, #0xffff584; CHECK-SME-NEXT: and z1.s, z1.s, #0xffff585; CHECK-SME-NEXT: umlalb z0.d, z1.s, z2.s586; CHECK-SME-NEXT: umlalt z0.d, z1.s, z2.s587; CHECK-SME-NEXT: ret588entry:589 %a.wide = zext <vscale x 4 x i16> %a to <vscale x 4 x i64>590 %b.wide = zext <vscale x 4 x i16> %b to <vscale x 4 x i64>591 %mult = mul nuw nsw <vscale x 4 x i64> %a.wide, %b.wide592 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv4i64(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %mult)593 ret <vscale x 2 x i64> %partial.reduce594}595 596define <vscale x 2 x i64> @not_usdot(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {597; CHECK-SVE2-LABEL: not_usdot:598; CHECK-SVE2: // %bb.0: // %entry599; CHECK-SVE2-NEXT: uunpklo z3.s, z1.h600; CHECK-SVE2-NEXT: sunpklo z4.s, z2.h601; CHECK-SVE2-NEXT: ptrue p0.d602; CHECK-SVE2-NEXT: uunpkhi z1.s, z1.h603; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h604; CHECK-SVE2-NEXT: uunpklo z5.d, z3.s605; CHECK-SVE2-NEXT: sunpklo z6.d, z4.s606; CHECK-SVE2-NEXT: uunpkhi z3.d, z3.s607; CHECK-SVE2-NEXT: sunpkhi z4.d, z4.s608; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d609; CHECK-SVE2-NEXT: uunpklo z5.d, z1.s610; CHECK-SVE2-NEXT: sunpklo z6.d, z2.s611; CHECK-SVE2-NEXT: uunpkhi z1.d, z1.s612; CHECK-SVE2-NEXT: sunpkhi z2.d, z2.s613; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d614; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d615; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d616; CHECK-SVE2-NEXT: ret617;618; CHECK-SVE2-I8MM-LABEL: not_usdot:619; CHECK-SVE2-I8MM: // %bb.0: // %entry620; CHECK-SVE2-I8MM-NEXT: uunpklo z3.s, z1.h621; CHECK-SVE2-I8MM-NEXT: sunpklo z4.s, z2.h622; CHECK-SVE2-I8MM-NEXT: ptrue p0.d623; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.s, z1.h624; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.s, z2.h625; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z3.s626; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z4.s627; CHECK-SVE2-I8MM-NEXT: uunpkhi z3.d, z3.s628; CHECK-SVE2-I8MM-NEXT: sunpkhi z4.d, z4.s629; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d630; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z1.s631; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z2.s632; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.d, z1.s633; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.d, z2.s634; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d635; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d636; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d637; CHECK-SVE2-I8MM-NEXT: ret638;639; CHECK-SME-LABEL: not_usdot:640; CHECK-SME: // %bb.0: // %entry641; CHECK-SME-NEXT: uunpklo z3.s, z1.h642; CHECK-SME-NEXT: sunpklo z4.s, z2.h643; CHECK-SME-NEXT: ptrue p0.d644; CHECK-SME-NEXT: uunpkhi z1.s, z1.h645; CHECK-SME-NEXT: sunpkhi z2.s, z2.h646; CHECK-SME-NEXT: uunpklo z5.d, z3.s647; CHECK-SME-NEXT: sunpklo z6.d, z4.s648; CHECK-SME-NEXT: uunpkhi z3.d, z3.s649; CHECK-SME-NEXT: sunpkhi z4.d, z4.s650; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d651; CHECK-SME-NEXT: uunpklo z5.d, z1.s652; CHECK-SME-NEXT: sunpklo z6.d, z2.s653; CHECK-SME-NEXT: uunpkhi z1.d, z1.s654; CHECK-SME-NEXT: sunpkhi z2.d, z2.s655; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d656; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d657; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d658; CHECK-SME-NEXT: ret659entry:660 %a.wide = zext <vscale x 8 x i16> %a to <vscale x 8 x i64>661 %b.wide = sext <vscale x 8 x i16> %b to <vscale x 8 x i64>662 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide663 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)664 ret <vscale x 2 x i64> %partial.reduce665}666 667define <vscale x 2 x i64> @not_sudot(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {668; CHECK-SVE2-LABEL: not_sudot:669; CHECK-SVE2: // %bb.0: // %entry670; CHECK-SVE2-NEXT: sunpklo z3.s, z1.h671; CHECK-SVE2-NEXT: uunpklo z4.s, z2.h672; CHECK-SVE2-NEXT: ptrue p0.d673; CHECK-SVE2-NEXT: sunpkhi z1.s, z1.h674; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h675; CHECK-SVE2-NEXT: sunpklo z5.d, z3.s676; CHECK-SVE2-NEXT: uunpklo z6.d, z4.s677; CHECK-SVE2-NEXT: sunpkhi z3.d, z3.s678; CHECK-SVE2-NEXT: uunpkhi z4.d, z4.s679; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d680; CHECK-SVE2-NEXT: sunpklo z5.d, z1.s681; CHECK-SVE2-NEXT: uunpklo z6.d, z2.s682; CHECK-SVE2-NEXT: sunpkhi z1.d, z1.s683; CHECK-SVE2-NEXT: uunpkhi z2.d, z2.s684; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d685; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d686; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d687; CHECK-SVE2-NEXT: ret688;689; CHECK-SVE2-I8MM-LABEL: not_sudot:690; CHECK-SVE2-I8MM: // %bb.0: // %entry691; CHECK-SVE2-I8MM-NEXT: sunpklo z3.s, z1.h692; CHECK-SVE2-I8MM-NEXT: uunpklo z4.s, z2.h693; CHECK-SVE2-I8MM-NEXT: ptrue p0.d694; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.s, z1.h695; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.s, z2.h696; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z3.s697; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z4.s698; CHECK-SVE2-I8MM-NEXT: sunpkhi z3.d, z3.s699; CHECK-SVE2-I8MM-NEXT: uunpkhi z4.d, z4.s700; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d701; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z1.s702; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z2.s703; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.d, z1.s704; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.d, z2.s705; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d706; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d707; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d708; CHECK-SVE2-I8MM-NEXT: ret709;710; CHECK-SME-LABEL: not_sudot:711; CHECK-SME: // %bb.0: // %entry712; CHECK-SME-NEXT: sunpklo z3.s, z1.h713; CHECK-SME-NEXT: uunpklo z4.s, z2.h714; CHECK-SME-NEXT: ptrue p0.d715; CHECK-SME-NEXT: sunpkhi z1.s, z1.h716; CHECK-SME-NEXT: uunpkhi z2.s, z2.h717; CHECK-SME-NEXT: sunpklo z5.d, z3.s718; CHECK-SME-NEXT: uunpklo z6.d, z4.s719; CHECK-SME-NEXT: sunpkhi z3.d, z3.s720; CHECK-SME-NEXT: uunpkhi z4.d, z4.s721; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d722; CHECK-SME-NEXT: sunpklo z5.d, z1.s723; CHECK-SME-NEXT: uunpklo z6.d, z2.s724; CHECK-SME-NEXT: sunpkhi z1.d, z1.s725; CHECK-SME-NEXT: uunpkhi z2.d, z2.s726; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d727; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d728; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d729; CHECK-SME-NEXT: ret730entry:731 %a.wide = sext <vscale x 8 x i16> %a to <vscale x 8 x i64>732 %b.wide = zext <vscale x 8 x i16> %b to <vscale x 8 x i64>733 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide734 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)735 ret <vscale x 2 x i64> %partial.reduce736}737 738define <vscale x 2 x i64> @udot_different_types(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b){739; CHECK-SVE2-LABEL: udot_different_types:740; CHECK-SVE2: // %bb.0: // %entry741; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff742; CHECK-SVE2-NEXT: uunpklo z3.s, z1.h743; CHECK-SVE2-NEXT: ptrue p0.d744; CHECK-SVE2-NEXT: uunpkhi z1.s, z1.h745; CHECK-SVE2-NEXT: uunpklo z4.s, z2.h746; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h747; CHECK-SVE2-NEXT: uunpklo z5.d, z3.s748; CHECK-SVE2-NEXT: uunpkhi z3.d, z3.s749; CHECK-SVE2-NEXT: uunpklo z6.d, z4.s750; CHECK-SVE2-NEXT: uunpkhi z4.d, z4.s751; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d752; CHECK-SVE2-NEXT: uunpklo z5.d, z1.s753; CHECK-SVE2-NEXT: uunpklo z6.d, z2.s754; CHECK-SVE2-NEXT: uunpkhi z1.d, z1.s755; CHECK-SVE2-NEXT: uunpkhi z2.d, z2.s756; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d757; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d758; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d759; CHECK-SVE2-NEXT: ret760;761; CHECK-SVE2-I8MM-LABEL: udot_different_types:762; CHECK-SVE2-I8MM: // %bb.0: // %entry763; CHECK-SVE2-I8MM-NEXT: and z2.h, z2.h, #0xff764; CHECK-SVE2-I8MM-NEXT: uunpklo z3.s, z1.h765; CHECK-SVE2-I8MM-NEXT: ptrue p0.d766; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.s, z1.h767; CHECK-SVE2-I8MM-NEXT: uunpklo z4.s, z2.h768; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.s, z2.h769; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z3.s770; CHECK-SVE2-I8MM-NEXT: uunpkhi z3.d, z3.s771; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z4.s772; CHECK-SVE2-I8MM-NEXT: uunpkhi z4.d, z4.s773; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d774; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z1.s775; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z2.s776; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.d, z1.s777; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.d, z2.s778; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d779; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d780; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d781; CHECK-SVE2-I8MM-NEXT: ret782;783; CHECK-SME-LABEL: udot_different_types:784; CHECK-SME: // %bb.0: // %entry785; CHECK-SME-NEXT: and z2.h, z2.h, #0xff786; CHECK-SME-NEXT: uunpklo z3.s, z1.h787; CHECK-SME-NEXT: ptrue p0.d788; CHECK-SME-NEXT: uunpkhi z1.s, z1.h789; CHECK-SME-NEXT: uunpklo z4.s, z2.h790; CHECK-SME-NEXT: uunpkhi z2.s, z2.h791; CHECK-SME-NEXT: uunpklo z5.d, z3.s792; CHECK-SME-NEXT: uunpkhi z3.d, z3.s793; CHECK-SME-NEXT: uunpklo z6.d, z4.s794; CHECK-SME-NEXT: uunpkhi z4.d, z4.s795; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d796; CHECK-SME-NEXT: uunpklo z5.d, z1.s797; CHECK-SME-NEXT: uunpklo z6.d, z2.s798; CHECK-SME-NEXT: uunpkhi z1.d, z1.s799; CHECK-SME-NEXT: uunpkhi z2.d, z2.s800; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d801; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d802; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d803; CHECK-SME-NEXT: ret804entry:805 %a.wide = zext <vscale x 8 x i16> %a to <vscale x 8 x i64>806 %b.wide = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>807 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide808 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)809 ret <vscale x 2 x i64> %partial.reduce810}811 812define <vscale x 2 x i64> @sdot_different_types(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b){813; CHECK-SVE2-LABEL: sdot_different_types:814; CHECK-SVE2: // %bb.0: // %entry815; CHECK-SVE2-NEXT: ptrue p0.h816; CHECK-SVE2-NEXT: sunpklo z3.s, z1.h817; CHECK-SVE2-NEXT: sunpkhi z1.s, z1.h818; CHECK-SVE2-NEXT: sxtb z2.h, p0/m, z2.h819; CHECK-SVE2-NEXT: ptrue p0.d820; CHECK-SVE2-NEXT: sunpklo z5.d, z3.s821; CHECK-SVE2-NEXT: sunpkhi z3.d, z3.s822; CHECK-SVE2-NEXT: sunpklo z4.s, z2.h823; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h824; CHECK-SVE2-NEXT: sunpklo z6.d, z4.s825; CHECK-SVE2-NEXT: sunpkhi z4.d, z4.s826; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d827; CHECK-SVE2-NEXT: sunpklo z5.d, z1.s828; CHECK-SVE2-NEXT: sunpklo z6.d, z2.s829; CHECK-SVE2-NEXT: sunpkhi z1.d, z1.s830; CHECK-SVE2-NEXT: sunpkhi z2.d, z2.s831; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d832; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d833; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d834; CHECK-SVE2-NEXT: ret835;836; CHECK-SVE2-I8MM-LABEL: sdot_different_types:837; CHECK-SVE2-I8MM: // %bb.0: // %entry838; CHECK-SVE2-I8MM-NEXT: ptrue p0.h839; CHECK-SVE2-I8MM-NEXT: sunpklo z3.s, z1.h840; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.s, z1.h841; CHECK-SVE2-I8MM-NEXT: sxtb z2.h, p0/m, z2.h842; CHECK-SVE2-I8MM-NEXT: ptrue p0.d843; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z3.s844; CHECK-SVE2-I8MM-NEXT: sunpkhi z3.d, z3.s845; CHECK-SVE2-I8MM-NEXT: sunpklo z4.s, z2.h846; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.s, z2.h847; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z4.s848; CHECK-SVE2-I8MM-NEXT: sunpkhi z4.d, z4.s849; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d850; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z1.s851; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z2.s852; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.d, z1.s853; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.d, z2.s854; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d855; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d856; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d857; CHECK-SVE2-I8MM-NEXT: ret858;859; CHECK-SME-LABEL: sdot_different_types:860; CHECK-SME: // %bb.0: // %entry861; CHECK-SME-NEXT: ptrue p0.h862; CHECK-SME-NEXT: sunpklo z3.s, z1.h863; CHECK-SME-NEXT: sunpkhi z1.s, z1.h864; CHECK-SME-NEXT: sxtb z2.h, p0/m, z2.h865; CHECK-SME-NEXT: ptrue p0.d866; CHECK-SME-NEXT: sunpklo z5.d, z3.s867; CHECK-SME-NEXT: sunpkhi z3.d, z3.s868; CHECK-SME-NEXT: sunpklo z4.s, z2.h869; CHECK-SME-NEXT: sunpkhi z2.s, z2.h870; CHECK-SME-NEXT: sunpklo z6.d, z4.s871; CHECK-SME-NEXT: sunpkhi z4.d, z4.s872; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d873; CHECK-SME-NEXT: sunpklo z5.d, z1.s874; CHECK-SME-NEXT: sunpklo z6.d, z2.s875; CHECK-SME-NEXT: sunpkhi z1.d, z1.s876; CHECK-SME-NEXT: sunpkhi z2.d, z2.s877; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d878; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d879; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d880; CHECK-SME-NEXT: ret881entry:882 %a.wide = sext <vscale x 8 x i16> %a to <vscale x 8 x i64>883 %b.wide = sext <vscale x 8 x i8> %b to <vscale x 8 x i64>884 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide885 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)886 ret <vscale x 2 x i64> %partial.reduce887}888 889define <vscale x 2 x i64> @usdot_different_types(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b){890; CHECK-SVE2-LABEL: usdot_different_types:891; CHECK-SVE2: // %bb.0: // %entry892; CHECK-SVE2-NEXT: ptrue p0.h893; CHECK-SVE2-NEXT: uunpklo z3.s, z1.h894; CHECK-SVE2-NEXT: uunpkhi z1.s, z1.h895; CHECK-SVE2-NEXT: sxtb z2.h, p0/m, z2.h896; CHECK-SVE2-NEXT: ptrue p0.d897; CHECK-SVE2-NEXT: uunpklo z5.d, z3.s898; CHECK-SVE2-NEXT: uunpkhi z3.d, z3.s899; CHECK-SVE2-NEXT: sunpklo z4.s, z2.h900; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h901; CHECK-SVE2-NEXT: sunpklo z6.d, z4.s902; CHECK-SVE2-NEXT: sunpkhi z4.d, z4.s903; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d904; CHECK-SVE2-NEXT: uunpklo z5.d, z1.s905; CHECK-SVE2-NEXT: sunpklo z6.d, z2.s906; CHECK-SVE2-NEXT: uunpkhi z1.d, z1.s907; CHECK-SVE2-NEXT: sunpkhi z2.d, z2.s908; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d909; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d910; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d911; CHECK-SVE2-NEXT: ret912;913; CHECK-SVE2-I8MM-LABEL: usdot_different_types:914; CHECK-SVE2-I8MM: // %bb.0: // %entry915; CHECK-SVE2-I8MM-NEXT: ptrue p0.h916; CHECK-SVE2-I8MM-NEXT: uunpklo z3.s, z1.h917; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.s, z1.h918; CHECK-SVE2-I8MM-NEXT: sxtb z2.h, p0/m, z2.h919; CHECK-SVE2-I8MM-NEXT: ptrue p0.d920; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z3.s921; CHECK-SVE2-I8MM-NEXT: uunpkhi z3.d, z3.s922; CHECK-SVE2-I8MM-NEXT: sunpklo z4.s, z2.h923; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.s, z2.h924; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z4.s925; CHECK-SVE2-I8MM-NEXT: sunpkhi z4.d, z4.s926; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d927; CHECK-SVE2-I8MM-NEXT: uunpklo z5.d, z1.s928; CHECK-SVE2-I8MM-NEXT: sunpklo z6.d, z2.s929; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.d, z1.s930; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.d, z2.s931; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d932; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d933; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d934; CHECK-SVE2-I8MM-NEXT: ret935;936; CHECK-SME-LABEL: usdot_different_types:937; CHECK-SME: // %bb.0: // %entry938; CHECK-SME-NEXT: ptrue p0.h939; CHECK-SME-NEXT: uunpklo z3.s, z1.h940; CHECK-SME-NEXT: uunpkhi z1.s, z1.h941; CHECK-SME-NEXT: sxtb z2.h, p0/m, z2.h942; CHECK-SME-NEXT: ptrue p0.d943; CHECK-SME-NEXT: uunpklo z5.d, z3.s944; CHECK-SME-NEXT: uunpkhi z3.d, z3.s945; CHECK-SME-NEXT: sunpklo z4.s, z2.h946; CHECK-SME-NEXT: sunpkhi z2.s, z2.h947; CHECK-SME-NEXT: sunpklo z6.d, z4.s948; CHECK-SME-NEXT: sunpkhi z4.d, z4.s949; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d950; CHECK-SME-NEXT: uunpklo z5.d, z1.s951; CHECK-SME-NEXT: sunpklo z6.d, z2.s952; CHECK-SME-NEXT: uunpkhi z1.d, z1.s953; CHECK-SME-NEXT: sunpkhi z2.d, z2.s954; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d955; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d956; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d957; CHECK-SME-NEXT: ret958entry:959 %a.wide = zext <vscale x 8 x i16> %a to <vscale x 8 x i64>960 %b.wide = sext <vscale x 8 x i8> %b to <vscale x 8 x i64>961 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide962 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)963 ret <vscale x 2 x i64> %partial.reduce964}965 966define <vscale x 2 x i64> @sudot_different_types(<vscale x 2 x i64> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b){967; CHECK-SVE2-LABEL: sudot_different_types:968; CHECK-SVE2: // %bb.0: // %entry969; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff970; CHECK-SVE2-NEXT: sunpklo z3.s, z1.h971; CHECK-SVE2-NEXT: ptrue p0.d972; CHECK-SVE2-NEXT: sunpkhi z1.s, z1.h973; CHECK-SVE2-NEXT: uunpklo z4.s, z2.h974; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h975; CHECK-SVE2-NEXT: sunpklo z5.d, z3.s976; CHECK-SVE2-NEXT: sunpkhi z3.d, z3.s977; CHECK-SVE2-NEXT: uunpklo z6.d, z4.s978; CHECK-SVE2-NEXT: uunpkhi z4.d, z4.s979; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d980; CHECK-SVE2-NEXT: sunpklo z5.d, z1.s981; CHECK-SVE2-NEXT: uunpklo z6.d, z2.s982; CHECK-SVE2-NEXT: sunpkhi z1.d, z1.s983; CHECK-SVE2-NEXT: uunpkhi z2.d, z2.s984; CHECK-SVE2-NEXT: mla z0.d, p0/m, z3.d, z4.d985; CHECK-SVE2-NEXT: mla z0.d, p0/m, z5.d, z6.d986; CHECK-SVE2-NEXT: mla z0.d, p0/m, z1.d, z2.d987; CHECK-SVE2-NEXT: ret988;989; CHECK-SVE2-I8MM-LABEL: sudot_different_types:990; CHECK-SVE2-I8MM: // %bb.0: // %entry991; CHECK-SVE2-I8MM-NEXT: and z2.h, z2.h, #0xff992; CHECK-SVE2-I8MM-NEXT: sunpklo z3.s, z1.h993; CHECK-SVE2-I8MM-NEXT: ptrue p0.d994; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.s, z1.h995; CHECK-SVE2-I8MM-NEXT: uunpklo z4.s, z2.h996; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.s, z2.h997; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z3.s998; CHECK-SVE2-I8MM-NEXT: sunpkhi z3.d, z3.s999; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z4.s1000; CHECK-SVE2-I8MM-NEXT: uunpkhi z4.d, z4.s1001; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d1002; CHECK-SVE2-I8MM-NEXT: sunpklo z5.d, z1.s1003; CHECK-SVE2-I8MM-NEXT: uunpklo z6.d, z2.s1004; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.d, z1.s1005; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.d, z2.s1006; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z3.d, z4.d1007; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z5.d, z6.d1008; CHECK-SVE2-I8MM-NEXT: mla z0.d, p0/m, z1.d, z2.d1009; CHECK-SVE2-I8MM-NEXT: ret1010;1011; CHECK-SME-LABEL: sudot_different_types:1012; CHECK-SME: // %bb.0: // %entry1013; CHECK-SME-NEXT: and z2.h, z2.h, #0xff1014; CHECK-SME-NEXT: sunpklo z3.s, z1.h1015; CHECK-SME-NEXT: ptrue p0.d1016; CHECK-SME-NEXT: sunpkhi z1.s, z1.h1017; CHECK-SME-NEXT: uunpklo z4.s, z2.h1018; CHECK-SME-NEXT: uunpkhi z2.s, z2.h1019; CHECK-SME-NEXT: sunpklo z5.d, z3.s1020; CHECK-SME-NEXT: sunpkhi z3.d, z3.s1021; CHECK-SME-NEXT: uunpklo z6.d, z4.s1022; CHECK-SME-NEXT: uunpkhi z4.d, z4.s1023; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d1024; CHECK-SME-NEXT: sunpklo z5.d, z1.s1025; CHECK-SME-NEXT: uunpklo z6.d, z2.s1026; CHECK-SME-NEXT: sunpkhi z1.d, z1.s1027; CHECK-SME-NEXT: uunpkhi z2.d, z2.s1028; CHECK-SME-NEXT: mla z0.d, p0/m, z3.d, z4.d1029; CHECK-SME-NEXT: mla z0.d, p0/m, z5.d, z6.d1030; CHECK-SME-NEXT: mla z0.d, p0/m, z1.d, z2.d1031; CHECK-SME-NEXT: ret1032entry:1033 %a.wide = sext <vscale x 8 x i16> %a to <vscale x 8 x i64>1034 %b.wide = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>1035 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide1036 %partial.reduce = tail call <vscale x 2 x i64> @llvm.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> %acc, <vscale x 8 x i64> %mult)1037 ret <vscale x 2 x i64> %partial.reduce1038}1039 1040define <vscale x 2 x i16> @udot_nxv8i8_promote (<vscale x 2 x i16> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b){1041; CHECK-SVE2-LABEL: udot_nxv8i8_promote:1042; CHECK-SVE2: // %bb.0: // %entry1043; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff1044; CHECK-SVE2-NEXT: and z1.h, z1.h, #0xff1045; CHECK-SVE2-NEXT: udot z0.d, z1.h, z2.h1046; CHECK-SVE2-NEXT: ret1047;1048; CHECK-SVE2-I8MM-LABEL: udot_nxv8i8_promote:1049; CHECK-SVE2-I8MM: // %bb.0: // %entry1050; CHECK-SVE2-I8MM-NEXT: and z2.h, z2.h, #0xff1051; CHECK-SVE2-I8MM-NEXT: and z1.h, z1.h, #0xff1052; CHECK-SVE2-I8MM-NEXT: udot z0.d, z1.h, z2.h1053; CHECK-SVE2-I8MM-NEXT: ret1054;1055; CHECK-SME-LABEL: udot_nxv8i8_promote:1056; CHECK-SME: // %bb.0: // %entry1057; CHECK-SME-NEXT: and z2.h, z2.h, #0xff1058; CHECK-SME-NEXT: and z1.h, z1.h, #0xff1059; CHECK-SME-NEXT: udot z0.d, z1.h, z2.h1060; CHECK-SME-NEXT: ret1061entry:1062 %a.wide = zext <vscale x 8 x i8> %a to <vscale x 8 x i16>1063 %b.wide = zext <vscale x 8 x i8> %b to <vscale x 8 x i16>1064 %mult = mul nuw nsw <vscale x 8 x i16> %a.wide, %b.wide1065 %partial.reduce = tail call <vscale x 2 x i16> @llvm.vector.partial.reduce.add.nxv2i16.nxv8i16(<vscale x 2 x i16> %acc, <vscale x 8 x i16> %mult)1066 ret <vscale x 2 x i16> %partial.reduce1067}1068 1069define <vscale x 2 x i16> @sdot_nxv8i8_promote (<vscale x 2 x i16> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b){1070; CHECK-SVE2-LABEL: sdot_nxv8i8_promote:1071; CHECK-SVE2: // %bb.0: // %entry1072; CHECK-SVE2-NEXT: ptrue p0.h1073; CHECK-SVE2-NEXT: sxtb z2.h, p0/m, z2.h1074; CHECK-SVE2-NEXT: sxtb z1.h, p0/m, z1.h1075; CHECK-SVE2-NEXT: sdot z0.d, z1.h, z2.h1076; CHECK-SVE2-NEXT: ret1077;1078; CHECK-SVE2-I8MM-LABEL: sdot_nxv8i8_promote:1079; CHECK-SVE2-I8MM: // %bb.0: // %entry1080; CHECK-SVE2-I8MM-NEXT: ptrue p0.h1081; CHECK-SVE2-I8MM-NEXT: sxtb z2.h, p0/m, z2.h1082; CHECK-SVE2-I8MM-NEXT: sxtb z1.h, p0/m, z1.h1083; CHECK-SVE2-I8MM-NEXT: sdot z0.d, z1.h, z2.h1084; CHECK-SVE2-I8MM-NEXT: ret1085;1086; CHECK-SME-LABEL: sdot_nxv8i8_promote:1087; CHECK-SME: // %bb.0: // %entry1088; CHECK-SME-NEXT: ptrue p0.h1089; CHECK-SME-NEXT: sxtb z2.h, p0/m, z2.h1090; CHECK-SME-NEXT: sxtb z1.h, p0/m, z1.h1091; CHECK-SME-NEXT: sdot z0.d, z1.h, z2.h1092; CHECK-SME-NEXT: ret1093entry:1094 %a.wide = sext <vscale x 8 x i8> %a to <vscale x 8 x i16>1095 %b.wide = sext <vscale x 8 x i8> %b to <vscale x 8 x i16>1096 %mult = mul nuw nsw <vscale x 8 x i16> %a.wide, %b.wide1097 %partial.reduce = tail call <vscale x 2 x i16> @llvm.vector.partial.reduce.add.nxv2i16.nxv8i16(<vscale x 2 x i16> %acc, <vscale x 8 x i16> %mult)1098 ret <vscale x 2 x i16> %partial.reduce1099}1100 1101define <vscale x 4 x i64> @partial_reduce_only_split_acc(<vscale x 4 x i64> %acc, <vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {1102; CHECK-SVE2-LABEL: partial_reduce_only_split_acc:1103; CHECK-SVE2: // %bb.0: // %entry1104; CHECK-SVE2-NEXT: and z3.h, z3.h, #0xff1105; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff1106; CHECK-SVE2-NEXT: udot z0.d, z2.h, z3.h1107; CHECK-SVE2-NEXT: ret1108;1109; CHECK-SVE2-I8MM-LABEL: partial_reduce_only_split_acc:1110; CHECK-SVE2-I8MM: // %bb.0: // %entry1111; CHECK-SVE2-I8MM-NEXT: and z3.h, z3.h, #0xff1112; CHECK-SVE2-I8MM-NEXT: and z2.h, z2.h, #0xff1113; CHECK-SVE2-I8MM-NEXT: udot z0.d, z2.h, z3.h1114; CHECK-SVE2-I8MM-NEXT: ret1115;1116; CHECK-SME-LABEL: partial_reduce_only_split_acc:1117; CHECK-SME: // %bb.0: // %entry1118; CHECK-SME-NEXT: and z3.h, z3.h, #0xff1119; CHECK-SME-NEXT: and z2.h, z2.h, #0xff1120; CHECK-SME-NEXT: udot z0.d, z2.h, z3.h1121; CHECK-SME-NEXT: ret1122entry:1123 %a.wide = zext <vscale x 8 x i8> %a to <vscale x 8 x i64>1124 %b.wide = zext <vscale x 8 x i8> %b to <vscale x 8 x i64>1125 %mult = mul nuw nsw <vscale x 8 x i64> %a.wide, %b.wide1126 %partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv8i64(1127 <vscale x 4 x i64> %acc, <vscale x 8 x i64> %mult)1128 ret <vscale x 4 x i64> %partial.reduce1129}1130 1131define <vscale x 4 x i32> @sdot_imm(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a) {1132; CHECK-SVE2-LABEL: sdot_imm:1133; CHECK-SVE2: // %bb.0: // %entry1134; CHECK-SVE2-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1135; CHECK-SVE2-NEXT: sdot z0.s, z1.b, z2.b1136; CHECK-SVE2-NEXT: ret1137;1138; CHECK-SVE2-I8MM-LABEL: sdot_imm:1139; CHECK-SVE2-I8MM: // %bb.0: // %entry1140; CHECK-SVE2-I8MM-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1141; CHECK-SVE2-I8MM-NEXT: sdot z0.s, z1.b, z2.b1142; CHECK-SVE2-I8MM-NEXT: ret1143;1144; CHECK-SME-LABEL: sdot_imm:1145; CHECK-SME: // %bb.0: // %entry1146; CHECK-SME-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1147; CHECK-SME-NEXT: sdot z0.s, z1.b, z2.b1148; CHECK-SME-NEXT: ret1149entry:1150 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>1151 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, splat(i32 -1)1152 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)1153 ret <vscale x 4 x i32> %partial.reduce1154}1155 1156define <vscale x 4 x i32> @sdot_imm_does_not_fit(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a) {1157; CHECK-SVE2-LABEL: sdot_imm_does_not_fit:1158; CHECK-SVE2: // %bb.0: // %entry1159; CHECK-SVE2-NEXT: sunpklo z2.h, z1.b1160; CHECK-SVE2-NEXT: sunpkhi z1.h, z1.b1161; CHECK-SVE2-NEXT: sunpklo z3.s, z2.h1162; CHECK-SVE2-NEXT: sunpkhi z2.s, z2.h1163; CHECK-SVE2-NEXT: sunpklo z4.s, z1.h1164; CHECK-SVE2-NEXT: sunpkhi z1.s, z1.h1165; CHECK-SVE2-NEXT: lsl z4.s, z4.s, #81166; CHECK-SVE2-NEXT: lsl z2.s, z2.s, #81167; CHECK-SVE2-NEXT: lsl z3.s, z3.s, #81168; CHECK-SVE2-NEXT: lsl z1.s, z1.s, #81169; CHECK-SVE2-NEXT: add z0.s, z0.s, z3.s1170; CHECK-SVE2-NEXT: add z2.s, z2.s, z4.s1171; CHECK-SVE2-NEXT: add z0.s, z0.s, z2.s1172; CHECK-SVE2-NEXT: add z0.s, z0.s, z1.s1173; CHECK-SVE2-NEXT: ret1174;1175; CHECK-SVE2-I8MM-LABEL: sdot_imm_does_not_fit:1176; CHECK-SVE2-I8MM: // %bb.0: // %entry1177; CHECK-SVE2-I8MM-NEXT: sunpklo z2.h, z1.b1178; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.h, z1.b1179; CHECK-SVE2-I8MM-NEXT: sunpklo z3.s, z2.h1180; CHECK-SVE2-I8MM-NEXT: sunpkhi z2.s, z2.h1181; CHECK-SVE2-I8MM-NEXT: sunpklo z4.s, z1.h1182; CHECK-SVE2-I8MM-NEXT: sunpkhi z1.s, z1.h1183; CHECK-SVE2-I8MM-NEXT: lsl z4.s, z4.s, #81184; CHECK-SVE2-I8MM-NEXT: lsl z2.s, z2.s, #81185; CHECK-SVE2-I8MM-NEXT: lsl z3.s, z3.s, #81186; CHECK-SVE2-I8MM-NEXT: lsl z1.s, z1.s, #81187; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z3.s1188; CHECK-SVE2-I8MM-NEXT: add z2.s, z2.s, z4.s1189; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z2.s1190; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z1.s1191; CHECK-SVE2-I8MM-NEXT: ret1192;1193; CHECK-SME-LABEL: sdot_imm_does_not_fit:1194; CHECK-SME: // %bb.0: // %entry1195; CHECK-SME-NEXT: sunpklo z2.h, z1.b1196; CHECK-SME-NEXT: sunpkhi z1.h, z1.b1197; CHECK-SME-NEXT: sunpklo z3.s, z2.h1198; CHECK-SME-NEXT: sunpkhi z2.s, z2.h1199; CHECK-SME-NEXT: sunpklo z4.s, z1.h1200; CHECK-SME-NEXT: sunpkhi z1.s, z1.h1201; CHECK-SME-NEXT: lsl z4.s, z4.s, #81202; CHECK-SME-NEXT: lsl z2.s, z2.s, #81203; CHECK-SME-NEXT: lsl z3.s, z3.s, #81204; CHECK-SME-NEXT: lsl z1.s, z1.s, #81205; CHECK-SME-NEXT: add z0.s, z0.s, z3.s1206; CHECK-SME-NEXT: add z2.s, z2.s, z4.s1207; CHECK-SME-NEXT: add z0.s, z0.s, z2.s1208; CHECK-SME-NEXT: add z0.s, z0.s, z1.s1209; CHECK-SME-NEXT: ret1210entry:1211 %a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>1212 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, splat(i32 256)1213 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)1214 ret <vscale x 4 x i32> %partial.reduce1215}1216 1217define <vscale x 4 x i32> @udot_imm(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a) {1218; CHECK-SVE2-LABEL: udot_imm:1219; CHECK-SVE2: // %bb.0: // %entry1220; CHECK-SVE2-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1221; CHECK-SVE2-NEXT: udot z0.s, z1.b, z2.b1222; CHECK-SVE2-NEXT: ret1223;1224; CHECK-SVE2-I8MM-LABEL: udot_imm:1225; CHECK-SVE2-I8MM: // %bb.0: // %entry1226; CHECK-SVE2-I8MM-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1227; CHECK-SVE2-I8MM-NEXT: udot z0.s, z1.b, z2.b1228; CHECK-SVE2-I8MM-NEXT: ret1229;1230; CHECK-SME-LABEL: udot_imm:1231; CHECK-SME: // %bb.0: // %entry1232; CHECK-SME-NEXT: mov z2.b, #-1 // =0xffffffffffffffff1233; CHECK-SME-NEXT: udot z0.s, z1.b, z2.b1234; CHECK-SME-NEXT: ret1235entry:1236 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>1237 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, splat(i32 255)1238 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)1239 ret <vscale x 4 x i32> %partial.reduce1240}1241 1242define <vscale x 4 x i32> @udot_imm_does_not_fit(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a) {1243; CHECK-SVE2-LABEL: udot_imm_does_not_fit:1244; CHECK-SVE2: // %bb.0: // %entry1245; CHECK-SVE2-NEXT: uunpklo z2.h, z1.b1246; CHECK-SVE2-NEXT: uunpkhi z1.h, z1.b1247; CHECK-SVE2-NEXT: uunpklo z3.s, z2.h1248; CHECK-SVE2-NEXT: uunpkhi z2.s, z2.h1249; CHECK-SVE2-NEXT: uunpklo z4.s, z1.h1250; CHECK-SVE2-NEXT: uunpkhi z1.s, z1.h1251; CHECK-SVE2-NEXT: lsl z4.s, z4.s, #81252; CHECK-SVE2-NEXT: lsl z2.s, z2.s, #81253; CHECK-SVE2-NEXT: lsl z3.s, z3.s, #81254; CHECK-SVE2-NEXT: lsl z1.s, z1.s, #81255; CHECK-SVE2-NEXT: add z0.s, z0.s, z3.s1256; CHECK-SVE2-NEXT: add z2.s, z2.s, z4.s1257; CHECK-SVE2-NEXT: add z0.s, z0.s, z2.s1258; CHECK-SVE2-NEXT: add z0.s, z0.s, z1.s1259; CHECK-SVE2-NEXT: ret1260;1261; CHECK-SVE2-I8MM-LABEL: udot_imm_does_not_fit:1262; CHECK-SVE2-I8MM: // %bb.0: // %entry1263; CHECK-SVE2-I8MM-NEXT: uunpklo z2.h, z1.b1264; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.h, z1.b1265; CHECK-SVE2-I8MM-NEXT: uunpklo z3.s, z2.h1266; CHECK-SVE2-I8MM-NEXT: uunpkhi z2.s, z2.h1267; CHECK-SVE2-I8MM-NEXT: uunpklo z4.s, z1.h1268; CHECK-SVE2-I8MM-NEXT: uunpkhi z1.s, z1.h1269; CHECK-SVE2-I8MM-NEXT: lsl z4.s, z4.s, #81270; CHECK-SVE2-I8MM-NEXT: lsl z2.s, z2.s, #81271; CHECK-SVE2-I8MM-NEXT: lsl z3.s, z3.s, #81272; CHECK-SVE2-I8MM-NEXT: lsl z1.s, z1.s, #81273; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z3.s1274; CHECK-SVE2-I8MM-NEXT: add z2.s, z2.s, z4.s1275; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z2.s1276; CHECK-SVE2-I8MM-NEXT: add z0.s, z0.s, z1.s1277; CHECK-SVE2-I8MM-NEXT: ret1278;1279; CHECK-SME-LABEL: udot_imm_does_not_fit:1280; CHECK-SME: // %bb.0: // %entry1281; CHECK-SME-NEXT: uunpklo z2.h, z1.b1282; CHECK-SME-NEXT: uunpkhi z1.h, z1.b1283; CHECK-SME-NEXT: uunpklo z3.s, z2.h1284; CHECK-SME-NEXT: uunpkhi z2.s, z2.h1285; CHECK-SME-NEXT: uunpklo z4.s, z1.h1286; CHECK-SME-NEXT: uunpkhi z1.s, z1.h1287; CHECK-SME-NEXT: lsl z4.s, z4.s, #81288; CHECK-SME-NEXT: lsl z2.s, z2.s, #81289; CHECK-SME-NEXT: lsl z3.s, z3.s, #81290; CHECK-SME-NEXT: lsl z1.s, z1.s, #81291; CHECK-SME-NEXT: add z0.s, z0.s, z3.s1292; CHECK-SME-NEXT: add z2.s, z2.s, z4.s1293; CHECK-SME-NEXT: add z0.s, z0.s, z2.s1294; CHECK-SME-NEXT: add z0.s, z0.s, z1.s1295; CHECK-SME-NEXT: ret1296entry:1297 %a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>1298 %mult = mul nuw nsw <vscale x 16 x i32> %a.wide, splat(i32 256)1299 %partial.reduce = tail call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> %acc, <vscale x 16 x i32> %mult)1300 ret <vscale x 4 x i32> %partial.reduce1301}1302