1036 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT3; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NODOT4; RUN: llc -mtriple=riscv32 -mattr=+v,+experimental-zvqdotq -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT325; RUN: llc -mtriple=riscv64 -mattr=+v,+experimental-zvqdotq -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,DOT,DOT646 7define i32 @vqdot_vv(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {8; NODOT-LABEL: vqdot_vv:9; NODOT: # %bb.0: # %entry10; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma11; NODOT-NEXT: vsext.vf2 v16, v812; NODOT-NEXT: vsext.vf2 v20, v1013; NODOT-NEXT: vwmul.vv v8, v16, v2014; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma15; NODOT-NEXT: vmv.s.x v16, zero16; NODOT-NEXT: vredsum.vs v8, v8, v1617; NODOT-NEXT: vmv.x.s a0, v818; NODOT-NEXT: ret19;20; DOT-LABEL: vqdot_vv:21; DOT: # %bb.0: # %entry22; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma23; DOT-NEXT: vmv.v.i v12, 024; DOT-NEXT: vqdot.vv v12, v8, v1025; DOT-NEXT: vmv.s.x v8, zero26; DOT-NEXT: vredsum.vs v8, v12, v827; DOT-NEXT: vmv.x.s a0, v828; DOT-NEXT: ret29entry:30 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>31 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>32 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext33 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)34 ret i32 %res35}36 37define i32 @vqdot_vx_constant(<vscale x 16 x i8> %a) {38; CHECK-LABEL: vqdot_vx_constant:39; CHECK: # %bb.0: # %entry40; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma41; CHECK-NEXT: vsext.vf2 v16, v842; CHECK-NEXT: li a0, 2343; CHECK-NEXT: vwmul.vx v8, v16, a044; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma45; CHECK-NEXT: vmv.s.x v16, zero46; CHECK-NEXT: vredsum.vs v8, v8, v1647; CHECK-NEXT: vmv.x.s a0, v848; CHECK-NEXT: ret49entry:50 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>51 %mul = mul <vscale x 16 x i32> %a.sext, splat (i32 23)52 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)53 ret i32 %res54}55 56define i32 @vqdot_vx_constant_swapped(<vscale x 16 x i8> %a) {57; CHECK-LABEL: vqdot_vx_constant_swapped:58; CHECK: # %bb.0: # %entry59; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma60; CHECK-NEXT: vsext.vf2 v16, v861; CHECK-NEXT: li a0, 2362; CHECK-NEXT: vwmul.vx v8, v16, a063; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma64; CHECK-NEXT: vmv.s.x v16, zero65; CHECK-NEXT: vredsum.vs v8, v8, v1666; CHECK-NEXT: vmv.x.s a0, v867; CHECK-NEXT: ret68entry:69 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>70 %mul = mul <vscale x 16 x i32> splat (i32 23), %a.sext71 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)72 ret i32 %res73}74 75define i32 @vqdotu_vv(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {76; NODOT-LABEL: vqdotu_vv:77; NODOT: # %bb.0: # %entry78; NODOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma79; NODOT-NEXT: vwmulu.vv v12, v8, v1080; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma81; NODOT-NEXT: vmv.s.x v8, zero82; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma83; NODOT-NEXT: vwredsumu.vs v8, v12, v884; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma85; NODOT-NEXT: vmv.x.s a0, v886; NODOT-NEXT: ret87;88; DOT-LABEL: vqdotu_vv:89; DOT: # %bb.0: # %entry90; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma91; DOT-NEXT: vmv.v.i v12, 092; DOT-NEXT: vqdotu.vv v12, v8, v1093; DOT-NEXT: vmv.s.x v8, zero94; DOT-NEXT: vredsum.vs v8, v12, v895; DOT-NEXT: vmv.x.s a0, v896; DOT-NEXT: ret97entry:98 %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>99 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>100 %mul = mul <vscale x 16 x i32> %a.zext, %b.zext101 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)102 ret i32 %res103}104 105define i32 @vqdotu_vx_constant(<vscale x 16 x i8> %a) {106; CHECK-LABEL: vqdotu_vx_constant:107; CHECK: # %bb.0: # %entry108; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma109; CHECK-NEXT: vzext.vf2 v16, v8110; CHECK-NEXT: li a0, 123111; CHECK-NEXT: vwmulu.vx v8, v16, a0112; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma113; CHECK-NEXT: vmv.s.x v16, zero114; CHECK-NEXT: vredsum.vs v8, v8, v16115; CHECK-NEXT: vmv.x.s a0, v8116; CHECK-NEXT: ret117entry:118 %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>119 %mul = mul <vscale x 16 x i32> %a.zext, splat (i32 123)120 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)121 ret i32 %res122}123 124define i32 @vqdotsu_vv(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {125; NODOT-LABEL: vqdotsu_vv:126; NODOT: # %bb.0: # %entry127; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma128; NODOT-NEXT: vsext.vf2 v16, v8129; NODOT-NEXT: vzext.vf2 v20, v10130; NODOT-NEXT: vwmulsu.vv v8, v16, v20131; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma132; NODOT-NEXT: vmv.s.x v16, zero133; NODOT-NEXT: vredsum.vs v8, v8, v16134; NODOT-NEXT: vmv.x.s a0, v8135; NODOT-NEXT: ret136;137; DOT-LABEL: vqdotsu_vv:138; DOT: # %bb.0: # %entry139; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma140; DOT-NEXT: vmv.v.i v12, 0141; DOT-NEXT: vqdotsu.vv v12, v8, v10142; DOT-NEXT: vmv.s.x v8, zero143; DOT-NEXT: vredsum.vs v8, v12, v8144; DOT-NEXT: vmv.x.s a0, v8145; DOT-NEXT: ret146entry:147 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>148 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>149 %mul = mul <vscale x 16 x i32> %a.sext, %b.zext150 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)151 ret i32 %res152}153 154define i32 @vqdotsu_vv_swapped(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {155; NODOT-LABEL: vqdotsu_vv_swapped:156; NODOT: # %bb.0: # %entry157; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma158; NODOT-NEXT: vsext.vf2 v16, v8159; NODOT-NEXT: vzext.vf2 v20, v10160; NODOT-NEXT: vwmulsu.vv v8, v16, v20161; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma162; NODOT-NEXT: vmv.s.x v16, zero163; NODOT-NEXT: vredsum.vs v8, v8, v16164; NODOT-NEXT: vmv.x.s a0, v8165; NODOT-NEXT: ret166;167; DOT-LABEL: vqdotsu_vv_swapped:168; DOT: # %bb.0: # %entry169; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma170; DOT-NEXT: vmv.v.i v12, 0171; DOT-NEXT: vqdotsu.vv v12, v8, v10172; DOT-NEXT: vmv.s.x v8, zero173; DOT-NEXT: vredsum.vs v8, v12, v8174; DOT-NEXT: vmv.x.s a0, v8175; DOT-NEXT: ret176entry:177 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>178 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>179 %mul = mul <vscale x 16 x i32> %b.zext, %a.sext180 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)181 ret i32 %res182}183 184define i32 @vdotqsu_vx_constant(<vscale x 16 x i8> %a) {185; CHECK-LABEL: vdotqsu_vx_constant:186; CHECK: # %bb.0: # %entry187; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma188; CHECK-NEXT: vsext.vf2 v16, v8189; CHECK-NEXT: li a0, 123190; CHECK-NEXT: vwmul.vx v8, v16, a0191; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma192; CHECK-NEXT: vmv.s.x v16, zero193; CHECK-NEXT: vredsum.vs v8, v8, v16194; CHECK-NEXT: vmv.x.s a0, v8195; CHECK-NEXT: ret196entry:197 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>198 %mul = mul <vscale x 16 x i32> %a.sext, splat (i32 123)199 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)200 ret i32 %res201}202 203define i32 @vdotqus_vx_constant(<vscale x 16 x i8> %a) {204; CHECK-LABEL: vdotqus_vx_constant:205; CHECK: # %bb.0: # %entry206; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma207; CHECK-NEXT: vzext.vf2 v16, v8208; CHECK-NEXT: li a0, -23209; CHECK-NEXT: vmv.v.x v20, a0210; CHECK-NEXT: vwmulsu.vv v8, v20, v16211; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma212; CHECK-NEXT: vmv.s.x v16, zero213; CHECK-NEXT: vredsum.vs v8, v8, v16214; CHECK-NEXT: vmv.x.s a0, v8215; CHECK-NEXT: ret216entry:217 %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>218 %mul = mul <vscale x 16 x i32> %a.zext, splat (i32 -23)219 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)220 ret i32 %res221}222 223define i32 @reduce_of_sext(<vscale x 16 x i8> %a) {224; NODOT-LABEL: reduce_of_sext:225; NODOT: # %bb.0: # %entry226; NODOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma227; NODOT-NEXT: vsext.vf4 v16, v8228; NODOT-NEXT: vmv.s.x v8, zero229; NODOT-NEXT: vredsum.vs v8, v16, v8230; NODOT-NEXT: vmv.x.s a0, v8231; NODOT-NEXT: ret232;233; DOT-LABEL: reduce_of_sext:234; DOT: # %bb.0: # %entry235; DOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma236; DOT-NEXT: vmv.v.i v10, 1237; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma238; DOT-NEXT: vmv.v.i v12, 0239; DOT-NEXT: vqdot.vv v12, v8, v10240; DOT-NEXT: vmv.s.x v8, zero241; DOT-NEXT: vredsum.vs v8, v12, v8242; DOT-NEXT: vmv.x.s a0, v8243; DOT-NEXT: ret244entry:245 %a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>246 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %a.ext)247 ret i32 %res248}249 250define i32 @reduce_of_zext(<vscale x 16 x i8> %a) {251; NODOT-LABEL: reduce_of_zext:252; NODOT: # %bb.0: # %entry253; NODOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma254; NODOT-NEXT: vzext.vf4 v16, v8255; NODOT-NEXT: vmv.s.x v8, zero256; NODOT-NEXT: vredsum.vs v8, v16, v8257; NODOT-NEXT: vmv.x.s a0, v8258; NODOT-NEXT: ret259;260; DOT-LABEL: reduce_of_zext:261; DOT: # %bb.0: # %entry262; DOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma263; DOT-NEXT: vmv.v.i v10, 1264; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma265; DOT-NEXT: vmv.v.i v12, 0266; DOT-NEXT: vqdotu.vv v12, v8, v10267; DOT-NEXT: vmv.s.x v8, zero268; DOT-NEXT: vredsum.vs v8, v12, v8269; DOT-NEXT: vmv.x.s a0, v8270; DOT-NEXT: ret271entry:272 %a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>273 %res = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %a.ext)274 ret i32 %res275}276 277define i32 @vqdot_vv_accum(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i32> %x) {278; NODOT-LABEL: vqdot_vv_accum:279; NODOT: # %bb.0: # %entry280; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma281; NODOT-NEXT: vsext.vf2 v12, v8282; NODOT-NEXT: vsext.vf2 v24, v10283; NODOT-NEXT: vwmacc.vv v16, v12, v24284; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma285; NODOT-NEXT: vmv.s.x v8, zero286; NODOT-NEXT: vredsum.vs v8, v16, v8287; NODOT-NEXT: vmv.x.s a0, v8288; NODOT-NEXT: ret289;290; DOT-LABEL: vqdot_vv_accum:291; DOT: # %bb.0: # %entry292; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma293; DOT-NEXT: vmv.s.x v12, zero294; DOT-NEXT: vqdot.vv v16, v8, v10295; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma296; DOT-NEXT: vredsum.vs v8, v16, v12297; DOT-NEXT: vmv.x.s a0, v8298; DOT-NEXT: ret299entry:300 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>301 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>302 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext303 %add = add <vscale x 16 x i32> %mul, %x304 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %add)305 ret i32 %sum306}307 308define i32 @vqdotu_vv_accum(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i32> %x) {309; NODOT-LABEL: vqdotu_vv_accum:310; NODOT: # %bb.0: # %entry311; NODOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma312; NODOT-NEXT: vwmulu.vv v12, v8, v10313; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma314; NODOT-NEXT: vwaddu.wv v16, v16, v12315; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma316; NODOT-NEXT: vmv.s.x v8, zero317; NODOT-NEXT: vredsum.vs v8, v16, v8318; NODOT-NEXT: vmv.x.s a0, v8319; NODOT-NEXT: ret320;321; DOT-LABEL: vqdotu_vv_accum:322; DOT: # %bb.0: # %entry323; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma324; DOT-NEXT: vmv.s.x v12, zero325; DOT-NEXT: vqdotu.vv v16, v8, v10326; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma327; DOT-NEXT: vredsum.vs v8, v16, v12328; DOT-NEXT: vmv.x.s a0, v8329; DOT-NEXT: ret330entry:331 %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>332 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>333 %mul = mul <vscale x 16 x i32> %a.zext, %b.zext334 %add = add <vscale x 16 x i32> %mul, %x335 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %add)336 ret i32 %sum337}338 339define i32 @vqdotsu_vv_accum(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i32> %x) {340; NODOT-LABEL: vqdotsu_vv_accum:341; NODOT: # %bb.0: # %entry342; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma343; NODOT-NEXT: vsext.vf2 v12, v8344; NODOT-NEXT: vzext.vf2 v24, v10345; NODOT-NEXT: vwmaccsu.vv v16, v12, v24346; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma347; NODOT-NEXT: vmv.s.x v8, zero348; NODOT-NEXT: vredsum.vs v8, v16, v8349; NODOT-NEXT: vmv.x.s a0, v8350; NODOT-NEXT: ret351;352; DOT-LABEL: vqdotsu_vv_accum:353; DOT: # %bb.0: # %entry354; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma355; DOT-NEXT: vmv.s.x v12, zero356; DOT-NEXT: vqdotsu.vv v16, v8, v10357; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma358; DOT-NEXT: vredsum.vs v8, v16, v12359; DOT-NEXT: vmv.x.s a0, v8360; DOT-NEXT: ret361entry:362 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>363 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>364 %mul = mul <vscale x 16 x i32> %a.sext, %b.zext365 %add = add <vscale x 16 x i32> %mul, %x366 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %add)367 ret i32 %sum368}369 370define i32 @vqdot_vv_scalar_add(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 %x) {371; NODOT-LABEL: vqdot_vv_scalar_add:372; NODOT: # %bb.0: # %entry373; NODOT-NEXT: vsetvli a1, zero, e16, m4, ta, ma374; NODOT-NEXT: vsext.vf2 v16, v8375; NODOT-NEXT: vsext.vf2 v20, v10376; NODOT-NEXT: vwmul.vv v8, v16, v20377; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma378; NODOT-NEXT: vmv.s.x v16, a0379; NODOT-NEXT: vredsum.vs v8, v8, v16380; NODOT-NEXT: vmv.x.s a0, v8381; NODOT-NEXT: ret382;383; DOT-LABEL: vqdot_vv_scalar_add:384; DOT: # %bb.0: # %entry385; DOT-NEXT: vsetvli a1, zero, e32, m2, ta, ma386; DOT-NEXT: vmv.v.i v12, 0387; DOT-NEXT: vqdot.vv v12, v8, v10388; DOT-NEXT: vmv.s.x v8, a0389; DOT-NEXT: vredsum.vs v8, v12, v8390; DOT-NEXT: vmv.x.s a0, v8391; DOT-NEXT: ret392entry:393 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>394 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>395 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext396 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)397 %add = add i32 %sum, %x398 ret i32 %add399}400 401define i32 @vqdotu_vv_scalar_add(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 %x) {402; NODOT-LABEL: vqdotu_vv_scalar_add:403; NODOT: # %bb.0: # %entry404; NODOT-NEXT: vsetvli a1, zero, e8, m2, ta, ma405; NODOT-NEXT: vwmulu.vv v12, v8, v10406; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma407; NODOT-NEXT: vmv.s.x v8, a0408; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma409; NODOT-NEXT: vwredsumu.vs v8, v12, v8410; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma411; NODOT-NEXT: vmv.x.s a0, v8412; NODOT-NEXT: ret413;414; DOT-LABEL: vqdotu_vv_scalar_add:415; DOT: # %bb.0: # %entry416; DOT-NEXT: vsetvli a1, zero, e32, m2, ta, ma417; DOT-NEXT: vmv.v.i v12, 0418; DOT-NEXT: vqdotu.vv v12, v8, v10419; DOT-NEXT: vmv.s.x v8, a0420; DOT-NEXT: vredsum.vs v8, v12, v8421; DOT-NEXT: vmv.x.s a0, v8422; DOT-NEXT: ret423entry:424 %a.zext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>425 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>426 %mul = mul <vscale x 16 x i32> %a.zext, %b.zext427 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)428 %add = add i32 %sum, %x429 ret i32 %add430}431 432define i32 @vqdotsu_vv_scalar_add(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, i32 %x) {433; NODOT-LABEL: vqdotsu_vv_scalar_add:434; NODOT: # %bb.0: # %entry435; NODOT-NEXT: vsetvli a1, zero, e16, m4, ta, ma436; NODOT-NEXT: vsext.vf2 v16, v8437; NODOT-NEXT: vzext.vf2 v20, v10438; NODOT-NEXT: vwmulsu.vv v8, v16, v20439; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma440; NODOT-NEXT: vmv.s.x v16, a0441; NODOT-NEXT: vredsum.vs v8, v8, v16442; NODOT-NEXT: vmv.x.s a0, v8443; NODOT-NEXT: ret444;445; DOT-LABEL: vqdotsu_vv_scalar_add:446; DOT: # %bb.0: # %entry447; DOT-NEXT: vsetvli a1, zero, e32, m2, ta, ma448; DOT-NEXT: vmv.v.i v12, 0449; DOT-NEXT: vqdotsu.vv v12, v8, v10450; DOT-NEXT: vmv.s.x v8, a0451; DOT-NEXT: vredsum.vs v8, v12, v8452; DOT-NEXT: vmv.x.s a0, v8453; DOT-NEXT: ret454entry:455 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>456 %b.zext = zext <vscale x 16 x i8> %b to <vscale x 16 x i32>457 %mul = mul <vscale x 16 x i32> %a.sext, %b.zext458 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul)459 %add = add i32 %sum, %x460 ret i32 %add461}462 463define i32 @vqdot_vv_split(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d) {464; NODOT-LABEL: vqdot_vv_split:465; NODOT: # %bb.0: # %entry466; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma467; NODOT-NEXT: vsext.vf2 v16, v8468; NODOT-NEXT: vsext.vf2 v20, v10469; NODOT-NEXT: vsext.vf2 v24, v12470; NODOT-NEXT: vsext.vf2 v28, v14471; NODOT-NEXT: vwmul.vv v8, v16, v20472; NODOT-NEXT: vwmacc.vv v8, v24, v28473; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma474; NODOT-NEXT: vmv.s.x v16, zero475; NODOT-NEXT: vredsum.vs v8, v8, v16476; NODOT-NEXT: vmv.x.s a0, v8477; NODOT-NEXT: ret478;479; DOT-LABEL: vqdot_vv_split:480; DOT: # %bb.0: # %entry481; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma482; DOT-NEXT: vmv.v.i v16, 0483; DOT-NEXT: vqdot.vv v16, v8, v10484; DOT-NEXT: vqdot.vv v16, v12, v14485; DOT-NEXT: vmv.s.x v8, zero486; DOT-NEXT: vredsum.vs v8, v16, v8487; DOT-NEXT: vmv.x.s a0, v8488; DOT-NEXT: ret489entry:490 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>491 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>492 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext493 %c.sext = sext <vscale x 16 x i8> %c to <vscale x 16 x i32>494 %d.sext = sext <vscale x 16 x i8> %d to <vscale x 16 x i32>495 %mul2 = mul <vscale x 16 x i32> %c.sext, %d.sext496 %add = add <vscale x 16 x i32> %mul, %mul2497 %sum = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %add)498 ret i32 %sum499}500 501 502define <vscale x 1 x i32> @partial_reduce_nf2(<vscale x 4 x i8> %a, <vscale x 4 x i8> %b) {503; NODOT-LABEL: partial_reduce_nf2:504; NODOT: # %bb.0: # %entry505; NODOT-NEXT: vsetvli a0, zero, e16, m1, ta, ma506; NODOT-NEXT: vsext.vf2 v10, v8507; NODOT-NEXT: vsext.vf2 v11, v9508; NODOT-NEXT: csrr a0, vlenb509; NODOT-NEXT: vwmul.vv v8, v10, v11510; NODOT-NEXT: srli a0, a0, 3511; NODOT-NEXT: vsetvli a1, zero, e32, m1, ta, ma512; NODOT-NEXT: vslidedown.vx v10, v9, a0513; NODOT-NEXT: vslidedown.vx v11, v8, a0514; NODOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma515; NODOT-NEXT: vadd.vv v8, v10, v8516; NODOT-NEXT: vadd.vv v9, v11, v9517; NODOT-NEXT: vadd.vv v8, v9, v8518; NODOT-NEXT: ret519;520; DOT-LABEL: partial_reduce_nf2:521; DOT: # %bb.0: # %entry522; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma523; DOT-NEXT: vmv.v.i v10, 0524; DOT-NEXT: vqdot.vv v10, v8, v9525; DOT-NEXT: vmv1r.v v8, v10526; DOT-NEXT: ret527entry:528 %a.sext = sext <vscale x 4 x i8> %a to <vscale x 4 x i32>529 %b.sext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>530 %mul = mul <vscale x 4 x i32> %a.sext, %b.sext531 %res = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add(<vscale x 1 x i32> zeroinitializer, <vscale x 4 x i32> %mul)532 ret <vscale x 1 x i32> %res533}534 535define <vscale x 2 x i32> @partial_reduce_m1(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b) {536; NODOT-LABEL: partial_reduce_m1:537; NODOT: # %bb.0: # %entry538; NODOT-NEXT: vsetvli a0, zero, e16, m2, ta, ma539; NODOT-NEXT: vsext.vf2 v12, v8540; NODOT-NEXT: vsext.vf2 v14, v9541; NODOT-NEXT: vwmul.vv v8, v12, v14542; NODOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma543; NODOT-NEXT: vadd.vv v8, v11, v8544; NODOT-NEXT: vadd.vv v9, v9, v10545; NODOT-NEXT: vadd.vv v8, v9, v8546; NODOT-NEXT: ret547;548; DOT-LABEL: partial_reduce_m1:549; DOT: # %bb.0: # %entry550; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma551; DOT-NEXT: vmv.v.i v10, 0552; DOT-NEXT: vqdot.vv v10, v8, v9553; DOT-NEXT: vmv.v.v v8, v10554; DOT-NEXT: ret555entry:556 %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i32>557 %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i32>558 %mul = mul <vscale x 8 x i32> %a.sext, %b.sext559 %res = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add(<vscale x 2 x i32> zeroinitializer, <vscale x 8 x i32> %mul)560 ret <vscale x 2 x i32> %res561}562 563define <vscale x 4 x i32> @partial_reduce_m2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {564; NODOT-LABEL: partial_reduce_m2:565; NODOT: # %bb.0: # %entry566; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma567; NODOT-NEXT: vsext.vf2 v16, v8568; NODOT-NEXT: vsext.vf2 v20, v10569; NODOT-NEXT: vwmul.vv v8, v16, v20570; NODOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma571; NODOT-NEXT: vadd.vv v8, v14, v8572; NODOT-NEXT: vadd.vv v10, v10, v12573; NODOT-NEXT: vadd.vv v8, v10, v8574; NODOT-NEXT: ret575;576; DOT-LABEL: partial_reduce_m2:577; DOT: # %bb.0: # %entry578; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma579; DOT-NEXT: vmv.v.i v12, 0580; DOT-NEXT: vqdot.vv v12, v8, v10581; DOT-NEXT: vmv.v.v v8, v12582; DOT-NEXT: ret583entry:584 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>585 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>586 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext587 %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i32> %mul)588 ret <vscale x 4 x i32> %res589}590 591define <vscale x 8 x i32> @partial_reduce_m4(<vscale x 32 x i8> %a, <vscale x 32 x i8> %b) {592; NODOT-LABEL: partial_reduce_m4:593; NODOT: # %bb.0: # %entry594; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma595; NODOT-NEXT: vsext.vf2 v24, v8596; NODOT-NEXT: vsext.vf2 v16, v10597; NODOT-NEXT: vsext.vf2 v28, v12598; NODOT-NEXT: vsext.vf2 v20, v14599; NODOT-NEXT: vwmul.vv v8, v16, v20600; NODOT-NEXT: vwmul.vv v16, v24, v28601; NODOT-NEXT: vsetvli a0, zero, e32, m4, ta, ma602; NODOT-NEXT: vadd.vv v16, v20, v16603; NODOT-NEXT: vadd.vv v8, v12, v8604; NODOT-NEXT: vadd.vv v8, v8, v16605; NODOT-NEXT: ret606;607; DOT-LABEL: partial_reduce_m4:608; DOT: # %bb.0: # %entry609; DOT-NEXT: vsetvli a0, zero, e32, m4, ta, ma610; DOT-NEXT: vmv.v.i v16, 0611; DOT-NEXT: vqdot.vv v16, v8, v12612; DOT-NEXT: vmv.v.v v8, v16613; DOT-NEXT: ret614entry:615 %a.sext = sext <vscale x 32 x i8> %a to <vscale x 32 x i32>616 %b.sext = sext <vscale x 32 x i8> %b to <vscale x 32 x i32>617 %mul = mul <vscale x 32 x i32> %a.sext, %b.sext618 %res = call <vscale x 8 x i32> @llvm.vector.partial.reduce.add(<vscale x 8 x i32> zeroinitializer, <vscale x 32 x i32> %mul)619 ret <vscale x 8 x i32> %res620}621 622define <vscale x 16 x i32> @partial_reduce_m8(<vscale x 64 x i8> %a, <vscale x 64 x i8> %b) {623; NODOT-LABEL: partial_reduce_m8:624; NODOT: # %bb.0: # %entry625; NODOT-NEXT: addi sp, sp, -16626; NODOT-NEXT: .cfi_def_cfa_offset 16627; NODOT-NEXT: csrr a0, vlenb628; NODOT-NEXT: slli a0, a0, 2629; NODOT-NEXT: sub sp, sp, a0630; NODOT-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb631; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma632; NODOT-NEXT: vsext.vf2 v24, v10633; NODOT-NEXT: addi a0, sp, 16634; NODOT-NEXT: vs4r.v v24, (a0) # vscale x 32-byte Folded Spill635; NODOT-NEXT: vsext.vf2 v0, v8636; NODOT-NEXT: vsext.vf2 v8, v18637; NODOT-NEXT: vsext.vf2 v4, v16638; NODOT-NEXT: vwmul.vv v24, v0, v4639; NODOT-NEXT: vl4r.v v16, (a0) # vscale x 32-byte Folded Reload640; NODOT-NEXT: vwmacc.vv v24, v16, v8641; NODOT-NEXT: vsext.vf2 v8, v12642; NODOT-NEXT: vsext.vf2 v16, v20643; NODOT-NEXT: vwmacc.vv v24, v8, v16644; NODOT-NEXT: vsext.vf2 v8, v14645; NODOT-NEXT: vsext.vf2 v12, v22646; NODOT-NEXT: vwmacc.vv v24, v8, v12647; NODOT-NEXT: vmv8r.v v8, v24648; NODOT-NEXT: csrr a0, vlenb649; NODOT-NEXT: slli a0, a0, 2650; NODOT-NEXT: add sp, sp, a0651; NODOT-NEXT: .cfi_def_cfa sp, 16652; NODOT-NEXT: addi sp, sp, 16653; NODOT-NEXT: .cfi_def_cfa_offset 0654; NODOT-NEXT: ret655;656; DOT-LABEL: partial_reduce_m8:657; DOT: # %bb.0: # %entry658; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma659; DOT-NEXT: vmv.v.i v24, 0660; DOT-NEXT: vqdot.vv v24, v8, v16661; DOT-NEXT: vmv.v.v v8, v24662; DOT-NEXT: ret663entry:664 %a.sext = sext <vscale x 64 x i8> %a to <vscale x 64 x i32>665 %b.sext = sext <vscale x 64 x i8> %b to <vscale x 64 x i32>666 %mul = mul <vscale x 64 x i32> %a.sext, %b.sext667 %res = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add(<vscale x 16 x i32> zeroinitializer, <vscale x 64 x i32> %mul)668 ret <vscale x 16 x i32> %res669}670 671define <vscale x 32 x i32> @partial_reduce_m16(<vscale x 128 x i8> %a, <vscale x 128 x i8> %b) {672; NODOT-LABEL: partial_reduce_m16:673; NODOT: # %bb.0: # %entry674; NODOT-NEXT: addi sp, sp, -16675; NODOT-NEXT: .cfi_def_cfa_offset 16676; NODOT-NEXT: csrr a1, vlenb677; NODOT-NEXT: slli a1, a1, 3678; NODOT-NEXT: mv a2, a1679; NODOT-NEXT: slli a1, a1, 1680; NODOT-NEXT: add a1, a1, a2681; NODOT-NEXT: sub sp, sp, a1682; NODOT-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb683; NODOT-NEXT: csrr a1, vlenb684; NODOT-NEXT: slli a1, a1, 4685; NODOT-NEXT: add a1, sp, a1686; NODOT-NEXT: addi a1, a1, 16687; NODOT-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill688; NODOT-NEXT: addi a1, sp, 16689; NODOT-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill690; NODOT-NEXT: vl8r.v v16, (a0)691; NODOT-NEXT: csrr a1, vlenb692; NODOT-NEXT: slli a1, a1, 3693; NODOT-NEXT: add a1, sp, a1694; NODOT-NEXT: addi a1, a1, 16695; NODOT-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill696; NODOT-NEXT: vsetvli a1, zero, e16, m4, ta, ma697; NODOT-NEXT: vsext.vf2 v4, v8698; NODOT-NEXT: vsext.vf2 v0, v16699; NODOT-NEXT: vwmul.vv v24, v4, v0700; NODOT-NEXT: vsext.vf2 v4, v10701; NODOT-NEXT: vsext.vf2 v8, v18702; NODOT-NEXT: vwmacc.vv v24, v4, v8703; NODOT-NEXT: csrr a1, vlenb704; NODOT-NEXT: slli a1, a1, 3705; NODOT-NEXT: add a0, a0, a1706; NODOT-NEXT: vsext.vf2 v0, v12707; NODOT-NEXT: vl8r.v v8, (a0)708; NODOT-NEXT: csrr a0, vlenb709; NODOT-NEXT: slli a0, a0, 3710; NODOT-NEXT: add a0, sp, a0711; NODOT-NEXT: addi a0, a0, 16712; NODOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload713; NODOT-NEXT: vsext.vf2 v4, v20714; NODOT-NEXT: vwmacc.vv v24, v0, v4715; NODOT-NEXT: csrr a0, vlenb716; NODOT-NEXT: slli a0, a0, 4717; NODOT-NEXT: add a0, sp, a0718; NODOT-NEXT: addi a0, a0, 16719; NODOT-NEXT: vl8r.v v0, (a0) # vscale x 64-byte Folded Reload720; NODOT-NEXT: vsext.vf2 v20, v0721; NODOT-NEXT: vsext.vf2 v16, v8722; NODOT-NEXT: vwmul.vv v0, v20, v16723; NODOT-NEXT: csrr a0, vlenb724; NODOT-NEXT: slli a0, a0, 4725; NODOT-NEXT: add a0, sp, a0726; NODOT-NEXT: addi a0, a0, 16727; NODOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload728; NODOT-NEXT: vsext.vf2 v20, v18729; NODOT-NEXT: vsext.vf2 v16, v10730; NODOT-NEXT: vwmacc.vv v0, v20, v16731; NODOT-NEXT: csrr a0, vlenb732; NODOT-NEXT: slli a0, a0, 4733; NODOT-NEXT: add a0, sp, a0734; NODOT-NEXT: addi a0, a0, 16735; NODOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload736; NODOT-NEXT: vsext.vf2 v8, v20737; NODOT-NEXT: vsext.vf2 v16, v12738; NODOT-NEXT: vwmacc.vv v0, v8, v16739; NODOT-NEXT: csrr a0, vlenb740; NODOT-NEXT: slli a0, a0, 4741; NODOT-NEXT: add a0, sp, a0742; NODOT-NEXT: addi a0, a0, 16743; NODOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload744; NODOT-NEXT: vsext.vf2 v8, v22745; NODOT-NEXT: vsext.vf2 v16, v14746; NODOT-NEXT: vwmacc.vv v0, v8, v16747; NODOT-NEXT: addi a0, sp, 16748; NODOT-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload749; NODOT-NEXT: vsext.vf2 v8, v14750; NODOT-NEXT: csrr a0, vlenb751; NODOT-NEXT: slli a0, a0, 3752; NODOT-NEXT: add a0, sp, a0753; NODOT-NEXT: addi a0, a0, 16754; NODOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload755; NODOT-NEXT: vsext.vf2 v12, v22756; NODOT-NEXT: vwmacc.vv v24, v8, v12757; NODOT-NEXT: vmv8r.v v8, v24758; NODOT-NEXT: vmv8r.v v16, v0759; NODOT-NEXT: csrr a0, vlenb760; NODOT-NEXT: slli a0, a0, 3761; NODOT-NEXT: mv a1, a0762; NODOT-NEXT: slli a0, a0, 1763; NODOT-NEXT: add a0, a0, a1764; NODOT-NEXT: add sp, sp, a0765; NODOT-NEXT: .cfi_def_cfa sp, 16766; NODOT-NEXT: addi sp, sp, 16767; NODOT-NEXT: .cfi_def_cfa_offset 0768; NODOT-NEXT: ret769;770; DOT-LABEL: partial_reduce_m16:771; DOT: # %bb.0: # %entry772; DOT-NEXT: addi sp, sp, -16773; DOT-NEXT: .cfi_def_cfa_offset 16774; DOT-NEXT: csrr a1, vlenb775; DOT-NEXT: slli a1, a1, 3776; DOT-NEXT: mv a2, a1777; DOT-NEXT: slli a1, a1, 1778; DOT-NEXT: add a1, a1, a2779; DOT-NEXT: sub sp, sp, a1780; DOT-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 24 * vlenb781; DOT-NEXT: csrr a1, vlenb782; DOT-NEXT: slli a1, a1, 4783; DOT-NEXT: add a1, sp, a1784; DOT-NEXT: addi a1, a1, 16785; DOT-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill786; DOT-NEXT: csrr a1, vlenb787; DOT-NEXT: slli a1, a1, 3788; DOT-NEXT: add a1, sp, a1789; DOT-NEXT: addi a1, a1, 16790; DOT-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill791; DOT-NEXT: csrr a1, vlenb792; DOT-NEXT: slli a1, a1, 3793; DOT-NEXT: add a1, a0, a1794; DOT-NEXT: vl8r.v v8, (a0)795; DOT-NEXT: vl8r.v v16, (a1)796; DOT-NEXT: addi a0, sp, 16797; DOT-NEXT: vs8r.v v16, (a0) # vscale x 64-byte Folded Spill798; DOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma799; DOT-NEXT: vmv.v.i v24, 0800; DOT-NEXT: vmv.v.i v0, 0801; DOT-NEXT: csrr a0, vlenb802; DOT-NEXT: slli a0, a0, 3803; DOT-NEXT: add a0, sp, a0804; DOT-NEXT: addi a0, a0, 16805; DOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload806; DOT-NEXT: vqdot.vv v0, v16, v8807; DOT-NEXT: csrr a0, vlenb808; DOT-NEXT: slli a0, a0, 4809; DOT-NEXT: add a0, sp, a0810; DOT-NEXT: addi a0, a0, 16811; DOT-NEXT: vl8r.v v8, (a0) # vscale x 64-byte Folded Reload812; DOT-NEXT: addi a0, sp, 16813; DOT-NEXT: vl8r.v v16, (a0) # vscale x 64-byte Folded Reload814; DOT-NEXT: vqdot.vv v24, v8, v16815; DOT-NEXT: vmv.v.v v8, v0816; DOT-NEXT: vmv.v.v v16, v24817; DOT-NEXT: csrr a0, vlenb818; DOT-NEXT: slli a0, a0, 3819; DOT-NEXT: mv a1, a0820; DOT-NEXT: slli a0, a0, 1821; DOT-NEXT: add a0, a0, a1822; DOT-NEXT: add sp, sp, a0823; DOT-NEXT: .cfi_def_cfa sp, 16824; DOT-NEXT: addi sp, sp, 16825; DOT-NEXT: .cfi_def_cfa_offset 0826; DOT-NEXT: ret827entry:828 %a.sext = sext <vscale x 128 x i8> %a to <vscale x 128 x i32>829 %b.sext = sext <vscale x 128 x i8> %b to <vscale x 128 x i32>830 %mul = mul <vscale x 128 x i32> %a.sext, %b.sext831 %res = call <vscale x 32 x i32> @llvm.vector.partial.reduce.add(<vscale x 32 x i32> zeroinitializer, <vscale x 128 x i32> %mul)832 ret <vscale x 32 x i32> %res833}834 835define <vscale x 4 x i32> @partial_reduce_accum(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 4 x i32> %accum) {836; NODOT-LABEL: partial_reduce_accum:837; NODOT: # %bb.0: # %entry838; NODOT-NEXT: vsetvli a0, zero, e16, m4, ta, ma839; NODOT-NEXT: vsext.vf2 v24, v8840; NODOT-NEXT: vsext.vf2 v28, v10841; NODOT-NEXT: vwmul.vv v16, v24, v28842; NODOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma843; NODOT-NEXT: vadd.vv v8, v18, v20844; NODOT-NEXT: vadd.vv v10, v12, v16845; NODOT-NEXT: vadd.vv v10, v22, v10846; NODOT-NEXT: vadd.vv v8, v8, v10847; NODOT-NEXT: ret848;849; DOT-LABEL: partial_reduce_accum:850; DOT: # %bb.0: # %entry851; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma852; DOT-NEXT: vqdot.vv v12, v8, v10853; DOT-NEXT: vmv.v.v v8, v12854; DOT-NEXT: ret855entry:856 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>857 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>858 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext859 %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %accum, <vscale x 16 x i32> %mul)860 ret <vscale x 4 x i32> %res861}862 863define <vscale x 16 x i32> @partial_reduce_via_accum(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {864; CHECK-LABEL: partial_reduce_via_accum:865; CHECK: # %bb.0: # %entry866; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma867; CHECK-NEXT: vsext.vf2 v16, v8868; CHECK-NEXT: vsext.vf2 v20, v10869; CHECK-NEXT: vwmul.vv v8, v16, v20870; CHECK-NEXT: ret871entry:872 %a.sext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>873 %b.sext = sext <vscale x 16 x i8> %b to <vscale x 16 x i32>874 %mul = mul <vscale x 16 x i32> %a.sext, %b.sext875 %res = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nvx16i32.nvx16i32(<vscale x 16 x i32> %mul, <vscale x 16 x i32> zeroinitializer)876 ret <vscale x 16 x i32> %res877}878 879define <vscale x 1 x i32> @partial_reduce_vqdotu(<vscale x 4 x i8> %a, <vscale x 4 x i8> %b) {880; NODOT-LABEL: partial_reduce_vqdotu:881; NODOT: # %bb.0: # %entry882; NODOT-NEXT: vsetvli a0, zero, e8, mf2, ta, ma883; NODOT-NEXT: vwmulu.vv v10, v8, v9884; NODOT-NEXT: csrr a0, vlenb885; NODOT-NEXT: vsetvli zero, zero, e32, m2, ta, ma886; NODOT-NEXT: vzext.vf2 v8, v10887; NODOT-NEXT: srli a0, a0, 3888; NODOT-NEXT: vsetvli a1, zero, e32, m1, ta, ma889; NODOT-NEXT: vslidedown.vx v10, v9, a0890; NODOT-NEXT: vslidedown.vx v11, v8, a0891; NODOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma892; NODOT-NEXT: vadd.vv v8, v10, v8893; NODOT-NEXT: vadd.vv v9, v11, v9894; NODOT-NEXT: vadd.vv v8, v9, v8895; NODOT-NEXT: ret896;897; DOT-LABEL: partial_reduce_vqdotu:898; DOT: # %bb.0: # %entry899; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma900; DOT-NEXT: vmv.v.i v10, 0901; DOT-NEXT: vqdotu.vv v10, v8, v9902; DOT-NEXT: vmv1r.v v8, v10903; DOT-NEXT: ret904entry:905 %a.sext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>906 %b.sext = zext <vscale x 4 x i8> %b to <vscale x 4 x i32>907 %mul = mul <vscale x 4 x i32> %a.sext, %b.sext908 %res = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add(<vscale x 1 x i32> zeroinitializer, <vscale x 4 x i32> %mul)909 ret <vscale x 1 x i32> %res910}911 912define <vscale x 1 x i32> @partial_reduce_vqdotsu(<vscale x 4 x i8> %a, <vscale x 4 x i8> %b) {913; NODOT-LABEL: partial_reduce_vqdotsu:914; NODOT: # %bb.0: # %entry915; NODOT-NEXT: vsetvli a0, zero, e16, m1, ta, ma916; NODOT-NEXT: vsext.vf2 v10, v8917; NODOT-NEXT: vzext.vf2 v11, v9918; NODOT-NEXT: csrr a0, vlenb919; NODOT-NEXT: vwmulsu.vv v8, v10, v11920; NODOT-NEXT: srli a0, a0, 3921; NODOT-NEXT: vsetvli a1, zero, e32, m1, ta, ma922; NODOT-NEXT: vslidedown.vx v10, v9, a0923; NODOT-NEXT: vslidedown.vx v11, v8, a0924; NODOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma925; NODOT-NEXT: vadd.vv v8, v10, v8926; NODOT-NEXT: vadd.vv v9, v11, v9927; NODOT-NEXT: vadd.vv v8, v9, v8928; NODOT-NEXT: ret929;930; DOT-LABEL: partial_reduce_vqdotsu:931; DOT: # %bb.0: # %entry932; DOT-NEXT: vsetvli a0, zero, e32, mf2, ta, ma933; DOT-NEXT: vmv.v.i v10, 0934; DOT-NEXT: vqdotsu.vv v10, v8, v9935; DOT-NEXT: vmv1r.v v8, v10936; DOT-NEXT: ret937entry:938 %a.sext = sext <vscale x 4 x i8> %a to <vscale x 4 x i32>939 %b.sext = zext <vscale x 4 x i8> %b to <vscale x 4 x i32>940 %mul = mul <vscale x 4 x i32> %a.sext, %b.sext941 %res = call <vscale x 1 x i32> @llvm.vector.partial.reduce.add(<vscale x 1 x i32> zeroinitializer, <vscale x 4 x i32> %mul)942 ret <vscale x 1 x i32> %res943}944 945 946define <vscale x 4 x i32> @partial_of_sext(<vscale x 16 x i8> %a) {947; NODOT-LABEL: partial_of_sext:948; NODOT: # %bb.0: # %entry949; NODOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma950; NODOT-NEXT: vsext.vf4 v16, v8951; NODOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma952; NODOT-NEXT: vadd.vv v8, v22, v16953; NODOT-NEXT: vadd.vv v10, v18, v20954; NODOT-NEXT: vadd.vv v8, v10, v8955; NODOT-NEXT: ret956;957; DOT-LABEL: partial_of_sext:958; DOT: # %bb.0: # %entry959; DOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma960; DOT-NEXT: vmv.v.i v12, 1961; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma962; DOT-NEXT: vmv.v.i v10, 0963; DOT-NEXT: vqdot.vv v10, v8, v12964; DOT-NEXT: vmv.v.v v8, v10965; DOT-NEXT: ret966entry:967 %a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i32>968 %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i32> %a.ext)969 ret <vscale x 4 x i32> %res970}971 972define <vscale x 4 x i32> @partial_of_zext(<vscale x 16 x i8> %a) {973; NODOT-LABEL: partial_of_zext:974; NODOT: # %bb.0: # %entry975; NODOT-NEXT: vsetvli a0, zero, e32, m8, ta, ma976; NODOT-NEXT: vzext.vf4 v16, v8977; NODOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma978; NODOT-NEXT: vadd.vv v8, v22, v16979; NODOT-NEXT: vadd.vv v10, v18, v20980; NODOT-NEXT: vadd.vv v8, v10, v8981; NODOT-NEXT: ret982;983; DOT-LABEL: partial_of_zext:984; DOT: # %bb.0: # %entry985; DOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma986; DOT-NEXT: vmv.v.i v12, 1987; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma988; DOT-NEXT: vmv.v.i v10, 0989; DOT-NEXT: vqdotu.vv v10, v8, v12990; DOT-NEXT: vmv.v.v v8, v10991; DOT-NEXT: ret992entry:993 %a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i32>994 %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i32> %a.ext)995 ret <vscale x 4 x i32> %res996}997 998define <vscale x 2 x i32> @partial_reduce_select(<vscale x 8 x i8> %a, <vscale x 8 x i8> %b, <vscale x 8 x i1> %m) {999; NODOT-LABEL: partial_reduce_select:1000; NODOT: # %bb.0: # %entry1001; NODOT-NEXT: vsetvli a0, zero, e16, m2, ta, ma1002; NODOT-NEXT: vsext.vf2 v12, v81003; NODOT-NEXT: vsext.vf2 v14, v91004; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma1005; NODOT-NEXT: vmv.v.i v8, 01006; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, mu1007; NODOT-NEXT: vwmul.vv v8, v12, v14, v0.t1008; NODOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma1009; NODOT-NEXT: vadd.vv v8, v11, v81010; NODOT-NEXT: vadd.vv v9, v9, v101011; NODOT-NEXT: vadd.vv v8, v9, v81012; NODOT-NEXT: ret1013;1014; DOT-LABEL: partial_reduce_select:1015; DOT: # %bb.0: # %entry1016; DOT-NEXT: vsetvli a0, zero, e8, m1, ta, ma1017; DOT-NEXT: vmv.v.i v10, 01018; DOT-NEXT: vmerge.vvm v10, v10, v9, v01019; DOT-NEXT: vsetvli a0, zero, e32, m1, ta, ma1020; DOT-NEXT: vmv.v.i v9, 01021; DOT-NEXT: vqdot.vv v9, v8, v101022; DOT-NEXT: vmv.v.v v8, v91023; DOT-NEXT: ret1024entry:1025 %a.sext = sext <vscale x 8 x i8> %a to <vscale x 8 x i32>1026 %b.sext = sext <vscale x 8 x i8> %b to <vscale x 8 x i32>1027 %mul = mul <vscale x 8 x i32> %a.sext, %b.sext1028 %sel = select <vscale x 8 x i1> %m, <vscale x 8 x i32> %mul, <vscale x 8 x i32> zeroinitializer1029 %res = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add(<vscale x 2 x i32> zeroinitializer, <vscale x 8 x i32> %sel)1030 ret <vscale x 2 x i32> %res1031}1032 1033;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:1034; DOT32: {{.*}}1035; DOT64: {{.*}}1036