1532 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK3 4define arm_aapcs_vfpcc i32 @add_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y) {5; CHECK-LABEL: add_v4i32_v4i32:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: vmlav.u32 r0, q0, q18; CHECK-NEXT: bx lr9entry:10 %m = mul <4 x i32> %x, %y11 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)12 ret i32 %z13}14 15define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {16; CHECK-LABEL: add_v4i32_v4i64_zext:17; CHECK: @ %bb.0: @ %entry18; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q119; CHECK-NEXT: bx lr20entry:21 %xx = zext <4 x i32> %x to <4 x i64>22 %yy = zext <4 x i32> %y to <4 x i64>23 %m = mul <4 x i64> %xx, %yy24 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)25 ret i64 %z26}27 28define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y) {29; CHECK-LABEL: add_v4i32_v4i64_sext:30; CHECK: @ %bb.0: @ %entry31; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q132; CHECK-NEXT: bx lr33entry:34 %xx = sext <4 x i32> %x to <4 x i64>35 %yy = sext <4 x i32> %y to <4 x i64>36 %m = mul <4 x i64> %xx, %yy37 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)38 ret i64 %z39}40 41define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) {42; CHECK-LABEL: add_v2i32_v2i64_zext:43; CHECK: @ %bb.0: @ %entry44; CHECK-NEXT: vmullb.u32 q2, q0, q145; CHECK-NEXT: vmov r0, r1, d546; CHECK-NEXT: vmov r2, r3, d447; CHECK-NEXT: adds r0, r0, r248; CHECK-NEXT: adcs r1, r349; CHECK-NEXT: bx lr50entry:51 %xx = zext <2 x i32> %x to <2 x i64>52 %yy = zext <2 x i32> %y to <2 x i64>53 %m = mul <2 x i64> %xx, %yy54 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)55 ret i64 %z56}57 58define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) {59; CHECK-LABEL: add_v2i32_v2i64_sext:60; CHECK: @ %bb.0: @ %entry61; CHECK-NEXT: vmullb.s32 q2, q0, q162; CHECK-NEXT: vmov r0, r1, d563; CHECK-NEXT: vmov r2, r3, d464; CHECK-NEXT: adds r0, r0, r265; CHECK-NEXT: adcs r1, r366; CHECK-NEXT: bx lr67entry:68 %xx = sext <2 x i32> %x to <2 x i64>69 %yy = sext <2 x i32> %y to <2 x i64>70 %m = mul <2 x i64> %xx, %yy71 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)72 ret i64 %z73}74 75define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y) {76; CHECK-LABEL: add_v8i16_v8i32_zext:77; CHECK: @ %bb.0: @ %entry78; CHECK-NEXT: vmlav.u16 r0, q0, q179; CHECK-NEXT: bx lr80entry:81 %xx = zext <8 x i16> %x to <8 x i32>82 %yy = zext <8 x i16> %y to <8 x i32>83 %m = mul <8 x i32> %xx, %yy84 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)85 ret i32 %z86}87 88define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y) {89; CHECK-LABEL: add_v8i16_v8i32_sext:90; CHECK: @ %bb.0: @ %entry91; CHECK-NEXT: vmlav.s16 r0, q0, q192; CHECK-NEXT: bx lr93entry:94 %xx = sext <8 x i16> %x to <8 x i32>95 %yy = sext <8 x i16> %y to <8 x i32>96 %m = mul <8 x i32> %xx, %yy97 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)98 ret i32 %z99}100 101define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y) {102; CHECK-LABEL: add_v4i16_v4i32_zext:103; CHECK: @ %bb.0: @ %entry104; CHECK-NEXT: vmovlb.u16 q1, q1105; CHECK-NEXT: vmovlb.u16 q0, q0106; CHECK-NEXT: vmlav.u32 r0, q0, q1107; CHECK-NEXT: bx lr108entry:109 %xx = zext <4 x i16> %x to <4 x i32>110 %yy = zext <4 x i16> %y to <4 x i32>111 %m = mul <4 x i32> %xx, %yy112 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)113 ret i32 %z114}115 116define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y) {117; CHECK-LABEL: add_v4i16_v4i32_sext:118; CHECK: @ %bb.0: @ %entry119; CHECK-NEXT: vmovlb.s16 q1, q1120; CHECK-NEXT: vmovlb.s16 q0, q0121; CHECK-NEXT: vmlav.u32 r0, q0, q1122; CHECK-NEXT: bx lr123entry:124 %xx = sext <4 x i16> %x to <4 x i32>125 %yy = sext <4 x i16> %y to <4 x i32>126 %m = mul <4 x i32> %xx, %yy127 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)128 ret i32 %z129}130 131define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y) {132; CHECK-LABEL: add_v8i16_v8i16:133; CHECK: @ %bb.0: @ %entry134; CHECK-NEXT: vmlav.u16 r0, q0, q1135; CHECK-NEXT: uxth r0, r0136; CHECK-NEXT: bx lr137entry:138 %m = mul <8 x i16> %x, %y139 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)140 ret i16 %z141}142 143define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y) {144; CHECK-LABEL: add_v8i16_v8i64_zext:145; CHECK: @ %bb.0: @ %entry146; CHECK-NEXT: vmlalv.u16 r0, r1, q0, q1147; CHECK-NEXT: bx lr148entry:149 %xx = zext <8 x i16> %x to <8 x i64>150 %yy = zext <8 x i16> %y to <8 x i64>151 %m = mul <8 x i64> %xx, %yy152 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)153 ret i64 %z154}155 156define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y) {157; CHECK-LABEL: add_v8i16_v8i64_sext:158; CHECK: @ %bb.0: @ %entry159; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q1160; CHECK-NEXT: bx lr161entry:162 %xx = sext <8 x i16> %x to <8 x i64>163 %yy = sext <8 x i16> %y to <8 x i64>164 %m = mul <8 x i64> %xx, %yy165 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)166 ret i64 %z167}168 169define arm_aapcs_vfpcc i64 @add_v8i8i16_v8i64_zext(<8 x i16> %x, <8 x i8> %y) {170; CHECK-LABEL: add_v8i8i16_v8i64_zext:171; CHECK: @ %bb.0: @ %entry172; CHECK-NEXT: vmovlb.u8 q1, q1173; CHECK-NEXT: vmlalv.u16 r0, r1, q0, q1174; CHECK-NEXT: bx lr175entry:176 %xx = zext <8 x i16> %x to <8 x i64>177 %yy = zext <8 x i8> %y to <8 x i64>178 %m = mul <8 x i64> %xx, %yy179 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)180 ret i64 %z181}182 183define arm_aapcs_vfpcc i64 @add_v8i8i16_v8i64_sext(<8 x i16> %x, <8 x i8> %y) {184; CHECK-LABEL: add_v8i8i16_v8i64_sext:185; CHECK: @ %bb.0: @ %entry186; CHECK-NEXT: vmovlb.s8 q1, q1187; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q1188; CHECK-NEXT: bx lr189entry:190 %xx = sext <8 x i16> %x to <8 x i64>191 %yy = sext <8 x i8> %y to <8 x i64>192 %m = mul <8 x i64> %xx, %yy193 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)194 ret i64 %z195}196 197define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) {198; CHECK-LABEL: add_v4i16_v4i64_zext:199; CHECK: @ %bb.0: @ %entry200; CHECK-NEXT: vmovlb.u16 q1, q1201; CHECK-NEXT: vmovlb.u16 q0, q0202; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1203; CHECK-NEXT: bx lr204entry:205 %xx = zext <4 x i16> %x to <4 x i64>206 %yy = zext <4 x i16> %y to <4 x i64>207 %m = mul <4 x i64> %xx, %yy208 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)209 ret i64 %z210}211 212define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) {213; CHECK-LABEL: add_v4i16_v4i64_sext:214; CHECK: @ %bb.0: @ %entry215; CHECK-NEXT: vmovlb.s16 q1, q1216; CHECK-NEXT: vmovlb.s16 q0, q0217; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1218; CHECK-NEXT: bx lr219entry:220 %xx = sext <4 x i16> %x to <4 x i64>221 %yy = sext <4 x i16> %y to <4 x i64>222 %m = mul <4 x i64> %xx, %yy223 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)224 ret i64 %z225}226 227define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_zext(<8 x i16> %x, <8 x i16> %y) {228; CHECK-LABEL: add_v8i16_v8i32_v8i64_zext:229; CHECK: @ %bb.0: @ %entry230; CHECK-NEXT: vmlalv.u16 r0, r1, q0, q1231; CHECK-NEXT: bx lr232entry:233 %xx = zext <8 x i16> %x to <8 x i32>234 %yy = zext <8 x i16> %y to <8 x i32>235 %m = mul <8 x i32> %xx, %yy236 %ma = zext <8 x i32> %m to <8 x i64>237 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)238 ret i64 %z239}240 241define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_sext(<8 x i16> %x, <8 x i16> %y) {242; CHECK-LABEL: add_v8i16_v8i32_v8i64_sext:243; CHECK: @ %bb.0: @ %entry244; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q1245; CHECK-NEXT: bx lr246entry:247 %xx = sext <8 x i16> %x to <8 x i32>248 %yy = sext <8 x i16> %y to <8 x i32>249 %m = mul <8 x i32> %xx, %yy250 %ma = sext <8 x i32> %m to <8 x i64>251 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)252 ret i64 %z253}254 255define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_sextzext(<8 x i16> %x, <8 x i16> %y) {256; CHECK-LABEL: add_v8i16_v8i32_v8i64_sextzext:257; CHECK: @ %bb.0: @ %entry258; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q0259; CHECK-NEXT: bx lr260entry:261 %xx = sext <8 x i16> %x to <8 x i32>262 %m = mul <8 x i32> %xx, %xx263 %ma = zext <8 x i32> %m to <8 x i64>264 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)265 ret i64 %z266}267 268define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) {269; CHECK-LABEL: add_v2i16_v2i64_zext:270; CHECK: @ %bb.0: @ %entry271; CHECK-NEXT: vmov.i64 q2, #0xffff272; CHECK-NEXT: vand q1, q1, q2273; CHECK-NEXT: vand q0, q0, q2274; CHECK-NEXT: vmov r0, s6275; CHECK-NEXT: vmov r1, s2276; CHECK-NEXT: vmov r2, s4277; CHECK-NEXT: vmov r3, s0278; CHECK-NEXT: umull r0, r1, r1, r0279; CHECK-NEXT: umlal r0, r1, r3, r2280; CHECK-NEXT: bx lr281entry:282 %xx = zext <2 x i16> %x to <2 x i64>283 %yy = zext <2 x i16> %y to <2 x i64>284 %m = mul <2 x i64> %xx, %yy285 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)286 ret i64 %z287}288 289define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y) {290; CHECK-LABEL: add_v2i16_v2i64_sext:291; CHECK: @ %bb.0: @ %entry292; CHECK-NEXT: vmov r0, s6293; CHECK-NEXT: vmov r1, s2294; CHECK-NEXT: vmov r2, s4295; CHECK-NEXT: vmov r3, s0296; CHECK-NEXT: sxth r0, r0297; CHECK-NEXT: sxth r1, r1298; CHECK-NEXT: smull r0, r1, r1, r0299; CHECK-NEXT: sxth r2, r2300; CHECK-NEXT: sxth r3, r3301; CHECK-NEXT: smlal r0, r1, r3, r2302; CHECK-NEXT: bx lr303entry:304 %xx = sext <2 x i16> %x to <2 x i64>305 %yy = sext <2 x i16> %y to <2 x i64>306 %m = mul <2 x i64> %xx, %yy307 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)308 ret i64 %z309}310 311define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y) {312; CHECK-LABEL: add_v16i8_v16i32_zext:313; CHECK: @ %bb.0: @ %entry314; CHECK-NEXT: vmlav.u8 r0, q0, q1315; CHECK-NEXT: bx lr316entry:317 %xx = zext <16 x i8> %x to <16 x i32>318 %yy = zext <16 x i8> %y to <16 x i32>319 %m = mul <16 x i32> %xx, %yy320 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m)321 ret i32 %z322}323 324define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y) {325; CHECK-LABEL: add_v16i8_v16i32_sext:326; CHECK: @ %bb.0: @ %entry327; CHECK-NEXT: vmlav.s8 r0, q0, q1328; CHECK-NEXT: bx lr329entry:330 %xx = sext <16 x i8> %x to <16 x i32>331 %yy = sext <16 x i8> %y to <16 x i32>332 %m = mul <16 x i32> %xx, %yy333 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m)334 ret i32 %z335}336 337define arm_aapcs_vfpcc i32 @add_v8i8_v8i32_zext(<8 x i8> %x, <8 x i8> %y) {338; CHECK-LABEL: add_v8i8_v8i32_zext:339; CHECK: @ %bb.0: @ %entry340; CHECK-NEXT: vmovlb.u8 q1, q1341; CHECK-NEXT: vmovlb.u8 q0, q0342; CHECK-NEXT: vmlav.u16 r0, q0, q1343; CHECK-NEXT: bx lr344entry:345 %xx = zext <8 x i8> %x to <8 x i32>346 %yy = zext <8 x i8> %y to <8 x i32>347 %m = mul <8 x i32> %xx, %yy348 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)349 ret i32 %z350}351 352define arm_aapcs_vfpcc i32 @add_v8i8_v8i32_sext(<8 x i8> %x, <8 x i8> %y) {353; CHECK-LABEL: add_v8i8_v8i32_sext:354; CHECK: @ %bb.0: @ %entry355; CHECK-NEXT: vmovlb.s8 q1, q1356; CHECK-NEXT: vmovlb.s8 q0, q0357; CHECK-NEXT: vmlav.s16 r0, q0, q1358; CHECK-NEXT: bx lr359entry:360 %xx = sext <8 x i8> %x to <8 x i32>361 %yy = sext <8 x i8> %y to <8 x i32>362 %m = mul <8 x i32> %xx, %yy363 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)364 ret i32 %z365}366 367define arm_aapcs_vfpcc i32 @add_v8i8i16_v8i32_zext(<8 x i8> %x, <8 x i16> %y) {368; CHECK-LABEL: add_v8i8i16_v8i32_zext:369; CHECK: @ %bb.0: @ %entry370; CHECK-NEXT: vmovlb.u8 q0, q0371; CHECK-NEXT: vmlav.u16 r0, q0, q1372; CHECK-NEXT: bx lr373entry:374 %xx = zext <8 x i8> %x to <8 x i32>375 %yy = zext <8 x i16> %y to <8 x i32>376 %m = mul <8 x i32> %xx, %yy377 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)378 ret i32 %z379}380 381define arm_aapcs_vfpcc i32 @add_v8i8i16_v8i32_sext(<8 x i8> %x, <8 x i16> %y) {382; CHECK-LABEL: add_v8i8i16_v8i32_sext:383; CHECK: @ %bb.0: @ %entry384; CHECK-NEXT: vmovlb.s8 q0, q0385; CHECK-NEXT: vmlav.s16 r0, q0, q1386; CHECK-NEXT: bx lr387entry:388 %xx = sext <8 x i8> %x to <8 x i32>389 %yy = sext <8 x i16> %y to <8 x i32>390 %m = mul <8 x i32> %xx, %yy391 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)392 ret i32 %z393}394 395define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_zext(<16 x i8> %x, <16 x i8> %y) {396; CHECK-LABEL: add_v16i8_v16i16_v16i32_zext:397; CHECK: @ %bb.0: @ %entry398; CHECK-NEXT: vmlav.u8 r0, q0, q1399; CHECK-NEXT: bx lr400entry:401 %xx = zext <16 x i8> %x to <16 x i16>402 %yy = zext <16 x i8> %y to <16 x i16>403 %m = mul <16 x i16> %xx, %yy404 %ma = zext <16 x i16> %m to <16 x i32>405 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)406 ret i32 %z407}408 409define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_sext(<16 x i8> %x, <16 x i8> %y) {410; CHECK-LABEL: add_v16i8_v16i16_v16i32_sext:411; CHECK: @ %bb.0: @ %entry412; CHECK-NEXT: vmlav.s8 r0, q0, q1413; CHECK-NEXT: bx lr414entry:415 %xx = sext <16 x i8> %x to <16 x i16>416 %yy = sext <16 x i8> %y to <16 x i16>417 %m = mul <16 x i16> %xx, %yy418 %ma = sext <16 x i16> %m to <16 x i32>419 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)420 ret i32 %z421}422 423define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_sextzext(<16 x i8> %x, <16 x i8> %y) {424; CHECK-LABEL: add_v16i8_v16i16_v16i32_sextzext:425; CHECK: @ %bb.0: @ %entry426; CHECK-NEXT: vmlav.s8 r0, q0, q0427; CHECK-NEXT: bx lr428entry:429 %xx = sext <16 x i8> %x to <16 x i16>430 %m = mul <16 x i16> %xx, %xx431 %ma = zext <16 x i16> %m to <16 x i32>432 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)433 ret i32 %z434}435 436define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y) {437; CHECK-LABEL: add_v4i8_v4i32_zext:438; CHECK: @ %bb.0: @ %entry439; CHECK-NEXT: vmov.i32 q2, #0xff440; CHECK-NEXT: vand q1, q1, q2441; CHECK-NEXT: vand q0, q0, q2442; CHECK-NEXT: vmlav.u32 r0, q0, q1443; CHECK-NEXT: bx lr444entry:445 %xx = zext <4 x i8> %x to <4 x i32>446 %yy = zext <4 x i8> %y to <4 x i32>447 %m = mul <4 x i32> %xx, %yy448 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)449 ret i32 %z450}451 452define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y) {453; CHECK-LABEL: add_v4i8_v4i32_sext:454; CHECK: @ %bb.0: @ %entry455; CHECK-NEXT: vmovlb.s8 q1, q1456; CHECK-NEXT: vmovlb.s8 q0, q0457; CHECK-NEXT: vmovlb.s16 q1, q1458; CHECK-NEXT: vmovlb.s16 q0, q0459; CHECK-NEXT: vmlav.u32 r0, q0, q1460; CHECK-NEXT: bx lr461entry:462 %xx = sext <4 x i8> %x to <4 x i32>463 %yy = sext <4 x i8> %y to <4 x i32>464 %m = mul <4 x i32> %xx, %yy465 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)466 ret i32 %z467}468 469define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_szext(<4 x i8> %x, <4 x i8> %y) {470; CHECK-LABEL: add_v4i8_v4i32_szext:471; CHECK: @ %bb.0: @ %entry472; CHECK-NEXT: vmovlb.s8 q0, q0473; CHECK-NEXT: vmov.i32 q2, #0xff474; CHECK-NEXT: vand q1, q1, q2475; CHECK-NEXT: vmovlb.s16 q0, q0476; CHECK-NEXT: vmlav.u32 r0, q0, q1477; CHECK-NEXT: bx lr478entry:479 %xx = sext <4 x i8> %x to <4 x i32>480 %yy = zext <4 x i8> %y to <4 x i32>481 %m = mul <4 x i32> %xx, %yy482 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)483 ret i32 %z484}485 486define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y) {487; CHECK-LABEL: add_v16i8_v16i16_zext:488; CHECK: @ %bb.0: @ %entry489; CHECK-NEXT: vmlav.u8 r0, q0, q1490; CHECK-NEXT: uxth r0, r0491; CHECK-NEXT: bx lr492entry:493 %xx = zext <16 x i8> %x to <16 x i16>494 %yy = zext <16 x i8> %y to <16 x i16>495 %m = mul <16 x i16> %xx, %yy496 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %m)497 ret i16 %z498}499 500define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y) {501; CHECK-LABEL: add_v16i8_v16i16_sext:502; CHECK: @ %bb.0: @ %entry503; CHECK-NEXT: vmlav.s8 r0, q0, q1504; CHECK-NEXT: sxth r0, r0505; CHECK-NEXT: bx lr506entry:507 %xx = sext <16 x i8> %x to <16 x i16>508 %yy = sext <16 x i8> %y to <16 x i16>509 %m = mul <16 x i16> %xx, %yy510 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %m)511 ret i16 %z512}513 514define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_szext(<16 x i8> %x, <16 x i8> %y) {515; CHECK-LABEL: add_v16i8_v16i16_szext:516; CHECK: @ %bb.0: @ %entry517; CHECK-NEXT: .pad #32518; CHECK-NEXT: sub sp, #32519; CHECK-NEXT: add r0, sp, #16520; CHECK-NEXT: mov r1, sp521; CHECK-NEXT: vstrw.32 q1, [r0]522; CHECK-NEXT: vstrw.32 q0, [r1]523; CHECK-NEXT: vldrb.u16 q0, [r0, #8]524; CHECK-NEXT: vldrb.s16 q1, [r1, #8]525; CHECK-NEXT: vmlav.u16 r2, q1, q0526; CHECK-NEXT: vldrb.u16 q0, [r0]527; CHECK-NEXT: vldrb.s16 q1, [r1]528; CHECK-NEXT: vmlava.u16 r2, q1, q0529; CHECK-NEXT: sxth r0, r2530; CHECK-NEXT: add sp, #32531; CHECK-NEXT: bx lr532entry:533 %xx = sext <16 x i8> %x to <16 x i16>534 %yy = zext <16 x i8> %y to <16 x i16>535 %m = mul <16 x i16> %xx, %yy536 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %m)537 ret i16 %z538}539 540define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) {541; CHECK-LABEL: add_v8i8_v8i16_zext:542; CHECK: @ %bb.0: @ %entry543; CHECK-NEXT: vmovlb.u8 q1, q1544; CHECK-NEXT: vmovlb.u8 q0, q0545; CHECK-NEXT: vmlav.u16 r0, q0, q1546; CHECK-NEXT: uxth r0, r0547; CHECK-NEXT: bx lr548entry:549 %xx = zext <8 x i8> %x to <8 x i16>550 %yy = zext <8 x i8> %y to <8 x i16>551 %m = mul <8 x i16> %xx, %yy552 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)553 ret i16 %z554}555 556define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) {557; CHECK-LABEL: add_v8i8_v8i16_sext:558; CHECK: @ %bb.0: @ %entry559; CHECK-NEXT: vmovlb.s8 q1, q1560; CHECK-NEXT: vmovlb.s8 q0, q0561; CHECK-NEXT: vmlav.u16 r0, q0, q1562; CHECK-NEXT: sxth r0, r0563; CHECK-NEXT: bx lr564entry:565 %xx = sext <8 x i8> %x to <8 x i16>566 %yy = sext <8 x i8> %y to <8 x i16>567 %m = mul <8 x i16> %xx, %yy568 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)569 ret i16 %z570}571 572define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y) {573; CHECK-LABEL: add_v16i8_v16i8:574; CHECK: @ %bb.0: @ %entry575; CHECK-NEXT: vmlav.u8 r0, q0, q1576; CHECK-NEXT: uxtb r0, r0577; CHECK-NEXT: bx lr578entry:579 %m = mul <16 x i8> %x, %y580 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %m)581 ret i8 %z582}583 584define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {585; CHECK-LABEL: add_v16i8_v16i64_zext:586; CHECK: @ %bb.0: @ %entry587; CHECK-NEXT: .pad #32588; CHECK-NEXT: sub sp, #32589; CHECK-NEXT: add r2, sp, #16590; CHECK-NEXT: mov r3, sp591; CHECK-NEXT: vstrw.32 q1, [r2]592; CHECK-NEXT: vstrw.32 q0, [r3]593; CHECK-NEXT: vldrb.u16 q0, [r2]594; CHECK-NEXT: vldrb.u16 q1, [r3]595; CHECK-NEXT: vmlalv.u16 r0, r1, q1, q0596; CHECK-NEXT: vldrb.u16 q0, [r2, #8]597; CHECK-NEXT: vldrb.u16 q1, [r3, #8]598; CHECK-NEXT: vmlalva.u16 r0, r1, q1, q0599; CHECK-NEXT: add sp, #32600; CHECK-NEXT: bx lr601entry:602 %xx = zext <16 x i8> %x to <16 x i64>603 %yy = zext <16 x i8> %y to <16 x i64>604 %m = mul <16 x i64> %xx, %yy605 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)606 ret i64 %z607}608 609define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) {610; CHECK-LABEL: add_v16i8_v16i64_sext:611; CHECK: @ %bb.0: @ %entry612; CHECK-NEXT: .pad #32613; CHECK-NEXT: sub sp, #32614; CHECK-NEXT: add r2, sp, #16615; CHECK-NEXT: mov r3, sp616; CHECK-NEXT: vstrw.32 q1, [r2]617; CHECK-NEXT: vstrw.32 q0, [r3]618; CHECK-NEXT: vldrb.s16 q0, [r2]619; CHECK-NEXT: vldrb.s16 q1, [r3]620; CHECK-NEXT: vmlalv.s16 r0, r1, q1, q0621; CHECK-NEXT: vldrb.s16 q0, [r2, #8]622; CHECK-NEXT: vldrb.s16 q1, [r3, #8]623; CHECK-NEXT: vmlalva.s16 r0, r1, q1, q0624; CHECK-NEXT: add sp, #32625; CHECK-NEXT: bx lr626entry:627 %xx = sext <16 x i8> %x to <16 x i64>628 %yy = sext <16 x i8> %y to <16 x i64>629 %m = mul <16 x i64> %xx, %yy630 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)631 ret i64 %z632}633 634define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext_load(ptr %xp, ptr %yp) {635; CHECK-LABEL: add_v16i8_v16i64_zext_load:636; CHECK: @ %bb.0: @ %entry637; CHECK-NEXT: vldrb.u16 q0, [r1]638; CHECK-NEXT: vldrb.u16 q1, [r0]639; CHECK-NEXT: vmlalv.u16 r2, r3, q1, q0640; CHECK-NEXT: vldrb.u16 q0, [r1, #8]641; CHECK-NEXT: vldrb.u16 q1, [r0, #8]642; CHECK-NEXT: vmlalva.u16 r2, r3, q1, q0643; CHECK-NEXT: mov r0, r2644; CHECK-NEXT: mov r1, r3645; CHECK-NEXT: bx lr646entry:647 %x = load <16 x i8>, ptr %xp648 %y = load <16 x i8>, ptr %yp649 %xx = zext <16 x i8> %x to <16 x i64>650 %yy = zext <16 x i8> %y to <16 x i64>651 %m = mul <16 x i64> %xx, %yy652 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)653 ret i64 %z654}655 656define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext_load(ptr %xp, ptr %yp) {657; CHECK-LABEL: add_v16i8_v16i64_sext_load:658; CHECK: @ %bb.0: @ %entry659; CHECK-NEXT: vldrb.s16 q0, [r1]660; CHECK-NEXT: vldrb.s16 q1, [r0]661; CHECK-NEXT: vmlalv.s16 r2, r3, q1, q0662; CHECK-NEXT: vldrb.s16 q0, [r1, #8]663; CHECK-NEXT: vldrb.s16 q1, [r0, #8]664; CHECK-NEXT: vmlalva.s16 r2, r3, q1, q0665; CHECK-NEXT: mov r0, r2666; CHECK-NEXT: mov r1, r3667; CHECK-NEXT: bx lr668entry:669 %x = load <16 x i8>, ptr %xp670 %y = load <16 x i8>, ptr %yp671 %xx = sext <16 x i8> %x to <16 x i64>672 %yy = sext <16 x i8> %y to <16 x i64>673 %m = mul <16 x i64> %xx, %yy674 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)675 ret i64 %z676}677 678define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) {679; CHECK-LABEL: add_v8i8_v8i64_zext:680; CHECK: @ %bb.0: @ %entry681; CHECK-NEXT: vmovlb.u8 q1, q1682; CHECK-NEXT: vmovlb.u8 q0, q0683; CHECK-NEXT: vmlalv.u16 r0, r1, q0, q1684; CHECK-NEXT: bx lr685entry:686 %xx = zext <8 x i8> %x to <8 x i64>687 %yy = zext <8 x i8> %y to <8 x i64>688 %m = mul <8 x i64> %xx, %yy689 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)690 ret i64 %z691}692 693define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) {694; CHECK-LABEL: add_v8i8_v8i64_sext:695; CHECK: @ %bb.0: @ %entry696; CHECK-NEXT: vmovlb.s8 q1, q1697; CHECK-NEXT: vmovlb.s8 q0, q0698; CHECK-NEXT: vmlalv.s16 r0, r1, q0, q1699; CHECK-NEXT: bx lr700entry:701 %xx = sext <8 x i8> %x to <8 x i64>702 %yy = sext <8 x i8> %y to <8 x i64>703 %m = mul <8 x i64> %xx, %yy704 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)705 ret i64 %z706}707 708define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) {709; CHECK-LABEL: add_v4i8_v4i64_zext:710; CHECK: @ %bb.0: @ %entry711; CHECK-NEXT: vmov.i32 q2, #0xff712; CHECK-NEXT: vand q1, q1, q2713; CHECK-NEXT: vand q0, q0, q2714; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1715; CHECK-NEXT: bx lr716entry:717 %xx = zext <4 x i8> %x to <4 x i64>718 %yy = zext <4 x i8> %y to <4 x i64>719 %m = mul <4 x i64> %xx, %yy720 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)721 ret i64 %z722}723 724define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) {725; CHECK-LABEL: add_v4i8_v4i64_sext:726; CHECK: @ %bb.0: @ %entry727; CHECK-NEXT: vmovlb.s8 q1, q1728; CHECK-NEXT: vmovlb.s8 q0, q0729; CHECK-NEXT: vmovlb.s16 q1, q1730; CHECK-NEXT: vmovlb.s16 q0, q0731; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1732; CHECK-NEXT: bx lr733entry:734 %xx = sext <4 x i8> %x to <4 x i64>735 %yy = sext <4 x i8> %y to <4 x i64>736 %m = mul <4 x i64> %xx, %yy737 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)738 ret i64 %z739}740 741define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i64_zext(<4 x i8> %x, <4 x i16> %y) {742; CHECK-LABEL: add_v4i8i16_v4i64_zext:743; CHECK: @ %bb.0: @ %entry744; CHECK-NEXT: vmov.i32 q2, #0xff745; CHECK-NEXT: vmovlb.u16 q1, q1746; CHECK-NEXT: vand q0, q0, q2747; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1748; CHECK-NEXT: bx lr749entry:750 %xx = zext <4 x i8> %x to <4 x i64>751 %yy = zext <4 x i16> %y to <4 x i64>752 %m = mul <4 x i64> %xx, %yy753 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)754 ret i64 %z755}756 757define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i64_sext(<4 x i8> %x, <4 x i16> %y) {758; CHECK-LABEL: add_v4i8i16_v4i64_sext:759; CHECK: @ %bb.0: @ %entry760; CHECK-NEXT: vmovlb.s8 q0, q0761; CHECK-NEXT: vmovlb.s16 q1, q1762; CHECK-NEXT: vmovlb.s16 q0, q0763; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1764; CHECK-NEXT: bx lr765entry:766 %xx = sext <4 x i8> %x to <4 x i64>767 %yy = sext <4 x i16> %y to <4 x i64>768 %m = mul <4 x i64> %xx, %yy769 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)770 ret i64 %z771}772 773define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i32_v4i64_zext(<4 x i8> %x, <4 x i16> %y) {774; CHECK-LABEL: add_v4i8i16_v4i32_v4i64_zext:775; CHECK: @ %bb.0: @ %entry776; CHECK-NEXT: vmov.i32 q2, #0xff777; CHECK-NEXT: vmovlb.u16 q1, q1778; CHECK-NEXT: vand q0, q0, q2779; CHECK-NEXT: vmlalv.u32 r0, r1, q0, q1780; CHECK-NEXT: bx lr781entry:782 %xx = zext <4 x i8> %x to <4 x i32>783 %yy = zext <4 x i16> %y to <4 x i32>784 %mm = mul <4 x i32> %xx, %yy785 %m = zext <4 x i32> %mm to <4 x i64>786 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)787 ret i64 %z788}789 790define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i32_v4i64_sext(<4 x i8> %x, <4 x i16> %y) {791; CHECK-LABEL: add_v4i8i16_v4i32_v4i64_sext:792; CHECK: @ %bb.0: @ %entry793; CHECK-NEXT: vmovlb.s8 q0, q0794; CHECK-NEXT: vmovlb.s16 q1, q1795; CHECK-NEXT: vmovlb.s16 q0, q0796; CHECK-NEXT: vmlalv.s32 r0, r1, q0, q1797; CHECK-NEXT: bx lr798entry:799 %xx = sext <4 x i8> %x to <4 x i32>800 %yy = sext <4 x i16> %y to <4 x i32>801 %mm = mul <4 x i32> %xx, %yy802 %m = sext <4 x i32> %mm to <4 x i64>803 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)804 ret i64 %z805}806 807define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) {808; CHECK-LABEL: add_v2i8_v2i64_zext:809; CHECK: @ %bb.0: @ %entry810; CHECK-NEXT: vmov.i64 q2, #0xff811; CHECK-NEXT: vand q1, q1, q2812; CHECK-NEXT: vand q0, q0, q2813; CHECK-NEXT: vmov r0, s6814; CHECK-NEXT: vmov r1, s2815; CHECK-NEXT: vmov r2, s4816; CHECK-NEXT: vmov r3, s0817; CHECK-NEXT: umull r0, r1, r1, r0818; CHECK-NEXT: umull r2, r3, r3, r2819; CHECK-NEXT: add r0, r2820; CHECK-NEXT: orrs r1, r3821; CHECK-NEXT: bx lr822entry:823 %xx = zext <2 x i8> %x to <2 x i64>824 %yy = zext <2 x i8> %y to <2 x i64>825 %m = mul <2 x i64> %xx, %yy826 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)827 ret i64 %z828}829 830define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y) {831; CHECK-LABEL: add_v2i8_v2i64_sext:832; CHECK: @ %bb.0: @ %entry833; CHECK-NEXT: vmov r0, s6834; CHECK-NEXT: vmov r1, s2835; CHECK-NEXT: vmov r2, s4836; CHECK-NEXT: vmov r3, s0837; CHECK-NEXT: sxtb r0, r0838; CHECK-NEXT: sxtb r1, r1839; CHECK-NEXT: smull r0, r1, r1, r0840; CHECK-NEXT: sxtb r2, r2841; CHECK-NEXT: sxtb r3, r3842; CHECK-NEXT: smlal r0, r1, r3, r2843; CHECK-NEXT: bx lr844entry:845 %xx = sext <2 x i8> %x to <2 x i64>846 %yy = sext <2 x i8> %y to <2 x i64>847 %m = mul <2 x i64> %xx, %yy848 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)849 ret i64 %z850}851 852define arm_aapcs_vfpcc i64 @add_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y) {853; CHECK-LABEL: add_v2i64_v2i64:854; CHECK: @ %bb.0: @ %entry855; CHECK-NEXT: .save {r4, r5, r7, lr}856; CHECK-NEXT: push {r4, r5, r7, lr}857; CHECK-NEXT: vmov r0, lr, d3858; CHECK-NEXT: vmov r2, r3, d1859; CHECK-NEXT: umull r12, r1, r2, r0860; CHECK-NEXT: mla r1, r2, lr, r1861; CHECK-NEXT: mla lr, r3, r0, r1862; CHECK-NEXT: vmov r0, r2, d2863; CHECK-NEXT: vmov r3, r1, d0864; CHECK-NEXT: umull r4, r5, r3, r0865; CHECK-NEXT: mla r2, r3, r2, r5866; CHECK-NEXT: mla r1, r1, r0, r2867; CHECK-NEXT: adds.w r0, r4, r12868; CHECK-NEXT: adc.w r1, r1, lr869; CHECK-NEXT: pop {r4, r5, r7, pc}870entry:871 %m = mul <2 x i64> %x, %y872 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)873 ret i64 %z874}875 876define arm_aapcs_vfpcc i32 @add_v4i32_v4i32_acc(<4 x i32> %x, <4 x i32> %y, i32 %a) {877; CHECK-LABEL: add_v4i32_v4i32_acc:878; CHECK: @ %bb.0: @ %entry879; CHECK-NEXT: vmlava.u32 r0, q0, q1880; CHECK-NEXT: bx lr881entry:882 %m = mul <4 x i32> %x, %y883 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)884 %r = add i32 %z, %a885 ret i32 %r886}887 888define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, <4 x i32> %y, i64 %a) {889; CHECK-LABEL: add_v4i32_v4i64_acc_zext:890; CHECK: @ %bb.0: @ %entry891; CHECK-NEXT: vmlalva.u32 r0, r1, q0, q1892; CHECK-NEXT: bx lr893entry:894 %xx = zext <4 x i32> %x to <4 x i64>895 %yy = zext <4 x i32> %y to <4 x i64>896 %m = mul <4 x i64> %xx, %yy897 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)898 %r = add i64 %z, %a899 ret i64 %r900}901 902define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, <4 x i32> %y, i64 %a) {903; CHECK-LABEL: add_v4i32_v4i64_acc_sext:904; CHECK: @ %bb.0: @ %entry905; CHECK-NEXT: vmlalva.s32 r0, r1, q0, q1906; CHECK-NEXT: bx lr907entry:908 %xx = sext <4 x i32> %x to <4 x i64>909 %yy = sext <4 x i32> %y to <4 x i64>910 %m = mul <4 x i64> %xx, %yy911 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %m)912 %r = add i64 %z, %a913 ret i64 %r914}915 916define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, <2 x i32> %y, i64 %a) {917; CHECK-LABEL: add_v2i32_v2i64_acc_zext:918; CHECK: @ %bb.0: @ %entry919; CHECK-NEXT: .save {r7, lr}920; CHECK-NEXT: push {r7, lr}921; CHECK-NEXT: vmullb.u32 q2, q0, q1922; CHECK-NEXT: vmov lr, r12, d5923; CHECK-NEXT: vmov r3, r2, d4924; CHECK-NEXT: adds.w r3, r3, lr925; CHECK-NEXT: adc.w r2, r2, r12926; CHECK-NEXT: adds r0, r0, r3927; CHECK-NEXT: adcs r1, r2928; CHECK-NEXT: pop {r7, pc}929entry:930 %xx = zext <2 x i32> %x to <2 x i64>931 %yy = zext <2 x i32> %y to <2 x i64>932 %m = mul <2 x i64> %xx, %yy933 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)934 %r = add i64 %z, %a935 ret i64 %r936}937 938define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, <2 x i32> %y, i64 %a) {939; CHECK-LABEL: add_v2i32_v2i64_acc_sext:940; CHECK: @ %bb.0: @ %entry941; CHECK-NEXT: .save {r7, lr}942; CHECK-NEXT: push {r7, lr}943; CHECK-NEXT: vmullb.s32 q2, q0, q1944; CHECK-NEXT: vmov lr, r12, d5945; CHECK-NEXT: vmov r3, r2, d4946; CHECK-NEXT: adds.w r3, r3, lr947; CHECK-NEXT: adc.w r2, r2, r12948; CHECK-NEXT: adds r0, r0, r3949; CHECK-NEXT: adcs r1, r2950; CHECK-NEXT: pop {r7, pc}951entry:952 %xx = sext <2 x i32> %x to <2 x i64>953 %yy = sext <2 x i32> %y to <2 x i64>954 %m = mul <2 x i64> %xx, %yy955 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)956 %r = add i64 %z, %a957 ret i64 %r958}959 960define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, <8 x i16> %y, i32 %a) {961; CHECK-LABEL: add_v8i16_v8i32_acc_zext:962; CHECK: @ %bb.0: @ %entry963; CHECK-NEXT: vmlava.u16 r0, q0, q1964; CHECK-NEXT: bx lr965entry:966 %xx = zext <8 x i16> %x to <8 x i32>967 %yy = zext <8 x i16> %y to <8 x i32>968 %m = mul <8 x i32> %xx, %yy969 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)970 %r = add i32 %z, %a971 ret i32 %r972}973 974define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, <8 x i16> %y, i32 %a) {975; CHECK-LABEL: add_v8i16_v8i32_acc_sext:976; CHECK: @ %bb.0: @ %entry977; CHECK-NEXT: vmlava.s16 r0, q0, q1978; CHECK-NEXT: bx lr979entry:980 %xx = sext <8 x i16> %x to <8 x i32>981 %yy = sext <8 x i16> %y to <8 x i32>982 %m = mul <8 x i32> %xx, %yy983 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m)984 %r = add i32 %z, %a985 ret i32 %r986}987 988define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, <4 x i16> %y, i32 %a) {989; CHECK-LABEL: add_v4i16_v4i32_acc_zext:990; CHECK: @ %bb.0: @ %entry991; CHECK-NEXT: vmovlb.u16 q1, q1992; CHECK-NEXT: vmovlb.u16 q0, q0993; CHECK-NEXT: vmlava.u32 r0, q0, q1994; CHECK-NEXT: bx lr995entry:996 %xx = zext <4 x i16> %x to <4 x i32>997 %yy = zext <4 x i16> %y to <4 x i32>998 %m = mul <4 x i32> %xx, %yy999 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)1000 %r = add i32 %z, %a1001 ret i32 %r1002}1003 1004define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, <4 x i16> %y, i32 %a) {1005; CHECK-LABEL: add_v4i16_v4i32_acc_sext:1006; CHECK: @ %bb.0: @ %entry1007; CHECK-NEXT: vmovlb.s16 q1, q11008; CHECK-NEXT: vmovlb.s16 q0, q01009; CHECK-NEXT: vmlava.u32 r0, q0, q11010; CHECK-NEXT: bx lr1011entry:1012 %xx = sext <4 x i16> %x to <4 x i32>1013 %yy = sext <4 x i16> %y to <4 x i32>1014 %m = mul <4 x i32> %xx, %yy1015 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)1016 %r = add i32 %z, %a1017 ret i32 %r1018}1019 1020define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, <8 x i16> %y, i16 %a) {1021; CHECK-LABEL: add_v8i16_v8i16_acc:1022; CHECK: @ %bb.0: @ %entry1023; CHECK-NEXT: vmlava.u16 r0, q0, q11024; CHECK-NEXT: uxth r0, r01025; CHECK-NEXT: bx lr1026entry:1027 %m = mul <8 x i16> %x, %y1028 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)1029 %r = add i16 %z, %a1030 ret i16 %r1031}1032 1033define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %y, i64 %a) {1034; CHECK-LABEL: add_v8i16_v8i64_acc_zext:1035; CHECK: @ %bb.0: @ %entry1036; CHECK-NEXT: vmlalva.u16 r0, r1, q0, q11037; CHECK-NEXT: bx lr1038entry:1039 %xx = zext <8 x i16> %x to <8 x i64>1040 %yy = zext <8 x i16> %y to <8 x i64>1041 %m = mul <8 x i64> %xx, %yy1042 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)1043 %r = add i64 %z, %a1044 ret i64 %r1045}1046 1047define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %y, i64 %a) {1048; CHECK-LABEL: add_v8i16_v8i64_acc_sext:1049; CHECK: @ %bb.0: @ %entry1050; CHECK-NEXT: vmlalva.s16 r0, r1, q0, q11051; CHECK-NEXT: bx lr1052entry:1053 %xx = sext <8 x i16> %x to <8 x i64>1054 %yy = sext <8 x i16> %y to <8 x i64>1055 %m = mul <8 x i64> %xx, %yy1056 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %m)1057 %r = add i64 %z, %a1058 ret i64 %r1059}1060 1061define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %y, i64 %a) {1062; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_zext:1063; CHECK: @ %bb.0: @ %entry1064; CHECK-NEXT: vmlalva.u16 r0, r1, q0, q11065; CHECK-NEXT: bx lr1066entry:1067 %xx = zext <8 x i16> %x to <8 x i32>1068 %yy = zext <8 x i16> %y to <8 x i32>1069 %m = mul <8 x i32> %xx, %yy1070 %ma = zext <8 x i32> %m to <8 x i64>1071 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)1072 %r = add i64 %z, %a1073 ret i64 %r1074}1075 1076define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %y, i64 %a) {1077; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_sext:1078; CHECK: @ %bb.0: @ %entry1079; CHECK-NEXT: vmlalva.s16 r0, r1, q0, q11080; CHECK-NEXT: bx lr1081entry:1082 %xx = sext <8 x i16> %x to <8 x i32>1083 %yy = sext <8 x i16> %y to <8 x i32>1084 %m = mul <8 x i32> %xx, %yy1085 %ma = sext <8 x i32> %m to <8 x i64>1086 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)1087 %r = add i64 %z, %a1088 ret i64 %r1089}1090 1091define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_sextzext(<8 x i16> %x, <8 x i16> %y, i64 %a) {1092; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_sextzext:1093; CHECK: @ %bb.0: @ %entry1094; CHECK-NEXT: vmlalva.s16 r0, r1, q0, q01095; CHECK-NEXT: bx lr1096entry:1097 %xx = sext <8 x i16> %x to <8 x i32>1098 %m = mul <8 x i32> %xx, %xx1099 %ma = zext <8 x i32> %m to <8 x i64>1100 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %ma)1101 %r = add i64 %z, %a1102 ret i64 %r1103}1104 1105define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %y, i64 %a) {1106; CHECK-LABEL: add_v2i16_v2i64_acc_zext:1107; CHECK: @ %bb.0: @ %entry1108; CHECK-NEXT: .save {r7, lr}1109; CHECK-NEXT: push {r7, lr}1110; CHECK-NEXT: vmov.i64 q2, #0xffff1111; CHECK-NEXT: vand q1, q1, q21112; CHECK-NEXT: vand q0, q0, q21113; CHECK-NEXT: vmov r2, s61114; CHECK-NEXT: vmov r3, s21115; CHECK-NEXT: vmov r12, s41116; CHECK-NEXT: umull r2, lr, r3, r21117; CHECK-NEXT: vmov r3, s01118; CHECK-NEXT: umlal r2, lr, r3, r121119; CHECK-NEXT: adds r0, r0, r21120; CHECK-NEXT: adc.w r1, r1, lr1121; CHECK-NEXT: pop {r7, pc}1122entry:1123 %xx = zext <2 x i16> %x to <2 x i64>1124 %yy = zext <2 x i16> %y to <2 x i64>1125 %m = mul <2 x i64> %xx, %yy1126 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)1127 %r = add i64 %z, %a1128 ret i64 %r1129}1130 1131define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %y, i64 %a) {1132; CHECK-LABEL: add_v2i16_v2i64_acc_sext:1133; CHECK: @ %bb.0: @ %entry1134; CHECK-NEXT: .save {r7, lr}1135; CHECK-NEXT: push {r7, lr}1136; CHECK-NEXT: vmov r2, s61137; CHECK-NEXT: vmov r3, s21138; CHECK-NEXT: sxth r2, r21139; CHECK-NEXT: sxth r3, r31140; CHECK-NEXT: smull r2, r12, r3, r21141; CHECK-NEXT: vmov r3, s41142; CHECK-NEXT: sxth.w lr, r31143; CHECK-NEXT: vmov r3, s01144; CHECK-NEXT: sxth r3, r31145; CHECK-NEXT: smlal r2, r12, r3, lr1146; CHECK-NEXT: adds r0, r0, r21147; CHECK-NEXT: adc.w r1, r1, r121148; CHECK-NEXT: pop {r7, pc}1149entry:1150 %xx = sext <2 x i16> %x to <2 x i64>1151 %yy = sext <2 x i16> %y to <2 x i64>1152 %m = mul <2 x i64> %xx, %yy1153 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)1154 %r = add i64 %z, %a1155 ret i64 %r1156}1157 1158define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %y, i32 %a) {1159; CHECK-LABEL: add_v16i8_v16i32_acc_zext:1160; CHECK: @ %bb.0: @ %entry1161; CHECK-NEXT: vmlava.u8 r0, q0, q11162; CHECK-NEXT: bx lr1163entry:1164 %xx = zext <16 x i8> %x to <16 x i32>1165 %yy = zext <16 x i8> %y to <16 x i32>1166 %m = mul <16 x i32> %xx, %yy1167 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m)1168 %r = add i32 %z, %a1169 ret i32 %r1170}1171 1172define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %y, i32 %a) {1173; CHECK-LABEL: add_v16i8_v16i32_acc_sext:1174; CHECK: @ %bb.0: @ %entry1175; CHECK-NEXT: vmlava.s8 r0, q0, q11176; CHECK-NEXT: bx lr1177entry:1178 %xx = sext <16 x i8> %x to <16 x i32>1179 %yy = sext <16 x i8> %y to <16 x i32>1180 %m = mul <16 x i32> %xx, %yy1181 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m)1182 %r = add i32 %z, %a1183 ret i32 %r1184}1185 1186define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %y, i32 %a) {1187; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_zext:1188; CHECK: @ %bb.0: @ %entry1189; CHECK-NEXT: vmlava.u8 r0, q0, q11190; CHECK-NEXT: bx lr1191entry:1192 %xx = zext <16 x i8> %x to <16 x i16>1193 %yy = zext <16 x i8> %y to <16 x i16>1194 %m = mul <16 x i16> %xx, %yy1195 %ma = zext <16 x i16> %m to <16 x i32>1196 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)1197 %r = add i32 %z, %a1198 ret i32 %r1199}1200 1201define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %y, i32 %a) {1202; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_sext:1203; CHECK: @ %bb.0: @ %entry1204; CHECK-NEXT: vmlava.s8 r0, q0, q11205; CHECK-NEXT: bx lr1206entry:1207 %xx = sext <16 x i8> %x to <16 x i16>1208 %yy = sext <16 x i8> %y to <16 x i16>1209 %m = mul <16 x i16> %xx, %yy1210 %ma = sext <16 x i16> %m to <16 x i32>1211 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)1212 %r = add i32 %z, %a1213 ret i32 %r1214}1215 1216define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_sextzext(<16 x i8> %x, i32 %a) {1217; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_sextzext:1218; CHECK: @ %bb.0: @ %entry1219; CHECK-NEXT: vmlava.s8 r0, q0, q01220; CHECK-NEXT: bx lr1221entry:1222 %xx = sext <16 x i8> %x to <16 x i16>1223 %m = mul <16 x i16> %xx, %xx1224 %ma = zext <16 x i16> %m to <16 x i32>1225 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %ma)1226 %r = add i32 %z, %a1227 ret i32 %r1228}1229 1230define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, <4 x i8> %y, i32 %a) {1231; CHECK-LABEL: add_v4i8_v4i32_acc_zext:1232; CHECK: @ %bb.0: @ %entry1233; CHECK-NEXT: vmov.i32 q2, #0xff1234; CHECK-NEXT: vand q1, q1, q21235; CHECK-NEXT: vand q0, q0, q21236; CHECK-NEXT: vmlava.u32 r0, q0, q11237; CHECK-NEXT: bx lr1238entry:1239 %xx = zext <4 x i8> %x to <4 x i32>1240 %yy = zext <4 x i8> %y to <4 x i32>1241 %m = mul <4 x i32> %xx, %yy1242 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)1243 %r = add i32 %z, %a1244 ret i32 %r1245}1246 1247define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, <4 x i8> %y, i32 %a) {1248; CHECK-LABEL: add_v4i8_v4i32_acc_sext:1249; CHECK: @ %bb.0: @ %entry1250; CHECK-NEXT: vmovlb.s8 q1, q11251; CHECK-NEXT: vmovlb.s8 q0, q01252; CHECK-NEXT: vmovlb.s16 q1, q11253; CHECK-NEXT: vmovlb.s16 q0, q01254; CHECK-NEXT: vmlava.u32 r0, q0, q11255; CHECK-NEXT: bx lr1256entry:1257 %xx = sext <4 x i8> %x to <4 x i32>1258 %yy = sext <4 x i8> %y to <4 x i32>1259 %m = mul <4 x i32> %xx, %yy1260 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m)1261 %r = add i32 %z, %a1262 ret i32 %r1263}1264 1265define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16 x i8> %y, i16 %a) {1266; CHECK-LABEL: add_v16i8_v16i16_acc_zext:1267; CHECK: @ %bb.0: @ %entry1268; CHECK-NEXT: vmlava.u8 r0, q0, q11269; CHECK-NEXT: uxth r0, r01270; CHECK-NEXT: bx lr1271entry:1272 %xx = zext <16 x i8> %x to <16 x i16>1273 %yy = zext <16 x i8> %y to <16 x i16>1274 %m = mul <16 x i16> %xx, %yy1275 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %m)1276 %r = add i16 %z, %a1277 ret i16 %r1278}1279 1280define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16 x i8> %y, i16 %a) {1281; CHECK-LABEL: add_v16i8_v16i16_acc_sext:1282; CHECK: @ %bb.0: @ %entry1283; CHECK-NEXT: vmlava.s8 r0, q0, q11284; CHECK-NEXT: sxth r0, r01285; CHECK-NEXT: bx lr1286entry:1287 %xx = sext <16 x i8> %x to <16 x i16>1288 %yy = sext <16 x i8> %y to <16 x i16>1289 %m = mul <16 x i16> %xx, %yy1290 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %m)1291 %r = add i16 %z, %a1292 ret i16 %r1293}1294 1295define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, <8 x i8> %y, i16 %a) {1296; CHECK-LABEL: add_v8i8_v8i16_acc_zext:1297; CHECK: @ %bb.0: @ %entry1298; CHECK-NEXT: vmovlb.u8 q1, q11299; CHECK-NEXT: vmovlb.u8 q0, q01300; CHECK-NEXT: vmlava.u16 r0, q0, q11301; CHECK-NEXT: uxth r0, r01302; CHECK-NEXT: bx lr1303entry:1304 %xx = zext <8 x i8> %x to <8 x i16>1305 %yy = zext <8 x i8> %y to <8 x i16>1306 %m = mul <8 x i16> %xx, %yy1307 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)1308 %r = add i16 %z, %a1309 ret i16 %r1310}1311 1312define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, <8 x i8> %y, i16 %a) {1313; CHECK-LABEL: add_v8i8_v8i16_acc_sext:1314; CHECK: @ %bb.0: @ %entry1315; CHECK-NEXT: vmovlb.s8 q1, q11316; CHECK-NEXT: vmovlb.s8 q0, q01317; CHECK-NEXT: vmlava.u16 r0, q0, q11318; CHECK-NEXT: sxth r0, r01319; CHECK-NEXT: bx lr1320entry:1321 %xx = sext <8 x i8> %x to <8 x i16>1322 %yy = sext <8 x i8> %y to <8 x i16>1323 %m = mul <8 x i16> %xx, %yy1324 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %m)1325 %r = add i16 %z, %a1326 ret i16 %r1327}1328 1329define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, <16 x i8> %y, i8 %a) {1330; CHECK-LABEL: add_v16i8_v16i8_acc:1331; CHECK: @ %bb.0: @ %entry1332; CHECK-NEXT: vmlava.u8 r0, q0, q11333; CHECK-NEXT: uxtb r0, r01334; CHECK-NEXT: bx lr1335entry:1336 %m = mul <16 x i8> %x, %y1337 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %m)1338 %r = add i8 %z, %a1339 ret i8 %r1340}1341 1342define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y, i64 %a) {1343; CHECK-LABEL: add_v16i8_v16i64_acc_zext:1344; CHECK: @ %bb.0: @ %entry1345; CHECK-NEXT: .pad #321346; CHECK-NEXT: sub sp, #321347; CHECK-NEXT: add r2, sp, #161348; CHECK-NEXT: mov r3, sp1349; CHECK-NEXT: vstrw.32 q1, [r2]1350; CHECK-NEXT: vstrw.32 q0, [r3]1351; CHECK-NEXT: vldrb.u16 q0, [r2]1352; CHECK-NEXT: vldrb.u16 q1, [r3]1353; CHECK-NEXT: vmlalva.u16 r0, r1, q1, q01354; CHECK-NEXT: vldrb.u16 q0, [r2, #8]1355; CHECK-NEXT: vldrb.u16 q1, [r3, #8]1356; CHECK-NEXT: vmlalva.u16 r0, r1, q1, q01357; CHECK-NEXT: add sp, #321358; CHECK-NEXT: bx lr1359entry:1360 %xx = zext <16 x i8> %x to <16 x i64>1361 %yy = zext <16 x i8> %y to <16 x i64>1362 %m = mul <16 x i64> %xx, %yy1363 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)1364 %r = add i64 %z, %a1365 ret i64 %r1366}1367 1368define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %y, i64 %a) {1369; CHECK-LABEL: add_v16i8_v16i64_acc_sext:1370; CHECK: @ %bb.0: @ %entry1371; CHECK-NEXT: .pad #321372; CHECK-NEXT: sub sp, #321373; CHECK-NEXT: add r2, sp, #161374; CHECK-NEXT: mov r3, sp1375; CHECK-NEXT: vstrw.32 q1, [r2]1376; CHECK-NEXT: vstrw.32 q0, [r3]1377; CHECK-NEXT: vldrb.s16 q0, [r2]1378; CHECK-NEXT: vldrb.s16 q1, [r3]1379; CHECK-NEXT: vmlalva.s16 r0, r1, q1, q01380; CHECK-NEXT: vldrb.s16 q0, [r2, #8]1381; CHECK-NEXT: vldrb.s16 q1, [r3, #8]1382; CHECK-NEXT: vmlalva.s16 r0, r1, q1, q01383; CHECK-NEXT: add sp, #321384; CHECK-NEXT: bx lr1385entry:1386 %xx = sext <16 x i8> %x to <16 x i64>1387 %yy = sext <16 x i8> %y to <16 x i64>1388 %m = mul <16 x i64> %xx, %yy1389 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)1390 %r = add i64 %z, %a1391 ret i64 %r1392}1393 1394define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext_load(ptr %xp, ptr %yp, i64 %a) {1395; CHECK-LABEL: add_v16i8_v16i64_acc_zext_load:1396; CHECK: @ %bb.0: @ %entry1397; CHECK-NEXT: vldrb.u16 q0, [r1]1398; CHECK-NEXT: vldrb.u16 q1, [r0]1399; CHECK-NEXT: vmlalva.u16 r2, r3, q1, q01400; CHECK-NEXT: vldrb.u16 q0, [r1, #8]1401; CHECK-NEXT: vldrb.u16 q1, [r0, #8]1402; CHECK-NEXT: vmlalva.u16 r2, r3, q1, q01403; CHECK-NEXT: mov r0, r21404; CHECK-NEXT: mov r1, r31405; CHECK-NEXT: bx lr1406entry:1407 %x = load <16 x i8>, ptr %xp1408 %y = load <16 x i8>, ptr %yp1409 %xx = zext <16 x i8> %x to <16 x i64>1410 %yy = zext <16 x i8> %y to <16 x i64>1411 %m = mul <16 x i64> %xx, %yy1412 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)1413 %r = add i64 %z, %a1414 ret i64 %r1415}1416 1417define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext_load(ptr %xp, ptr %yp, i64 %a) {1418; CHECK-LABEL: add_v16i8_v16i64_acc_sext_load:1419; CHECK: @ %bb.0: @ %entry1420; CHECK-NEXT: vldrb.s16 q0, [r1]1421; CHECK-NEXT: vldrb.s16 q1, [r0]1422; CHECK-NEXT: vmlalva.s16 r2, r3, q1, q01423; CHECK-NEXT: vldrb.s16 q0, [r1, #8]1424; CHECK-NEXT: vldrb.s16 q1, [r0, #8]1425; CHECK-NEXT: vmlalva.s16 r2, r3, q1, q01426; CHECK-NEXT: mov r0, r21427; CHECK-NEXT: mov r1, r31428; CHECK-NEXT: bx lr1429entry:1430 %x = load <16 x i8>, ptr %xp1431 %y = load <16 x i8>, ptr %yp1432 %xx = sext <16 x i8> %x to <16 x i64>1433 %yy = sext <16 x i8> %y to <16 x i64>1434 %m = mul <16 x i64> %xx, %yy1435 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %m)1436 %r = add i64 %z, %a1437 ret i64 %r1438}1439 1440define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %y, i64 %a) {1441; CHECK-LABEL: add_v2i8_v2i64_acc_zext:1442; CHECK: @ %bb.0: @ %entry1443; CHECK-NEXT: .save {r7, lr}1444; CHECK-NEXT: push {r7, lr}1445; CHECK-NEXT: vmov.i64 q2, #0xff1446; CHECK-NEXT: vand q1, q1, q21447; CHECK-NEXT: vand q0, q0, q21448; CHECK-NEXT: vmov r2, s61449; CHECK-NEXT: vmov r3, s21450; CHECK-NEXT: umull r12, lr, r3, r21451; CHECK-NEXT: vmov r2, s41452; CHECK-NEXT: vmov r3, s01453; CHECK-NEXT: umull r2, r3, r3, r21454; CHECK-NEXT: add r2, r121455; CHECK-NEXT: orr.w r3, r3, lr1456; CHECK-NEXT: adds r0, r0, r21457; CHECK-NEXT: adcs r1, r31458; CHECK-NEXT: pop {r7, pc}1459entry:1460 %xx = zext <2 x i8> %x to <2 x i64>1461 %yy = zext <2 x i8> %y to <2 x i64>1462 %m = mul <2 x i64> %xx, %yy1463 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)1464 %r = add i64 %z, %a1465 ret i64 %r1466}1467 1468define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %y, i64 %a) {1469; CHECK-LABEL: add_v2i8_v2i64_acc_sext:1470; CHECK: @ %bb.0: @ %entry1471; CHECK-NEXT: .save {r7, lr}1472; CHECK-NEXT: push {r7, lr}1473; CHECK-NEXT: vmov r2, s61474; CHECK-NEXT: vmov r3, s21475; CHECK-NEXT: sxtb r2, r21476; CHECK-NEXT: sxtb r3, r31477; CHECK-NEXT: smull r2, r12, r3, r21478; CHECK-NEXT: vmov r3, s41479; CHECK-NEXT: sxtb.w lr, r31480; CHECK-NEXT: vmov r3, s01481; CHECK-NEXT: sxtb r3, r31482; CHECK-NEXT: smlal r2, r12, r3, lr1483; CHECK-NEXT: adds r0, r0, r21484; CHECK-NEXT: adc.w r1, r1, r121485; CHECK-NEXT: pop {r7, pc}1486entry:1487 %xx = sext <2 x i8> %x to <2 x i64>1488 %yy = sext <2 x i8> %y to <2 x i64>1489 %m = mul <2 x i64> %xx, %yy1490 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)1491 %r = add i64 %z, %a1492 ret i64 %r1493}1494 1495define arm_aapcs_vfpcc i64 @add_v2i64_v2i64_acc(<2 x i64> %x, <2 x i64> %y, i64 %a) {1496; CHECK-LABEL: add_v2i64_v2i64_acc:1497; CHECK: @ %bb.0: @ %entry1498; CHECK-NEXT: .save {r4, r5, r6, r7, r8, lr}1499; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, lr}1500; CHECK-NEXT: vmov r2, r12, d31501; CHECK-NEXT: vmov r3, lr, d11502; CHECK-NEXT: vmov r4, r6, d01503; CHECK-NEXT: umull r8, r5, r3, r21504; CHECK-NEXT: mla r3, r3, r12, r51505; CHECK-NEXT: mla r12, lr, r2, r31506; CHECK-NEXT: vmov r3, r5, d21507; CHECK-NEXT: umull r7, r2, r4, r31508; CHECK-NEXT: mla r2, r4, r5, r21509; CHECK-NEXT: mla r2, r6, r3, r21510; CHECK-NEXT: adds.w r3, r7, r81511; CHECK-NEXT: adc.w r2, r2, r121512; CHECK-NEXT: adds r0, r0, r31513; CHECK-NEXT: adcs r1, r21514; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, pc}1515entry:1516 %m = mul <2 x i64> %x, %y1517 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %m)1518 %r = add i64 %z, %a1519 ret i64 %r1520}1521 1522declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)1523declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)1524declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)1525declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)1526declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)1527declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)1528declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)1529declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)1530declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)1531declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)1532