3007 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK3 4define arm_aapcs_vfpcc i32 @add_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {5; CHECK-LABEL: add_v4i32_v4i32:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: vpt.i32 eq, q2, zr8; CHECK-NEXT: vmlavt.u32 r0, q0, q19; CHECK-NEXT: bx lr10entry:11 %c = icmp eq <4 x i32> %b, zeroinitializer12 %m = mul <4 x i32> %x, %y13 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer14 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)15 ret i32 %z16}17 18define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {19; CHECK-LABEL: add_v4i32_v4i64_zext:20; CHECK: @ %bb.0: @ %entry21; CHECK-NEXT: vpt.i32 eq, q2, zr22; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q123; CHECK-NEXT: bx lr24entry:25 %c = icmp eq <4 x i32> %b, zeroinitializer26 %xx = zext <4 x i32> %x to <4 x i64>27 %yy = zext <4 x i32> %y to <4 x i64>28 %m = mul <4 x i64> %xx, %yy29 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer30 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)31 ret i64 %z32}33 34define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b) {35; CHECK-LABEL: add_v4i32_v4i64_sext:36; CHECK: @ %bb.0: @ %entry37; CHECK-NEXT: vpt.i32 eq, q2, zr38; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q139; CHECK-NEXT: bx lr40entry:41 %c = icmp eq <4 x i32> %b, zeroinitializer42 %xx = sext <4 x i32> %x to <4 x i64>43 %yy = sext <4 x i32> %y to <4 x i64>44 %m = mul <4 x i64> %xx, %yy45 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer46 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)47 ret i64 %z48}49 50define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b) {51; CHECK-LABEL: add_v2i32_v2i64_zext:52; CHECK: @ %bb.0: @ %entry53; CHECK-NEXT: vmov r0, s854; CHECK-NEXT: movs r1, #055; CHECK-NEXT: vmullb.u32 q3, q0, q156; CHECK-NEXT: vmov.i32 q0, #0x057; CHECK-NEXT: cmp r0, #058; CHECK-NEXT: csetm r0, eq59; CHECK-NEXT: bfi r1, r0, #0, #860; CHECK-NEXT: vmov r0, s1061; CHECK-NEXT: cmp r0, #062; CHECK-NEXT: csetm r0, eq63; CHECK-NEXT: bfi r1, r0, #8, #864; CHECK-NEXT: vmsr p0, r165; CHECK-NEXT: vpsel q0, q3, q066; CHECK-NEXT: vmov r0, r1, d167; CHECK-NEXT: vmov r2, r3, d068; CHECK-NEXT: adds r0, r0, r269; CHECK-NEXT: adcs r1, r370; CHECK-NEXT: bx lr71entry:72 %c = icmp eq <2 x i32> %b, zeroinitializer73 %xx = zext <2 x i32> %x to <2 x i64>74 %yy = zext <2 x i32> %y to <2 x i64>75 %m = mul <2 x i64> %xx, %yy76 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer77 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)78 ret i64 %z79}80 81define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b) {82; CHECK-LABEL: add_v2i32_v2i64_sext:83; CHECK: @ %bb.0: @ %entry84; CHECK-NEXT: vmov r0, s885; CHECK-NEXT: movs r1, #086; CHECK-NEXT: vmullb.s32 q3, q0, q187; CHECK-NEXT: vmov.i32 q0, #0x088; CHECK-NEXT: cmp r0, #089; CHECK-NEXT: csetm r0, eq90; CHECK-NEXT: bfi r1, r0, #0, #891; CHECK-NEXT: vmov r0, s1092; CHECK-NEXT: cmp r0, #093; CHECK-NEXT: csetm r0, eq94; CHECK-NEXT: bfi r1, r0, #8, #895; CHECK-NEXT: vmsr p0, r196; CHECK-NEXT: vpsel q0, q3, q097; CHECK-NEXT: vmov r0, r1, d198; CHECK-NEXT: vmov r2, r3, d099; CHECK-NEXT: adds r0, r0, r2100; CHECK-NEXT: adcs r1, r3101; CHECK-NEXT: bx lr102entry:103 %c = icmp eq <2 x i32> %b, zeroinitializer104 %xx = sext <2 x i32> %x to <2 x i64>105 %yy = sext <2 x i32> %y to <2 x i64>106 %m = mul <2 x i64> %xx, %yy107 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer108 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)109 ret i64 %z110}111 112define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {113; CHECK-LABEL: add_v8i16_v8i32_zext:114; CHECK: @ %bb.0: @ %entry115; CHECK-NEXT: vpt.i16 eq, q2, zr116; CHECK-NEXT: vmlavt.u16 r0, q0, q1117; CHECK-NEXT: bx lr118entry:119 %c = icmp eq <8 x i16> %b, zeroinitializer120 %xx = zext <8 x i16> %x to <8 x i32>121 %yy = zext <8 x i16> %y to <8 x i32>122 %m = mul <8 x i32> %xx, %yy123 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer124 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)125 ret i32 %z126}127 128define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {129; CHECK-LABEL: add_v8i16_v8i32_sext:130; CHECK: @ %bb.0: @ %entry131; CHECK-NEXT: vpt.i16 eq, q2, zr132; CHECK-NEXT: vmlavt.s16 r0, q0, q1133; CHECK-NEXT: bx lr134entry:135 %c = icmp eq <8 x i16> %b, zeroinitializer136 %xx = sext <8 x i16> %x to <8 x i32>137 %yy = sext <8 x i16> %y to <8 x i32>138 %m = mul <8 x i32> %xx, %yy139 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer140 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)141 ret i32 %z142}143 144define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {145; CHECK-LABEL: add_v4i16_v4i32_zext:146; CHECK: @ %bb.0: @ %entry147; CHECK-NEXT: vmovlb.u16 q2, q2148; CHECK-NEXT: vmovlb.u16 q1, q1149; CHECK-NEXT: vmovlb.u16 q0, q0150; CHECK-NEXT: vpt.i32 eq, q2, zr151; CHECK-NEXT: vmlavt.u32 r0, q0, q1152; CHECK-NEXT: bx lr153entry:154 %c = icmp eq <4 x i16> %b, zeroinitializer155 %xx = zext <4 x i16> %x to <4 x i32>156 %yy = zext <4 x i16> %y to <4 x i32>157 %m = mul <4 x i32> %xx, %yy158 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer159 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)160 ret i32 %z161}162 163define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {164; CHECK-LABEL: add_v4i16_v4i32_sext:165; CHECK: @ %bb.0: @ %entry166; CHECK-NEXT: vmovlb.u16 q2, q2167; CHECK-NEXT: vmovlb.s16 q1, q1168; CHECK-NEXT: vmovlb.s16 q0, q0169; CHECK-NEXT: vpt.i32 eq, q2, zr170; CHECK-NEXT: vmlavt.u32 r0, q0, q1171; CHECK-NEXT: bx lr172entry:173 %c = icmp eq <4 x i16> %b, zeroinitializer174 %xx = sext <4 x i16> %x to <4 x i32>175 %yy = sext <4 x i16> %y to <4 x i32>176 %m = mul <4 x i32> %xx, %yy177 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer178 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)179 ret i32 %z180}181 182define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {183; CHECK-LABEL: add_v8i16_v8i16:184; CHECK: @ %bb.0: @ %entry185; CHECK-NEXT: vpt.i16 eq, q2, zr186; CHECK-NEXT: vmlavt.u16 r0, q0, q1187; CHECK-NEXT: uxth r0, r0188; CHECK-NEXT: bx lr189entry:190 %c = icmp eq <8 x i16> %b, zeroinitializer191 %m = mul <8 x i16> %x, %y192 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer193 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)194 ret i16 %z195}196 197define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {198; CHECK-LABEL: add_v8i16_v8i64_zext:199; CHECK: @ %bb.0: @ %entry200; CHECK-NEXT: vpt.i16 eq, q2, zr201; CHECK-NEXT: vmlalvt.u16 r0, r1, q0, q1202; CHECK-NEXT: bx lr203entry:204 %c = icmp eq <8 x i16> %b, zeroinitializer205 %xx = zext <8 x i16> %x to <8 x i64>206 %yy = zext <8 x i16> %y to <8 x i64>207 %m = mul <8 x i64> %xx, %yy208 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer209 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)210 ret i64 %z211}212 213define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {214; CHECK-LABEL: add_v8i16_v8i64_sext:215; CHECK: @ %bb.0: @ %entry216; CHECK-NEXT: vpt.i16 eq, q2, zr217; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q1218; CHECK-NEXT: bx lr219entry:220 %c = icmp eq <8 x i16> %b, zeroinitializer221 %xx = sext <8 x i16> %x to <8 x i64>222 %yy = sext <8 x i16> %y to <8 x i64>223 %m = mul <8 x i64> %xx, %yy224 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer225 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)226 ret i64 %z227}228 229define arm_aapcs_vfpcc i64 @add_v8i8i16_v8i64_zext(<8 x i16> %x, <8 x i8> %y, <8 x i16> %b) {230; CHECK-LABEL: add_v8i8i16_v8i64_zext:231; CHECK: @ %bb.0: @ %entry232; CHECK-NEXT: vmovlb.u8 q1, q1233; CHECK-NEXT: vpt.i16 eq, q2, zr234; CHECK-NEXT: vmlalvt.u16 r0, r1, q0, q1235; CHECK-NEXT: bx lr236entry:237 %c = icmp eq <8 x i16> %b, zeroinitializer238 %xx = zext <8 x i16> %x to <8 x i64>239 %yy = zext <8 x i8> %y to <8 x i64>240 %m = mul <8 x i64> %xx, %yy241 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer242 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)243 ret i64 %z244}245 246define arm_aapcs_vfpcc i64 @add_v8i8i16_v8i64_sext(<8 x i16> %x, <8 x i8> %y, <8 x i16> %b) {247; CHECK-LABEL: add_v8i8i16_v8i64_sext:248; CHECK: @ %bb.0: @ %entry249; CHECK-NEXT: vmovlb.s8 q1, q1250; CHECK-NEXT: vpt.i16 eq, q2, zr251; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q1252; CHECK-NEXT: bx lr253entry:254 %c = icmp eq <8 x i16> %b, zeroinitializer255 %xx = sext <8 x i16> %x to <8 x i64>256 %yy = sext <8 x i8> %y to <8 x i64>257 %m = mul <8 x i64> %xx, %yy258 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer259 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)260 ret i64 %z261}262 263define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {264; CHECK-LABEL: add_v8i16_v8i32_v8i64_zext:265; CHECK: @ %bb.0: @ %entry266; CHECK-NEXT: vpt.i16 eq, q2, zr267; CHECK-NEXT: vmlalvt.u16 r0, r1, q0, q1268; CHECK-NEXT: bx lr269entry:270 %c = icmp eq <8 x i16> %b, zeroinitializer271 %xx = zext <8 x i16> %x to <8 x i32>272 %yy = zext <8 x i16> %y to <8 x i32>273 %m = mul <8 x i32> %xx, %yy274 %ma = zext <8 x i32> %m to <8 x i64>275 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer276 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)277 ret i64 %z278}279 280define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {281; CHECK-LABEL: add_v8i16_v8i32_v8i64_sext:282; CHECK: @ %bb.0: @ %entry283; CHECK-NEXT: vpt.i16 eq, q2, zr284; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q1285; CHECK-NEXT: bx lr286entry:287 %c = icmp eq <8 x i16> %b, zeroinitializer288 %xx = sext <8 x i16> %x to <8 x i32>289 %yy = sext <8 x i16> %y to <8 x i32>290 %m = mul <8 x i32> %xx, %yy291 %ma = sext <8 x i32> %m to <8 x i64>292 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer293 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)294 ret i64 %z295}296 297define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_sextzext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b) {298; CHECK-LABEL: add_v8i16_v8i32_v8i64_sextzext:299; CHECK: @ %bb.0: @ %entry300; CHECK-NEXT: vpt.i16 eq, q2, zr301; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q0302; CHECK-NEXT: bx lr303entry:304 %c = icmp eq <8 x i16> %b, zeroinitializer305 %xx = sext <8 x i16> %x to <8 x i32>306 %m = mul <8 x i32> %xx, %xx307 %ma = zext <8 x i32> %m to <8 x i64>308 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer309 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)310 ret i64 %z311}312 313define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {314; CHECK-LABEL: add_v4i16_v4i64_zext:315; CHECK: @ %bb.0: @ %entry316; CHECK-NEXT: vmovlb.u16 q2, q2317; CHECK-NEXT: vmovlb.u16 q1, q1318; CHECK-NEXT: vmovlb.u16 q0, q0319; CHECK-NEXT: vpt.i32 eq, q2, zr320; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q1321; CHECK-NEXT: bx lr322entry:323 %c = icmp eq <4 x i16> %b, zeroinitializer324 %xx = zext <4 x i16> %x to <4 x i64>325 %yy = zext <4 x i16> %y to <4 x i64>326 %m = mul <4 x i64> %xx, %yy327 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer328 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)329 ret i64 %z330}331 332define arm_aapcs_vfpcc i64 @add_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b) {333; CHECK-LABEL: add_v4i16_v4i64_sext:334; CHECK: @ %bb.0: @ %entry335; CHECK-NEXT: vmovlb.u16 q2, q2336; CHECK-NEXT: vmovlb.s16 q1, q1337; CHECK-NEXT: vmovlb.s16 q0, q0338; CHECK-NEXT: vpt.i32 eq, q2, zr339; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q1340; CHECK-NEXT: bx lr341entry:342 %c = icmp eq <4 x i16> %b, zeroinitializer343 %xx = sext <4 x i16> %x to <4 x i64>344 %yy = sext <4 x i16> %y to <4 x i64>345 %m = mul <4 x i64> %xx, %yy346 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer347 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)348 ret i64 %z349}350 351define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b) {352; CHECK-LABEL: add_v2i16_v2i64_zext:353; CHECK: @ %bb.0: @ %entry354; CHECK-NEXT: vmov.i64 q3, #0xffff355; CHECK-NEXT: vand q1, q1, q3356; CHECK-NEXT: vand q0, q0, q3357; CHECK-NEXT: vmov r0, s6358; CHECK-NEXT: vmov r1, s2359; CHECK-NEXT: vmov r2, s4360; CHECK-NEXT: vand q1, q2, q3361; CHECK-NEXT: vmov r3, s0362; CHECK-NEXT: umull r0, r1, r1, r0363; CHECK-NEXT: umull r2, r3, r3, r2364; CHECK-NEXT: vmov q0[2], q0[0], r2, r0365; CHECK-NEXT: vmov r0, s4366; CHECK-NEXT: vmov q0[3], q0[1], r3, r1367; CHECK-NEXT: movs r1, #0368; CHECK-NEXT: cmp r0, #0369; CHECK-NEXT: csetm r0, eq370; CHECK-NEXT: bfi r1, r0, #0, #8371; CHECK-NEXT: vmov r0, s6372; CHECK-NEXT: vmov.i32 q1, #0x0373; CHECK-NEXT: cmp r0, #0374; CHECK-NEXT: csetm r0, eq375; CHECK-NEXT: bfi r1, r0, #8, #8376; CHECK-NEXT: vmsr p0, r1377; CHECK-NEXT: vpsel q0, q0, q1378; CHECK-NEXT: vmov r0, r1, d1379; CHECK-NEXT: vmov r2, r3, d0380; CHECK-NEXT: adds r0, r0, r2381; CHECK-NEXT: adcs r1, r3382; CHECK-NEXT: bx lr383entry:384 %c = icmp eq <2 x i16> %b, zeroinitializer385 %xx = zext <2 x i16> %x to <2 x i64>386 %yy = zext <2 x i16> %y to <2 x i64>387 %m = mul <2 x i64> %xx, %yy388 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer389 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)390 ret i64 %z391}392 393define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b) {394; CHECK-LABEL: add_v2i16_v2i64_sext:395; CHECK: @ %bb.0: @ %entry396; CHECK-NEXT: vmov.i32 q3, #0xffff397; CHECK-NEXT: movs r1, #0398; CHECK-NEXT: vand q2, q2, q3399; CHECK-NEXT: vmov r2, s4400; CHECK-NEXT: vmov r0, s8401; CHECK-NEXT: vmov r3, s0402; CHECK-NEXT: cmp r0, #0403; CHECK-NEXT: sxth r2, r2404; CHECK-NEXT: csetm r0, eq405; CHECK-NEXT: bfi r1, r0, #0, #8406; CHECK-NEXT: vmov r0, s10407; CHECK-NEXT: sxth r3, r3408; CHECK-NEXT: smull r2, r3, r3, r2409; CHECK-NEXT: cmp r0, #0410; CHECK-NEXT: csetm r0, eq411; CHECK-NEXT: bfi r1, r0, #8, #8412; CHECK-NEXT: vmov r0, s6413; CHECK-NEXT: vmsr p0, r1414; CHECK-NEXT: vmov r1, s2415; CHECK-NEXT: vmov.i32 q1, #0x0416; CHECK-NEXT: sxth r0, r0417; CHECK-NEXT: sxth r1, r1418; CHECK-NEXT: smull r0, r1, r1, r0419; CHECK-NEXT: vmov q0[2], q0[0], r2, r0420; CHECK-NEXT: vmov q0[3], q0[1], r3, r1421; CHECK-NEXT: vpsel q0, q0, q1422; CHECK-NEXT: vmov r0, r1, d1423; CHECK-NEXT: vmov r2, r3, d0424; CHECK-NEXT: adds r0, r0, r2425; CHECK-NEXT: adcs r1, r3426; CHECK-NEXT: bx lr427entry:428 %c = icmp eq <2 x i16> %b, zeroinitializer429 %xx = sext <2 x i16> %x to <2 x i64>430 %yy = sext <2 x i16> %y to <2 x i64>431 %m = mul <2 x i64> %xx, %yy432 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer433 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)434 ret i64 %z435}436 437define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {438; CHECK-LABEL: add_v16i8_v16i32_zext:439; CHECK: @ %bb.0: @ %entry440; CHECK-NEXT: vpt.i8 eq, q2, zr441; CHECK-NEXT: vmlavt.u8 r0, q0, q1442; CHECK-NEXT: bx lr443entry:444 %c = icmp eq <16 x i8> %b, zeroinitializer445 %xx = zext <16 x i8> %x to <16 x i32>446 %yy = zext <16 x i8> %y to <16 x i32>447 %m = mul <16 x i32> %xx, %yy448 %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer449 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)450 ret i32 %z451}452 453define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {454; CHECK-LABEL: add_v16i8_v16i32_sext:455; CHECK: @ %bb.0: @ %entry456; CHECK-NEXT: vpt.i8 eq, q2, zr457; CHECK-NEXT: vmlavt.s8 r0, q0, q1458; CHECK-NEXT: bx lr459entry:460 %c = icmp eq <16 x i8> %b, zeroinitializer461 %xx = sext <16 x i8> %x to <16 x i32>462 %yy = sext <16 x i8> %y to <16 x i32>463 %m = mul <16 x i32> %xx, %yy464 %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer465 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)466 ret i32 %z467}468 469define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {470; CHECK-LABEL: add_v16i8_v16i16_v16i32_zext:471; CHECK: @ %bb.0: @ %entry472; CHECK-NEXT: vpt.i8 eq, q2, zr473; CHECK-NEXT: vmlavt.u8 r0, q0, q1474; CHECK-NEXT: bx lr475entry:476 %c = icmp eq <16 x i8> %b, zeroinitializer477 %xx = zext <16 x i8> %x to <16 x i16>478 %yy = zext <16 x i8> %y to <16 x i16>479 %m = mul <16 x i16> %xx, %yy480 %ma = zext <16 x i16> %m to <16 x i32>481 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer482 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)483 ret i32 %z484}485 486define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {487; CHECK-LABEL: add_v16i8_v16i16_v16i32_sext:488; CHECK: @ %bb.0: @ %entry489; CHECK-NEXT: vpt.i8 eq, q2, zr490; CHECK-NEXT: vmlavt.s8 r0, q0, q1491; CHECK-NEXT: bx lr492entry:493 %c = icmp eq <16 x i8> %b, zeroinitializer494 %xx = sext <16 x i8> %x to <16 x i16>495 %yy = sext <16 x i8> %y to <16 x i16>496 %m = mul <16 x i16> %xx, %yy497 %ma = sext <16 x i16> %m to <16 x i32>498 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer499 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)500 ret i32 %z501}502 503define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_sextzext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {504; CHECK-LABEL: add_v16i8_v16i16_v16i32_sextzext:505; CHECK: @ %bb.0: @ %entry506; CHECK-NEXT: vpt.i8 eq, q2, zr507; CHECK-NEXT: vmlavt.s8 r0, q0, q0508; CHECK-NEXT: bx lr509entry:510 %c = icmp eq <16 x i8> %b, zeroinitializer511 %xx = sext <16 x i8> %x to <16 x i16>512 %m = mul <16 x i16> %xx, %xx513 %ma = zext <16 x i16> %m to <16 x i32>514 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer515 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)516 ret i32 %z517}518 519define arm_aapcs_vfpcc i32 @add_v8i8_v8i32_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {520; CHECK-LABEL: add_v8i8_v8i32_zext:521; CHECK: @ %bb.0: @ %entry522; CHECK-NEXT: vmovlb.u8 q2, q2523; CHECK-NEXT: vmovlb.u8 q1, q1524; CHECK-NEXT: vmovlb.u8 q0, q0525; CHECK-NEXT: vpt.i16 eq, q2, zr526; CHECK-NEXT: vmlavt.u16 r0, q0, q1527; CHECK-NEXT: bx lr528entry:529 %c = icmp eq <8 x i8> %b, zeroinitializer530 %xx = zext <8 x i8> %x to <8 x i32>531 %yy = zext <8 x i8> %y to <8 x i32>532 %m = mul <8 x i32> %xx, %yy533 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer534 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)535 ret i32 %z536}537 538define arm_aapcs_vfpcc i32 @add_v8i8_v8i32_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {539; CHECK-LABEL: add_v8i8_v8i32_sext:540; CHECK: @ %bb.0: @ %entry541; CHECK-NEXT: vmovlb.u8 q2, q2542; CHECK-NEXT: vmovlb.s8 q1, q1543; CHECK-NEXT: vmovlb.s8 q0, q0544; CHECK-NEXT: vpt.i16 eq, q2, zr545; CHECK-NEXT: vmlavt.s16 r0, q0, q1546; CHECK-NEXT: bx lr547entry:548 %c = icmp eq <8 x i8> %b, zeroinitializer549 %xx = sext <8 x i8> %x to <8 x i32>550 %yy = sext <8 x i8> %y to <8 x i32>551 %m = mul <8 x i32> %xx, %yy552 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer553 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)554 ret i32 %z555}556 557define arm_aapcs_vfpcc i32 @add_v8i8i16_v8i32_zext(<8 x i8> %x, <8 x i16> %y, <8 x i8> %b) {558; CHECK-LABEL: add_v8i8i16_v8i32_zext:559; CHECK: @ %bb.0: @ %entry560; CHECK-NEXT: vmovlb.u8 q2, q2561; CHECK-NEXT: vmovlb.u8 q0, q0562; CHECK-NEXT: vpt.i16 eq, q2, zr563; CHECK-NEXT: vmlavt.u16 r0, q0, q1564; CHECK-NEXT: bx lr565entry:566 %c = icmp eq <8 x i8> %b, zeroinitializer567 %xx = zext <8 x i8> %x to <8 x i32>568 %yy = zext <8 x i16> %y to <8 x i32>569 %m = mul <8 x i32> %xx, %yy570 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer571 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)572 ret i32 %z573}574 575define arm_aapcs_vfpcc i32 @add_v8i8i16_v8i32_sext(<8 x i8> %x, <8 x i16> %y, <8 x i8> %b) {576; CHECK-LABEL: add_v8i8i16_v8i32_sext:577; CHECK: @ %bb.0: @ %entry578; CHECK-NEXT: vmovlb.u8 q2, q2579; CHECK-NEXT: vmovlb.s8 q0, q0580; CHECK-NEXT: vpt.i16 eq, q2, zr581; CHECK-NEXT: vmlavt.s16 r0, q0, q1582; CHECK-NEXT: bx lr583entry:584 %c = icmp eq <8 x i8> %b, zeroinitializer585 %xx = sext <8 x i8> %x to <8 x i32>586 %yy = sext <8 x i16> %y to <8 x i32>587 %m = mul <8 x i32> %xx, %yy588 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer589 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)590 ret i32 %z591}592 593define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {594; CHECK-LABEL: add_v4i8_v4i32_zext:595; CHECK: @ %bb.0: @ %entry596; CHECK-NEXT: vmov.i32 q3, #0xff597; CHECK-NEXT: vand q2, q2, q3598; CHECK-NEXT: vand q1, q1, q3599; CHECK-NEXT: vand q0, q0, q3600; CHECK-NEXT: vpt.i32 eq, q2, zr601; CHECK-NEXT: vmlavt.u32 r0, q0, q1602; CHECK-NEXT: bx lr603entry:604 %c = icmp eq <4 x i8> %b, zeroinitializer605 %xx = zext <4 x i8> %x to <4 x i32>606 %yy = zext <4 x i8> %y to <4 x i32>607 %m = mul <4 x i32> %xx, %yy608 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer609 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)610 ret i32 %z611}612 613define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {614; CHECK-LABEL: add_v4i8_v4i32_sext:615; CHECK: @ %bb.0: @ %entry616; CHECK-NEXT: vmov.i32 q3, #0xff617; CHECK-NEXT: vmovlb.s8 q1, q1618; CHECK-NEXT: vmovlb.s8 q0, q0619; CHECK-NEXT: vand q2, q2, q3620; CHECK-NEXT: vmovlb.s16 q1, q1621; CHECK-NEXT: vmovlb.s16 q0, q0622; CHECK-NEXT: vpt.i32 eq, q2, zr623; CHECK-NEXT: vmlavt.u32 r0, q0, q1624; CHECK-NEXT: bx lr625entry:626 %c = icmp eq <4 x i8> %b, zeroinitializer627 %xx = sext <4 x i8> %x to <4 x i32>628 %yy = sext <4 x i8> %y to <4 x i32>629 %m = mul <4 x i32> %xx, %yy630 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer631 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)632 ret i32 %z633}634 635define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_szext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {636; CHECK-LABEL: add_v4i8_v4i32_szext:637; CHECK: @ %bb.0: @ %entry638; CHECK-NEXT: vmov.i32 q3, #0xff639; CHECK-NEXT: vmovlb.s8 q0, q0640; CHECK-NEXT: vand q2, q2, q3641; CHECK-NEXT: vand q1, q1, q3642; CHECK-NEXT: vmovlb.s16 q0, q0643; CHECK-NEXT: vpt.i32 eq, q2, zr644; CHECK-NEXT: vmlavt.u32 r0, q0, q1645; CHECK-NEXT: bx lr646entry:647 %c = icmp eq <4 x i8> %b, zeroinitializer648 %xx = sext <4 x i8> %x to <4 x i32>649 %yy = zext <4 x i8> %y to <4 x i32>650 %m = mul <4 x i32> %xx, %yy651 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer652 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)653 ret i32 %z654}655 656define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {657; CHECK-LABEL: add_v16i8_v16i16_zext:658; CHECK: @ %bb.0: @ %entry659; CHECK-NEXT: vpt.i8 eq, q2, zr660; CHECK-NEXT: vmlavt.u8 r0, q0, q1661; CHECK-NEXT: uxth r0, r0662; CHECK-NEXT: bx lr663entry:664 %c = icmp eq <16 x i8> %b, zeroinitializer665 %xx = zext <16 x i8> %x to <16 x i16>666 %yy = zext <16 x i8> %y to <16 x i16>667 %m = mul <16 x i16> %xx, %yy668 %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer669 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)670 ret i16 %z671}672 673define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {674; CHECK-LABEL: add_v16i8_v16i16_sext:675; CHECK: @ %bb.0: @ %entry676; CHECK-NEXT: vpt.i8 eq, q2, zr677; CHECK-NEXT: vmlavt.s8 r0, q0, q1678; CHECK-NEXT: sxth r0, r0679; CHECK-NEXT: bx lr680entry:681 %c = icmp eq <16 x i8> %b, zeroinitializer682 %xx = sext <16 x i8> %x to <16 x i16>683 %yy = sext <16 x i8> %y to <16 x i16>684 %m = mul <16 x i16> %xx, %yy685 %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer686 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)687 ret i16 %z688}689 690define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_szext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {691; CHECK-LABEL: add_v16i8_v16i16_szext:692; CHECK: @ %bb.0: @ %entry693; CHECK-NEXT: .pad #32694; CHECK-NEXT: sub sp, #32695; CHECK-NEXT: add r0, sp, #16696; CHECK-NEXT: mov r1, sp697; CHECK-NEXT: vstrw.32 q1, [r0]698; CHECK-NEXT: vstrw.32 q0, [r1]699; CHECK-NEXT: vcmp.i8 eq, q2, zr700; CHECK-NEXT: vmov.i8 q0, #0x0701; CHECK-NEXT: vmov.i8 q1, #0xff702; CHECK-NEXT: vldrb.s16 q2, [r1, #8]703; CHECK-NEXT: vpsel q0, q1, q0704; CHECK-NEXT: vmov.u8 r2, q0[8]705; CHECK-NEXT: vmov.u8 r3, q0[0]706; CHECK-NEXT: vmov.16 q1[0], r2707; CHECK-NEXT: vmov.u8 r2, q0[9]708; CHECK-NEXT: vmov.16 q1[1], r2709; CHECK-NEXT: vmov.u8 r2, q0[10]710; CHECK-NEXT: vmov.16 q1[2], r2711; CHECK-NEXT: vmov.u8 r2, q0[11]712; CHECK-NEXT: vmov.16 q1[3], r2713; CHECK-NEXT: vmov.u8 r2, q0[12]714; CHECK-NEXT: vmov.16 q1[4], r2715; CHECK-NEXT: vmov.u8 r2, q0[13]716; CHECK-NEXT: vmov.16 q1[5], r2717; CHECK-NEXT: vmov.u8 r2, q0[14]718; CHECK-NEXT: vmov.16 q1[6], r2719; CHECK-NEXT: vmov.u8 r2, q0[15]720; CHECK-NEXT: vmov.16 q1[7], r2721; CHECK-NEXT: vcmp.i16 ne, q1, zr722; CHECK-NEXT: vldrb.u16 q1, [r0, #8]723; CHECK-NEXT: vpst724; CHECK-NEXT: vmlavt.u16 r2, q2, q1725; CHECK-NEXT: vmov.16 q1[0], r3726; CHECK-NEXT: vmov.u8 r3, q0[1]727; CHECK-NEXT: vmov.16 q1[1], r3728; CHECK-NEXT: vmov.u8 r3, q0[2]729; CHECK-NEXT: vmov.16 q1[2], r3730; CHECK-NEXT: vmov.u8 r3, q0[3]731; CHECK-NEXT: vmov.16 q1[3], r3732; CHECK-NEXT: vmov.u8 r3, q0[4]733; CHECK-NEXT: vmov.16 q1[4], r3734; CHECK-NEXT: vmov.u8 r3, q0[5]735; CHECK-NEXT: vmov.16 q1[5], r3736; CHECK-NEXT: vmov.u8 r3, q0[6]737; CHECK-NEXT: vmov.16 q1[6], r3738; CHECK-NEXT: vmov.u8 r3, q0[7]739; CHECK-NEXT: vmov.16 q1[7], r3740; CHECK-NEXT: vldrb.u16 q0, [r0]741; CHECK-NEXT: vcmp.i16 ne, q1, zr742; CHECK-NEXT: vldrb.s16 q1, [r1]743; CHECK-NEXT: vpst744; CHECK-NEXT: vmlavat.u16 r2, q1, q0745; CHECK-NEXT: sxth r0, r2746; CHECK-NEXT: add sp, #32747; CHECK-NEXT: bx lr748entry:749 %c = icmp eq <16 x i8> %b, zeroinitializer750 %xx = sext <16 x i8> %x to <16 x i16>751 %yy = zext <16 x i8> %y to <16 x i16>752 %m = mul <16 x i16> %xx, %yy753 %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer754 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)755 ret i16 %z756}757 758define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {759; CHECK-LABEL: add_v8i8_v8i16_zext:760; CHECK: @ %bb.0: @ %entry761; CHECK-NEXT: vmovlb.u8 q2, q2762; CHECK-NEXT: vmovlb.u8 q1, q1763; CHECK-NEXT: vmovlb.u8 q0, q0764; CHECK-NEXT: vpt.i16 eq, q2, zr765; CHECK-NEXT: vmlavt.u16 r0, q0, q1766; CHECK-NEXT: uxth r0, r0767; CHECK-NEXT: bx lr768entry:769 %c = icmp eq <8 x i8> %b, zeroinitializer770 %xx = zext <8 x i8> %x to <8 x i16>771 %yy = zext <8 x i8> %y to <8 x i16>772 %m = mul <8 x i16> %xx, %yy773 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer774 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)775 ret i16 %z776}777 778define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {779; CHECK-LABEL: add_v8i8_v8i16_sext:780; CHECK: @ %bb.0: @ %entry781; CHECK-NEXT: vmovlb.u8 q2, q2782; CHECK-NEXT: vmovlb.s8 q1, q1783; CHECK-NEXT: vmovlb.s8 q0, q0784; CHECK-NEXT: vpt.i16 eq, q2, zr785; CHECK-NEXT: vmlavt.u16 r0, q0, q1786; CHECK-NEXT: sxth r0, r0787; CHECK-NEXT: bx lr788entry:789 %c = icmp eq <8 x i8> %b, zeroinitializer790 %xx = sext <8 x i8> %x to <8 x i16>791 %yy = sext <8 x i8> %y to <8 x i16>792 %m = mul <8 x i16> %xx, %yy793 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer794 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)795 ret i16 %z796}797 798define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {799; CHECK-LABEL: add_v16i8_v16i8:800; CHECK: @ %bb.0: @ %entry801; CHECK-NEXT: vpt.i8 eq, q2, zr802; CHECK-NEXT: vmlavt.u8 r0, q0, q1803; CHECK-NEXT: uxtb r0, r0804; CHECK-NEXT: bx lr805entry:806 %c = icmp eq <16 x i8> %b, zeroinitializer807 %m = mul <16 x i8> %x, %y808 %s = select <16 x i1> %c, <16 x i8> %m, <16 x i8> zeroinitializer809 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %s)810 ret i8 %z811}812 813define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {814; CHECK-LABEL: add_v16i8_v16i64_zext:815; CHECK: @ %bb.0: @ %entry816; CHECK-NEXT: .save {r7, lr}817; CHECK-NEXT: push {r7, lr}818; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}819; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}820; CHECK-NEXT: vmov q3, q0821; CHECK-NEXT: vcmp.i8 eq, q2, zr822; CHECK-NEXT: vmov.i8 q0, #0x0823; CHECK-NEXT: vmov.i8 q2, #0xff824; CHECK-NEXT: vpsel q6, q2, q0825; CHECK-NEXT: vmov.i8 q2, #0xff826; CHECK-NEXT: vmov.u8 r0, q6[0]827; CHECK-NEXT: vmov.i8 q4, #0x0828; CHECK-NEXT: vmov.16 q0[0], r0829; CHECK-NEXT: vmov.u8 r0, q6[1]830; CHECK-NEXT: vmov.16 q0[1], r0831; CHECK-NEXT: vmov.u8 r0, q6[2]832; CHECK-NEXT: vmov.16 q0[2], r0833; CHECK-NEXT: vmov.u8 r0, q6[3]834; CHECK-NEXT: vmov.16 q0[3], r0835; CHECK-NEXT: vmov.u8 r0, q6[4]836; CHECK-NEXT: vmov.16 q0[4], r0837; CHECK-NEXT: vmov.u8 r0, q6[5]838; CHECK-NEXT: vmov.16 q0[5], r0839; CHECK-NEXT: vmov.u8 r0, q6[6]840; CHECK-NEXT: vmov.16 q0[6], r0841; CHECK-NEXT: vmov.u8 r0, q6[7]842; CHECK-NEXT: vmov.16 q0[7], r0843; CHECK-NEXT: vmov.u8 r2, q3[0]844; CHECK-NEXT: vcmp.i16 ne, q0, zr845; CHECK-NEXT: vpsel q7, q2, q4846; CHECK-NEXT: vmov.u16 r0, q7[2]847; CHECK-NEXT: vmov.u16 r1, q7[0]848; CHECK-NEXT: vmov q0[2], q0[0], r1, r0849; CHECK-NEXT: vmov.u16 r0, q7[3]850; CHECK-NEXT: vmov.u16 r1, q7[1]851; CHECK-NEXT: vmov q0[3], q0[1], r1, r0852; CHECK-NEXT: vcmp.i32 ne, q0, zr853; CHECK-NEXT: vpsel q0, q2, q4854; CHECK-NEXT: vmov r0, r1, d0855; CHECK-NEXT: vmov q2[2], q2[0], r0, r1856; CHECK-NEXT: vmov q2[3], q2[1], r0, r1857; CHECK-NEXT: vmov.u8 r0, q1[1]858; CHECK-NEXT: vmov.u8 r1, q1[0]859; CHECK-NEXT: vcmp.i32 ne, q2, zr860; CHECK-NEXT: vmov q5[2], q5[0], r1, r0861; CHECK-NEXT: vmov.u8 r1, q3[1]862; CHECK-NEXT: vmov.i64 q2, #0xff863; CHECK-NEXT: vmov q4[2], q4[0], r2, r1864; CHECK-NEXT: vand q5, q5, q2865; CHECK-NEXT: vand q4, q4, q2866; CHECK-NEXT: vmov r0, s22867; CHECK-NEXT: vmov r1, s18868; CHECK-NEXT: vmov r2, s20869; CHECK-NEXT: vmov.i32 q5, #0x0870; CHECK-NEXT: vmov r3, s16871; CHECK-NEXT: umull r0, r1, r1, r0872; CHECK-NEXT: umull r2, r3, r3, r2873; CHECK-NEXT: vmov q4[2], q4[0], r2, r0874; CHECK-NEXT: vmov q4[3], q4[1], r3, r1875; CHECK-NEXT: vpsel q4, q4, q5876; CHECK-NEXT: vmov r0, r1, d9877; CHECK-NEXT: vmov r2, r3, d8878; CHECK-NEXT: adds.w r12, r2, r0879; CHECK-NEXT: vmov.u8 r0, q3[2]880; CHECK-NEXT: adc.w lr, r3, r1881; CHECK-NEXT: vmov r2, r3, d1882; CHECK-NEXT: vmov q0[2], q0[0], r2, r3883; CHECK-NEXT: vmov q0[3], q0[1], r2, r3884; CHECK-NEXT: vmov.u8 r2, q1[3]885; CHECK-NEXT: vmov.u8 r3, q1[2]886; CHECK-NEXT: vcmp.i32 ne, q0, zr887; CHECK-NEXT: vmov q0[2], q0[0], r3, r2888; CHECK-NEXT: vmov.u8 r3, q3[3]889; CHECK-NEXT: vmov q4[2], q4[0], r0, r3890; CHECK-NEXT: vand q0, q0, q2891; CHECK-NEXT: vand q4, q4, q2892; CHECK-NEXT: vmov r2, s2893; CHECK-NEXT: vmov r0, s18894; CHECK-NEXT: vmov r1, s16895; CHECK-NEXT: vmov.i8 q4, #0xff896; CHECK-NEXT: vmov r3, s0897; CHECK-NEXT: umull r0, r2, r0, r2898; CHECK-NEXT: umull r1, r3, r1, r3899; CHECK-NEXT: vmov q0[2], q0[0], r1, r0900; CHECK-NEXT: vmov q0[3], q0[1], r3, r2901; CHECK-NEXT: vpsel q0, q0, q5902; CHECK-NEXT: vmov r0, r1, d0903; CHECK-NEXT: vmov r2, r3, d1904; CHECK-NEXT: adds.w r0, r0, r12905; CHECK-NEXT: adc.w r1, r1, lr906; CHECK-NEXT: adds.w r12, r0, r2907; CHECK-NEXT: adc.w lr, r1, r3908; CHECK-NEXT: vmov.u16 r2, q7[6]909; CHECK-NEXT: vmov.u16 r3, q7[4]910; CHECK-NEXT: vmov.u8 r0, q3[4]911; CHECK-NEXT: vmov q0[2], q0[0], r3, r2912; CHECK-NEXT: vmov.u16 r2, q7[7]913; CHECK-NEXT: vmov.u16 r3, q7[5]914; CHECK-NEXT: vmov q0[3], q0[1], r3, r2915; CHECK-NEXT: vcmp.i32 ne, q0, zr916; CHECK-NEXT: vmov.i8 q0, #0x0917; CHECK-NEXT: vpsel q0, q4, q0918; CHECK-NEXT: vmov r2, r3, d0919; CHECK-NEXT: vmov q4[2], q4[0], r2, r3920; CHECK-NEXT: vmov q4[3], q4[1], r2, r3921; CHECK-NEXT: vmov.u8 r2, q1[5]922; CHECK-NEXT: vmov.u8 r3, q1[4]923; CHECK-NEXT: vcmp.i32 ne, q4, zr924; CHECK-NEXT: vmov q4[2], q4[0], r3, r2925; CHECK-NEXT: vmov.u8 r3, q3[5]926; CHECK-NEXT: vmov q7[2], q7[0], r0, r3927; CHECK-NEXT: vand q4, q4, q2928; CHECK-NEXT: vand q7, q7, q2929; CHECK-NEXT: vmov r2, s18930; CHECK-NEXT: vmov r0, s30931; CHECK-NEXT: vmov r1, s28932; CHECK-NEXT: vmov.i8 q7, #0xff933; CHECK-NEXT: vmov r3, s16934; CHECK-NEXT: umull r0, r2, r0, r2935; CHECK-NEXT: umull r1, r3, r1, r3936; CHECK-NEXT: vmov q4[2], q4[0], r1, r0937; CHECK-NEXT: vmov q4[3], q4[1], r3, r2938; CHECK-NEXT: vpsel q4, q4, q5939; CHECK-NEXT: vmov r0, r1, d8940; CHECK-NEXT: vmov r2, r3, d9941; CHECK-NEXT: adds.w r0, r0, r12942; CHECK-NEXT: adc.w r1, r1, lr943; CHECK-NEXT: adds.w r12, r0, r2944; CHECK-NEXT: adc.w lr, r1, r3945; CHECK-NEXT: vmov r2, r3, d1946; CHECK-NEXT: vmov q0[2], q0[0], r2, r3947; CHECK-NEXT: vmov.u8 r0, q3[6]948; CHECK-NEXT: vmov q0[3], q0[1], r2, r3949; CHECK-NEXT: vmov.u8 r2, q1[7]950; CHECK-NEXT: vmov.u8 r3, q1[6]951; CHECK-NEXT: vcmp.i32 ne, q0, zr952; CHECK-NEXT: vmov q0[2], q0[0], r3, r2953; CHECK-NEXT: vmov.u8 r3, q3[7]954; CHECK-NEXT: vmov q4[2], q4[0], r0, r3955; CHECK-NEXT: vand q0, q0, q2956; CHECK-NEXT: vand q4, q4, q2957; CHECK-NEXT: vmov r2, s2958; CHECK-NEXT: vmov r0, s18959; CHECK-NEXT: vmov r1, s16960; CHECK-NEXT: vmov.i8 q4, #0x0961; CHECK-NEXT: vmov r3, s0962; CHECK-NEXT: umull r0, r2, r0, r2963; CHECK-NEXT: umull r1, r3, r1, r3964; CHECK-NEXT: vmov q0[2], q0[0], r1, r0965; CHECK-NEXT: vmov q0[3], q0[1], r3, r2966; CHECK-NEXT: vpsel q0, q0, q5967; CHECK-NEXT: vmov r0, r1, d0968; CHECK-NEXT: vmov r2, r3, d1969; CHECK-NEXT: adds.w r0, r0, r12970; CHECK-NEXT: adc.w r1, r1, lr971; CHECK-NEXT: adds.w r12, r0, r2972; CHECK-NEXT: vmov.u8 r2, q6[8]973; CHECK-NEXT: adc.w lr, r1, r3974; CHECK-NEXT: vmov.16 q0[0], r2975; CHECK-NEXT: vmov.u8 r2, q6[9]976; CHECK-NEXT: vmov.16 q0[1], r2977; CHECK-NEXT: vmov.u8 r2, q6[10]978; CHECK-NEXT: vmov.16 q0[2], r2979; CHECK-NEXT: vmov.u8 r2, q6[11]980; CHECK-NEXT: vmov.16 q0[3], r2981; CHECK-NEXT: vmov.u8 r2, q6[12]982; CHECK-NEXT: vmov.16 q0[4], r2983; CHECK-NEXT: vmov.u8 r2, q6[13]984; CHECK-NEXT: vmov.16 q0[5], r2985; CHECK-NEXT: vmov.u8 r2, q6[14]986; CHECK-NEXT: vmov.16 q0[6], r2987; CHECK-NEXT: vmov.u8 r2, q6[15]988; CHECK-NEXT: vmov.16 q0[7], r2989; CHECK-NEXT: vmov.u8 r0, q3[8]990; CHECK-NEXT: vcmp.i16 ne, q0, zr991; CHECK-NEXT: vpsel q6, q7, q4992; CHECK-NEXT: vmov.u16 r2, q6[2]993; CHECK-NEXT: vmov.u16 r3, q6[0]994; CHECK-NEXT: vmov q0[2], q0[0], r3, r2995; CHECK-NEXT: vmov.u16 r2, q6[3]996; CHECK-NEXT: vmov.u16 r3, q6[1]997; CHECK-NEXT: vmov q0[3], q0[1], r3, r2998; CHECK-NEXT: vcmp.i32 ne, q0, zr999; CHECK-NEXT: vpsel q0, q7, q41000; CHECK-NEXT: vmov r2, r3, d01001; CHECK-NEXT: vmov q4[2], q4[0], r2, r31002; CHECK-NEXT: vmov q4[3], q4[1], r2, r31003; CHECK-NEXT: vmov.u8 r2, q1[9]1004; CHECK-NEXT: vmov.u8 r3, q1[8]1005; CHECK-NEXT: vcmp.i32 ne, q4, zr1006; CHECK-NEXT: vmov q4[2], q4[0], r3, r21007; CHECK-NEXT: vmov.u8 r3, q3[9]1008; CHECK-NEXT: vmov q7[2], q7[0], r0, r31009; CHECK-NEXT: vand q4, q4, q21010; CHECK-NEXT: vand q7, q7, q21011; CHECK-NEXT: vmov r2, s181012; CHECK-NEXT: vmov r0, s301013; CHECK-NEXT: vmov r3, s161014; CHECK-NEXT: vmov r1, s281015; CHECK-NEXT: umull r0, r2, r0, r21016; CHECK-NEXT: umull r1, r3, r1, r31017; CHECK-NEXT: vmov q4[2], q4[0], r1, r01018; CHECK-NEXT: vmov q4[3], q4[1], r3, r21019; CHECK-NEXT: vpsel q4, q4, q51020; CHECK-NEXT: vmov r0, r1, d81021; CHECK-NEXT: vmov r2, r3, d91022; CHECK-NEXT: adds.w r0, r0, r121023; CHECK-NEXT: adc.w r1, r1, lr1024; CHECK-NEXT: adds.w r12, r0, r21025; CHECK-NEXT: adc.w lr, r1, r31026; CHECK-NEXT: vmov r2, r3, d11027; CHECK-NEXT: vmov q0[2], q0[0], r2, r31028; CHECK-NEXT: vmov.u8 r0, q3[10]1029; CHECK-NEXT: vmov q0[3], q0[1], r2, r31030; CHECK-NEXT: vmov.u8 r2, q1[11]1031; CHECK-NEXT: vmov.u8 r3, q1[10]1032; CHECK-NEXT: vcmp.i32 ne, q0, zr1033; CHECK-NEXT: vmov q0[2], q0[0], r3, r21034; CHECK-NEXT: vmov.u8 r3, q3[11]1035; CHECK-NEXT: vmov q4[2], q4[0], r0, r31036; CHECK-NEXT: vand q0, q0, q21037; CHECK-NEXT: vand q4, q4, q21038; CHECK-NEXT: vmov r2, s21039; CHECK-NEXT: vmov r0, s181040; CHECK-NEXT: vmov r1, s161041; CHECK-NEXT: vmov.i8 q4, #0x01042; CHECK-NEXT: vmov r3, s01043; CHECK-NEXT: umull r0, r2, r0, r21044; CHECK-NEXT: umull r1, r3, r1, r31045; CHECK-NEXT: vmov q0[2], q0[0], r1, r01046; CHECK-NEXT: vmov q0[3], q0[1], r3, r21047; CHECK-NEXT: vpsel q0, q0, q51048; CHECK-NEXT: vmov r0, r1, d01049; CHECK-NEXT: vmov r2, r3, d11050; CHECK-NEXT: adds.w r0, r0, r121051; CHECK-NEXT: adc.w r1, r1, lr1052; CHECK-NEXT: adds.w r12, r0, r21053; CHECK-NEXT: adc.w lr, r1, r31054; CHECK-NEXT: vmov.u16 r2, q6[6]1055; CHECK-NEXT: vmov.u16 r3, q6[4]1056; CHECK-NEXT: vmov.u8 r0, q3[12]1057; CHECK-NEXT: vmov q0[2], q0[0], r3, r21058; CHECK-NEXT: vmov.u16 r2, q6[7]1059; CHECK-NEXT: vmov.u16 r3, q6[5]1060; CHECK-NEXT: vmov q0[3], q0[1], r3, r21061; CHECK-NEXT: vcmp.i32 ne, q0, zr1062; CHECK-NEXT: vmov.i8 q0, #0xff1063; CHECK-NEXT: vpsel q0, q0, q41064; CHECK-NEXT: vmov r2, r3, d01065; CHECK-NEXT: vmov q4[2], q4[0], r2, r31066; CHECK-NEXT: vmov q4[3], q4[1], r2, r31067; CHECK-NEXT: vmov.u8 r2, q1[13]1068; CHECK-NEXT: vmov.u8 r3, q1[12]1069; CHECK-NEXT: vcmp.i32 ne, q4, zr1070; CHECK-NEXT: vmov q4[2], q4[0], r3, r21071; CHECK-NEXT: vmov.u8 r3, q3[13]1072; CHECK-NEXT: vmov q6[2], q6[0], r0, r31073; CHECK-NEXT: vand q4, q4, q21074; CHECK-NEXT: vand q6, q6, q21075; CHECK-NEXT: vmov r2, s181076; CHECK-NEXT: vmov r0, s261077; CHECK-NEXT: vmov r3, s161078; CHECK-NEXT: vmov r1, s241079; CHECK-NEXT: umull r0, r2, r0, r21080; CHECK-NEXT: umull r1, r3, r1, r31081; CHECK-NEXT: vmov q4[2], q4[0], r1, r01082; CHECK-NEXT: vmov q4[3], q4[1], r3, r21083; CHECK-NEXT: vpsel q4, q4, q51084; CHECK-NEXT: vmov r0, r1, d81085; CHECK-NEXT: vmov r2, r3, d91086; CHECK-NEXT: adds.w r0, r0, r121087; CHECK-NEXT: adc.w r1, r1, lr1088; CHECK-NEXT: adds.w r12, r0, r21089; CHECK-NEXT: adc.w lr, r1, r31090; CHECK-NEXT: vmov r2, r3, d11091; CHECK-NEXT: vmov q0[2], q0[0], r2, r31092; CHECK-NEXT: vmov.u8 r0, q3[14]1093; CHECK-NEXT: vmov q0[3], q0[1], r2, r31094; CHECK-NEXT: vmov.u8 r2, q1[15]1095; CHECK-NEXT: vmov.u8 r3, q1[14]1096; CHECK-NEXT: vcmp.i32 ne, q0, zr1097; CHECK-NEXT: vmov q0[2], q0[0], r3, r21098; CHECK-NEXT: vmov.u8 r3, q3[15]1099; CHECK-NEXT: vmov q1[2], q1[0], r0, r31100; CHECK-NEXT: vand q0, q0, q21101; CHECK-NEXT: vand q1, q1, q21102; CHECK-NEXT: vmov r2, s21103; CHECK-NEXT: vmov r0, s61104; CHECK-NEXT: vmov r3, s01105; CHECK-NEXT: vmov r1, s41106; CHECK-NEXT: umull r0, r2, r0, r21107; CHECK-NEXT: umull r1, r3, r1, r31108; CHECK-NEXT: vmov q0[2], q0[0], r1, r01109; CHECK-NEXT: vmov q0[3], q0[1], r3, r21110; CHECK-NEXT: vpsel q0, q0, q51111; CHECK-NEXT: vmov r0, r1, d01112; CHECK-NEXT: vmov r2, r3, d11113; CHECK-NEXT: adds.w r0, r0, r121114; CHECK-NEXT: adc.w r1, r1, lr1115; CHECK-NEXT: adds r0, r0, r21116; CHECK-NEXT: adcs r1, r31117; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}1118; CHECK-NEXT: pop {r7, pc}1119entry:1120 %c = icmp eq <16 x i8> %b, zeroinitializer1121 %xx = zext <16 x i8> %x to <16 x i64>1122 %yy = zext <16 x i8> %y to <16 x i64>1123 %m = mul <16 x i64> %xx, %yy1124 %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer1125 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %s)1126 ret i64 %z1127}1128 1129define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b) {1130; CHECK-LABEL: add_v16i8_v16i64_sext:1131; CHECK: @ %bb.0: @ %entry1132; CHECK-NEXT: .save {r7, lr}1133; CHECK-NEXT: push {r7, lr}1134; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}1135; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}1136; CHECK-NEXT: vmov q3, q01137; CHECK-NEXT: vcmp.i8 eq, q2, zr1138; CHECK-NEXT: vmov.i8 q0, #0x01139; CHECK-NEXT: vmov.i8 q2, #0xff1140; CHECK-NEXT: vpsel q5, q2, q01141; CHECK-NEXT: vmov.i8 q0, #0x01142; CHECK-NEXT: vmov.u8 r0, q5[0]1143; CHECK-NEXT: vmov.s8 r2, q1[0]1144; CHECK-NEXT: vmov.16 q4[0], r01145; CHECK-NEXT: vmov.u8 r0, q5[1]1146; CHECK-NEXT: vmov.16 q4[1], r01147; CHECK-NEXT: vmov.u8 r0, q5[2]1148; CHECK-NEXT: vmov.16 q4[2], r01149; CHECK-NEXT: vmov.u8 r0, q5[3]1150; CHECK-NEXT: vmov.16 q4[3], r01151; CHECK-NEXT: vmov.u8 r0, q5[4]1152; CHECK-NEXT: vmov.16 q4[4], r01153; CHECK-NEXT: vmov.u8 r0, q5[5]1154; CHECK-NEXT: vmov.16 q4[5], r01155; CHECK-NEXT: vmov.u8 r0, q5[6]1156; CHECK-NEXT: vmov.16 q4[6], r01157; CHECK-NEXT: vmov.u8 r0, q5[7]1158; CHECK-NEXT: vmov.16 q4[7], r01159; CHECK-NEXT: vmov.s8 r3, q3[0]1160; CHECK-NEXT: vcmp.i16 ne, q4, zr1161; CHECK-NEXT: smull r2, r3, r3, r21162; CHECK-NEXT: vpsel q6, q2, q01163; CHECK-NEXT: vmov.u16 r0, q6[2]1164; CHECK-NEXT: vmov.u16 r1, q6[0]1165; CHECK-NEXT: vmov q4[2], q4[0], r1, r01166; CHECK-NEXT: vmov.u16 r0, q6[3]1167; CHECK-NEXT: vmov.u16 r1, q6[1]1168; CHECK-NEXT: vmov q4[3], q4[1], r1, r01169; CHECK-NEXT: vcmp.i32 ne, q4, zr1170; CHECK-NEXT: vpsel q7, q2, q01171; CHECK-NEXT: vmov r0, r1, d141172; CHECK-NEXT: vmov q4[2], q4[0], r0, r11173; CHECK-NEXT: vmov q4[3], q4[1], r0, r11174; CHECK-NEXT: vmov.s8 r0, q1[1]1175; CHECK-NEXT: vmov.s8 r1, q3[1]1176; CHECK-NEXT: vcmp.i32 ne, q4, zr1177; CHECK-NEXT: smull r0, r1, r1, r01178; CHECK-NEXT: vmov.i32 q4, #0x01179; CHECK-NEXT: vmov q0[2], q0[0], r2, r01180; CHECK-NEXT: vmov q0[3], q0[1], r3, r11181; CHECK-NEXT: vpsel q0, q0, q41182; CHECK-NEXT: vmov r0, r1, d11183; CHECK-NEXT: vmov r2, r3, d01184; CHECK-NEXT: adds.w r12, r2, r01185; CHECK-NEXT: vmov.s8 r0, q1[2]1186; CHECK-NEXT: adc.w lr, r3, r11187; CHECK-NEXT: vmov r2, r3, d151188; CHECK-NEXT: vmov q0[2], q0[0], r2, r31189; CHECK-NEXT: vmov.s8 r1, q3[2]1190; CHECK-NEXT: vmov q0[3], q0[1], r2, r31191; CHECK-NEXT: vmov.s8 r2, q1[3]1192; CHECK-NEXT: vmov.s8 r3, q3[3]1193; CHECK-NEXT: smull r0, r1, r1, r01194; CHECK-NEXT: vcmp.i32 ne, q0, zr1195; CHECK-NEXT: vmov.i8 q7, #0x01196; CHECK-NEXT: smull r2, r3, r3, r21197; CHECK-NEXT: vmov q0[2], q0[0], r0, r21198; CHECK-NEXT: vmov q0[3], q0[1], r1, r31199; CHECK-NEXT: vpsel q0, q0, q41200; CHECK-NEXT: vmov r0, r1, d01201; CHECK-NEXT: vmov r2, r3, d11202; CHECK-NEXT: adds.w r0, r0, r121203; CHECK-NEXT: adc.w r1, r1, lr1204; CHECK-NEXT: adds.w r12, r0, r21205; CHECK-NEXT: adc.w lr, r1, r31206; CHECK-NEXT: vmov.u16 r2, q6[6]1207; CHECK-NEXT: vmov.u16 r3, q6[4]1208; CHECK-NEXT: vmov.s8 r0, q1[4]1209; CHECK-NEXT: vmov q0[2], q0[0], r3, r21210; CHECK-NEXT: vmov.u16 r2, q6[7]1211; CHECK-NEXT: vmov.u16 r3, q6[5]1212; CHECK-NEXT: vmov.s8 r1, q3[4]1213; CHECK-NEXT: vmov q0[3], q0[1], r3, r21214; CHECK-NEXT: smull r0, r1, r1, r01215; CHECK-NEXT: vcmp.i32 ne, q0, zr1216; CHECK-NEXT: vmov.i8 q0, #0x01217; CHECK-NEXT: vpsel q6, q2, q01218; CHECK-NEXT: vmov r2, r3, d121219; CHECK-NEXT: vmov q0[2], q0[0], r2, r31220; CHECK-NEXT: vmov q0[3], q0[1], r2, r31221; CHECK-NEXT: vmov.s8 r2, q1[5]1222; CHECK-NEXT: vmov.s8 r3, q3[5]1223; CHECK-NEXT: vcmp.i32 ne, q0, zr1224; CHECK-NEXT: smull r2, r3, r3, r21225; CHECK-NEXT: vmov q0[2], q0[0], r0, r21226; CHECK-NEXT: vmov q0[3], q0[1], r1, r31227; CHECK-NEXT: vpsel q0, q0, q41228; CHECK-NEXT: vmov r0, r1, d01229; CHECK-NEXT: vmov r2, r3, d11230; CHECK-NEXT: adds.w r0, r0, r121231; CHECK-NEXT: adc.w r1, r1, lr1232; CHECK-NEXT: adds.w r12, r0, r21233; CHECK-NEXT: adc.w lr, r1, r31234; CHECK-NEXT: vmov r2, r3, d131235; CHECK-NEXT: vmov q0[2], q0[0], r2, r31236; CHECK-NEXT: vmov.s8 r0, q1[6]1237; CHECK-NEXT: vmov q0[3], q0[1], r2, r31238; CHECK-NEXT: vmov.s8 r1, q3[6]1239; CHECK-NEXT: vmov.s8 r2, q1[7]1240; CHECK-NEXT: vmov.s8 r3, q3[7]1241; CHECK-NEXT: smull r2, r3, r3, r21242; CHECK-NEXT: vcmp.i32 ne, q0, zr1243; CHECK-NEXT: smull r0, r1, r1, r01244; CHECK-NEXT: vmov q0[2], q0[0], r0, r21245; CHECK-NEXT: vmov q0[3], q0[1], r1, r31246; CHECK-NEXT: vpsel q0, q0, q41247; CHECK-NEXT: vmov r0, r1, d01248; CHECK-NEXT: vmov r2, r3, d11249; CHECK-NEXT: adds.w r0, r0, r121250; CHECK-NEXT: adc.w r1, r1, lr1251; CHECK-NEXT: adds.w r12, r0, r21252; CHECK-NEXT: vmov.u8 r2, q5[8]1253; CHECK-NEXT: adc.w lr, r1, r31254; CHECK-NEXT: vmov.16 q6[0], r21255; CHECK-NEXT: vmov.u8 r2, q5[9]1256; CHECK-NEXT: vmov.16 q6[1], r21257; CHECK-NEXT: vmov.u8 r2, q5[10]1258; CHECK-NEXT: vmov.16 q6[2], r21259; CHECK-NEXT: vmov.u8 r2, q5[11]1260; CHECK-NEXT: vmov.16 q6[3], r21261; CHECK-NEXT: vmov.u8 r2, q5[12]1262; CHECK-NEXT: vmov.16 q6[4], r21263; CHECK-NEXT: vmov.u8 r2, q5[13]1264; CHECK-NEXT: vmov.16 q6[5], r21265; CHECK-NEXT: vmov.u8 r2, q5[14]1266; CHECK-NEXT: vmov.16 q6[6], r21267; CHECK-NEXT: vmov.u8 r2, q5[15]1268; CHECK-NEXT: vmov.16 q6[7], r21269; CHECK-NEXT: vmov.s8 r0, q1[8]1270; CHECK-NEXT: vcmp.i16 ne, q6, zr1271; CHECK-NEXT: vmov.i8 q6, #0x01272; CHECK-NEXT: vpsel q5, q2, q61273; CHECK-NEXT: vmov.s8 r1, q3[8]1274; CHECK-NEXT: vmov.u16 r2, q5[2]1275; CHECK-NEXT: vmov.u16 r3, q5[0]1276; CHECK-NEXT: vmov q0[2], q0[0], r3, r21277; CHECK-NEXT: vmov.u16 r2, q5[3]1278; CHECK-NEXT: vmov.u16 r3, q5[1]1279; CHECK-NEXT: smull r0, r1, r1, r01280; CHECK-NEXT: vmov q0[3], q0[1], r3, r21281; CHECK-NEXT: vcmp.i32 ne, q0, zr1282; CHECK-NEXT: vpsel q6, q2, q61283; CHECK-NEXT: vmov r2, r3, d121284; CHECK-NEXT: vmov q0[2], q0[0], r2, r31285; CHECK-NEXT: vmov q0[3], q0[1], r2, r31286; CHECK-NEXT: vmov.s8 r2, q1[9]1287; CHECK-NEXT: vmov.s8 r3, q3[9]1288; CHECK-NEXT: vcmp.i32 ne, q0, zr1289; CHECK-NEXT: smull r2, r3, r3, r21290; CHECK-NEXT: vmov q0[2], q0[0], r0, r21291; CHECK-NEXT: vmov q0[3], q0[1], r1, r31292; CHECK-NEXT: vpsel q0, q0, q41293; CHECK-NEXT: vmov r0, r1, d01294; CHECK-NEXT: vmov r2, r3, d11295; CHECK-NEXT: adds.w r0, r0, r121296; CHECK-NEXT: adc.w r1, r1, lr1297; CHECK-NEXT: adds.w r12, r0, r21298; CHECK-NEXT: adc.w lr, r1, r31299; CHECK-NEXT: vmov r2, r3, d131300; CHECK-NEXT: vmov q0[2], q0[0], r2, r31301; CHECK-NEXT: vmov.s8 r0, q1[10]1302; CHECK-NEXT: vmov q0[3], q0[1], r2, r31303; CHECK-NEXT: vmov.s8 r1, q3[10]1304; CHECK-NEXT: vmov.s8 r2, q1[11]1305; CHECK-NEXT: vmov.s8 r3, q3[11]1306; CHECK-NEXT: smull r2, r3, r3, r21307; CHECK-NEXT: vcmp.i32 ne, q0, zr1308; CHECK-NEXT: smull r0, r1, r1, r01309; CHECK-NEXT: vmov q0[2], q0[0], r0, r21310; CHECK-NEXT: vmov q0[3], q0[1], r1, r31311; CHECK-NEXT: vpsel q0, q0, q41312; CHECK-NEXT: vmov r0, r1, d01313; CHECK-NEXT: vmov r2, r3, d11314; CHECK-NEXT: adds.w r0, r0, r121315; CHECK-NEXT: adc.w r1, r1, lr1316; CHECK-NEXT: adds.w r12, r0, r21317; CHECK-NEXT: adc.w lr, r1, r31318; CHECK-NEXT: vmov.u16 r2, q5[6]1319; CHECK-NEXT: vmov.u16 r3, q5[4]1320; CHECK-NEXT: vmov.s8 r0, q1[12]1321; CHECK-NEXT: vmov q0[2], q0[0], r3, r21322; CHECK-NEXT: vmov.u16 r2, q5[7]1323; CHECK-NEXT: vmov.u16 r3, q5[5]1324; CHECK-NEXT: vmov.s8 r1, q3[12]1325; CHECK-NEXT: vmov q0[3], q0[1], r3, r21326; CHECK-NEXT: smull r0, r1, r1, r01327; CHECK-NEXT: vcmp.i32 ne, q0, zr1328; CHECK-NEXT: vpsel q2, q2, q71329; CHECK-NEXT: vmov r2, r3, d41330; CHECK-NEXT: vmov q0[2], q0[0], r2, r31331; CHECK-NEXT: vmov q0[3], q0[1], r2, r31332; CHECK-NEXT: vmov.s8 r2, q1[13]1333; CHECK-NEXT: vmov.s8 r3, q3[13]1334; CHECK-NEXT: vcmp.i32 ne, q0, zr1335; CHECK-NEXT: smull r2, r3, r3, r21336; CHECK-NEXT: vmov q0[2], q0[0], r0, r21337; CHECK-NEXT: vmov q0[3], q0[1], r1, r31338; CHECK-NEXT: vpsel q0, q0, q41339; CHECK-NEXT: vmov r0, r1, d01340; CHECK-NEXT: vmov r2, r3, d11341; CHECK-NEXT: adds.w r0, r0, r121342; CHECK-NEXT: adc.w r1, r1, lr1343; CHECK-NEXT: adds.w r12, r0, r21344; CHECK-NEXT: adc.w lr, r1, r31345; CHECK-NEXT: vmov r2, r3, d51346; CHECK-NEXT: vmov q0[2], q0[0], r2, r31347; CHECK-NEXT: vmov.s8 r0, q1[14]1348; CHECK-NEXT: vmov q0[3], q0[1], r2, r31349; CHECK-NEXT: vmov.s8 r1, q3[14]1350; CHECK-NEXT: vmov.s8 r2, q1[15]1351; CHECK-NEXT: vmov.s8 r3, q3[15]1352; CHECK-NEXT: smull r2, r3, r3, r21353; CHECK-NEXT: vcmp.i32 ne, q0, zr1354; CHECK-NEXT: smull r0, r1, r1, r01355; CHECK-NEXT: vmov q0[2], q0[0], r0, r21356; CHECK-NEXT: vmov q0[3], q0[1], r1, r31357; CHECK-NEXT: vpsel q0, q0, q41358; CHECK-NEXT: vmov r0, r1, d01359; CHECK-NEXT: vmov r2, r3, d11360; CHECK-NEXT: adds.w r0, r0, r121361; CHECK-NEXT: adc.w r1, r1, lr1362; CHECK-NEXT: adds r0, r0, r21363; CHECK-NEXT: adcs r1, r31364; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}1365; CHECK-NEXT: pop {r7, pc}1366entry:1367 %c = icmp eq <16 x i8> %b, zeroinitializer1368 %xx = sext <16 x i8> %x to <16 x i64>1369 %yy = sext <16 x i8> %y to <16 x i64>1370 %m = mul <16 x i64> %xx, %yy1371 %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer1372 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %s)1373 ret i64 %z1374}1375 1376define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {1377; CHECK-LABEL: add_v8i8_v8i64_zext:1378; CHECK: @ %bb.0: @ %entry1379; CHECK-NEXT: vmovlb.u8 q2, q21380; CHECK-NEXT: vmovlb.u8 q1, q11381; CHECK-NEXT: vmovlb.u8 q0, q01382; CHECK-NEXT: vpt.i16 eq, q2, zr1383; CHECK-NEXT: vmlalvt.u16 r0, r1, q0, q11384; CHECK-NEXT: bx lr1385entry:1386 %c = icmp eq <8 x i8> %b, zeroinitializer1387 %xx = zext <8 x i8> %x to <8 x i64>1388 %yy = zext <8 x i8> %y to <8 x i64>1389 %m = mul <8 x i64> %xx, %yy1390 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer1391 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1392 ret i64 %z1393}1394 1395define arm_aapcs_vfpcc i64 @add_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b) {1396; CHECK-LABEL: add_v8i8_v8i64_sext:1397; CHECK: @ %bb.0: @ %entry1398; CHECK-NEXT: vmovlb.u8 q2, q21399; CHECK-NEXT: vmovlb.s8 q1, q11400; CHECK-NEXT: vmovlb.s8 q0, q01401; CHECK-NEXT: vpt.i16 eq, q2, zr1402; CHECK-NEXT: vmlalvt.s16 r0, r1, q0, q11403; CHECK-NEXT: bx lr1404entry:1405 %c = icmp eq <8 x i8> %b, zeroinitializer1406 %xx = sext <8 x i8> %x to <8 x i64>1407 %yy = sext <8 x i8> %y to <8 x i64>1408 %m = mul <8 x i64> %xx, %yy1409 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer1410 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1411 ret i64 %z1412}1413 1414define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {1415; CHECK-LABEL: add_v4i8_v4i64_zext:1416; CHECK: @ %bb.0: @ %entry1417; CHECK-NEXT: vmov.i32 q3, #0xff1418; CHECK-NEXT: vand q2, q2, q31419; CHECK-NEXT: vand q1, q1, q31420; CHECK-NEXT: vand q0, q0, q31421; CHECK-NEXT: vpt.i32 eq, q2, zr1422; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q11423; CHECK-NEXT: bx lr1424entry:1425 %c = icmp eq <4 x i8> %b, zeroinitializer1426 %xx = zext <4 x i8> %x to <4 x i64>1427 %yy = zext <4 x i8> %y to <4 x i64>1428 %m = mul <4 x i64> %xx, %yy1429 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1430 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1431 ret i64 %z1432}1433 1434define arm_aapcs_vfpcc i64 @add_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b) {1435; CHECK-LABEL: add_v4i8_v4i64_sext:1436; CHECK: @ %bb.0: @ %entry1437; CHECK-NEXT: vmov.i32 q3, #0xff1438; CHECK-NEXT: vmovlb.s8 q1, q11439; CHECK-NEXT: vmovlb.s8 q0, q01440; CHECK-NEXT: vand q2, q2, q31441; CHECK-NEXT: vmovlb.s16 q1, q11442; CHECK-NEXT: vmovlb.s16 q0, q01443; CHECK-NEXT: vpt.i32 eq, q2, zr1444; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q11445; CHECK-NEXT: bx lr1446entry:1447 %c = icmp eq <4 x i8> %b, zeroinitializer1448 %xx = sext <4 x i8> %x to <4 x i64>1449 %yy = sext <4 x i8> %y to <4 x i64>1450 %m = mul <4 x i64> %xx, %yy1451 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1452 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1453 ret i64 %z1454}1455 1456define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i64_zext(<4 x i8> %x, <4 x i16> %y, <4 x i8> %b) {1457; CHECK-LABEL: add_v4i8i16_v4i64_zext:1458; CHECK: @ %bb.0: @ %entry1459; CHECK-NEXT: vmov.i32 q3, #0xff1460; CHECK-NEXT: vmovlb.u16 q1, q11461; CHECK-NEXT: vand q2, q2, q31462; CHECK-NEXT: vand q0, q0, q31463; CHECK-NEXT: vpt.i32 eq, q2, zr1464; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q11465; CHECK-NEXT: bx lr1466entry:1467 %c = icmp eq <4 x i8> %b, zeroinitializer1468 %xx = zext <4 x i8> %x to <4 x i64>1469 %yy = zext <4 x i16> %y to <4 x i64>1470 %m = mul <4 x i64> %xx, %yy1471 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1472 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1473 ret i64 %z1474}1475 1476define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i64_sext(<4 x i8> %x, <4 x i16> %y, <4 x i8> %b) {1477; CHECK-LABEL: add_v4i8i16_v4i64_sext:1478; CHECK: @ %bb.0: @ %entry1479; CHECK-NEXT: vmov.i32 q3, #0xff1480; CHECK-NEXT: vmovlb.s8 q0, q01481; CHECK-NEXT: vand q2, q2, q31482; CHECK-NEXT: vmovlb.s16 q1, q11483; CHECK-NEXT: vmovlb.s16 q0, q01484; CHECK-NEXT: vpt.i32 eq, q2, zr1485; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q11486; CHECK-NEXT: bx lr1487entry:1488 %c = icmp eq <4 x i8> %b, zeroinitializer1489 %xx = sext <4 x i8> %x to <4 x i64>1490 %yy = sext <4 x i16> %y to <4 x i64>1491 %m = mul <4 x i64> %xx, %yy1492 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1493 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1494 ret i64 %z1495}1496 1497define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i32_v4i64_zext(<4 x i8> %x, <4 x i16> %y, <4 x i8> %b) {1498; CHECK-LABEL: add_v4i8i16_v4i32_v4i64_zext:1499; CHECK: @ %bb.0: @ %entry1500; CHECK-NEXT: vmov.i32 q3, #0xff1501; CHECK-NEXT: vmovlb.u16 q1, q11502; CHECK-NEXT: vand q2, q2, q31503; CHECK-NEXT: vand q0, q0, q31504; CHECK-NEXT: vpt.i32 eq, q2, zr1505; CHECK-NEXT: vmlalvt.u32 r0, r1, q0, q11506; CHECK-NEXT: bx lr1507entry:1508 %c = icmp eq <4 x i8> %b, zeroinitializer1509 %xx = zext <4 x i8> %x to <4 x i32>1510 %yy = zext <4 x i16> %y to <4 x i32>1511 %mm = mul <4 x i32> %xx, %yy1512 %m = zext <4 x i32> %mm to <4 x i64>1513 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1514 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1515 ret i64 %z1516}1517 1518define arm_aapcs_vfpcc i64 @add_v4i8i16_v4i32_v4i64_sext(<4 x i8> %x, <4 x i16> %y, <4 x i8> %b) {1519; CHECK-LABEL: add_v4i8i16_v4i32_v4i64_sext:1520; CHECK: @ %bb.0: @ %entry1521; CHECK-NEXT: vmov.i32 q3, #0xff1522; CHECK-NEXT: vmovlb.s8 q0, q01523; CHECK-NEXT: vand q2, q2, q31524; CHECK-NEXT: vmovlb.s16 q1, q11525; CHECK-NEXT: vmovlb.s16 q0, q01526; CHECK-NEXT: vpt.i32 eq, q2, zr1527; CHECK-NEXT: vmlalvt.s32 r0, r1, q0, q11528; CHECK-NEXT: bx lr1529entry:1530 %c = icmp eq <4 x i8> %b, zeroinitializer1531 %xx = sext <4 x i8> %x to <4 x i32>1532 %yy = sext <4 x i16> %y to <4 x i32>1533 %mm = mul <4 x i32> %xx, %yy1534 %m = sext <4 x i32> %mm to <4 x i64>1535 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1536 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1537 ret i64 %z1538}1539 1540define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b) {1541; CHECK-LABEL: add_v2i8_v2i64_zext:1542; CHECK: @ %bb.0: @ %entry1543; CHECK-NEXT: vmov.i64 q3, #0xff1544; CHECK-NEXT: vand q1, q1, q31545; CHECK-NEXT: vand q0, q0, q31546; CHECK-NEXT: vmov r0, s61547; CHECK-NEXT: vmov r1, s21548; CHECK-NEXT: vmov r2, s41549; CHECK-NEXT: vand q1, q2, q31550; CHECK-NEXT: vmov r3, s01551; CHECK-NEXT: umull r0, r1, r1, r01552; CHECK-NEXT: umull r2, r3, r3, r21553; CHECK-NEXT: vmov q0[2], q0[0], r2, r01554; CHECK-NEXT: vmov r0, s41555; CHECK-NEXT: vmov q0[3], q0[1], r3, r11556; CHECK-NEXT: movs r1, #01557; CHECK-NEXT: cmp r0, #01558; CHECK-NEXT: csetm r0, eq1559; CHECK-NEXT: bfi r1, r0, #0, #81560; CHECK-NEXT: vmov r0, s61561; CHECK-NEXT: vmov.i32 q1, #0x01562; CHECK-NEXT: cmp r0, #01563; CHECK-NEXT: csetm r0, eq1564; CHECK-NEXT: bfi r1, r0, #8, #81565; CHECK-NEXT: vmsr p0, r11566; CHECK-NEXT: vpsel q0, q0, q11567; CHECK-NEXT: vmov r0, r1, d11568; CHECK-NEXT: vmov r2, r3, d01569; CHECK-NEXT: adds r0, r0, r21570; CHECK-NEXT: adcs r1, r31571; CHECK-NEXT: bx lr1572entry:1573 %c = icmp eq <2 x i8> %b, zeroinitializer1574 %xx = zext <2 x i8> %x to <2 x i64>1575 %yy = zext <2 x i8> %y to <2 x i64>1576 %m = mul <2 x i64> %xx, %yy1577 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer1578 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)1579 ret i64 %z1580}1581 1582define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b) {1583; CHECK-LABEL: add_v2i8_v2i64_sext:1584; CHECK: @ %bb.0: @ %entry1585; CHECK-NEXT: vmov.i32 q3, #0xff1586; CHECK-NEXT: movs r1, #01587; CHECK-NEXT: vand q2, q2, q31588; CHECK-NEXT: vmov r2, s41589; CHECK-NEXT: vmov r0, s81590; CHECK-NEXT: vmov r3, s01591; CHECK-NEXT: cmp r0, #01592; CHECK-NEXT: sxtb r2, r21593; CHECK-NEXT: csetm r0, eq1594; CHECK-NEXT: bfi r1, r0, #0, #81595; CHECK-NEXT: vmov r0, s101596; CHECK-NEXT: sxtb r3, r31597; CHECK-NEXT: smull r2, r3, r3, r21598; CHECK-NEXT: cmp r0, #01599; CHECK-NEXT: csetm r0, eq1600; CHECK-NEXT: bfi r1, r0, #8, #81601; CHECK-NEXT: vmov r0, s61602; CHECK-NEXT: vmsr p0, r11603; CHECK-NEXT: vmov r1, s21604; CHECK-NEXT: vmov.i32 q1, #0x01605; CHECK-NEXT: sxtb r0, r01606; CHECK-NEXT: sxtb r1, r11607; CHECK-NEXT: smull r0, r1, r1, r01608; CHECK-NEXT: vmov q0[2], q0[0], r2, r01609; CHECK-NEXT: vmov q0[3], q0[1], r3, r11610; CHECK-NEXT: vpsel q0, q0, q11611; CHECK-NEXT: vmov r0, r1, d11612; CHECK-NEXT: vmov r2, r3, d01613; CHECK-NEXT: adds r0, r0, r21614; CHECK-NEXT: adcs r1, r31615; CHECK-NEXT: bx lr1616entry:1617 %c = icmp eq <2 x i8> %b, zeroinitializer1618 %xx = sext <2 x i8> %x to <2 x i64>1619 %yy = sext <2 x i8> %y to <2 x i64>1620 %m = mul <2 x i64> %xx, %yy1621 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer1622 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)1623 ret i64 %z1624}1625 1626define arm_aapcs_vfpcc i64 @add_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %b) {1627; CHECK-LABEL: add_v2i64_v2i64:1628; CHECK: @ %bb.0: @ %entry1629; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, lr}1630; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, lr}1631; CHECK-NEXT: vmov r0, r12, d31632; CHECK-NEXT: vmov r2, lr, d11633; CHECK-NEXT: vmov r4, r9, d21634; CHECK-NEXT: vmov.i32 q1, #0x01635; CHECK-NEXT: vmov r6, r7, d01636; CHECK-NEXT: umull r1, r8, r2, r01637; CHECK-NEXT: umull r3, r5, r6, r41638; CHECK-NEXT: vmov q0[2], q0[0], r3, r11639; CHECK-NEXT: mla r1, r2, r12, r81640; CHECK-NEXT: mla r0, lr, r0, r11641; CHECK-NEXT: mla r1, r6, r9, r51642; CHECK-NEXT: mla r1, r7, r4, r11643; CHECK-NEXT: vmov q0[3], q0[1], r1, r01644; CHECK-NEXT: vmov r0, r1, d41645; CHECK-NEXT: orrs r0, r11646; CHECK-NEXT: mov.w r1, #01647; CHECK-NEXT: csetm r0, eq1648; CHECK-NEXT: bfi r1, r0, #0, #81649; CHECK-NEXT: vmov r0, r2, d51650; CHECK-NEXT: orrs r0, r21651; CHECK-NEXT: csetm r0, eq1652; CHECK-NEXT: bfi r1, r0, #8, #81653; CHECK-NEXT: vmsr p0, r11654; CHECK-NEXT: vpsel q0, q0, q11655; CHECK-NEXT: vmov r0, r1, d11656; CHECK-NEXT: vmov r2, r3, d01657; CHECK-NEXT: adds r0, r0, r21658; CHECK-NEXT: adcs r1, r31659; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, pc}1660entry:1661 %c = icmp eq <2 x i64> %b, zeroinitializer1662 %m = mul <2 x i64> %x, %y1663 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer1664 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)1665 ret i64 %z1666}1667 1668define arm_aapcs_vfpcc i32 @add_v4i32_v4i32_acc(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i32 %a) {1669; CHECK-LABEL: add_v4i32_v4i32_acc:1670; CHECK: @ %bb.0: @ %entry1671; CHECK-NEXT: vpt.i32 eq, q2, zr1672; CHECK-NEXT: vmlavat.u32 r0, q0, q11673; CHECK-NEXT: bx lr1674entry:1675 %c = icmp eq <4 x i32> %b, zeroinitializer1676 %m = mul <4 x i32> %x, %y1677 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer1678 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)1679 %r = add i32 %z, %a1680 ret i32 %r1681}1682 1683define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i64 %a) {1684; CHECK-LABEL: add_v4i32_v4i64_acc_zext:1685; CHECK: @ %bb.0: @ %entry1686; CHECK-NEXT: vpt.i32 eq, q2, zr1687; CHECK-NEXT: vmlalvat.u32 r0, r1, q0, q11688; CHECK-NEXT: bx lr1689entry:1690 %c = icmp eq <4 x i32> %b, zeroinitializer1691 %xx = zext <4 x i32> %x to <4 x i64>1692 %yy = zext <4 x i32> %y to <4 x i64>1693 %m = mul <4 x i64> %xx, %yy1694 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1695 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1696 %r = add i64 %z, %a1697 ret i64 %r1698}1699 1700define arm_aapcs_vfpcc i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, <4 x i32> %y, <4 x i32> %b, i64 %a) {1701; CHECK-LABEL: add_v4i32_v4i64_acc_sext:1702; CHECK: @ %bb.0: @ %entry1703; CHECK-NEXT: vpt.i32 eq, q2, zr1704; CHECK-NEXT: vmlalvat.s32 r0, r1, q0, q11705; CHECK-NEXT: bx lr1706entry:1707 %c = icmp eq <4 x i32> %b, zeroinitializer1708 %xx = sext <4 x i32> %x to <4 x i64>1709 %yy = sext <4 x i32> %y to <4 x i64>1710 %m = mul <4 x i64> %xx, %yy1711 %s = select <4 x i1> %c, <4 x i64> %m, <4 x i64> zeroinitializer1712 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %s)1713 %r = add i64 %z, %a1714 ret i64 %r1715}1716 1717define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b, i64 %a) {1718; CHECK-LABEL: add_v2i32_v2i64_acc_zext:1719; CHECK: @ %bb.0: @ %entry1720; CHECK-NEXT: .save {r7, lr}1721; CHECK-NEXT: push {r7, lr}1722; CHECK-NEXT: vmov r2, s81723; CHECK-NEXT: movs r3, #01724; CHECK-NEXT: vmullb.u32 q3, q0, q11725; CHECK-NEXT: vmov.i32 q0, #0x01726; CHECK-NEXT: cmp r2, #01727; CHECK-NEXT: csetm r2, eq1728; CHECK-NEXT: bfi r3, r2, #0, #81729; CHECK-NEXT: vmov r2, s101730; CHECK-NEXT: cmp r2, #01731; CHECK-NEXT: csetm r2, eq1732; CHECK-NEXT: bfi r3, r2, #8, #81733; CHECK-NEXT: vmsr p0, r31734; CHECK-NEXT: vpsel q0, q3, q01735; CHECK-NEXT: vmov lr, r12, d11736; CHECK-NEXT: vmov r3, r2, d01737; CHECK-NEXT: adds.w r3, r3, lr1738; CHECK-NEXT: adc.w r2, r2, r121739; CHECK-NEXT: adds r0, r0, r31740; CHECK-NEXT: adcs r1, r21741; CHECK-NEXT: pop {r7, pc}1742entry:1743 %c = icmp eq <2 x i32> %b, zeroinitializer1744 %xx = zext <2 x i32> %x to <2 x i64>1745 %yy = zext <2 x i32> %y to <2 x i64>1746 %m = mul <2 x i64> %xx, %yy1747 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer1748 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)1749 %r = add i64 %z, %a1750 ret i64 %r1751}1752 1753define arm_aapcs_vfpcc i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, <2 x i32> %y, <2 x i32> %b, i64 %a) {1754; CHECK-LABEL: add_v2i32_v2i64_acc_sext:1755; CHECK: @ %bb.0: @ %entry1756; CHECK-NEXT: .save {r7, lr}1757; CHECK-NEXT: push {r7, lr}1758; CHECK-NEXT: vmov r2, s81759; CHECK-NEXT: movs r3, #01760; CHECK-NEXT: vmullb.s32 q3, q0, q11761; CHECK-NEXT: vmov.i32 q0, #0x01762; CHECK-NEXT: cmp r2, #01763; CHECK-NEXT: csetm r2, eq1764; CHECK-NEXT: bfi r3, r2, #0, #81765; CHECK-NEXT: vmov r2, s101766; CHECK-NEXT: cmp r2, #01767; CHECK-NEXT: csetm r2, eq1768; CHECK-NEXT: bfi r3, r2, #8, #81769; CHECK-NEXT: vmsr p0, r31770; CHECK-NEXT: vpsel q0, q3, q01771; CHECK-NEXT: vmov lr, r12, d11772; CHECK-NEXT: vmov r3, r2, d01773; CHECK-NEXT: adds.w r3, r3, lr1774; CHECK-NEXT: adc.w r2, r2, r121775; CHECK-NEXT: adds r0, r0, r31776; CHECK-NEXT: adcs r1, r21777; CHECK-NEXT: pop {r7, pc}1778entry:1779 %c = icmp eq <2 x i32> %b, zeroinitializer1780 %xx = sext <2 x i32> %x to <2 x i64>1781 %yy = sext <2 x i32> %y to <2 x i64>1782 %m = mul <2 x i64> %xx, %yy1783 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer1784 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)1785 %r = add i64 %z, %a1786 ret i64 %r1787}1788 1789define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i32 %a) {1790; CHECK-LABEL: add_v8i16_v8i32_acc_zext:1791; CHECK: @ %bb.0: @ %entry1792; CHECK-NEXT: vpt.i16 eq, q2, zr1793; CHECK-NEXT: vmlavat.u16 r0, q0, q11794; CHECK-NEXT: bx lr1795entry:1796 %c = icmp eq <8 x i16> %b, zeroinitializer1797 %xx = zext <8 x i16> %x to <8 x i32>1798 %yy = zext <8 x i16> %y to <8 x i32>1799 %m = mul <8 x i32> %xx, %yy1800 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer1801 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)1802 %r = add i32 %z, %a1803 ret i32 %r1804}1805 1806define arm_aapcs_vfpcc i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i32 %a) {1807; CHECK-LABEL: add_v8i16_v8i32_acc_sext:1808; CHECK: @ %bb.0: @ %entry1809; CHECK-NEXT: vpt.i16 eq, q2, zr1810; CHECK-NEXT: vmlavat.s16 r0, q0, q11811; CHECK-NEXT: bx lr1812entry:1813 %c = icmp eq <8 x i16> %b, zeroinitializer1814 %xx = sext <8 x i16> %x to <8 x i32>1815 %yy = sext <8 x i16> %y to <8 x i32>1816 %m = mul <8 x i32> %xx, %yy1817 %s = select <8 x i1> %c, <8 x i32> %m, <8 x i32> zeroinitializer1818 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %s)1819 %r = add i32 %z, %a1820 ret i32 %r1821}1822 1823define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b, i32 %a) {1824; CHECK-LABEL: add_v4i16_v4i32_acc_zext:1825; CHECK: @ %bb.0: @ %entry1826; CHECK-NEXT: vmovlb.u16 q2, q21827; CHECK-NEXT: vmovlb.u16 q1, q11828; CHECK-NEXT: vmovlb.u16 q0, q01829; CHECK-NEXT: vpt.i32 eq, q2, zr1830; CHECK-NEXT: vmlavat.u32 r0, q0, q11831; CHECK-NEXT: bx lr1832entry:1833 %c = icmp eq <4 x i16> %b, zeroinitializer1834 %xx = zext <4 x i16> %x to <4 x i32>1835 %yy = zext <4 x i16> %y to <4 x i32>1836 %m = mul <4 x i32> %xx, %yy1837 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer1838 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)1839 %r = add i32 %z, %a1840 ret i32 %r1841}1842 1843define arm_aapcs_vfpcc i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, <4 x i16> %y, <4 x i16> %b, i32 %a) {1844; CHECK-LABEL: add_v4i16_v4i32_acc_sext:1845; CHECK: @ %bb.0: @ %entry1846; CHECK-NEXT: vmovlb.u16 q2, q21847; CHECK-NEXT: vmovlb.s16 q1, q11848; CHECK-NEXT: vmovlb.s16 q0, q01849; CHECK-NEXT: vpt.i32 eq, q2, zr1850; CHECK-NEXT: vmlavat.u32 r0, q0, q11851; CHECK-NEXT: bx lr1852entry:1853 %c = icmp eq <4 x i16> %b, zeroinitializer1854 %xx = sext <4 x i16> %x to <4 x i32>1855 %yy = sext <4 x i16> %y to <4 x i32>1856 %m = mul <4 x i32> %xx, %yy1857 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer1858 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)1859 %r = add i32 %z, %a1860 ret i32 %r1861}1862 1863define arm_aapcs_vfpcc zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i16 %a) {1864; CHECK-LABEL: add_v8i16_v8i16_acc:1865; CHECK: @ %bb.0: @ %entry1866; CHECK-NEXT: vpt.i16 eq, q2, zr1867; CHECK-NEXT: vmlavat.u16 r0, q0, q11868; CHECK-NEXT: uxth r0, r01869; CHECK-NEXT: bx lr1870entry:1871 %c = icmp eq <8 x i16> %b, zeroinitializer1872 %m = mul <8 x i16> %x, %y1873 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer1874 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)1875 %r = add i16 %z, %a1876 ret i16 %r1877}1878 1879define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {1880; CHECK-LABEL: add_v8i16_v8i64_acc_zext:1881; CHECK: @ %bb.0: @ %entry1882; CHECK-NEXT: vpt.i16 eq, q2, zr1883; CHECK-NEXT: vmlalvat.u16 r0, r1, q0, q11884; CHECK-NEXT: bx lr1885entry:1886 %c = icmp eq <8 x i16> %b, zeroinitializer1887 %xx = zext <8 x i16> %x to <8 x i64>1888 %yy = zext <8 x i16> %y to <8 x i64>1889 %m = mul <8 x i64> %xx, %yy1890 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer1891 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1892 %r = add i64 %z, %a1893 ret i64 %r1894}1895 1896define arm_aapcs_vfpcc i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {1897; CHECK-LABEL: add_v8i16_v8i64_acc_sext:1898; CHECK: @ %bb.0: @ %entry1899; CHECK-NEXT: vpt.i16 eq, q2, zr1900; CHECK-NEXT: vmlalvat.s16 r0, r1, q0, q11901; CHECK-NEXT: bx lr1902entry:1903 %c = icmp eq <8 x i16> %b, zeroinitializer1904 %xx = sext <8 x i16> %x to <8 x i64>1905 %yy = sext <8 x i16> %y to <8 x i64>1906 %m = mul <8 x i64> %xx, %yy1907 %s = select <8 x i1> %c, <8 x i64> %m, <8 x i64> zeroinitializer1908 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1909 %r = add i64 %z, %a1910 ret i64 %r1911}1912 1913define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_zext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {1914; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_zext:1915; CHECK: @ %bb.0: @ %entry1916; CHECK-NEXT: vpt.i16 eq, q2, zr1917; CHECK-NEXT: vmlalvat.u16 r0, r1, q0, q11918; CHECK-NEXT: bx lr1919entry:1920 %c = icmp eq <8 x i16> %b, zeroinitializer1921 %xx = zext <8 x i16> %x to <8 x i32>1922 %yy = zext <8 x i16> %y to <8 x i32>1923 %m = mul <8 x i32> %xx, %yy1924 %ma = zext <8 x i32> %m to <8 x i64>1925 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer1926 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1927 %r = add i64 %z, %a1928 ret i64 %r1929}1930 1931define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_sext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {1932; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_sext:1933; CHECK: @ %bb.0: @ %entry1934; CHECK-NEXT: vpt.i16 eq, q2, zr1935; CHECK-NEXT: vmlalvat.s16 r0, r1, q0, q11936; CHECK-NEXT: bx lr1937entry:1938 %c = icmp eq <8 x i16> %b, zeroinitializer1939 %xx = sext <8 x i16> %x to <8 x i32>1940 %yy = sext <8 x i16> %y to <8 x i32>1941 %m = mul <8 x i32> %xx, %yy1942 %ma = sext <8 x i32> %m to <8 x i64>1943 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer1944 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1945 %r = add i64 %z, %a1946 ret i64 %r1947}1948 1949define arm_aapcs_vfpcc i64 @add_v8i16_v8i32_v8i64_acc_sextzext(<8 x i16> %x, <8 x i16> %y, <8 x i16> %b, i64 %a) {1950; CHECK-LABEL: add_v8i16_v8i32_v8i64_acc_sextzext:1951; CHECK: @ %bb.0: @ %entry1952; CHECK-NEXT: vpt.i16 eq, q2, zr1953; CHECK-NEXT: vmlalvat.s16 r0, r1, q0, q01954; CHECK-NEXT: bx lr1955entry:1956 %c = icmp eq <8 x i16> %b, zeroinitializer1957 %xx = sext <8 x i16> %x to <8 x i32>1958 %m = mul <8 x i32> %xx, %xx1959 %ma = zext <8 x i32> %m to <8 x i64>1960 %s = select <8 x i1> %c, <8 x i64> %ma, <8 x i64> zeroinitializer1961 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %s)1962 %r = add i64 %z, %a1963 ret i64 %r1964}1965 1966define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b, i64 %a) {1967; CHECK-LABEL: add_v2i16_v2i64_acc_zext:1968; CHECK: @ %bb.0: @ %entry1969; CHECK-NEXT: .save {r7, lr}1970; CHECK-NEXT: push {r7, lr}1971; CHECK-NEXT: vmov.i64 q3, #0xffff1972; CHECK-NEXT: vand q1, q1, q31973; CHECK-NEXT: vand q0, q0, q31974; CHECK-NEXT: vmov r2, s61975; CHECK-NEXT: vmov r3, s21976; CHECK-NEXT: umull lr, r12, r3, r21977; CHECK-NEXT: vmov r3, s41978; CHECK-NEXT: vmov r2, s01979; CHECK-NEXT: vand q1, q2, q31980; CHECK-NEXT: umull r2, r3, r2, r31981; CHECK-NEXT: vmov q0[2], q0[0], r2, lr1982; CHECK-NEXT: vmov r2, s41983; CHECK-NEXT: vmov q0[3], q0[1], r3, r121984; CHECK-NEXT: movs r3, #01985; CHECK-NEXT: cmp r2, #01986; CHECK-NEXT: csetm r2, eq1987; CHECK-NEXT: bfi r3, r2, #0, #81988; CHECK-NEXT: vmov r2, s61989; CHECK-NEXT: vmov.i32 q1, #0x01990; CHECK-NEXT: cmp r2, #01991; CHECK-NEXT: csetm r2, eq1992; CHECK-NEXT: bfi r3, r2, #8, #81993; CHECK-NEXT: vmsr p0, r31994; CHECK-NEXT: vpsel q0, q0, q11995; CHECK-NEXT: vmov lr, r12, d11996; CHECK-NEXT: vmov r3, r2, d01997; CHECK-NEXT: adds.w r3, r3, lr1998; CHECK-NEXT: adc.w r2, r2, r121999; CHECK-NEXT: adds r0, r0, r32000; CHECK-NEXT: adcs r1, r22001; CHECK-NEXT: pop {r7, pc}2002entry:2003 %c = icmp eq <2 x i16> %b, zeroinitializer2004 %xx = zext <2 x i16> %x to <2 x i64>2005 %yy = zext <2 x i16> %y to <2 x i64>2006 %m = mul <2 x i64> %xx, %yy2007 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer2008 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)2009 %r = add i64 %z, %a2010 ret i64 %r2011}2012 2013define arm_aapcs_vfpcc i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, <2 x i16> %y, <2 x i16> %b, i64 %a) {2014; CHECK-LABEL: add_v2i16_v2i64_acc_sext:2015; CHECK: @ %bb.0: @ %entry2016; CHECK-NEXT: .save {r7, lr}2017; CHECK-NEXT: push {r7, lr}2018; CHECK-NEXT: vmov.i32 q3, #0xffff2019; CHECK-NEXT: movs r3, #02020; CHECK-NEXT: vand q2, q2, q32021; CHECK-NEXT: vmov r2, s82022; CHECK-NEXT: cmp r2, #02023; CHECK-NEXT: csetm r2, eq2024; CHECK-NEXT: bfi r3, r2, #0, #82025; CHECK-NEXT: vmov r2, s102026; CHECK-NEXT: cmp r2, #02027; CHECK-NEXT: csetm r2, eq2028; CHECK-NEXT: bfi r3, r2, #8, #82029; CHECK-NEXT: vmov r2, s62030; CHECK-NEXT: vmsr p0, r32031; CHECK-NEXT: vmov r3, s22032; CHECK-NEXT: sxth r2, r22033; CHECK-NEXT: sxth r3, r32034; CHECK-NEXT: smull lr, r12, r3, r22035; CHECK-NEXT: vmov r3, s42036; CHECK-NEXT: vmov r2, s02037; CHECK-NEXT: vmov.i32 q1, #0x02038; CHECK-NEXT: sxth r3, r32039; CHECK-NEXT: sxth r2, r22040; CHECK-NEXT: smull r2, r3, r2, r32041; CHECK-NEXT: vmov q0[2], q0[0], r2, lr2042; CHECK-NEXT: vmov q0[3], q0[1], r3, r122043; CHECK-NEXT: vpsel q0, q0, q12044; CHECK-NEXT: vmov lr, r12, d12045; CHECK-NEXT: vmov r3, r2, d02046; CHECK-NEXT: adds.w r3, r3, lr2047; CHECK-NEXT: adc.w r2, r2, r122048; CHECK-NEXT: adds r0, r0, r32049; CHECK-NEXT: adcs r1, r22050; CHECK-NEXT: pop {r7, pc}2051entry:2052 %c = icmp eq <2 x i16> %b, zeroinitializer2053 %xx = sext <2 x i16> %x to <2 x i64>2054 %yy = sext <2 x i16> %y to <2 x i64>2055 %m = mul <2 x i64> %xx, %yy2056 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer2057 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)2058 %r = add i64 %z, %a2059 ret i64 %r2060}2061 2062define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {2063; CHECK-LABEL: add_v16i8_v16i32_acc_zext:2064; CHECK: @ %bb.0: @ %entry2065; CHECK-NEXT: vpt.i8 eq, q2, zr2066; CHECK-NEXT: vmlavat.u8 r0, q0, q12067; CHECK-NEXT: bx lr2068entry:2069 %c = icmp eq <16 x i8> %b, zeroinitializer2070 %xx = zext <16 x i8> %x to <16 x i32>2071 %yy = zext <16 x i8> %y to <16 x i32>2072 %m = mul <16 x i32> %xx, %yy2073 %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer2074 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)2075 %r = add i32 %z, %a2076 ret i32 %r2077}2078 2079define arm_aapcs_vfpcc i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {2080; CHECK-LABEL: add_v16i8_v16i32_acc_sext:2081; CHECK: @ %bb.0: @ %entry2082; CHECK-NEXT: vpt.i8 eq, q2, zr2083; CHECK-NEXT: vmlavat.s8 r0, q0, q12084; CHECK-NEXT: bx lr2085entry:2086 %c = icmp eq <16 x i8> %b, zeroinitializer2087 %xx = sext <16 x i8> %x to <16 x i32>2088 %yy = sext <16 x i8> %y to <16 x i32>2089 %m = mul <16 x i32> %xx, %yy2090 %s = select <16 x i1> %c, <16 x i32> %m, <16 x i32> zeroinitializer2091 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)2092 %r = add i32 %z, %a2093 ret i32 %r2094}2095 2096define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {2097; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_zext:2098; CHECK: @ %bb.0: @ %entry2099; CHECK-NEXT: vpt.i8 eq, q2, zr2100; CHECK-NEXT: vmlavat.u8 r0, q0, q12101; CHECK-NEXT: bx lr2102entry:2103 %c = icmp eq <16 x i8> %b, zeroinitializer2104 %xx = zext <16 x i8> %x to <16 x i16>2105 %yy = zext <16 x i8> %y to <16 x i16>2106 %m = mul <16 x i16> %xx, %yy2107 %ma = zext <16 x i16> %m to <16 x i32>2108 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer2109 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)2110 %r = add i32 %z, %a2111 ret i32 %r2112}2113 2114define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {2115; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_sext:2116; CHECK: @ %bb.0: @ %entry2117; CHECK-NEXT: vpt.i8 eq, q2, zr2118; CHECK-NEXT: vmlavat.s8 r0, q0, q12119; CHECK-NEXT: bx lr2120entry:2121 %c = icmp eq <16 x i8> %b, zeroinitializer2122 %xx = sext <16 x i8> %x to <16 x i16>2123 %yy = sext <16 x i8> %y to <16 x i16>2124 %m = mul <16 x i16> %xx, %yy2125 %ma = sext <16 x i16> %m to <16 x i32>2126 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer2127 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)2128 %r = add i32 %z, %a2129 ret i32 %r2130}2131 2132define arm_aapcs_vfpcc i32 @add_v16i8_v16i16_v16i32_acc_sextzext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i32 %a) {2133; CHECK-LABEL: add_v16i8_v16i16_v16i32_acc_sextzext:2134; CHECK: @ %bb.0: @ %entry2135; CHECK-NEXT: vpt.i8 eq, q2, zr2136; CHECK-NEXT: vmlavat.s8 r0, q0, q02137; CHECK-NEXT: bx lr2138entry:2139 %c = icmp eq <16 x i8> %b, zeroinitializer2140 %xx = sext <16 x i8> %x to <16 x i16>2141 %m = mul <16 x i16> %xx, %xx2142 %ma = zext <16 x i16> %m to <16 x i32>2143 %s = select <16 x i1> %c, <16 x i32> %ma, <16 x i32> zeroinitializer2144 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %s)2145 %r = add i32 %z, %a2146 ret i32 %r2147}2148 2149define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b, i32 %a) {2150; CHECK-LABEL: add_v4i8_v4i32_acc_zext:2151; CHECK: @ %bb.0: @ %entry2152; CHECK-NEXT: vmov.i32 q3, #0xff2153; CHECK-NEXT: vand q2, q2, q32154; CHECK-NEXT: vand q1, q1, q32155; CHECK-NEXT: vand q0, q0, q32156; CHECK-NEXT: vpt.i32 eq, q2, zr2157; CHECK-NEXT: vmlavat.u32 r0, q0, q12158; CHECK-NEXT: bx lr2159entry:2160 %c = icmp eq <4 x i8> %b, zeroinitializer2161 %xx = zext <4 x i8> %x to <4 x i32>2162 %yy = zext <4 x i8> %y to <4 x i32>2163 %m = mul <4 x i32> %xx, %yy2164 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer2165 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)2166 %r = add i32 %z, %a2167 ret i32 %r2168}2169 2170define arm_aapcs_vfpcc i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, <4 x i8> %y, <4 x i8> %b, i32 %a) {2171; CHECK-LABEL: add_v4i8_v4i32_acc_sext:2172; CHECK: @ %bb.0: @ %entry2173; CHECK-NEXT: vmov.i32 q3, #0xff2174; CHECK-NEXT: vmovlb.s8 q1, q12175; CHECK-NEXT: vmovlb.s8 q0, q02176; CHECK-NEXT: vand q2, q2, q32177; CHECK-NEXT: vmovlb.s16 q1, q12178; CHECK-NEXT: vmovlb.s16 q0, q02179; CHECK-NEXT: vpt.i32 eq, q2, zr2180; CHECK-NEXT: vmlavat.u32 r0, q0, q12181; CHECK-NEXT: bx lr2182entry:2183 %c = icmp eq <4 x i8> %b, zeroinitializer2184 %xx = sext <4 x i8> %x to <4 x i32>2185 %yy = sext <4 x i8> %y to <4 x i32>2186 %m = mul <4 x i32> %xx, %yy2187 %s = select <4 x i1> %c, <4 x i32> %m, <4 x i32> zeroinitializer2188 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %s)2189 %r = add i32 %z, %a2190 ret i32 %r2191}2192 2193define arm_aapcs_vfpcc zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i16 %a) {2194; CHECK-LABEL: add_v16i8_v16i16_acc_zext:2195; CHECK: @ %bb.0: @ %entry2196; CHECK-NEXT: vpt.i8 eq, q2, zr2197; CHECK-NEXT: vmlavat.u8 r0, q0, q12198; CHECK-NEXT: uxth r0, r02199; CHECK-NEXT: bx lr2200entry:2201 %c = icmp eq <16 x i8> %b, zeroinitializer2202 %xx = zext <16 x i8> %x to <16 x i16>2203 %yy = zext <16 x i8> %y to <16 x i16>2204 %m = mul <16 x i16> %xx, %yy2205 %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer2206 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)2207 %r = add i16 %z, %a2208 ret i16 %r2209}2210 2211define arm_aapcs_vfpcc signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i16 %a) {2212; CHECK-LABEL: add_v16i8_v16i16_acc_sext:2213; CHECK: @ %bb.0: @ %entry2214; CHECK-NEXT: vpt.i8 eq, q2, zr2215; CHECK-NEXT: vmlavat.s8 r0, q0, q12216; CHECK-NEXT: sxth r0, r02217; CHECK-NEXT: bx lr2218entry:2219 %c = icmp eq <16 x i8> %b, zeroinitializer2220 %xx = sext <16 x i8> %x to <16 x i16>2221 %yy = sext <16 x i8> %y to <16 x i16>2222 %m = mul <16 x i16> %xx, %yy2223 %s = select <16 x i1> %c, <16 x i16> %m, <16 x i16> zeroinitializer2224 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)2225 %r = add i16 %z, %a2226 ret i16 %r2227}2228 2229define arm_aapcs_vfpcc zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b, i16 %a) {2230; CHECK-LABEL: add_v8i8_v8i16_acc_zext:2231; CHECK: @ %bb.0: @ %entry2232; CHECK-NEXT: vmovlb.u8 q2, q22233; CHECK-NEXT: vmovlb.u8 q1, q12234; CHECK-NEXT: vmovlb.u8 q0, q02235; CHECK-NEXT: vpt.i16 eq, q2, zr2236; CHECK-NEXT: vmlavat.u16 r0, q0, q12237; CHECK-NEXT: uxth r0, r02238; CHECK-NEXT: bx lr2239entry:2240 %c = icmp eq <8 x i8> %b, zeroinitializer2241 %xx = zext <8 x i8> %x to <8 x i16>2242 %yy = zext <8 x i8> %y to <8 x i16>2243 %m = mul <8 x i16> %xx, %yy2244 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer2245 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)2246 %r = add i16 %z, %a2247 ret i16 %r2248}2249 2250define arm_aapcs_vfpcc signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, <8 x i8> %y, <8 x i8> %b, i16 %a) {2251; CHECK-LABEL: add_v8i8_v8i16_acc_sext:2252; CHECK: @ %bb.0: @ %entry2253; CHECK-NEXT: vmovlb.u8 q2, q22254; CHECK-NEXT: vmovlb.s8 q1, q12255; CHECK-NEXT: vmovlb.s8 q0, q02256; CHECK-NEXT: vpt.i16 eq, q2, zr2257; CHECK-NEXT: vmlavat.u16 r0, q0, q12258; CHECK-NEXT: sxth r0, r02259; CHECK-NEXT: bx lr2260entry:2261 %c = icmp eq <8 x i8> %b, zeroinitializer2262 %xx = sext <8 x i8> %x to <8 x i16>2263 %yy = sext <8 x i8> %y to <8 x i16>2264 %m = mul <8 x i16> %xx, %yy2265 %s = select <8 x i1> %c, <8 x i16> %m, <8 x i16> zeroinitializer2266 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %s)2267 %r = add i16 %z, %a2268 ret i16 %r2269}2270 2271define arm_aapcs_vfpcc zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i8 %a) {2272; CHECK-LABEL: add_v16i8_v16i8_acc:2273; CHECK: @ %bb.0: @ %entry2274; CHECK-NEXT: vpt.i8 eq, q2, zr2275; CHECK-NEXT: vmlavat.u8 r0, q0, q12276; CHECK-NEXT: uxtb r0, r02277; CHECK-NEXT: bx lr2278entry:2279 %c = icmp eq <16 x i8> %b, zeroinitializer2280 %m = mul <16 x i8> %x, %y2281 %s = select <16 x i1> %c, <16 x i8> %m, <16 x i8> zeroinitializer2282 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %s)2283 %r = add i8 %z, %a2284 ret i8 %r2285}2286 2287define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i64 %a) {2288; CHECK-LABEL: add_v16i8_v16i64_acc_zext:2289; CHECK: @ %bb.0: @ %entry2290; CHECK-NEXT: .save {r4, r5, r7, lr}2291; CHECK-NEXT: push {r4, r5, r7, lr}2292; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}2293; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}2294; CHECK-NEXT: vmov q3, q02295; CHECK-NEXT: vcmp.i8 eq, q2, zr2296; CHECK-NEXT: vmov.i8 q0, #0x02297; CHECK-NEXT: vmov.i8 q2, #0xff2298; CHECK-NEXT: vpsel q6, q2, q02299; CHECK-NEXT: vmov.i8 q2, #0xff2300; CHECK-NEXT: vmov.u8 r2, q6[0]2301; CHECK-NEXT: vmov.i8 q4, #0x02302; CHECK-NEXT: vmov.16 q0[0], r22303; CHECK-NEXT: vmov.u8 r2, q6[1]2304; CHECK-NEXT: vmov.16 q0[1], r22305; CHECK-NEXT: vmov.u8 r2, q6[2]2306; CHECK-NEXT: vmov.16 q0[2], r22307; CHECK-NEXT: vmov.u8 r2, q6[3]2308; CHECK-NEXT: vmov.16 q0[3], r22309; CHECK-NEXT: vmov.u8 r2, q6[4]2310; CHECK-NEXT: vmov.16 q0[4], r22311; CHECK-NEXT: vmov.u8 r2, q6[5]2312; CHECK-NEXT: vmov.16 q0[5], r22313; CHECK-NEXT: vmov.u8 r2, q6[6]2314; CHECK-NEXT: vmov.16 q0[6], r22315; CHECK-NEXT: vmov.u8 r2, q6[7]2316; CHECK-NEXT: vmov.16 q0[7], r22317; CHECK-NEXT: vmov.u8 r4, q3[2]2318; CHECK-NEXT: vcmp.i16 ne, q0, zr2319; CHECK-NEXT: vpsel q7, q2, q42320; CHECK-NEXT: vmov.u16 r2, q7[2]2321; CHECK-NEXT: vmov.u16 r3, q7[0]2322; CHECK-NEXT: vmov q0[2], q0[0], r3, r22323; CHECK-NEXT: vmov.u16 r2, q7[3]2324; CHECK-NEXT: vmov.u16 r3, q7[1]2325; CHECK-NEXT: vmov q0[3], q0[1], r3, r22326; CHECK-NEXT: vcmp.i32 ne, q0, zr2327; CHECK-NEXT: vpsel q0, q2, q42328; CHECK-NEXT: vmov r2, r3, d02329; CHECK-NEXT: vmov q2[2], q2[0], r2, r32330; CHECK-NEXT: vmov q2[3], q2[1], r2, r32331; CHECK-NEXT: vmov.u8 r2, q1[1]2332; CHECK-NEXT: vmov.u8 r3, q1[0]2333; CHECK-NEXT: vcmp.i32 ne, q2, zr2334; CHECK-NEXT: vmov q5[2], q5[0], r3, r22335; CHECK-NEXT: vmov.u8 r3, q3[1]2336; CHECK-NEXT: vmov.u8 r2, q3[0]2337; CHECK-NEXT: vmov.i64 q2, #0xff2338; CHECK-NEXT: vmov q4[2], q4[0], r2, r32339; CHECK-NEXT: vand q5, q5, q22340; CHECK-NEXT: vand q4, q4, q22341; CHECK-NEXT: vmov r12, s222342; CHECK-NEXT: vmov r2, s182343; CHECK-NEXT: vmov r3, s202344; CHECK-NEXT: vmov.i32 q5, #0x02345; CHECK-NEXT: umull lr, r12, r2, r122346; CHECK-NEXT: vmov r2, s162347; CHECK-NEXT: umull r2, r3, r2, r32348; CHECK-NEXT: vmov q4[2], q4[0], r2, lr2349; CHECK-NEXT: vmov q4[3], q4[1], r3, r122350; CHECK-NEXT: vpsel q4, q4, q52351; CHECK-NEXT: vmov lr, r12, d92352; CHECK-NEXT: vmov r3, r2, d82353; CHECK-NEXT: adds.w lr, lr, r32354; CHECK-NEXT: adc.w r12, r12, r22355; CHECK-NEXT: vmov r2, r3, d12356; CHECK-NEXT: vmov q0[2], q0[0], r2, r32357; CHECK-NEXT: vmov q0[3], q0[1], r2, r32358; CHECK-NEXT: vmov.u8 r2, q1[3]2359; CHECK-NEXT: vmov.u8 r3, q1[2]2360; CHECK-NEXT: vcmp.i32 ne, q0, zr2361; CHECK-NEXT: vmov q0[2], q0[0], r3, r22362; CHECK-NEXT: vmov.u8 r3, q3[3]2363; CHECK-NEXT: vmov q4[2], q4[0], r4, r32364; CHECK-NEXT: vand q0, q0, q22365; CHECK-NEXT: vand q4, q4, q22366; CHECK-NEXT: vmov r2, s22367; CHECK-NEXT: vmov r3, s182368; CHECK-NEXT: vmov r5, s162369; CHECK-NEXT: vmov.i8 q4, #0xff2370; CHECK-NEXT: vmov r4, s02371; CHECK-NEXT: umull r2, r3, r3, r22372; CHECK-NEXT: umull r4, r5, r5, r42373; CHECK-NEXT: vmov q0[2], q0[0], r4, r22374; CHECK-NEXT: vmov q0[3], q0[1], r5, r32375; CHECK-NEXT: vpsel q0, q0, q52376; CHECK-NEXT: vmov r2, r3, d02377; CHECK-NEXT: vmov r5, r4, d12378; CHECK-NEXT: adds.w r2, r2, lr2379; CHECK-NEXT: adc.w r3, r3, r122380; CHECK-NEXT: adds.w r12, r2, r52381; CHECK-NEXT: adc.w lr, r3, r42382; CHECK-NEXT: vmov.u16 r5, q7[6]2383; CHECK-NEXT: vmov.u16 r4, q7[4]2384; CHECK-NEXT: vmov.u8 r2, q3[4]2385; CHECK-NEXT: vmov q0[2], q0[0], r4, r52386; CHECK-NEXT: vmov.u16 r5, q7[7]2387; CHECK-NEXT: vmov.u16 r4, q7[5]2388; CHECK-NEXT: vmov q0[3], q0[1], r4, r52389; CHECK-NEXT: vcmp.i32 ne, q0, zr2390; CHECK-NEXT: vmov.i8 q0, #0x02391; CHECK-NEXT: vpsel q0, q4, q02392; CHECK-NEXT: vmov r5, r4, d02393; CHECK-NEXT: vmov q4[2], q4[0], r5, r42394; CHECK-NEXT: vmov q4[3], q4[1], r5, r42395; CHECK-NEXT: vmov.u8 r5, q1[5]2396; CHECK-NEXT: vmov.u8 r4, q1[4]2397; CHECK-NEXT: vcmp.i32 ne, q4, zr2398; CHECK-NEXT: vmov q4[2], q4[0], r4, r52399; CHECK-NEXT: vmov.u8 r4, q3[5]2400; CHECK-NEXT: vmov q7[2], q7[0], r2, r42401; CHECK-NEXT: vand q4, q4, q22402; CHECK-NEXT: vand q7, q7, q22403; CHECK-NEXT: vmov r5, s182404; CHECK-NEXT: vmov r2, s302405; CHECK-NEXT: vmov r3, s282406; CHECK-NEXT: vmov.i8 q7, #0xff2407; CHECK-NEXT: vmov r4, s162408; CHECK-NEXT: umull r2, r5, r2, r52409; CHECK-NEXT: umull r3, r4, r3, r42410; CHECK-NEXT: vmov q4[2], q4[0], r3, r22411; CHECK-NEXT: vmov q4[3], q4[1], r4, r52412; CHECK-NEXT: vpsel q4, q4, q52413; CHECK-NEXT: vmov r2, r3, d82414; CHECK-NEXT: vmov r5, r4, d92415; CHECK-NEXT: adds.w r2, r2, r122416; CHECK-NEXT: adc.w r3, r3, lr2417; CHECK-NEXT: adds.w r12, r2, r52418; CHECK-NEXT: adc.w lr, r3, r42419; CHECK-NEXT: vmov r5, r4, d12420; CHECK-NEXT: vmov q0[2], q0[0], r5, r42421; CHECK-NEXT: vmov.u8 r2, q3[6]2422; CHECK-NEXT: vmov q0[3], q0[1], r5, r42423; CHECK-NEXT: vmov.u8 r5, q1[7]2424; CHECK-NEXT: vmov.u8 r4, q1[6]2425; CHECK-NEXT: vcmp.i32 ne, q0, zr2426; CHECK-NEXT: vmov q0[2], q0[0], r4, r52427; CHECK-NEXT: vmov.u8 r4, q3[7]2428; CHECK-NEXT: vmov q4[2], q4[0], r2, r42429; CHECK-NEXT: vand q0, q0, q22430; CHECK-NEXT: vand q4, q4, q22431; CHECK-NEXT: vmov r5, s22432; CHECK-NEXT: vmov r2, s182433; CHECK-NEXT: vmov r3, s162434; CHECK-NEXT: vmov.i8 q4, #0x02435; CHECK-NEXT: vmov r4, s02436; CHECK-NEXT: umull r2, r5, r2, r52437; CHECK-NEXT: umull r3, r4, r3, r42438; CHECK-NEXT: vmov q0[2], q0[0], r3, r22439; CHECK-NEXT: vmov q0[3], q0[1], r4, r52440; CHECK-NEXT: vpsel q0, q0, q52441; CHECK-NEXT: vmov r2, r3, d02442; CHECK-NEXT: vmov r5, r4, d12443; CHECK-NEXT: adds.w r2, r2, r122444; CHECK-NEXT: adc.w r3, r3, lr2445; CHECK-NEXT: adds.w r12, r2, r52446; CHECK-NEXT: vmov.u8 r5, q6[8]2447; CHECK-NEXT: adc.w lr, r3, r42448; CHECK-NEXT: vmov.16 q0[0], r52449; CHECK-NEXT: vmov.u8 r5, q6[9]2450; CHECK-NEXT: vmov.16 q0[1], r52451; CHECK-NEXT: vmov.u8 r5, q6[10]2452; CHECK-NEXT: vmov.16 q0[2], r52453; CHECK-NEXT: vmov.u8 r5, q6[11]2454; CHECK-NEXT: vmov.16 q0[3], r52455; CHECK-NEXT: vmov.u8 r5, q6[12]2456; CHECK-NEXT: vmov.16 q0[4], r52457; CHECK-NEXT: vmov.u8 r5, q6[13]2458; CHECK-NEXT: vmov.16 q0[5], r52459; CHECK-NEXT: vmov.u8 r5, q6[14]2460; CHECK-NEXT: vmov.16 q0[6], r52461; CHECK-NEXT: vmov.u8 r5, q6[15]2462; CHECK-NEXT: vmov.16 q0[7], r52463; CHECK-NEXT: vmov.u8 r2, q3[8]2464; CHECK-NEXT: vcmp.i16 ne, q0, zr2465; CHECK-NEXT: vpsel q6, q7, q42466; CHECK-NEXT: vmov.u16 r5, q6[2]2467; CHECK-NEXT: vmov.u16 r4, q6[0]2468; CHECK-NEXT: vmov q0[2], q0[0], r4, r52469; CHECK-NEXT: vmov.u16 r5, q6[3]2470; CHECK-NEXT: vmov.u16 r4, q6[1]2471; CHECK-NEXT: vmov q0[3], q0[1], r4, r52472; CHECK-NEXT: vcmp.i32 ne, q0, zr2473; CHECK-NEXT: vpsel q0, q7, q42474; CHECK-NEXT: vmov r5, r4, d02475; CHECK-NEXT: vmov q4[2], q4[0], r5, r42476; CHECK-NEXT: vmov q4[3], q4[1], r5, r42477; CHECK-NEXT: vmov.u8 r5, q1[9]2478; CHECK-NEXT: vmov.u8 r4, q1[8]2479; CHECK-NEXT: vcmp.i32 ne, q4, zr2480; CHECK-NEXT: vmov q4[2], q4[0], r4, r52481; CHECK-NEXT: vmov.u8 r4, q3[9]2482; CHECK-NEXT: vmov q7[2], q7[0], r2, r42483; CHECK-NEXT: vand q4, q4, q22484; CHECK-NEXT: vand q7, q7, q22485; CHECK-NEXT: vmov r5, s182486; CHECK-NEXT: vmov r2, s302487; CHECK-NEXT: vmov r4, s162488; CHECK-NEXT: vmov r3, s282489; CHECK-NEXT: umull r2, r5, r2, r52490; CHECK-NEXT: umull r3, r4, r3, r42491; CHECK-NEXT: vmov q4[2], q4[0], r3, r22492; CHECK-NEXT: vmov q4[3], q4[1], r4, r52493; CHECK-NEXT: vpsel q4, q4, q52494; CHECK-NEXT: vmov r2, r3, d82495; CHECK-NEXT: vmov r5, r4, d92496; CHECK-NEXT: adds.w r2, r2, r122497; CHECK-NEXT: adc.w r3, r3, lr2498; CHECK-NEXT: adds.w r12, r2, r52499; CHECK-NEXT: adc.w lr, r3, r42500; CHECK-NEXT: vmov r5, r4, d12501; CHECK-NEXT: vmov q0[2], q0[0], r5, r42502; CHECK-NEXT: vmov.u8 r2, q3[10]2503; CHECK-NEXT: vmov q0[3], q0[1], r5, r42504; CHECK-NEXT: vmov.u8 r5, q1[11]2505; CHECK-NEXT: vmov.u8 r4, q1[10]2506; CHECK-NEXT: vcmp.i32 ne, q0, zr2507; CHECK-NEXT: vmov q0[2], q0[0], r4, r52508; CHECK-NEXT: vmov.u8 r4, q3[11]2509; CHECK-NEXT: vmov q4[2], q4[0], r2, r42510; CHECK-NEXT: vand q0, q0, q22511; CHECK-NEXT: vand q4, q4, q22512; CHECK-NEXT: vmov r5, s22513; CHECK-NEXT: vmov r2, s182514; CHECK-NEXT: vmov r3, s162515; CHECK-NEXT: vmov.i8 q4, #0x02516; CHECK-NEXT: vmov r4, s02517; CHECK-NEXT: umull r2, r5, r2, r52518; CHECK-NEXT: umull r3, r4, r3, r42519; CHECK-NEXT: vmov q0[2], q0[0], r3, r22520; CHECK-NEXT: vmov q0[3], q0[1], r4, r52521; CHECK-NEXT: vpsel q0, q0, q52522; CHECK-NEXT: vmov r2, r3, d02523; CHECK-NEXT: vmov r5, r4, d12524; CHECK-NEXT: adds.w r2, r2, r122525; CHECK-NEXT: adc.w r3, r3, lr2526; CHECK-NEXT: adds.w r12, r2, r52527; CHECK-NEXT: adc.w lr, r3, r42528; CHECK-NEXT: vmov.u16 r5, q6[6]2529; CHECK-NEXT: vmov.u16 r4, q6[4]2530; CHECK-NEXT: vmov.u8 r2, q3[12]2531; CHECK-NEXT: vmov q0[2], q0[0], r4, r52532; CHECK-NEXT: vmov.u16 r5, q6[7]2533; CHECK-NEXT: vmov.u16 r4, q6[5]2534; CHECK-NEXT: vmov q0[3], q0[1], r4, r52535; CHECK-NEXT: vcmp.i32 ne, q0, zr2536; CHECK-NEXT: vmov.i8 q0, #0xff2537; CHECK-NEXT: vpsel q0, q0, q42538; CHECK-NEXT: vmov r5, r4, d02539; CHECK-NEXT: vmov q4[2], q4[0], r5, r42540; CHECK-NEXT: vmov q4[3], q4[1], r5, r42541; CHECK-NEXT: vmov.u8 r5, q1[13]2542; CHECK-NEXT: vmov.u8 r4, q1[12]2543; CHECK-NEXT: vcmp.i32 ne, q4, zr2544; CHECK-NEXT: vmov q4[2], q4[0], r4, r52545; CHECK-NEXT: vmov.u8 r4, q3[13]2546; CHECK-NEXT: vmov q6[2], q6[0], r2, r42547; CHECK-NEXT: vand q4, q4, q22548; CHECK-NEXT: vand q6, q6, q22549; CHECK-NEXT: vmov r5, s182550; CHECK-NEXT: vmov r2, s262551; CHECK-NEXT: vmov r4, s162552; CHECK-NEXT: vmov r3, s242553; CHECK-NEXT: umull r2, r5, r2, r52554; CHECK-NEXT: umull r3, r4, r3, r42555; CHECK-NEXT: vmov q4[2], q4[0], r3, r22556; CHECK-NEXT: vmov q4[3], q4[1], r4, r52557; CHECK-NEXT: vpsel q4, q4, q52558; CHECK-NEXT: vmov r2, r3, d82559; CHECK-NEXT: vmov r5, r4, d92560; CHECK-NEXT: adds.w r2, r2, r122561; CHECK-NEXT: adc.w r3, r3, lr2562; CHECK-NEXT: adds.w r12, r2, r52563; CHECK-NEXT: adc.w lr, r3, r42564; CHECK-NEXT: vmov r5, r4, d12565; CHECK-NEXT: vmov q0[2], q0[0], r5, r42566; CHECK-NEXT: vmov.u8 r2, q3[14]2567; CHECK-NEXT: vmov q0[3], q0[1], r5, r42568; CHECK-NEXT: vmov.u8 r5, q1[15]2569; CHECK-NEXT: vmov.u8 r4, q1[14]2570; CHECK-NEXT: vcmp.i32 ne, q0, zr2571; CHECK-NEXT: vmov q0[2], q0[0], r4, r52572; CHECK-NEXT: vmov.u8 r4, q3[15]2573; CHECK-NEXT: vmov q1[2], q1[0], r2, r42574; CHECK-NEXT: vand q0, q0, q22575; CHECK-NEXT: vand q1, q1, q22576; CHECK-NEXT: vmov r5, s22577; CHECK-NEXT: vmov r2, s62578; CHECK-NEXT: vmov r4, s02579; CHECK-NEXT: vmov r3, s42580; CHECK-NEXT: umull r2, r5, r2, r52581; CHECK-NEXT: umull r3, r4, r3, r42582; CHECK-NEXT: vmov q0[2], q0[0], r3, r22583; CHECK-NEXT: vmov q0[3], q0[1], r4, r52584; CHECK-NEXT: vpsel q0, q0, q52585; CHECK-NEXT: vmov r2, r3, d02586; CHECK-NEXT: vmov r5, r4, d12587; CHECK-NEXT: adds.w r2, r2, r122588; CHECK-NEXT: adc.w r3, r3, lr2589; CHECK-NEXT: adds r2, r2, r52590; CHECK-NEXT: adcs r3, r42591; CHECK-NEXT: adds r0, r0, r22592; CHECK-NEXT: adcs r1, r32593; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}2594; CHECK-NEXT: pop {r4, r5, r7, pc}2595entry:2596 %c = icmp eq <16 x i8> %b, zeroinitializer2597 %xx = zext <16 x i8> %x to <16 x i64>2598 %yy = zext <16 x i8> %y to <16 x i64>2599 %m = mul <16 x i64> %xx, %yy2600 %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer2601 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %s)2602 %r = add i64 %z, %a2603 ret i64 %r2604}2605 2606define arm_aapcs_vfpcc i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, <16 x i8> %y, <16 x i8> %b, i64 %a) {2607; CHECK-LABEL: add_v16i8_v16i64_acc_sext:2608; CHECK: @ %bb.0: @ %entry2609; CHECK-NEXT: .save {r4, r5, r7, lr}2610; CHECK-NEXT: push {r4, r5, r7, lr}2611; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}2612; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}2613; CHECK-NEXT: vmov q3, q02614; CHECK-NEXT: vcmp.i8 eq, q2, zr2615; CHECK-NEXT: vmov.i8 q0, #0x02616; CHECK-NEXT: vmov.i8 q2, #0xff2617; CHECK-NEXT: vpsel q5, q2, q02618; CHECK-NEXT: vmov.i8 q0, #0x02619; CHECK-NEXT: vmov.u8 r2, q5[0]2620; CHECK-NEXT: vmov.s8 r4, q1[2]2621; CHECK-NEXT: vmov.16 q4[0], r22622; CHECK-NEXT: vmov.u8 r2, q5[1]2623; CHECK-NEXT: vmov.16 q4[1], r22624; CHECK-NEXT: vmov.u8 r2, q5[2]2625; CHECK-NEXT: vmov.16 q4[2], r22626; CHECK-NEXT: vmov.u8 r2, q5[3]2627; CHECK-NEXT: vmov.16 q4[3], r22628; CHECK-NEXT: vmov.u8 r2, q5[4]2629; CHECK-NEXT: vmov.16 q4[4], r22630; CHECK-NEXT: vmov.u8 r2, q5[5]2631; CHECK-NEXT: vmov.16 q4[5], r22632; CHECK-NEXT: vmov.u8 r2, q5[6]2633; CHECK-NEXT: vmov.16 q4[6], r22634; CHECK-NEXT: vmov.u8 r2, q5[7]2635; CHECK-NEXT: vmov.16 q4[7], r22636; CHECK-NEXT: vmov.s8 r5, q3[2]2637; CHECK-NEXT: vcmp.i16 ne, q4, zr2638; CHECK-NEXT: smull r4, r5, r5, r42639; CHECK-NEXT: vpsel q6, q2, q02640; CHECK-NEXT: vmov.u16 r2, q6[2]2641; CHECK-NEXT: vmov.u16 r3, q6[0]2642; CHECK-NEXT: vmov q4[2], q4[0], r3, r22643; CHECK-NEXT: vmov.u16 r2, q6[3]2644; CHECK-NEXT: vmov.u16 r3, q6[1]2645; CHECK-NEXT: vmov q4[3], q4[1], r3, r22646; CHECK-NEXT: vcmp.i32 ne, q4, zr2647; CHECK-NEXT: vpsel q7, q2, q02648; CHECK-NEXT: vmov r2, r3, d142649; CHECK-NEXT: vmov q4[2], q4[0], r2, r32650; CHECK-NEXT: vmov q4[3], q4[1], r2, r32651; CHECK-NEXT: vmov.s8 r2, q1[1]2652; CHECK-NEXT: vmov.s8 r3, q3[1]2653; CHECK-NEXT: vcmp.i32 ne, q4, zr2654; CHECK-NEXT: smull lr, r12, r3, r22655; CHECK-NEXT: vmov.s8 r3, q1[0]2656; CHECK-NEXT: vmov.s8 r2, q3[0]2657; CHECK-NEXT: vmov.i32 q4, #0x02658; CHECK-NEXT: smull r2, r3, r2, r32659; CHECK-NEXT: vmov q0[2], q0[0], r2, lr2660; CHECK-NEXT: vmov q0[3], q0[1], r3, r122661; CHECK-NEXT: vpsel q0, q0, q42662; CHECK-NEXT: vmov lr, r12, d12663; CHECK-NEXT: vmov r3, r2, d02664; CHECK-NEXT: adds.w lr, lr, r32665; CHECK-NEXT: adc.w r12, r12, r22666; CHECK-NEXT: vmov r2, r3, d152667; CHECK-NEXT: vmov q0[2], q0[0], r2, r32668; CHECK-NEXT: vmov.i8 q7, #0x02669; CHECK-NEXT: vmov q0[3], q0[1], r2, r32670; CHECK-NEXT: vmov.s8 r2, q1[3]2671; CHECK-NEXT: vmov.s8 r3, q3[3]2672; CHECK-NEXT: vcmp.i32 ne, q0, zr2673; CHECK-NEXT: smull r2, r3, r3, r22674; CHECK-NEXT: vmov q0[2], q0[0], r4, r22675; CHECK-NEXT: vmov q0[3], q0[1], r5, r32676; CHECK-NEXT: vpsel q0, q0, q42677; CHECK-NEXT: vmov r2, r3, d02678; CHECK-NEXT: vmov r5, r4, d12679; CHECK-NEXT: adds.w r2, r2, lr2680; CHECK-NEXT: adc.w r3, r3, r122681; CHECK-NEXT: adds.w r12, r2, r52682; CHECK-NEXT: adc.w lr, r3, r42683; CHECK-NEXT: vmov.u16 r5, q6[6]2684; CHECK-NEXT: vmov.u16 r4, q6[4]2685; CHECK-NEXT: vmov.s8 r2, q1[4]2686; CHECK-NEXT: vmov q0[2], q0[0], r4, r52687; CHECK-NEXT: vmov.u16 r5, q6[7]2688; CHECK-NEXT: vmov.u16 r4, q6[5]2689; CHECK-NEXT: vmov.s8 r3, q3[4]2690; CHECK-NEXT: vmov q0[3], q0[1], r4, r52691; CHECK-NEXT: smull r2, r3, r3, r22692; CHECK-NEXT: vcmp.i32 ne, q0, zr2693; CHECK-NEXT: vmov.i8 q0, #0x02694; CHECK-NEXT: vpsel q6, q2, q02695; CHECK-NEXT: vmov r5, r4, d122696; CHECK-NEXT: vmov q0[2], q0[0], r5, r42697; CHECK-NEXT: vmov q0[3], q0[1], r5, r42698; CHECK-NEXT: vmov.s8 r5, q1[5]2699; CHECK-NEXT: vmov.s8 r4, q3[5]2700; CHECK-NEXT: vcmp.i32 ne, q0, zr2701; CHECK-NEXT: smull r5, r4, r4, r52702; CHECK-NEXT: vmov q0[2], q0[0], r2, r52703; CHECK-NEXT: vmov q0[3], q0[1], r3, r42704; CHECK-NEXT: vpsel q0, q0, q42705; CHECK-NEXT: vmov r2, r3, d02706; CHECK-NEXT: vmov r5, r4, d12707; CHECK-NEXT: adds.w r2, r2, r122708; CHECK-NEXT: adc.w r3, r3, lr2709; CHECK-NEXT: adds.w r12, r2, r52710; CHECK-NEXT: adc.w lr, r3, r42711; CHECK-NEXT: vmov r5, r4, d132712; CHECK-NEXT: vmov q0[2], q0[0], r5, r42713; CHECK-NEXT: vmov.s8 r2, q1[6]2714; CHECK-NEXT: vmov q0[3], q0[1], r5, r42715; CHECK-NEXT: vmov.s8 r3, q3[6]2716; CHECK-NEXT: vmov.s8 r5, q1[7]2717; CHECK-NEXT: vmov.s8 r4, q3[7]2718; CHECK-NEXT: smull r5, r4, r4, r52719; CHECK-NEXT: vcmp.i32 ne, q0, zr2720; CHECK-NEXT: smull r2, r3, r3, r22721; CHECK-NEXT: vmov q0[2], q0[0], r2, r52722; CHECK-NEXT: vmov q0[3], q0[1], r3, r42723; CHECK-NEXT: vpsel q0, q0, q42724; CHECK-NEXT: vmov r2, r3, d02725; CHECK-NEXT: vmov r5, r4, d12726; CHECK-NEXT: adds.w r2, r2, r122727; CHECK-NEXT: adc.w r3, r3, lr2728; CHECK-NEXT: adds.w r12, r2, r52729; CHECK-NEXT: vmov.u8 r5, q5[8]2730; CHECK-NEXT: adc.w lr, r3, r42731; CHECK-NEXT: vmov.16 q6[0], r52732; CHECK-NEXT: vmov.u8 r5, q5[9]2733; CHECK-NEXT: vmov.16 q6[1], r52734; CHECK-NEXT: vmov.u8 r5, q5[10]2735; CHECK-NEXT: vmov.16 q6[2], r52736; CHECK-NEXT: vmov.u8 r5, q5[11]2737; CHECK-NEXT: vmov.16 q6[3], r52738; CHECK-NEXT: vmov.u8 r5, q5[12]2739; CHECK-NEXT: vmov.16 q6[4], r52740; CHECK-NEXT: vmov.u8 r5, q5[13]2741; CHECK-NEXT: vmov.16 q6[5], r52742; CHECK-NEXT: vmov.u8 r5, q5[14]2743; CHECK-NEXT: vmov.16 q6[6], r52744; CHECK-NEXT: vmov.u8 r5, q5[15]2745; CHECK-NEXT: vmov.16 q6[7], r52746; CHECK-NEXT: vmov.s8 r2, q1[8]2747; CHECK-NEXT: vcmp.i16 ne, q6, zr2748; CHECK-NEXT: vmov.i8 q6, #0x02749; CHECK-NEXT: vpsel q5, q2, q62750; CHECK-NEXT: vmov.s8 r3, q3[8]2751; CHECK-NEXT: vmov.u16 r5, q5[2]2752; CHECK-NEXT: vmov.u16 r4, q5[0]2753; CHECK-NEXT: vmov q0[2], q0[0], r4, r52754; CHECK-NEXT: vmov.u16 r5, q5[3]2755; CHECK-NEXT: vmov.u16 r4, q5[1]2756; CHECK-NEXT: smull r2, r3, r3, r22757; CHECK-NEXT: vmov q0[3], q0[1], r4, r52758; CHECK-NEXT: vcmp.i32 ne, q0, zr2759; CHECK-NEXT: vpsel q6, q2, q62760; CHECK-NEXT: vmov r5, r4, d122761; CHECK-NEXT: vmov q0[2], q0[0], r5, r42762; CHECK-NEXT: vmov q0[3], q0[1], r5, r42763; CHECK-NEXT: vmov.s8 r5, q1[9]2764; CHECK-NEXT: vmov.s8 r4, q3[9]2765; CHECK-NEXT: vcmp.i32 ne, q0, zr2766; CHECK-NEXT: smull r5, r4, r4, r52767; CHECK-NEXT: vmov q0[2], q0[0], r2, r52768; CHECK-NEXT: vmov q0[3], q0[1], r3, r42769; CHECK-NEXT: vpsel q0, q0, q42770; CHECK-NEXT: vmov r2, r3, d02771; CHECK-NEXT: vmov r5, r4, d12772; CHECK-NEXT: adds.w r2, r2, r122773; CHECK-NEXT: adc.w r3, r3, lr2774; CHECK-NEXT: adds.w r12, r2, r52775; CHECK-NEXT: adc.w lr, r3, r42776; CHECK-NEXT: vmov r5, r4, d132777; CHECK-NEXT: vmov q0[2], q0[0], r5, r42778; CHECK-NEXT: vmov.s8 r2, q1[10]2779; CHECK-NEXT: vmov q0[3], q0[1], r5, r42780; CHECK-NEXT: vmov.s8 r3, q3[10]2781; CHECK-NEXT: vmov.s8 r5, q1[11]2782; CHECK-NEXT: vmov.s8 r4, q3[11]2783; CHECK-NEXT: smull r5, r4, r4, r52784; CHECK-NEXT: vcmp.i32 ne, q0, zr2785; CHECK-NEXT: smull r2, r3, r3, r22786; CHECK-NEXT: vmov q0[2], q0[0], r2, r52787; CHECK-NEXT: vmov q0[3], q0[1], r3, r42788; CHECK-NEXT: vpsel q0, q0, q42789; CHECK-NEXT: vmov r2, r3, d02790; CHECK-NEXT: vmov r5, r4, d12791; CHECK-NEXT: adds.w r2, r2, r122792; CHECK-NEXT: adc.w r3, r3, lr2793; CHECK-NEXT: adds.w r12, r2, r52794; CHECK-NEXT: adc.w lr, r3, r42795; CHECK-NEXT: vmov.u16 r5, q5[6]2796; CHECK-NEXT: vmov.u16 r4, q5[4]2797; CHECK-NEXT: vmov.s8 r2, q1[12]2798; CHECK-NEXT: vmov q0[2], q0[0], r4, r52799; CHECK-NEXT: vmov.u16 r5, q5[7]2800; CHECK-NEXT: vmov.u16 r4, q5[5]2801; CHECK-NEXT: vmov.s8 r3, q3[12]2802; CHECK-NEXT: vmov q0[3], q0[1], r4, r52803; CHECK-NEXT: smull r2, r3, r3, r22804; CHECK-NEXT: vcmp.i32 ne, q0, zr2805; CHECK-NEXT: vpsel q2, q2, q72806; CHECK-NEXT: vmov r5, r4, d42807; CHECK-NEXT: vmov q0[2], q0[0], r5, r42808; CHECK-NEXT: vmov q0[3], q0[1], r5, r42809; CHECK-NEXT: vmov.s8 r5, q1[13]2810; CHECK-NEXT: vmov.s8 r4, q3[13]2811; CHECK-NEXT: vcmp.i32 ne, q0, zr2812; CHECK-NEXT: smull r5, r4, r4, r52813; CHECK-NEXT: vmov q0[2], q0[0], r2, r52814; CHECK-NEXT: vmov q0[3], q0[1], r3, r42815; CHECK-NEXT: vpsel q0, q0, q42816; CHECK-NEXT: vmov r2, r3, d02817; CHECK-NEXT: vmov r5, r4, d12818; CHECK-NEXT: adds.w r2, r2, r122819; CHECK-NEXT: adc.w r3, r3, lr2820; CHECK-NEXT: adds.w r12, r2, r52821; CHECK-NEXT: adc.w lr, r3, r42822; CHECK-NEXT: vmov r5, r4, d52823; CHECK-NEXT: vmov q0[2], q0[0], r5, r42824; CHECK-NEXT: vmov.s8 r2, q1[14]2825; CHECK-NEXT: vmov q0[3], q0[1], r5, r42826; CHECK-NEXT: vmov.s8 r3, q3[14]2827; CHECK-NEXT: vmov.s8 r5, q1[15]2828; CHECK-NEXT: vmov.s8 r4, q3[15]2829; CHECK-NEXT: smull r5, r4, r4, r52830; CHECK-NEXT: vcmp.i32 ne, q0, zr2831; CHECK-NEXT: smull r2, r3, r3, r22832; CHECK-NEXT: vmov q0[2], q0[0], r2, r52833; CHECK-NEXT: vmov q0[3], q0[1], r3, r42834; CHECK-NEXT: vpsel q0, q0, q42835; CHECK-NEXT: vmov r2, r3, d02836; CHECK-NEXT: vmov r5, r4, d12837; CHECK-NEXT: adds.w r2, r2, r122838; CHECK-NEXT: adc.w r3, r3, lr2839; CHECK-NEXT: adds r2, r2, r52840; CHECK-NEXT: adcs r3, r42841; CHECK-NEXT: adds r0, r0, r22842; CHECK-NEXT: adcs r1, r32843; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}2844; CHECK-NEXT: pop {r4, r5, r7, pc}2845entry:2846 %c = icmp eq <16 x i8> %b, zeroinitializer2847 %xx = sext <16 x i8> %x to <16 x i64>2848 %yy = sext <16 x i8> %y to <16 x i64>2849 %m = mul <16 x i64> %xx, %yy2850 %s = select <16 x i1> %c, <16 x i64> %m, <16 x i64> zeroinitializer2851 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %s)2852 %r = add i64 %z, %a2853 ret i64 %r2854}2855 2856define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b, i64 %a) {2857; CHECK-LABEL: add_v2i8_v2i64_acc_zext:2858; CHECK: @ %bb.0: @ %entry2859; CHECK-NEXT: .save {r7, lr}2860; CHECK-NEXT: push {r7, lr}2861; CHECK-NEXT: vmov.i64 q3, #0xff2862; CHECK-NEXT: vand q1, q1, q32863; CHECK-NEXT: vand q0, q0, q32864; CHECK-NEXT: vmov r2, s62865; CHECK-NEXT: vmov r3, s22866; CHECK-NEXT: umull lr, r12, r3, r22867; CHECK-NEXT: vmov r3, s42868; CHECK-NEXT: vmov r2, s02869; CHECK-NEXT: vand q1, q2, q32870; CHECK-NEXT: umull r2, r3, r2, r32871; CHECK-NEXT: vmov q0[2], q0[0], r2, lr2872; CHECK-NEXT: vmov r2, s42873; CHECK-NEXT: vmov q0[3], q0[1], r3, r122874; CHECK-NEXT: movs r3, #02875; CHECK-NEXT: cmp r2, #02876; CHECK-NEXT: csetm r2, eq2877; CHECK-NEXT: bfi r3, r2, #0, #82878; CHECK-NEXT: vmov r2, s62879; CHECK-NEXT: vmov.i32 q1, #0x02880; CHECK-NEXT: cmp r2, #02881; CHECK-NEXT: csetm r2, eq2882; CHECK-NEXT: bfi r3, r2, #8, #82883; CHECK-NEXT: vmsr p0, r32884; CHECK-NEXT: vpsel q0, q0, q12885; CHECK-NEXT: vmov lr, r12, d12886; CHECK-NEXT: vmov r3, r2, d02887; CHECK-NEXT: adds.w r3, r3, lr2888; CHECK-NEXT: adc.w r2, r2, r122889; CHECK-NEXT: adds r0, r0, r32890; CHECK-NEXT: adcs r1, r22891; CHECK-NEXT: pop {r7, pc}2892entry:2893 %c = icmp eq <2 x i8> %b, zeroinitializer2894 %xx = zext <2 x i8> %x to <2 x i64>2895 %yy = zext <2 x i8> %y to <2 x i64>2896 %m = mul <2 x i64> %xx, %yy2897 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer2898 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)2899 %r = add i64 %z, %a2900 ret i64 %r2901}2902 2903define arm_aapcs_vfpcc i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, <2 x i8> %y, <2 x i8> %b, i64 %a) {2904; CHECK-LABEL: add_v2i8_v2i64_acc_sext:2905; CHECK: @ %bb.0: @ %entry2906; CHECK-NEXT: .save {r7, lr}2907; CHECK-NEXT: push {r7, lr}2908; CHECK-NEXT: vmov.i32 q3, #0xff2909; CHECK-NEXT: movs r3, #02910; CHECK-NEXT: vand q2, q2, q32911; CHECK-NEXT: vmov r2, s82912; CHECK-NEXT: cmp r2, #02913; CHECK-NEXT: csetm r2, eq2914; CHECK-NEXT: bfi r3, r2, #0, #82915; CHECK-NEXT: vmov r2, s102916; CHECK-NEXT: cmp r2, #02917; CHECK-NEXT: csetm r2, eq2918; CHECK-NEXT: bfi r3, r2, #8, #82919; CHECK-NEXT: vmov r2, s62920; CHECK-NEXT: vmsr p0, r32921; CHECK-NEXT: vmov r3, s22922; CHECK-NEXT: sxtb r2, r22923; CHECK-NEXT: sxtb r3, r32924; CHECK-NEXT: smull lr, r12, r3, r22925; CHECK-NEXT: vmov r3, s42926; CHECK-NEXT: vmov r2, s02927; CHECK-NEXT: vmov.i32 q1, #0x02928; CHECK-NEXT: sxtb r3, r32929; CHECK-NEXT: sxtb r2, r22930; CHECK-NEXT: smull r2, r3, r2, r32931; CHECK-NEXT: vmov q0[2], q0[0], r2, lr2932; CHECK-NEXT: vmov q0[3], q0[1], r3, r122933; CHECK-NEXT: vpsel q0, q0, q12934; CHECK-NEXT: vmov lr, r12, d12935; CHECK-NEXT: vmov r3, r2, d02936; CHECK-NEXT: adds.w r3, r3, lr2937; CHECK-NEXT: adc.w r2, r2, r122938; CHECK-NEXT: adds r0, r0, r32939; CHECK-NEXT: adcs r1, r22940; CHECK-NEXT: pop {r7, pc}2941entry:2942 %c = icmp eq <2 x i8> %b, zeroinitializer2943 %xx = sext <2 x i8> %x to <2 x i64>2944 %yy = sext <2 x i8> %y to <2 x i64>2945 %m = mul <2 x i64> %xx, %yy2946 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer2947 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)2948 %r = add i64 %z, %a2949 ret i64 %r2950}2951 2952define arm_aapcs_vfpcc i64 @add_v2i64_v2i64_acc(<2 x i64> %x, <2 x i64> %y, <2 x i64> %b, i64 %a) {2953; CHECK-LABEL: add_v2i64_v2i64_acc:2954; CHECK: @ %bb.0: @ %entry2955; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}2956; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}2957; CHECK-NEXT: vmov r2, r12, d32958; CHECK-NEXT: vmov r3, lr, d12959; CHECK-NEXT: vmov r6, r9, d22960; CHECK-NEXT: vmov.i32 q1, #0x02961; CHECK-NEXT: vmov r5, r11, d02962; CHECK-NEXT: umull r10, r8, r3, r22963; CHECK-NEXT: umull r4, r7, r5, r62964; CHECK-NEXT: mla r3, r3, r12, r82965; CHECK-NEXT: vmov q0[2], q0[0], r4, r102966; CHECK-NEXT: mla r2, lr, r2, r32967; CHECK-NEXT: mla r3, r5, r9, r72968; CHECK-NEXT: mla r3, r11, r6, r32969; CHECK-NEXT: vmov q0[3], q0[1], r3, r22970; CHECK-NEXT: vmov r2, r3, d42971; CHECK-NEXT: orrs r2, r32972; CHECK-NEXT: mov.w r3, #02973; CHECK-NEXT: csetm r2, eq2974; CHECK-NEXT: bfi r3, r2, #0, #82975; CHECK-NEXT: vmov r2, r7, d52976; CHECK-NEXT: orrs r2, r72977; CHECK-NEXT: csetm r2, eq2978; CHECK-NEXT: bfi r3, r2, #8, #82979; CHECK-NEXT: vmsr p0, r32980; CHECK-NEXT: vpsel q0, q0, q12981; CHECK-NEXT: vmov r2, r3, d12982; CHECK-NEXT: vmov r7, r6, d02983; CHECK-NEXT: adds r2, r2, r72984; CHECK-NEXT: adcs r3, r62985; CHECK-NEXT: adds r0, r0, r22986; CHECK-NEXT: adcs r1, r32987; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}2988entry:2989 %c = icmp eq <2 x i64> %b, zeroinitializer2990 %m = mul <2 x i64> %x, %y2991 %s = select <2 x i1> %c, <2 x i64> %m, <2 x i64> zeroinitializer2992 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %s)2993 %r = add i64 %z, %a2994 ret i64 %r2995}2996 2997declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)2998declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)2999declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)3000declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)3001declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)3002declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)3003declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)3004declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)3005declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)3006declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)3007