1611 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -enable-arm-maskedldst %s -o - | FileCheck %s3 4define arm_aapcs_vfpcc <4 x i32> @gather_inc_mini_4i32(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <4 x i32> %offs) {5; CHECK-LABEL: gather_inc_mini_4i32:6; CHECK: @ %bb.0:7; CHECK-NEXT: movs r1, #48; CHECK-NEXT: vadd.i32 q1, q0, r19; CHECK-NEXT: vldrw.u32 q0, [r0, q1, uxtw #2]10; CHECK-NEXT: bx lr11 %1 = add <4 x i32> %offs, <i32 4, i32 4, i32 4, i32 4>12 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %113 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)14 ret <4 x i32> %wide.masked.gather15}16 17define arm_aapcs_vfpcc <4 x i32> @gather_inc_mini_4i32_i8(i32* noalias nocapture readonly %data, i32* noalias nocapture %dst, <4 x i32> %offs) {18; CHECK-LABEL: gather_inc_mini_4i32_i8:19; CHECK: @ %bb.0:20; CHECK-NEXT: movs r1, #1621; CHECK-NEXT: vadd.i32 q1, q0, r122; CHECK-NEXT: vldrw.u32 q0, [r0, q1]23; CHECK-NEXT: bx lr24 %1 = add <4 x i32> %offs, <i32 16, i32 16, i32 16, i32 16>25 %2 = getelementptr inbounds i8, i32* %data, <4 x i32> %126 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)27 ret <4 x i32> %wide.masked.gather28}29 30define arm_aapcs_vfpcc <4 x i32> @gather_inc_minipred_4i32(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <4 x i32> %offs) {31; CHECK-LABEL: gather_inc_minipred_4i32:32; CHECK: @ %bb.0:33; CHECK-NEXT: movs r1, #434; CHECK-NEXT: movw r2, #385535; CHECK-NEXT: vadd.i32 q1, q0, r136; CHECK-NEXT: vmsr p0, r237; CHECK-NEXT: vpst38; CHECK-NEXT: vldrwt.u32 q0, [r0, q1, uxtw #2]39; CHECK-NEXT: bx lr40 %1 = add <4 x i32> %offs, <i32 4, i32 4, i32 4, i32 4>41 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %142 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> undef)43 ret <4 x i32> %wide.masked.gather44}45 46define arm_aapcs_vfpcc <8 x i16> @gather_inc_mini_8i16(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <8 x i32> %offs) {47; CHECK-LABEL: gather_inc_mini_8i16:48; CHECK: @ %bb.0:49; CHECK-NEXT: .save {r4, r5, r6, lr}50; CHECK-NEXT: push {r4, r5, r6, lr}51; CHECK-NEXT: vshl.i32 q1, q1, #152; CHECK-NEXT: mov.w r12, #1653; CHECK-NEXT: vadd.i32 q1, q1, r054; CHECK-NEXT: vshl.i32 q0, q0, #155; CHECK-NEXT: vadd.i32 q1, q1, r1256; CHECK-NEXT: vadd.i32 q0, q0, r057; CHECK-NEXT: vmov r1, lr, d358; CHECK-NEXT: vadd.i32 q0, q0, r1259; CHECK-NEXT: vmov r0, r3, d160; CHECK-NEXT: vmov r2, r4, d261; CHECK-NEXT: ldrh r6, [r1]62; CHECK-NEXT: vmov r1, r5, d063; CHECK-NEXT: ldrh r0, [r0]64; CHECK-NEXT: ldrh r3, [r3]65; CHECK-NEXT: ldrh r2, [r2]66; CHECK-NEXT: ldrh r4, [r4]67; CHECK-NEXT: ldrh.w r12, [lr]68; CHECK-NEXT: ldrh r1, [r1]69; CHECK-NEXT: ldrh r5, [r5]70; CHECK-NEXT: vmov.16 q0[0], r171; CHECK-NEXT: vmov.16 q0[1], r572; CHECK-NEXT: vmov.16 q0[2], r073; CHECK-NEXT: vmov.16 q0[3], r374; CHECK-NEXT: vmov.16 q0[4], r275; CHECK-NEXT: vmov.16 q0[5], r476; CHECK-NEXT: vmov.16 q0[6], r677; CHECK-NEXT: vmov.16 q0[7], r1278; CHECK-NEXT: pop {r4, r5, r6, pc}79 %1 = add <8 x i32> %offs, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>80 %2 = getelementptr inbounds i16, ptr %data, <8 x i32> %181 %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %2, i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)82 ret <8 x i16> %wide.masked.gather83}84 85define arm_aapcs_vfpcc <8 x i16> @gather_inc_minipred_8i16(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <8 x i32> %offs) {86; CHECK-LABEL: gather_inc_minipred_8i16:87; CHECK: @ %bb.0:88; CHECK-NEXT: vshl.i32 q0, q0, #189; CHECK-NEXT: movs r1, #1690; CHECK-NEXT: vadd.i32 q0, q0, r091; CHECK-NEXT: vshl.i32 q1, q1, #192; CHECK-NEXT: vadd.i32 q0, q0, r193; CHECK-NEXT: vadd.i32 q1, q1, r094; CHECK-NEXT: vmov r2, s095; CHECK-NEXT: vadd.i32 q1, q1, r196; CHECK-NEXT: vmov r3, s297; CHECK-NEXT: vmov r0, s498; CHECK-NEXT: vmov r1, s699; CHECK-NEXT: ldrh r2, [r2]100; CHECK-NEXT: ldrh r3, [r3]101; CHECK-NEXT: vmov.16 q0[0], r2102; CHECK-NEXT: ldrh r0, [r0]103; CHECK-NEXT: vmov.16 q0[2], r3104; CHECK-NEXT: ldrh r1, [r1]105; CHECK-NEXT: vmov.16 q0[4], r0106; CHECK-NEXT: vmov.16 q0[6], r1107; CHECK-NEXT: bx lr108 %1 = add <8 x i32> %offs, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>109 %2 = getelementptr inbounds i16, ptr %data, <8 x i32> %1110 %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %2, i32 4, <8 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <8 x i16> undef)111 ret <8 x i16> %wide.masked.gather112}113 114define arm_aapcs_vfpcc <16 x i8> @gather_inc_mini_16i8(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <16 x i32> %offs) {115; CHECK-LABEL: gather_inc_mini_16i8:116; CHECK: @ %bb.0:117; CHECK-NEXT: .save {r4, r5, r6, r7, lr}118; CHECK-NEXT: push {r4, r5, r6, r7, lr}119; CHECK-NEXT: movs r5, #16120; CHECK-NEXT: vadd.i32 q3, q3, r0121; CHECK-NEXT: vadd.i32 q3, q3, r5122; CHECK-NEXT: vadd.i32 q0, q0, r0123; CHECK-NEXT: vmov r1, r2, d7124; CHECK-NEXT: vadd.i32 q1, q1, r0125; CHECK-NEXT: vmov r3, r4, d6126; CHECK-NEXT: vadd.i32 q3, q0, r5127; CHECK-NEXT: vadd.i32 q0, q2, r0128; CHECK-NEXT: vadd.i32 q1, q1, r5129; CHECK-NEXT: vadd.i32 q2, q0, r5130; CHECK-NEXT: ldrb.w r12, [r1]131; CHECK-NEXT: ldrb r1, [r3]132; CHECK-NEXT: ldrb.w lr, [r2]133; CHECK-NEXT: ldrb r3, [r4]134; CHECK-NEXT: vmov r2, r4, d6135; CHECK-NEXT: ldrb r2, [r2]136; CHECK-NEXT: ldrb r4, [r4]137; CHECK-NEXT: vmov.8 q0[0], r2138; CHECK-NEXT: vmov r2, r6, d5139; CHECK-NEXT: vmov.8 q0[1], r4140; CHECK-NEXT: ldrb r4, [r2]141; CHECK-NEXT: ldrb r2, [r6]142; CHECK-NEXT: vmov r6, r7, d7143; CHECK-NEXT: ldrb r0, [r6]144; CHECK-NEXT: ldrb r7, [r7]145; CHECK-NEXT: vmov.8 q0[2], r0146; CHECK-NEXT: vmov r0, r5, d2147; CHECK-NEXT: vmov.8 q0[3], r7148; CHECK-NEXT: ldrb r0, [r0]149; CHECK-NEXT: ldrb r5, [r5]150; CHECK-NEXT: vmov.8 q0[4], r0151; CHECK-NEXT: vmov.8 q0[5], r5152; CHECK-NEXT: vmov r0, r5, d3153; CHECK-NEXT: ldrb r0, [r0]154; CHECK-NEXT: ldrb r5, [r5]155; CHECK-NEXT: vmov.8 q0[6], r0156; CHECK-NEXT: vmov.8 q0[7], r5157; CHECK-NEXT: vmov r0, r5, d4158; CHECK-NEXT: ldrb r0, [r0]159; CHECK-NEXT: ldrb r5, [r5]160; CHECK-NEXT: vmov.8 q0[8], r0161; CHECK-NEXT: vmov.8 q0[9], r5162; CHECK-NEXT: vmov.8 q0[10], r4163; CHECK-NEXT: vmov.8 q0[11], r2164; CHECK-NEXT: vmov.8 q0[12], r1165; CHECK-NEXT: vmov.8 q0[13], r3166; CHECK-NEXT: vmov.8 q0[14], r12167; CHECK-NEXT: vmov.8 q0[15], lr168; CHECK-NEXT: pop {r4, r5, r6, r7, pc}169 %1 = add <16 x i32> %offs, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>170 %2 = getelementptr inbounds i8, ptr %data, <16 x i32> %1171 %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %2, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)172 ret <16 x i8> %wide.masked.gather173}174 175define arm_aapcs_vfpcc <16 x i8> @gather_inc_minipred_16i8(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, <16 x i32> %offs) {176; CHECK-LABEL: gather_inc_minipred_16i8:177; CHECK: @ %bb.0:178; CHECK-NEXT: .save {r4, r5, r7, lr}179; CHECK-NEXT: push {r4, r5, r7, lr}180; CHECK-NEXT: movs r1, #16181; CHECK-NEXT: vadd.i32 q1, q1, r0182; CHECK-NEXT: vadd.i32 q1, q1, r1183; CHECK-NEXT: vadd.i32 q2, q2, r0184; CHECK-NEXT: vmov r2, s4185; CHECK-NEXT: vadd.i32 q2, q2, r1186; CHECK-NEXT: vadd.i32 q0, q0, r0187; CHECK-NEXT: vmov r3, s10188; CHECK-NEXT: vadd.i32 q0, q0, r1189; CHECK-NEXT: vmov r4, s0190; CHECK-NEXT: vmov r5, s2191; CHECK-NEXT: ldrb.w r12, [r2]192; CHECK-NEXT: vmov r2, s8193; CHECK-NEXT: ldrb r3, [r3]194; CHECK-NEXT: ldrb r4, [r4]195; CHECK-NEXT: ldrb r5, [r5]196; CHECK-NEXT: vmov.8 q0[0], r4197; CHECK-NEXT: vmov.8 q0[2], r5198; CHECK-NEXT: vmov.8 q0[4], r12199; CHECK-NEXT: ldrb.w lr, [r2]200; CHECK-NEXT: vmov r2, s6201; CHECK-NEXT: vadd.i32 q1, q3, r0202; CHECK-NEXT: vadd.i32 q1, q1, r1203; CHECK-NEXT: vmov r0, s4204; CHECK-NEXT: vmov r1, s6205; CHECK-NEXT: ldrb r2, [r2]206; CHECK-NEXT: vmov.8 q0[6], r2207; CHECK-NEXT: vmov.8 q0[8], lr208; CHECK-NEXT: ldrb r0, [r0]209; CHECK-NEXT: vmov.8 q0[10], r3210; CHECK-NEXT: ldrb r1, [r1]211; CHECK-NEXT: vmov.8 q0[12], r0212; CHECK-NEXT: vmov.8 q0[14], r1213; CHECK-NEXT: pop {r4, r5, r7, pc}214 %1 = add <16 x i32> %offs, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>215 %2 = getelementptr inbounds i8, ptr %data, <16 x i32> %1216 %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %2, i32 2, <16 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>, <16 x i8> undef)217 ret <16 x i8> %wide.masked.gather218}219 220define arm_aapcs_vfpcc void @gather_pre_inc(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {221; CHECK-LABEL: gather_pre_inc:222; CHECK: @ %bb.0: @ %vector.ph223; CHECK-NEXT: adr r3, .LCPI7_0224; CHECK-NEXT: vldrw.u32 q0, [r3]225; CHECK-NEXT: vadd.i32 q0, q0, r0226; CHECK-NEXT: .LBB7_1: @ %vector.body227; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1228; CHECK-NEXT: vldrw.u32 q1, [q0, #96]!229; CHECK-NEXT: subs r2, #4230; CHECK-NEXT: vstrb.8 q1, [r1], #16231; CHECK-NEXT: bne .LBB7_1232; CHECK-NEXT: @ %bb.2: @ %end233; CHECK-NEXT: bx lr234; CHECK-NEXT: .p2align 4235; CHECK-NEXT: @ %bb.3:236; CHECK-NEXT: .LCPI7_0:237; CHECK-NEXT: .long 4294967224 @ 0xffffffb8238; CHECK-NEXT: .long 4294967248 @ 0xffffffd0239; CHECK-NEXT: .long 4294967272 @ 0xffffffe8240; CHECK-NEXT: .long 0 @ 0x0241vector.ph: ; preds = %for.body.preheader242 br label %vector.body243 244vector.body: ; preds = %vector.body, %vector.ph245 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]246 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]247 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>248 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>249 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1250 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)251 %3 = getelementptr inbounds i32, ptr %dst, i32 %index252 store <4 x i32> %wide.masked.gather, ptr %3, align 4253 %index.next = add i32 %index, 4254 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>255 %4 = icmp eq i32 %index.next, %n.vec256 br i1 %4, label %end, label %vector.body257 258end:259 ret void;260}261 262define arm_aapcs_vfpcc void @gather_pre_inc_i8(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {263; CHECK-LABEL: gather_pre_inc_i8:264; CHECK: @ %bb.0: @ %vector.ph265; CHECK-NEXT: adr r3, .LCPI8_0266; CHECK-NEXT: vldrw.u32 q0, [r3]267; CHECK-NEXT: vadd.i32 q0, q0, r0268; CHECK-NEXT: .LBB8_1: @ %vector.body269; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1270; CHECK-NEXT: vldrw.u32 q1, [q0, #24]!271; CHECK-NEXT: subs r2, #4272; CHECK-NEXT: vstrb.8 q1, [r1], #16273; CHECK-NEXT: bne .LBB8_1274; CHECK-NEXT: @ %bb.2: @ %end275; CHECK-NEXT: bx lr276; CHECK-NEXT: .p2align 4277; CHECK-NEXT: @ %bb.3:278; CHECK-NEXT: .LCPI8_0:279; CHECK-NEXT: .long 4294967278 @ 0xffffffee280; CHECK-NEXT: .long 4294967284 @ 0xfffffff4281; CHECK-NEXT: .long 4294967290 @ 0xfffffffa282; CHECK-NEXT: .long 0 @ 0x0283vector.ph: ; preds = %for.body.preheader284 br label %vector.body285 286vector.body: ; preds = %vector.body, %vector.ph287 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]288 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]289 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>290 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>291 %2 = getelementptr inbounds i8, ptr %data, <4 x i32> %1292 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)293 %3 = getelementptr inbounds i32, ptr %dst, i32 %index294 store <4 x i32> %wide.masked.gather, ptr %3, align 4295 %index.next = add i32 %index, 4296 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>297 %4 = icmp eq i32 %index.next, %n.vec298 br i1 %4, label %end, label %vector.body299 300end:301 ret void;302}303 304define arm_aapcs_vfpcc void @gather_post_inc(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec43) {305; CHECK-LABEL: gather_post_inc:306; CHECK: @ %bb.0: @ %vector.ph41307; CHECK-NEXT: adr r3, .LCPI9_0308; CHECK-NEXT: vldrw.u32 q0, [r3]309; CHECK-NEXT: vadd.i32 q0, q0, r0310; CHECK-NEXT: .LBB9_1: @ %vector.body39311; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1312; CHECK-NEXT: vldrw.u32 q1, [q0, #96]!313; CHECK-NEXT: subs r2, #4314; CHECK-NEXT: vstrb.8 q1, [r1], #16315; CHECK-NEXT: bne .LBB9_1316; CHECK-NEXT: @ %bb.2: @ %end317; CHECK-NEXT: bx lr318; CHECK-NEXT: .p2align 4319; CHECK-NEXT: @ %bb.3:320; CHECK-NEXT: .LCPI9_0:321; CHECK-NEXT: .long 4294967200 @ 0xffffffa0322; CHECK-NEXT: .long 4294967224 @ 0xffffffb8323; CHECK-NEXT: .long 4294967248 @ 0xffffffd0324; CHECK-NEXT: .long 4294967272 @ 0xffffffe8325vector.ph41: ; preds = %for.body6.preheader326 br label %vector.body39327 328vector.body39: ; preds = %vector.body39, %vector.ph41329 %index44 = phi i32 [ 0, %vector.ph41 ], [ %index.next45, %vector.body39 ]330 %vec.ind50 = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph41 ], [ %vec.ind.next51, %vector.body39 ]331 %0 = mul nuw nsw <4 x i32> %vec.ind50, <i32 3, i32 3, i32 3, i32 3>332 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0333 %wide.masked.gather55 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)334 %2 = getelementptr inbounds i32, ptr %dst, i32 %index44335 store <4 x i32> %wide.masked.gather55, ptr %2, align 4336 %index.next45 = add i32 %index44, 4337 %vec.ind.next51 = add <4 x i32> %vec.ind50, <i32 8, i32 8, i32 8, i32 8>338 %3 = icmp eq i32 %index.next45, %n.vec43339 br i1 %3, label %end, label %vector.body39340 341end:342 ret void;343}344 345define arm_aapcs_vfpcc void @gather_inc_v4i32_simple(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {346; CHECK-LABEL: gather_inc_v4i32_simple:347; CHECK: @ %bb.0: @ %entry348; CHECK-NEXT: cmp r2, #1349; CHECK-NEXT: it lt350; CHECK-NEXT: bxlt lr351; CHECK-NEXT: .LBB10_1: @ %vector.ph.preheader352; CHECK-NEXT: .save {r4, lr}353; CHECK-NEXT: push {r4, lr}354; CHECK-NEXT: bic r12, r2, #3355; CHECK-NEXT: movs r3, #1356; CHECK-NEXT: sub.w lr, r12, #4357; CHECK-NEXT: add.w r4, r3, lr, lsr #2358; CHECK-NEXT: adr r3, .LCPI10_0359; CHECK-NEXT: vldrw.u32 q0, [r3]360; CHECK-NEXT: vadd.i32 q0, q0, r0361; CHECK-NEXT: .LBB10_2: @ %vector.ph362; CHECK-NEXT: @ =>This Loop Header: Depth=1363; CHECK-NEXT: @ Child Loop BB10_3 Depth 2364; CHECK-NEXT: dls lr, r4365; CHECK-NEXT: mov r0, r1366; CHECK-NEXT: vmov q1, q0367; CHECK-NEXT: .LBB10_3: @ %vector.body368; CHECK-NEXT: @ Parent Loop BB10_2 Depth=1369; CHECK-NEXT: @ => This Inner Loop Header: Depth=2370; CHECK-NEXT: vldrw.u32 q2, [q1, #16]!371; CHECK-NEXT: vstrb.8 q2, [r0], #16372; CHECK-NEXT: le lr, .LBB10_3373; CHECK-NEXT: @ %bb.4: @ %middle.block374; CHECK-NEXT: @ in Loop: Header=BB10_2 Depth=1375; CHECK-NEXT: cmp r12, r2376; CHECK-NEXT: bne .LBB10_2377; CHECK-NEXT: @ %bb.5:378; CHECK-NEXT: pop.w {r4, lr}379; CHECK-NEXT: bx lr380; CHECK-NEXT: .p2align 4381; CHECK-NEXT: @ %bb.6:382; CHECK-NEXT: .LCPI10_0:383; CHECK-NEXT: .long 4294967280 @ 0xfffffff0384; CHECK-NEXT: .long 4294967284 @ 0xfffffff4385; CHECK-NEXT: .long 4294967288 @ 0xfffffff8386; CHECK-NEXT: .long 4294967292 @ 0xfffffffc387entry:388 %cmp22 = icmp sgt i32 %n, 0389 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup390 391vector.ph: ; preds = %for.body.preheader392 %n.vec = and i32 %n, -4393 br label %vector.body394 395vector.body: ; preds = %vector.body, %vector.ph396 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]397 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]398 %0 = getelementptr inbounds i32, ptr %data, <4 x i32> %vec.ind399 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %0, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)400 %1 = getelementptr inbounds i32, ptr %dst, i32 %index401 store <4 x i32> %wide.masked.gather, ptr %1, align 4402 %index.next = add i32 %index, 4403 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>404 %2 = icmp eq i32 %index.next, %n.vec405 br i1 %2, label %middle.block, label %vector.body406 407middle.block: ; preds = %vector.body408 %cmp.n = icmp eq i32 %n.vec, %n409 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph410 411for.cond.cleanup: ; preds = %for.body, %middle.block, %entry412 ret void413}414 415define arm_aapcs_vfpcc void @gather_inc_v4i32_complex(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {416; CHECK-LABEL: gather_inc_v4i32_complex:417; CHECK: @ %bb.0: @ %entry418; CHECK-NEXT: cmp r2, #1419; CHECK-NEXT: it lt420; CHECK-NEXT: bxlt lr421; CHECK-NEXT: .LBB11_1: @ %vector.ph.preheader422; CHECK-NEXT: .save {r4, r5, r7, lr}423; CHECK-NEXT: push {r4, r5, r7, lr}424; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}425; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}426; CHECK-NEXT: bic r12, r2, #3427; CHECK-NEXT: movs r3, #1428; CHECK-NEXT: sub.w lr, r12, #4429; CHECK-NEXT: adr r4, .LCPI11_1430; CHECK-NEXT: adr r5, .LCPI11_2431; CHECK-NEXT: vldrw.u32 q1, [r4]432; CHECK-NEXT: add.w r3, r3, lr, lsr #2433; CHECK-NEXT: adr.w lr, .LCPI11_0434; CHECK-NEXT: vldrw.u32 q0, [r5]435; CHECK-NEXT: vldrw.u32 q2, [lr]436; CHECK-NEXT: vadd.i32 q1, q1, r0437; CHECK-NEXT: vadd.i32 q0, q0, r0438; CHECK-NEXT: vadd.i32 q2, q2, r0439; CHECK-NEXT: .LBB11_2: @ %vector.ph440; CHECK-NEXT: @ =>This Loop Header: Depth=1441; CHECK-NEXT: @ Child Loop BB11_3 Depth 2442; CHECK-NEXT: dls lr, r3443; CHECK-NEXT: mov r0, r1444; CHECK-NEXT: vmov q3, q1445; CHECK-NEXT: vmov q4, q0446; CHECK-NEXT: vmov q5, q2447; CHECK-NEXT: .LBB11_3: @ %vector.body448; CHECK-NEXT: @ Parent Loop BB11_2 Depth=1449; CHECK-NEXT: @ => This Inner Loop Header: Depth=2450; CHECK-NEXT: vldrw.u32 q6, [q5, #48]!451; CHECK-NEXT: vldrw.u32 q7, [q3, #48]!452; CHECK-NEXT: vadd.i32 q6, q7, q6453; CHECK-NEXT: vldrw.u32 q7, [q4, #48]!454; CHECK-NEXT: vadd.i32 q6, q6, q7455; CHECK-NEXT: vstrb.8 q6, [r0], #16456; CHECK-NEXT: le lr, .LBB11_3457; CHECK-NEXT: @ %bb.4: @ %middle.block458; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=1459; CHECK-NEXT: cmp r12, r2460; CHECK-NEXT: bne .LBB11_2461; CHECK-NEXT: @ %bb.5:462; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}463; CHECK-NEXT: pop.w {r4, r5, r7, lr}464; CHECK-NEXT: bx lr465; CHECK-NEXT: .p2align 4466; CHECK-NEXT: @ %bb.6:467; CHECK-NEXT: .LCPI11_0:468; CHECK-NEXT: .long 4294967248 @ 0xffffffd0469; CHECK-NEXT: .long 4294967260 @ 0xffffffdc470; CHECK-NEXT: .long 4294967272 @ 0xffffffe8471; CHECK-NEXT: .long 4294967284 @ 0xfffffff4472; CHECK-NEXT: .LCPI11_1:473; CHECK-NEXT: .long 4294967252 @ 0xffffffd4474; CHECK-NEXT: .long 4294967264 @ 0xffffffe0475; CHECK-NEXT: .long 4294967276 @ 0xffffffec476; CHECK-NEXT: .long 4294967288 @ 0xfffffff8477; CHECK-NEXT: .LCPI11_2:478; CHECK-NEXT: .long 4294967256 @ 0xffffffd8479; CHECK-NEXT: .long 4294967268 @ 0xffffffe4480; CHECK-NEXT: .long 4294967280 @ 0xfffffff0481; CHECK-NEXT: .long 4294967292 @ 0xfffffffc482entry:483 %cmp22 = icmp sgt i32 %n, 0484 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup485 486vector.ph: ; preds = %for.body.preheader487 %n.vec = and i32 %n, -4488 br label %vector.body489 490vector.body: ; preds = %vector.body, %vector.ph491 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]492 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]493 %0 = mul nuw nsw <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>494 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0495 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)496 %2 = add nuw nsw <4 x i32> %0, <i32 1, i32 1, i32 1, i32 1>497 %3 = getelementptr inbounds i32, ptr %data, <4 x i32> %2498 %wide.masked.gather24 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)499 %4 = add nuw nsw <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>500 %5 = getelementptr inbounds i32, ptr %data, <4 x i32> %4501 %wide.masked.gather25 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %5, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)502 %6 = add nsw <4 x i32> %wide.masked.gather24, %wide.masked.gather503 %7 = add nsw <4 x i32> %6, %wide.masked.gather25504 %8 = getelementptr inbounds i32, ptr %dst, i32 %index505 store <4 x i32> %7, ptr %8, align 4506 %index.next = add i32 %index, 4507 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>508 %9 = icmp eq i32 %index.next, %n.vec509 br i1 %9, label %middle.block, label %vector.body510 511middle.block: ; preds = %vector.body512 %cmp.n = icmp eq i32 %n.vec, %n513 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph514 515for.cond.cleanup: ; preds = %for.body, %middle.block, %entry516 ret void517}518 519define arm_aapcs_vfpcc void @gather_inc_v4i32_large(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {520; CHECK-LABEL: gather_inc_v4i32_large:521; CHECK: @ %bb.0: @ %entry522; CHECK-NEXT: cmp r2, #1523; CHECK-NEXT: it lt524; CHECK-NEXT: bxlt lr525; CHECK-NEXT: .LBB12_1: @ %vector.ph.preheader526; CHECK-NEXT: .save {r4, lr}527; CHECK-NEXT: push {r4, lr}528; CHECK-NEXT: bic r12, r2, #3529; CHECK-NEXT: movs r3, #1530; CHECK-NEXT: sub.w lr, r12, #4531; CHECK-NEXT: add.w r4, r3, lr, lsr #2532; CHECK-NEXT: adr r3, .LCPI12_0533; CHECK-NEXT: vldrw.u32 q0, [r3]534; CHECK-NEXT: vadd.i32 q0, q0, r0535; CHECK-NEXT: .LBB12_2: @ %vector.ph536; CHECK-NEXT: @ =>This Loop Header: Depth=1537; CHECK-NEXT: @ Child Loop BB12_3 Depth 2538; CHECK-NEXT: dls lr, r4539; CHECK-NEXT: mov r0, r1540; CHECK-NEXT: vmov q1, q0541; CHECK-NEXT: .LBB12_3: @ %vector.body542; CHECK-NEXT: @ Parent Loop BB12_2 Depth=1543; CHECK-NEXT: @ => This Inner Loop Header: Depth=2544; CHECK-NEXT: vldrw.u32 q2, [q1, #508]!545; CHECK-NEXT: vstrb.8 q2, [r0], #16546; CHECK-NEXT: le lr, .LBB12_3547; CHECK-NEXT: @ %bb.4: @ %middle.block548; CHECK-NEXT: @ in Loop: Header=BB12_2 Depth=1549; CHECK-NEXT: cmp r12, r2550; CHECK-NEXT: bne .LBB12_2551; CHECK-NEXT: @ %bb.5:552; CHECK-NEXT: pop.w {r4, lr}553; CHECK-NEXT: bx lr554; CHECK-NEXT: .p2align 4555; CHECK-NEXT: @ %bb.6:556; CHECK-NEXT: .LCPI12_0:557; CHECK-NEXT: .long 4294966788 @ 0xfffffe04558; CHECK-NEXT: .long 4294966792 @ 0xfffffe08559; CHECK-NEXT: .long 4294966796 @ 0xfffffe0c560; CHECK-NEXT: .long 4294966800 @ 0xfffffe10561entry:562 %cmp22 = icmp sgt i32 %n, 0563 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup564 565vector.ph: ; preds = %for.body.preheader566 %n.vec = and i32 %n, -4567 br label %vector.body568 569vector.body: ; preds = %vector.body, %vector.ph570 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]571 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]572 %0 = getelementptr inbounds i32, ptr %data, <4 x i32> %vec.ind573 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %0, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)574 %1 = getelementptr inbounds i32, ptr %dst, i32 %index575 store <4 x i32> %wide.masked.gather, ptr %1, align 4576 %index.next = add i32 %index, 4577 %vec.ind.next = add <4 x i32> %vec.ind, <i32 127, i32 127, i32 127, i32 127>578 %2 = icmp eq i32 %index.next, %n.vec579 br i1 %2, label %middle.block, label %vector.body580 581middle.block: ; preds = %vector.body582 %cmp.n = icmp eq i32 %n.vec, %n583 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph584 585for.cond.cleanup: ; preds = %for.body, %middle.block, %entry586 ret void587}588 589; TODO: uneven - I think it's not possible to create such an example, because vec.ind will always be increased by a vector with 4 elements (=> x*4 = even)590 591; TODO: What is sxth?592define arm_aapcs_vfpcc void @gather_inc_v8i16_simple(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {593; CHECK-LABEL: gather_inc_v8i16_simple:594; CHECK: @ %bb.0: @ %entry595; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}596; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}597; CHECK-NEXT: .pad #28598; CHECK-NEXT: sub sp, #28599; CHECK-NEXT: cmp r2, #1600; CHECK-NEXT: strd r1, r2, [sp, #4] @ 8-byte Folded Spill601; CHECK-NEXT: blt .LBB13_5602; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader603; CHECK-NEXT: ldr r1, [sp, #8] @ 4-byte Reload604; CHECK-NEXT: movs r6, #1605; CHECK-NEXT: add r2, sp, #12606; CHECK-NEXT: mov.w r9, #8607; CHECK-NEXT: bic r1, r1, #7608; CHECK-NEXT: str r1, [sp] @ 4-byte Spill609; CHECK-NEXT: sub.w r3, r1, #8610; CHECK-NEXT: add.w r8, r6, r3, lsr #3611; CHECK-NEXT: adr r3, .LCPI13_0612; CHECK-NEXT: vldrw.u32 q0, [r3]613; CHECK-NEXT: .LBB13_2: @ %vector.ph614; CHECK-NEXT: @ =>This Loop Header: Depth=1615; CHECK-NEXT: @ Child Loop BB13_3 Depth 2616; CHECK-NEXT: dls lr, r8617; CHECK-NEXT: vmov q1, q0618; CHECK-NEXT: ldr r6, [sp, #4] @ 4-byte Reload619; CHECK-NEXT: .LBB13_3: @ %vector.body620; CHECK-NEXT: @ Parent Loop BB13_2 Depth=1621; CHECK-NEXT: @ => This Inner Loop Header: Depth=2622; CHECK-NEXT: vstrw.32 q1, [r2]623; CHECK-NEXT: mov r12, r2624; CHECK-NEXT: vldrh.s32 q2, [r2, #8]625; CHECK-NEXT: vadd.i16 q1, q1, r9626; CHECK-NEXT: vshl.i32 q2, q2, #1627; CHECK-NEXT: vadd.i32 q2, q2, r0628; CHECK-NEXT: vmov r7, r5, d5629; CHECK-NEXT: vmov r3, r4, d4630; CHECK-NEXT: vldrh.s32 q2, [r2]631; CHECK-NEXT: vshl.i32 q2, q2, #1632; CHECK-NEXT: vadd.i32 q2, q2, r0633; CHECK-NEXT: vmov r1, r10, d5634; CHECK-NEXT: ldrh r7, [r7]635; CHECK-NEXT: ldrh r4, [r4]636; CHECK-NEXT: ldrh r5, [r5]637; CHECK-NEXT: ldrh.w r2, [r10]638; CHECK-NEXT: ldrh.w r10, [r3]639; CHECK-NEXT: vmov r3, r11, d4640; CHECK-NEXT: ldrh r1, [r1]641; CHECK-NEXT: ldrh r3, [r3]642; CHECK-NEXT: ldrh.w r11, [r11]643; CHECK-NEXT: vmov.16 q2[0], r3644; CHECK-NEXT: vmov.16 q2[1], r11645; CHECK-NEXT: vmov.16 q2[2], r1646; CHECK-NEXT: vmov.16 q2[3], r2647; CHECK-NEXT: mov r2, r12648; CHECK-NEXT: vmov.16 q2[4], r10649; CHECK-NEXT: vmov.16 q2[5], r4650; CHECK-NEXT: vmov.16 q2[6], r7651; CHECK-NEXT: vmov.16 q2[7], r5652; CHECK-NEXT: vstrb.8 q2, [r6], #16653; CHECK-NEXT: le lr, .LBB13_3654; CHECK-NEXT: @ %bb.4: @ %middle.block655; CHECK-NEXT: @ in Loop: Header=BB13_2 Depth=1656; CHECK-NEXT: ldr r1, [sp, #8] @ 4-byte Reload657; CHECK-NEXT: ldr r3, [sp] @ 4-byte Reload658; CHECK-NEXT: cmp r3, r1659; CHECK-NEXT: bne .LBB13_2660; CHECK-NEXT: .LBB13_5: @ %for.cond.cleanup661; CHECK-NEXT: add sp, #28662; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}663; CHECK-NEXT: .p2align 4664; CHECK-NEXT: @ %bb.6:665; CHECK-NEXT: .LCPI13_0:666; CHECK-NEXT: .short 0 @ 0x0667; CHECK-NEXT: .short 1 @ 0x1668; CHECK-NEXT: .short 2 @ 0x2669; CHECK-NEXT: .short 3 @ 0x3670; CHECK-NEXT: .short 4 @ 0x4671; CHECK-NEXT: .short 5 @ 0x5672; CHECK-NEXT: .short 6 @ 0x6673; CHECK-NEXT: .short 7 @ 0x7674 675 676entry:677 %cmp22 = icmp sgt i32 %n, 0678 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup679 680vector.ph: ; preds = %for.body.preheader681 %n.vec = and i32 %n, -8682 br label %vector.body683 684vector.body: ; preds = %vector.body, %vector.ph685 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]686 %vec.ind = phi <8 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, %vector.ph ], [ %vec.ind.next, %vector.body ]687 %0 = getelementptr inbounds i16, ptr %data, <8 x i16> %vec.ind688 %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %0, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)689 %1 = getelementptr inbounds i16, ptr %dst, i32 %index690 store <8 x i16> %wide.masked.gather, ptr %1, align 2691 %index.next = add i32 %index, 8692 %vec.ind.next = add <8 x i16> %vec.ind, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>693 %2 = icmp eq i32 %index.next, %n.vec694 br i1 %2, label %middle.block, label %vector.body695 696middle.block: ; preds = %vector.body697 %cmp.n = icmp eq i32 %n.vec, %n698 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph699 700for.cond.cleanup: ; preds = %for.body, %middle.block, %entry701 ret void702}703 704; TODO: This looks absolutely terrifying :(705define arm_aapcs_vfpcc void @gather_inc_v8i16_complex(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {706; CHECK-LABEL: gather_inc_v8i16_complex:707; CHECK: @ %bb.0: @ %entry708; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}709; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}710; CHECK-NEXT: .pad #4711; CHECK-NEXT: sub sp, #4712; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}713; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}714; CHECK-NEXT: .pad #120715; CHECK-NEXT: sub sp, #120716; CHECK-NEXT: cmp r2, #1717; CHECK-NEXT: strd r1, r2, [sp, #64] @ 8-byte Folded Spill718; CHECK-NEXT: blt.w .LBB14_5719; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader720; CHECK-NEXT: ldr r1, [sp, #68] @ 4-byte Reload721; CHECK-NEXT: adr r3, .LCPI14_2722; CHECK-NEXT: vldrw.u32 q0, [r3]723; CHECK-NEXT: movs r2, #1724; CHECK-NEXT: bic r1, r1, #7725; CHECK-NEXT: str r1, [sp, #4] @ 4-byte Spill726; CHECK-NEXT: subs r1, #8727; CHECK-NEXT: vstrw.32 q0, [sp, #40] @ 16-byte Spill728; CHECK-NEXT: add.w r1, r2, r1, lsr #3729; CHECK-NEXT: str r1, [sp, #60] @ 4-byte Spill730; CHECK-NEXT: adr r1, .LCPI14_0731; CHECK-NEXT: adr r2, .LCPI14_1732; CHECK-NEXT: vldrw.u32 q0, [r1]733; CHECK-NEXT: vstrw.32 q0, [sp, #24] @ 16-byte Spill734; CHECK-NEXT: vldrw.u32 q0, [r2]735; CHECK-NEXT: add r2, sp, #104736; CHECK-NEXT: vstrw.32 q0, [sp, #8] @ 16-byte Spill737; CHECK-NEXT: .LBB14_2: @ %vector.ph738; CHECK-NEXT: @ =>This Loop Header: Depth=1739; CHECK-NEXT: @ Child Loop BB14_3 Depth 2740; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload741; CHECK-NEXT: add.w r10, sp, #88742; CHECK-NEXT: dls lr, r1743; CHECK-NEXT: ldr r7, [sp, #64] @ 4-byte Reload744; CHECK-NEXT: vldrw.u32 q4, [sp, #24] @ 16-byte Reload745; CHECK-NEXT: vldrw.u32 q5, [sp, #40] @ 16-byte Reload746; CHECK-NEXT: vldrw.u32 q6, [sp, #8] @ 16-byte Reload747; CHECK-NEXT: .LBB14_3: @ %vector.body748; CHECK-NEXT: @ Parent Loop BB14_2 Depth=1749; CHECK-NEXT: @ => This Inner Loop Header: Depth=2750; CHECK-NEXT: vstrw.32 q5, [r2]751; CHECK-NEXT: mov r8, r2752; CHECK-NEXT: vldrh.s32 q0, [r2, #8]753; CHECK-NEXT: vshl.i32 q0, q0, #1754; CHECK-NEXT: vadd.i32 q0, q0, r0755; CHECK-NEXT: vmov r1, r3, d0756; CHECK-NEXT: vmov r4, r5, d1757; CHECK-NEXT: vldrh.s32 q0, [r2]758; CHECK-NEXT: vshl.i32 q0, q0, #1759; CHECK-NEXT: vadd.i32 q2, q0, r0760; CHECK-NEXT: vmov r6, r2, d4761; CHECK-NEXT: ldrh r1, [r1]762; CHECK-NEXT: ldrh.w r12, [r4]763; CHECK-NEXT: add r4, sp, #72764; CHECK-NEXT: ldrh.w r11, [r5]765; CHECK-NEXT: ldrh r3, [r3]766; CHECK-NEXT: ldrh r5, [r6]767; CHECK-NEXT: ldrh r2, [r2]768; CHECK-NEXT: vstrw.32 q6, [r4]769; CHECK-NEXT: vldrh.s32 q0, [r4]770; CHECK-NEXT: vmov.16 q7[0], r5771; CHECK-NEXT: vmov.16 q7[1], r2772; CHECK-NEXT: vshl.i32 q0, q0, #1773; CHECK-NEXT: vadd.i32 q0, q0, r0774; CHECK-NEXT: vmov r6, r9, d0775; CHECK-NEXT: vmov r2, r5, d1776; CHECK-NEXT: vldrh.s32 q0, [r4, #8]777; CHECK-NEXT: vshl.i32 q0, q0, #1778; CHECK-NEXT: vadd.i32 q0, q0, r0779; CHECK-NEXT: ldrh r6, [r6]780; CHECK-NEXT: ldrh r2, [r2]781; CHECK-NEXT: vmov.16 q1[0], r6782; CHECK-NEXT: ldrh.w r6, [r9]783; CHECK-NEXT: ldrh r5, [r5]784; CHECK-NEXT: vmov.16 q1[1], r6785; CHECK-NEXT: vmov.16 q1[2], r2786; CHECK-NEXT: vmov r2, r6, d0787; CHECK-NEXT: vmov.16 q1[3], r5788; CHECK-NEXT: ldrh r2, [r2]789; CHECK-NEXT: ldrh r6, [r6]790; CHECK-NEXT: vmov.16 q1[4], r2791; CHECK-NEXT: vmov r2, r5, d1792; CHECK-NEXT: vmov.16 q1[5], r6793; CHECK-NEXT: mov r6, r10794; CHECK-NEXT: ldrh r2, [r2]795; CHECK-NEXT: ldrh r5, [r5]796; CHECK-NEXT: vstrw.32 q4, [r10]797; CHECK-NEXT: vldrh.s32 q0, [r6]798; CHECK-NEXT: vmov.16 q1[6], r2799; CHECK-NEXT: vmov.16 q1[7], r5800; CHECK-NEXT: vshl.i32 q0, q0, #1801; CHECK-NEXT: vadd.i32 q0, q0, r0802; CHECK-NEXT: vmov r2, r5, d0803; CHECK-NEXT: ldrh r2, [r2]804; CHECK-NEXT: ldrh r5, [r5]805; CHECK-NEXT: vmov.16 q3[0], r2806; CHECK-NEXT: vmov.16 q3[1], r5807; CHECK-NEXT: vmov r2, r5, d5808; CHECK-NEXT: vmov.i16 q2, #0x18809; CHECK-NEXT: vadd.i16 q6, q6, q2810; CHECK-NEXT: vadd.i16 q5, q5, q2811; CHECK-NEXT: vadd.i16 q4, q4, q2812; CHECK-NEXT: ldrh.w r9, [r2]813; CHECK-NEXT: vmov r2, r4, d1814; CHECK-NEXT: vldrh.s32 q0, [r6, #8]815; CHECK-NEXT: ldrh r5, [r5]816; CHECK-NEXT: vmov.16 q7[2], r9817; CHECK-NEXT: vshl.i32 q0, q0, #1818; CHECK-NEXT: vmov.16 q7[3], r5819; CHECK-NEXT: vadd.i32 q0, q0, r0820; CHECK-NEXT: vmov.16 q7[4], r1821; CHECK-NEXT: vmov.16 q7[5], r3822; CHECK-NEXT: vmov.16 q7[6], r12823; CHECK-NEXT: vmov.16 q7[7], r11824; CHECK-NEXT: ldrh r2, [r2]825; CHECK-NEXT: ldrh r4, [r4]826; CHECK-NEXT: vmov.16 q3[2], r2827; CHECK-NEXT: vmov.16 q3[3], r4828; CHECK-NEXT: vmov r2, r4, d0829; CHECK-NEXT: ldrh r2, [r2]830; CHECK-NEXT: ldrh r4, [r4]831; CHECK-NEXT: vmov.16 q3[4], r2832; CHECK-NEXT: vmov.16 q3[5], r4833; CHECK-NEXT: vmov r2, r4, d1834; CHECK-NEXT: ldrh r2, [r2]835; CHECK-NEXT: ldrh r4, [r4]836; CHECK-NEXT: vmov.16 q3[6], r2837; CHECK-NEXT: mov r2, r8838; CHECK-NEXT: vmov.16 q3[7], r4839; CHECK-NEXT: vadd.i16 q0, q3, q1840; CHECK-NEXT: vadd.i16 q0, q0, q7841; CHECK-NEXT: vstrb.8 q0, [r7], #16842; CHECK-NEXT: le lr, .LBB14_3843; CHECK-NEXT: @ %bb.4: @ %middle.block844; CHECK-NEXT: @ in Loop: Header=BB14_2 Depth=1845; CHECK-NEXT: ldr r1, [sp, #4] @ 4-byte Reload846; CHECK-NEXT: ldr r3, [sp, #68] @ 4-byte Reload847; CHECK-NEXT: cmp r1, r3848; CHECK-NEXT: bne.w .LBB14_2849; CHECK-NEXT: .LBB14_5: @ %for.cond.cleanup850; CHECK-NEXT: add sp, #120851; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}852; CHECK-NEXT: add sp, #4853; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}854; CHECK-NEXT: .p2align 4855; CHECK-NEXT: @ %bb.6:856; CHECK-NEXT: .LCPI14_0:857; CHECK-NEXT: .short 1 @ 0x1858; CHECK-NEXT: .short 4 @ 0x4859; CHECK-NEXT: .short 7 @ 0x7860; CHECK-NEXT: .short 10 @ 0xa861; CHECK-NEXT: .short 13 @ 0xd862; CHECK-NEXT: .short 16 @ 0x10863; CHECK-NEXT: .short 19 @ 0x13864; CHECK-NEXT: .short 22 @ 0x16865; CHECK-NEXT: .LCPI14_1:866; CHECK-NEXT: .short 0 @ 0x0867; CHECK-NEXT: .short 3 @ 0x3868; CHECK-NEXT: .short 6 @ 0x6869; CHECK-NEXT: .short 9 @ 0x9870; CHECK-NEXT: .short 12 @ 0xc871; CHECK-NEXT: .short 15 @ 0xf872; CHECK-NEXT: .short 18 @ 0x12873; CHECK-NEXT: .short 21 @ 0x15874; CHECK-NEXT: .LCPI14_2:875; CHECK-NEXT: .short 2 @ 0x2876; CHECK-NEXT: .short 5 @ 0x5877; CHECK-NEXT: .short 8 @ 0x8878; CHECK-NEXT: .short 11 @ 0xb879; CHECK-NEXT: .short 14 @ 0xe880; CHECK-NEXT: .short 17 @ 0x11881; CHECK-NEXT: .short 20 @ 0x14882; CHECK-NEXT: .short 23 @ 0x17883 884 885entry:886 %cmp22 = icmp sgt i32 %n, 0887 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup888 889vector.ph: ; preds = %for.body.preheader890 %n.vec = and i32 %n, -8891 br label %vector.body892 893vector.body: ; preds = %vector.body, %vector.ph894 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]895 %vec.ind = phi <8 x i16> [ <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, %vector.ph ], [ %vec.ind.next, %vector.body ]896 %0 = mul nuw nsw <8 x i16> %vec.ind, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>897 %1 = getelementptr inbounds i16, ptr %data, <8 x i16> %0898 %wide.masked.gather = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %1, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)899 %2 = add nuw nsw <8 x i16> %0, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>900 %3 = getelementptr inbounds i16, ptr %data, <8 x i16> %2901 %wide.masked.gather24 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %3, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)902 %4 = add nuw nsw <8 x i16> %0, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>903 %5 = getelementptr inbounds i16, ptr %data, <8 x i16> %4904 %wide.masked.gather25 = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %5, i32 2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> undef)905 %6 = add nsw <8 x i16> %wide.masked.gather24, %wide.masked.gather906 %7 = add nsw <8 x i16> %6, %wide.masked.gather25907 %8 = getelementptr inbounds i16, ptr %dst, i32 %index908 store <8 x i16> %7, ptr %8, align 2909 %index.next = add i32 %index, 8910 %vec.ind.next = add <8 x i16> %vec.ind, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>911 %9 = icmp eq i32 %index.next, %n.vec912 br i1 %9, label %middle.block, label %vector.body913 914middle.block: ; preds = %vector.body915 %cmp.n = icmp eq i32 %n.vec, %n916 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph917 918for.cond.cleanup: ; preds = %for.body, %middle.block, %entry919 ret void920}921 922 923define arm_aapcs_vfpcc void @gather_inc_v16i8_complex(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {924; CHECK-LABEL: gather_inc_v16i8_complex:925; CHECK: @ %bb.0: @ %entry926; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}927; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}928; CHECK-NEXT: .pad #4929; CHECK-NEXT: sub sp, #4930; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}931; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}932; CHECK-NEXT: .pad #312933; CHECK-NEXT: sub sp, #312934; CHECK-NEXT: cmp r2, #1935; CHECK-NEXT: str r1, [sp, #116] @ 4-byte Spill936; CHECK-NEXT: blt.w .LBB15_5937; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader938; CHECK-NEXT: adr r1, .LCPI15_0939; CHECK-NEXT: adr r6, .LCPI15_8940; CHECK-NEXT: vldrw.u32 q0, [r1]941; CHECK-NEXT: adr r1, .LCPI15_1942; CHECK-NEXT: adr r7, .LCPI15_7943; CHECK-NEXT: adr r3, .LCPI15_6944; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill945; CHECK-NEXT: vldrw.u32 q0, [r1]946; CHECK-NEXT: adr r1, .LCPI15_5947; CHECK-NEXT: bic r10, r2, #7948; CHECK-NEXT: vstrw.32 q0, [sp, #80] @ 16-byte Spill949; CHECK-NEXT: vldrw.u32 q0, [r6]950; CHECK-NEXT: adr r6, .LCPI15_9951; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill952; CHECK-NEXT: vldrw.u32 q0, [r7]953; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill954; CHECK-NEXT: vldrw.u32 q0, [r6]955; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill956; CHECK-NEXT: vldrw.u32 q0, [r1]957; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill958; CHECK-NEXT: vldrw.u32 q0, [r3]959; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill960; CHECK-NEXT: .LBB15_2: @ %vector.ph961; CHECK-NEXT: @ =>This Loop Header: Depth=1962; CHECK-NEXT: @ Child Loop BB15_3 Depth 2963; CHECK-NEXT: vldrw.u32 q2, [sp, #16] @ 16-byte Reload964; CHECK-NEXT: adr r1, .LCPI15_3965; CHECK-NEXT: vldrw.u32 q5, [r1]966; CHECK-NEXT: adr r1, .LCPI15_4967; CHECK-NEXT: vstrw.32 q2, [sp, #296] @ 16-byte Spill968; CHECK-NEXT: vldrw.u32 q2, [sp, #32] @ 16-byte Reload969; CHECK-NEXT: vldrw.u32 q6, [r1]970; CHECK-NEXT: adr r1, .LCPI15_2971; CHECK-NEXT: vstrw.32 q2, [sp, #280] @ 16-byte Spill972; CHECK-NEXT: vldrw.u32 q2, [sp, #48] @ 16-byte Reload973; CHECK-NEXT: vldrw.u32 q1, [r1]974; CHECK-NEXT: adr r1, .LCPI15_10975; CHECK-NEXT: vstrw.32 q2, [sp, #264] @ 16-byte Spill976; CHECK-NEXT: vldrw.u32 q2, [sp, #80] @ 16-byte Reload977; CHECK-NEXT: vldrw.u32 q3, [r1]978; CHECK-NEXT: adr r1, .LCPI15_11979; CHECK-NEXT: ldr.w r8, [sp, #116] @ 4-byte Reload980; CHECK-NEXT: vstrw.32 q2, [sp, #248] @ 16-byte Spill981; CHECK-NEXT: vldrw.u32 q2, [sp, #64] @ 16-byte Reload982; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload983; CHECK-NEXT: vldrw.u32 q7, [r1]984; CHECK-NEXT: vldrw.u32 q4, [sp] @ 16-byte Reload985; CHECK-NEXT: mov r11, r10986; CHECK-NEXT: vstrw.32 q2, [sp, #232] @ 16-byte Spill987; CHECK-NEXT: vstrw.32 q0, [sp, #216] @ 16-byte Spill988; CHECK-NEXT: .LBB15_3: @ %vector.body989; CHECK-NEXT: @ Parent Loop BB15_2 Depth=1990; CHECK-NEXT: @ => This Inner Loop Header: Depth=2991; CHECK-NEXT: vmov q0, q7992; CHECK-NEXT: vstrw.32 q7, [sp, #184] @ 16-byte Spill993; CHECK-NEXT: vadd.i32 q7, q5, r0994; CHECK-NEXT: vstrw.32 q5, [sp, #200] @ 16-byte Spill995; CHECK-NEXT: vadd.i32 q5, q0, r0996; CHECK-NEXT: vmov q0, q6997; CHECK-NEXT: vadd.i32 q6, q4, r0998; CHECK-NEXT: vmov r5, r4, d11999; CHECK-NEXT: vmov r1, lr, d121000; CHECK-NEXT: vadd.i32 q2, q1, r01001; CHECK-NEXT: vmov r6, r7, d151002; CHECK-NEXT: vstrw.32 q1, [sp, #152] @ 16-byte Spill1003; CHECK-NEXT: vmov q1, q31004; CHECK-NEXT: vstrw.32 q4, [sp, #168] @ 16-byte Spill1005; CHECK-NEXT: vldrw.u32 q4, [sp, #248] @ 16-byte Reload1006; CHECK-NEXT: vstrw.32 q0, [sp, #120] @ 16-byte Spill1007; CHECK-NEXT: vstrw.32 q3, [sp, #136] @ 16-byte Spill1008; CHECK-NEXT: vldrw.u32 q3, [sp, #184] @ 16-byte Reload1009; CHECK-NEXT: subs.w r11, r11, #161010; CHECK-NEXT: ldrb r5, [r5]1011; CHECK-NEXT: ldrb.w r9, [r1]1012; CHECK-NEXT: vmov r1, r3, d101013; CHECK-NEXT: ldrb r7, [r7]1014; CHECK-NEXT: ldrb r1, [r1]1015; CHECK-NEXT: vmov.8 q5[0], r11016; CHECK-NEXT: ldrb r1, [r3]1017; CHECK-NEXT: vmov.8 q5[1], r11018; CHECK-NEXT: vmov r1, r3, d141019; CHECK-NEXT: vmov.8 q5[2], r51020; CHECK-NEXT: ldrb r5, [r6]1021; CHECK-NEXT: ldrb r6, [r4]1022; CHECK-NEXT: vmov.8 q5[3], r61023; CHECK-NEXT: ldrb r1, [r1]1024; CHECK-NEXT: ldrb r3, [r3]1025; CHECK-NEXT: vmov.8 q7[0], r11026; CHECK-NEXT: vmov r6, r1, d41027; CHECK-NEXT: vmov.8 q7[1], r31028; CHECK-NEXT: vmov.8 q7[2], r51029; CHECK-NEXT: vmov.8 q7[3], r71030; CHECK-NEXT: ldrb.w r7, [lr]1031; CHECK-NEXT: vmov.8 q7[4], r91032; CHECK-NEXT: vmov.8 q7[5], r71033; CHECK-NEXT: ldrb r4, [r1]1034; CHECK-NEXT: vmov r1, r5, d51035; CHECK-NEXT: vadd.i32 q2, q1, r01036; CHECK-NEXT: vldrw.u32 q1, [sp, #280] @ 16-byte Reload1037; CHECK-NEXT: ldrb.w r12, [r1]1038; CHECK-NEXT: vmov r1, r3, d131039; CHECK-NEXT: ldrb r5, [r5]1040; CHECK-NEXT: vldrw.u32 q6, [sp, #232] @ 16-byte Reload1041; CHECK-NEXT: ldrb r1, [r1]1042; CHECK-NEXT: ldrb r3, [r3]1043; CHECK-NEXT: vmov.8 q7[6], r11044; CHECK-NEXT: vmov r1, r7, d41045; CHECK-NEXT: vmov.8 q7[7], r31046; CHECK-NEXT: ldrb r1, [r1]1047; CHECK-NEXT: ldrb r7, [r7]1048; CHECK-NEXT: vmov.8 q5[4], r11049; CHECK-NEXT: vmov r1, r3, d51050; CHECK-NEXT: vmov.8 q5[5], r71051; CHECK-NEXT: vadd.i32 q2, q1, r01052; CHECK-NEXT: vldrw.u32 q1, [sp, #296] @ 16-byte Reload1053; CHECK-NEXT: ldrb r1, [r1]1054; CHECK-NEXT: ldrb r3, [r3]1055; CHECK-NEXT: vmov.8 q5[6], r11056; CHECK-NEXT: ldrb r1, [r6]1057; CHECK-NEXT: vmov.8 q5[7], r31058; CHECK-NEXT: vmov r7, r6, d41059; CHECK-NEXT: vmov r3, lr, d51060; CHECK-NEXT: vmov.8 q5[8], r11061; CHECK-NEXT: vadd.i32 q2, q1, r01062; CHECK-NEXT: vmov.8 q5[9], r41063; CHECK-NEXT: vmov r4, r1, d41064; CHECK-NEXT: vmov.8 q5[10], r121065; CHECK-NEXT: vmov.8 q5[11], r51066; CHECK-NEXT: vldrw.u32 q1, [sp, #264] @ 16-byte Reload1067; CHECK-NEXT: ldrb r7, [r7]1068; CHECK-NEXT: ldrb r6, [r6]1069; CHECK-NEXT: ldrb r3, [r3]1070; CHECK-NEXT: ldrb r4, [r4]1071; CHECK-NEXT: ldrb r1, [r1]1072; CHECK-NEXT: vmov.8 q7[8], r41073; CHECK-NEXT: vmov r5, r4, d51074; CHECK-NEXT: vmov.8 q7[9], r11075; CHECK-NEXT: vadd.i32 q2, q0, r01076; CHECK-NEXT: vldrw.u32 q0, [sp, #216] @ 16-byte Reload1077; CHECK-NEXT: ldrb r5, [r5]1078; CHECK-NEXT: ldrb r4, [r4]1079; CHECK-NEXT: vmov.8 q7[10], r51080; CHECK-NEXT: vmov.8 q7[11], r41081; CHECK-NEXT: vmov.8 q7[12], r71082; CHECK-NEXT: vmov.8 q7[13], r61083; CHECK-NEXT: vmov.8 q7[14], r31084; CHECK-NEXT: vmov r1, r3, d41085; CHECK-NEXT: ldrb r1, [r1]1086; CHECK-NEXT: vmov.8 q5[12], r11087; CHECK-NEXT: ldrb r1, [r3]1088; CHECK-NEXT: vmov.8 q5[13], r11089; CHECK-NEXT: vmov r1, r3, d51090; CHECK-NEXT: vadd.i32 q2, q1, r01091; CHECK-NEXT: ldrb r1, [r1]1092; CHECK-NEXT: vmov.8 q5[14], r11093; CHECK-NEXT: ldrb r1, [r3]1094; CHECK-NEXT: vmov.8 q5[15], r11095; CHECK-NEXT: ldrb.w r1, [lr]1096; CHECK-NEXT: vmov.8 q7[15], r11097; CHECK-NEXT: vmov r1, r3, d41098; CHECK-NEXT: vadd.i8 q5, q7, q51099; CHECK-NEXT: ldrb r1, [r1]1100; CHECK-NEXT: ldrb r3, [r3]1101; CHECK-NEXT: vmov.8 q7[0], r11102; CHECK-NEXT: vmov.8 q7[1], r31103; CHECK-NEXT: vmov r1, r3, d51104; CHECK-NEXT: vadd.i32 q2, q4, r01105; CHECK-NEXT: ldrb r1, [r1]1106; CHECK-NEXT: vmov.8 q7[2], r11107; CHECK-NEXT: ldrb r1, [r3]1108; CHECK-NEXT: vmov.8 q7[3], r11109; CHECK-NEXT: vmov r1, r3, d41110; CHECK-NEXT: ldrb r1, [r1]1111; CHECK-NEXT: vmov.8 q7[4], r11112; CHECK-NEXT: ldrb r1, [r3]1113; CHECK-NEXT: vmov.8 q7[5], r11114; CHECK-NEXT: vmov r1, r3, d51115; CHECK-NEXT: vadd.i32 q2, q6, r01116; CHECK-NEXT: ldrb r1, [r1]1117; CHECK-NEXT: vmov.8 q7[6], r11118; CHECK-NEXT: ldrb r1, [r3]1119; CHECK-NEXT: vmov.8 q7[7], r11120; CHECK-NEXT: vmov r1, r3, d41121; CHECK-NEXT: ldrb r1, [r1]1122; CHECK-NEXT: vmov.8 q7[8], r11123; CHECK-NEXT: ldrb r1, [r3]1124; CHECK-NEXT: vmov.8 q7[9], r11125; CHECK-NEXT: vmov r1, r3, d51126; CHECK-NEXT: vadd.i32 q2, q0, r01127; CHECK-NEXT: ldrb r1, [r1]1128; CHECK-NEXT: vmov.8 q7[10], r11129; CHECK-NEXT: ldrb r1, [r3]1130; CHECK-NEXT: vmov.8 q7[11], r11131; CHECK-NEXT: vmov r1, r3, d41132; CHECK-NEXT: ldrb r1, [r1]1133; CHECK-NEXT: vmov.8 q7[12], r11134; CHECK-NEXT: ldrb r1, [r3]1135; CHECK-NEXT: vmov.8 q7[13], r11136; CHECK-NEXT: vmov r1, r3, d51137; CHECK-NEXT: ldrb r1, [r1]1138; CHECK-NEXT: vmov.8 q7[14], r11139; CHECK-NEXT: ldrb r1, [r3]1140; CHECK-NEXT: vmov.8 q7[15], r11141; CHECK-NEXT: vadd.i8 q2, q5, q71142; CHECK-NEXT: vldrw.u32 q5, [sp, #200] @ 16-byte Reload1143; CHECK-NEXT: vstrb.8 q2, [r8], #161144; CHECK-NEXT: vmov.i32 q2, #0x301145; CHECK-NEXT: vadd.i32 q6, q6, q21146; CHECK-NEXT: vadd.i32 q3, q3, q21147; CHECK-NEXT: vstrw.32 q6, [sp, #232] @ 16-byte Spill1148; CHECK-NEXT: vldrw.u32 q6, [sp, #296] @ 16-byte Reload1149; CHECK-NEXT: vadd.i32 q1, q1, q21150; CHECK-NEXT: vadd.i32 q4, q4, q21151; CHECK-NEXT: vadd.i32 q6, q6, q21152; CHECK-NEXT: vadd.i32 q0, q0, q21153; CHECK-NEXT: vmov q7, q31154; CHECK-NEXT: vldrw.u32 q3, [sp, #136] @ 16-byte Reload1155; CHECK-NEXT: vstrw.32 q1, [sp, #264] @ 16-byte Spill1156; CHECK-NEXT: vldrw.u32 q1, [sp, #152] @ 16-byte Reload1157; CHECK-NEXT: vstrw.32 q4, [sp, #248] @ 16-byte Spill1158; CHECK-NEXT: vldrw.u32 q4, [sp, #168] @ 16-byte Reload1159; CHECK-NEXT: vstrw.32 q6, [sp, #296] @ 16-byte Spill1160; CHECK-NEXT: vldrw.u32 q6, [sp, #120] @ 16-byte Reload1161; CHECK-NEXT: vstrw.32 q0, [sp, #216] @ 16-byte Spill1162; CHECK-NEXT: vldrw.u32 q0, [sp, #280] @ 16-byte Reload1163; CHECK-NEXT: vadd.i32 q5, q5, q21164; CHECK-NEXT: vadd.i32 q3, q3, q21165; CHECK-NEXT: vadd.i32 q0, q0, q21166; CHECK-NEXT: vadd.i32 q4, q4, q21167; CHECK-NEXT: vadd.i32 q1, q1, q21168; CHECK-NEXT: vadd.i32 q6, q6, q21169; CHECK-NEXT: vstrw.32 q0, [sp, #280] @ 16-byte Spill1170; CHECK-NEXT: bne.w .LBB15_31171; CHECK-NEXT: @ %bb.4: @ %middle.block1172; CHECK-NEXT: @ in Loop: Header=BB15_2 Depth=11173; CHECK-NEXT: cmp r10, r21174; CHECK-NEXT: bne.w .LBB15_21175; CHECK-NEXT: .LBB15_5: @ %for.cond.cleanup1176; CHECK-NEXT: add sp, #3121177; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}1178; CHECK-NEXT: add sp, #41179; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}1180; CHECK-NEXT: .p2align 41181; CHECK-NEXT: @ %bb.6:1182; CHECK-NEXT: .LCPI15_0:1183; CHECK-NEXT: .long 38 @ 0x261184; CHECK-NEXT: .long 41 @ 0x291185; CHECK-NEXT: .long 44 @ 0x2c1186; CHECK-NEXT: .long 47 @ 0x2f1187; CHECK-NEXT: .LCPI15_1:1188; CHECK-NEXT: .long 14 @ 0xe1189; CHECK-NEXT: .long 17 @ 0x111190; CHECK-NEXT: .long 20 @ 0x141191; CHECK-NEXT: .long 23 @ 0x171192; CHECK-NEXT: .LCPI15_2:1193; CHECK-NEXT: .long 24 @ 0x181194; CHECK-NEXT: .long 27 @ 0x1b1195; CHECK-NEXT: .long 30 @ 0x1e1196; CHECK-NEXT: .long 33 @ 0x211197; CHECK-NEXT: .LCPI15_3:1198; CHECK-NEXT: .long 1 @ 0x11199; CHECK-NEXT: .long 4 @ 0x41200; CHECK-NEXT: .long 7 @ 0x71201; CHECK-NEXT: .long 10 @ 0xa1202; CHECK-NEXT: .LCPI15_4:1203; CHECK-NEXT: .long 36 @ 0x241204; CHECK-NEXT: .long 39 @ 0x271205; CHECK-NEXT: .long 42 @ 0x2a1206; CHECK-NEXT: .long 45 @ 0x2d1207; CHECK-NEXT: .LCPI15_5:1208; CHECK-NEXT: .long 25 @ 0x191209; CHECK-NEXT: .long 28 @ 0x1c1210; CHECK-NEXT: .long 31 @ 0x1f1211; CHECK-NEXT: .long 34 @ 0x221212; CHECK-NEXT: .LCPI15_6:1213; CHECK-NEXT: .long 13 @ 0xd1214; CHECK-NEXT: .long 16 @ 0x101215; CHECK-NEXT: .long 19 @ 0x131216; CHECK-NEXT: .long 22 @ 0x161217; CHECK-NEXT: .LCPI15_7:1218; CHECK-NEXT: .long 2 @ 0x21219; CHECK-NEXT: .long 5 @ 0x51220; CHECK-NEXT: .long 8 @ 0x81221; CHECK-NEXT: .long 11 @ 0xb1222; CHECK-NEXT: .LCPI15_8:1223; CHECK-NEXT: .long 26 @ 0x1a1224; CHECK-NEXT: .long 29 @ 0x1d1225; CHECK-NEXT: .long 32 @ 0x201226; CHECK-NEXT: .long 35 @ 0x231227; CHECK-NEXT: .LCPI15_9:1228; CHECK-NEXT: .long 37 @ 0x251229; CHECK-NEXT: .long 40 @ 0x281230; CHECK-NEXT: .long 43 @ 0x2b1231; CHECK-NEXT: .long 46 @ 0x2e1232; CHECK-NEXT: .LCPI15_10:1233; CHECK-NEXT: .long 12 @ 0xc1234; CHECK-NEXT: .long 15 @ 0xf1235; CHECK-NEXT: .long 18 @ 0x121236; CHECK-NEXT: .long 21 @ 0x151237; CHECK-NEXT: .LCPI15_11:1238; CHECK-NEXT: .long 0 @ 0x01239; CHECK-NEXT: .long 3 @ 0x31240; CHECK-NEXT: .long 6 @ 0x61241; CHECK-NEXT: .long 9 @ 0x91242 1243 1244entry:1245 %cmp22 = icmp sgt i32 %n, 01246 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup1247 1248vector.ph: ; preds = %for.body.preheader1249 %n.vec = and i32 %n, -81250 br label %vector.body1251 1252vector.body: ; preds = %vector.body, %vector.ph1253 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]1254 %vec.ind = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, %vector.ph ], [ %vec.ind.next, %vector.body ]1255 %0 = mul nuw nsw <16 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>1256 %1 = getelementptr inbounds i8, ptr %data, <16 x i32> %01257 %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %1, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)1258 %2 = add nuw nsw <16 x i32> %0, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1259 %3 = getelementptr inbounds i8, ptr %data, <16 x i32> %21260 %wide.masked.gather24 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %3, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)1261 %4 = add nuw nsw <16 x i32> %0, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>1262 %5 = getelementptr inbounds i8, ptr %data, <16 x i32> %41263 %wide.masked.gather25 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %5, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)1264 %6 = add nsw <16 x i8> %wide.masked.gather24, %wide.masked.gather1265 %7 = add nsw <16 x i8> %6, %wide.masked.gather251266 %8 = getelementptr inbounds i8, ptr %dst, i32 %index1267 store <16 x i8> %7, ptr %8, align 21268 %index.next = add i32 %index, 161269 %vec.ind.next = add <16 x i32> %vec.ind, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1270 %9 = icmp eq i32 %index.next, %n.vec1271 br i1 %9, label %middle.block, label %vector.body1272 1273middle.block: ; preds = %vector.body1274 %cmp.n = icmp eq i32 %n.vec, %n1275 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph1276 1277for.cond.cleanup: ; preds = %for.body, %middle.block, %entry1278 ret void1279}1280 1281define arm_aapcs_vfpcc void @gather_inc_v16i8_simple(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {1282; CHECK-LABEL: gather_inc_v16i8_simple:1283; CHECK: @ %bb.0: @ %entry1284; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}1285; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}1286; CHECK-NEXT: .pad #41287; CHECK-NEXT: sub sp, #41288; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}1289; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}1290; CHECK-NEXT: .pad #641291; CHECK-NEXT: sub sp, #641292; CHECK-NEXT: cmp r2, #11293; CHECK-NEXT: strd r1, r2, [sp, #56] @ 8-byte Folded Spill1294; CHECK-NEXT: blt.w .LBB16_51295; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader1296; CHECK-NEXT: adr r5, .LCPI16_31297; CHECK-NEXT: adr r7, .LCPI16_11298; CHECK-NEXT: vldrw.u32 q0, [r5]1299; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload1300; CHECK-NEXT: adr r3, .LCPI16_01301; CHECK-NEXT: adr r6, .LCPI16_21302; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill1303; CHECK-NEXT: vldrw.u32 q0, [r7]1304; CHECK-NEXT: bic r9, r1, #71305; CHECK-NEXT: vldrw.u32 q3, [r3]1306; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill1307; CHECK-NEXT: vldrw.u32 q0, [r6]1308; CHECK-NEXT: mov.w lr, #161309; CHECK-NEXT: str.w r9, [sp, #52] @ 4-byte Spill1310; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill1311; CHECK-NEXT: .LBB16_2: @ %vector.ph1312; CHECK-NEXT: @ =>This Loop Header: Depth=11313; CHECK-NEXT: @ Child Loop BB16_3 Depth 21314; CHECK-NEXT: ldr.w r8, [sp, #56] @ 4-byte Reload1315; CHECK-NEXT: vldrw.u32 q5, [sp] @ 16-byte Reload1316; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload1317; CHECK-NEXT: vldrw.u32 q7, [sp, #32] @ 16-byte Reload1318; CHECK-NEXT: vmov q4, q31319; CHECK-NEXT: .LBB16_3: @ %vector.body1320; CHECK-NEXT: @ Parent Loop BB16_2 Depth=11321; CHECK-NEXT: @ => This Inner Loop Header: Depth=21322; CHECK-NEXT: vadd.i32 q1, q5, r01323; CHECK-NEXT: vadd.i32 q2, q4, r01324; CHECK-NEXT: vmov r7, r3, d31325; CHECK-NEXT: vadd.i32 q6, q0, lr1326; CHECK-NEXT: vmov r5, r6, d51327; CHECK-NEXT: subs.w r9, r9, #161328; CHECK-NEXT: vmov r4, r10, d21329; CHECK-NEXT: vadd.i32 q1, q7, lr1330; CHECK-NEXT: vadd.i32 q4, q4, lr1331; CHECK-NEXT: vadd.i32 q5, q5, lr1332; CHECK-NEXT: ldrb.w r11, [r3]1333; CHECK-NEXT: ldrb r3, [r7]1334; CHECK-NEXT: vmov r7, r12, d41335; CHECK-NEXT: vadd.i32 q2, q7, r01336; CHECK-NEXT: vadd.i32 q7, q0, r01337; CHECK-NEXT: ldrb r5, [r5]1338; CHECK-NEXT: ldrb r6, [r6]1339; CHECK-NEXT: ldrb r4, [r4]1340; CHECK-NEXT: ldrb.w r10, [r10]1341; CHECK-NEXT: ldrb r7, [r7]1342; CHECK-NEXT: ldrb.w r1, [r12]1343; CHECK-NEXT: vmov.8 q0[0], r71344; CHECK-NEXT: vmov.8 q0[1], r11345; CHECK-NEXT: vmov r1, r7, d151346; CHECK-NEXT: vmov.8 q0[2], r51347; CHECK-NEXT: vmov.8 q0[3], r61348; CHECK-NEXT: vmov.8 q0[4], r41349; CHECK-NEXT: vmov r4, r2, d41350; CHECK-NEXT: vmov.8 q0[5], r101351; CHECK-NEXT: vmov.8 q0[6], r31352; CHECK-NEXT: vmov.8 q0[7], r111353; CHECK-NEXT: ldrb r6, [r7]1354; CHECK-NEXT: vmov r5, r7, d51355; CHECK-NEXT: ldrb r1, [r1]1356; CHECK-NEXT: ldrb r2, [r2]1357; CHECK-NEXT: ldrb r3, [r5]1358; CHECK-NEXT: ldrb.w r12, [r7]1359; CHECK-NEXT: ldrb r5, [r4]1360; CHECK-NEXT: vmov r4, r7, d141361; CHECK-NEXT: vmov q7, q11362; CHECK-NEXT: ldrb r4, [r4]1363; CHECK-NEXT: ldrb r7, [r7]1364; CHECK-NEXT: vmov.8 q0[8], r41365; CHECK-NEXT: vmov.8 q0[9], r71366; CHECK-NEXT: vmov.8 q0[10], r11367; CHECK-NEXT: vmov.8 q0[11], r61368; CHECK-NEXT: vmov.8 q0[12], r51369; CHECK-NEXT: vmov.8 q0[13], r21370; CHECK-NEXT: vmov.8 q0[14], r31371; CHECK-NEXT: vmov.8 q0[15], r121372; CHECK-NEXT: vstrb.8 q0, [r8], #161373; CHECK-NEXT: vmov q0, q61374; CHECK-NEXT: bne .LBB16_31375; CHECK-NEXT: @ %bb.4: @ %middle.block1376; CHECK-NEXT: @ in Loop: Header=BB16_2 Depth=11377; CHECK-NEXT: ldr.w r9, [sp, #52] @ 4-byte Reload1378; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload1379; CHECK-NEXT: cmp r9, r11380; CHECK-NEXT: bne .LBB16_21381; CHECK-NEXT: .LBB16_5: @ %for.cond.cleanup1382; CHECK-NEXT: add sp, #641383; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}1384; CHECK-NEXT: add sp, #41385; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}1386; CHECK-NEXT: .p2align 41387; CHECK-NEXT: @ %bb.6:1388; CHECK-NEXT: .LCPI16_0:1389; CHECK-NEXT: .long 0 @ 0x01390; CHECK-NEXT: .long 1 @ 0x11391; CHECK-NEXT: .long 2 @ 0x21392; CHECK-NEXT: .long 3 @ 0x31393; CHECK-NEXT: .LCPI16_1:1394; CHECK-NEXT: .long 8 @ 0x81395; CHECK-NEXT: .long 9 @ 0x91396; CHECK-NEXT: .long 10 @ 0xa1397; CHECK-NEXT: .long 11 @ 0xb1398; CHECK-NEXT: .LCPI16_2:1399; CHECK-NEXT: .long 4 @ 0x41400; CHECK-NEXT: .long 5 @ 0x51401; CHECK-NEXT: .long 6 @ 0x61402; CHECK-NEXT: .long 7 @ 0x71403; CHECK-NEXT: .LCPI16_3:1404; CHECK-NEXT: .long 12 @ 0xc1405; CHECK-NEXT: .long 13 @ 0xd1406; CHECK-NEXT: .long 14 @ 0xe1407; CHECK-NEXT: .long 15 @ 0xf1408 1409 1410entry:1411 %cmp22 = icmp sgt i32 %n, 01412 br i1 %cmp22, label %vector.ph, label %for.cond.cleanup1413 1414vector.ph: ; preds = %for.body.preheader1415 %n.vec = and i32 %n, -81416 br label %vector.body1417 1418vector.body: ; preds = %vector.body, %vector.ph1419 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]1420 %vec.ind = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, %vector.ph ], [ %vec.ind.next, %vector.body ]1421 %0 = getelementptr inbounds i8, ptr %data, <16 x i32> %vec.ind1422 %wide.masked.gather = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %0, i32 2, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> undef)1423 %1 = getelementptr inbounds i8, ptr %dst, i32 %index1424 store <16 x i8> %wide.masked.gather, ptr %1, align 21425 %index.next = add i32 %index, 161426 %vec.ind.next = add <16 x i32> %vec.ind, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1427 %2 = icmp eq i32 %index.next, %n.vec1428 br i1 %2, label %middle.block, label %vector.body1429 1430middle.block: ; preds = %vector.body1431 %cmp.n = icmp eq i32 %n.vec, %n1432 br i1 %cmp.n, label %for.cond.cleanup, label %vector.ph1433 1434for.cond.cleanup: ; preds = %for.body, %middle.block, %entry1435 ret void1436}1437 1438define void @shl(ptr nocapture %x, ptr noalias nocapture readonly %y, i32 %n) {1439; CHECK-LABEL: shl:1440; CHECK: @ %bb.0: @ %entry1441; CHECK-NEXT: .save {r7, lr}1442; CHECK-NEXT: push {r7, lr}1443; CHECK-NEXT: cmp r2, #11444; CHECK-NEXT: it lt1445; CHECK-NEXT: poplt {r7, pc}1446; CHECK-NEXT: .LBB17_1: @ %vector.ph1447; CHECK-NEXT: adr r3, .LCPI17_01448; CHECK-NEXT: vldrw.u32 q0, [r3]1449; CHECK-NEXT: vadd.i32 q0, q0, r11450; CHECK-NEXT: dlstp.32 lr, r21451; CHECK-NEXT: .LBB17_2: @ %vector.body1452; CHECK-NEXT: @ =>This Inner Loop Header: Depth=11453; CHECK-NEXT: vldrw.u32 q1, [q0, #64]!1454; CHECK-NEXT: vstrw.32 q1, [r0], #161455; CHECK-NEXT: letp lr, .LBB17_21456; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup1457; CHECK-NEXT: pop {r7, pc}1458; CHECK-NEXT: .p2align 41459; CHECK-NEXT: @ %bb.4:1460; CHECK-NEXT: .LCPI17_0:1461; CHECK-NEXT: .long 4294967232 @ 0xffffffc01462; CHECK-NEXT: .long 4294967248 @ 0xffffffd01463; CHECK-NEXT: .long 4294967264 @ 0xffffffe01464; CHECK-NEXT: .long 4294967280 @ 0xfffffff01465entry:1466 %cmp6 = icmp sgt i32 %n, 01467 br i1 %cmp6, label %vector.ph, label %for.cond.cleanup1468 1469vector.ph: ; preds = %entry1470 %n.rnd.up = add i32 %n, 31471 %n.vec = and i32 %n.rnd.up, -41472 br label %vector.body1473 1474vector.body: ; preds = %vector.body, %vector.ph1475 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]1476 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]1477 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)1478 %0 = shl nsw <4 x i32> %vec.ind, <i32 2, i32 2, i32 2, i32 2>1479 %1 = getelementptr inbounds i32, ptr %y, <4 x i32> %01480 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)1481 %2 = getelementptr inbounds i32, ptr %x, i32 %index1482 call void @llvm.masked.store.v4i32.p0(<4 x i32> %wide.masked.gather, ptr %2, i32 4, <4 x i1> %active.lane.mask)1483 %index.next = add i32 %index, 41484 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>1485 %3 = icmp eq i32 %index.next, %n.vec1486 br i1 %3, label %for.cond.cleanup, label %vector.body1487 1488for.cond.cleanup: ; preds = %vector.body, %entry1489 ret void1490}1491 1492define void @shlor(ptr nocapture %x, ptr noalias nocapture readonly %y, i32 %n) {1493; CHECK-LABEL: shlor:1494; CHECK: @ %bb.0: @ %entry1495; CHECK-NEXT: .save {r4, r5, r6, lr}1496; CHECK-NEXT: push {r4, r5, r6, lr}1497; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}1498; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}1499; CHECK-NEXT: cmp r2, #11500; CHECK-NEXT: blt .LBB18_31501; CHECK-NEXT: @ %bb.1: @ %vector.ph1502; CHECK-NEXT: adr r3, .LCPI18_01503; CHECK-NEXT: adr r4, .LCPI18_11504; CHECK-NEXT: adr r5, .LCPI18_21505; CHECK-NEXT: adr r6, .LCPI18_31506; CHECK-NEXT: vldrw.u32 q0, [r6]1507; CHECK-NEXT: vldrw.u32 q1, [r5]1508; CHECK-NEXT: vldrw.u32 q2, [r4]1509; CHECK-NEXT: vldrw.u32 q3, [r3]1510; CHECK-NEXT: vadd.i32 q0, q0, r11511; CHECK-NEXT: vadd.i32 q1, q1, r11512; CHECK-NEXT: vadd.i32 q2, q2, r11513; CHECK-NEXT: vadd.i32 q3, q3, r11514; CHECK-NEXT: dlstp.32 lr, r21515; CHECK-NEXT: .LBB18_2: @ %vector.body1516; CHECK-NEXT: @ =>This Inner Loop Header: Depth=11517; CHECK-NEXT: vldrw.u32 q4, [q3, #128]!1518; CHECK-NEXT: vldrw.u32 q5, [q2, #128]!1519; CHECK-NEXT: vldrw.u32 q6, [q0, #128]!1520; CHECK-NEXT: vadd.i32 q4, q5, q41521; CHECK-NEXT: vldrw.u32 q5, [q1, #128]!1522; CHECK-NEXT: vadd.i32 q4, q4, q51523; CHECK-NEXT: vadd.i32 q4, q4, q61524; CHECK-NEXT: vstrw.32 q4, [r0], #161525; CHECK-NEXT: letp lr, .LBB18_21526; CHECK-NEXT: .LBB18_3: @ %for.cond.cleanup1527; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}1528; CHECK-NEXT: pop {r4, r5, r6, pc}1529; CHECK-NEXT: .p2align 41530; CHECK-NEXT: @ %bb.4:1531; CHECK-NEXT: .LCPI18_0:1532; CHECK-NEXT: .long 4294967168 @ 0xffffff801533; CHECK-NEXT: .long 4294967200 @ 0xffffffa01534; CHECK-NEXT: .long 4294967232 @ 0xffffffc01535; CHECK-NEXT: .long 4294967264 @ 0xffffffe01536; CHECK-NEXT: .LCPI18_1:1537; CHECK-NEXT: .long 4294967176 @ 0xffffff881538; CHECK-NEXT: .long 4294967208 @ 0xffffffa81539; CHECK-NEXT: .long 4294967240 @ 0xffffffc81540; CHECK-NEXT: .long 4294967272 @ 0xffffffe81541; CHECK-NEXT: .LCPI18_2:1542; CHECK-NEXT: .long 4294967184 @ 0xffffff901543; CHECK-NEXT: .long 4294967216 @ 0xffffffb01544; CHECK-NEXT: .long 4294967248 @ 0xffffffd01545; CHECK-NEXT: .long 4294967280 @ 0xfffffff01546; CHECK-NEXT: .LCPI18_3:1547; CHECK-NEXT: .long 4294967192 @ 0xffffff981548; CHECK-NEXT: .long 4294967224 @ 0xffffffb81549; CHECK-NEXT: .long 4294967256 @ 0xffffffd81550; CHECK-NEXT: .long 4294967288 @ 0xfffffff81551entry:1552 %cmp23 = icmp sgt i32 %n, 01553 br i1 %cmp23, label %vector.ph, label %for.cond.cleanup1554 1555vector.ph: ; preds = %entry1556 %n.rnd.up = add i32 %n, 31557 %n.vec = and i32 %n.rnd.up, -41558 br label %vector.body1559 1560vector.body: ; preds = %vector.body, %vector.ph1561 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]1562 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]1563 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n)1564 %0 = shl nsw <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>1565 %1 = getelementptr inbounds i32, ptr %y, <4 x i32> %01566 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)1567 %2 = or <4 x i32> %0, <i32 2, i32 2, i32 2, i32 2>1568 %3 = getelementptr inbounds i32, ptr %y, <4 x i32> %21569 %wide.masked.gather25 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)1570 %4 = add nsw <4 x i32> %wide.masked.gather25, %wide.masked.gather1571 %5 = or <4 x i32> %0, <i32 4, i32 4, i32 4, i32 4>1572 %6 = getelementptr inbounds i32, ptr %y, <4 x i32> %51573 %wide.masked.gather26 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %6, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)1574 %7 = add nsw <4 x i32> %4, %wide.masked.gather261575 %8 = or <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>1576 %9 = getelementptr inbounds i32, ptr %y, <4 x i32> %81577 %wide.masked.gather27 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %9, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)1578 %10 = add nsw <4 x i32> %7, %wide.masked.gather271579 %11 = getelementptr inbounds i32, ptr %x, i32 %index1580 call void @llvm.masked.store.v4i32.p0(<4 x i32> %10, ptr %11, i32 4, <4 x i1> %active.lane.mask)1581 %index.next = add i32 %index, 41582 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>1583 %12 = icmp eq i32 %index.next, %n.vec1584 br i1 %12, label %for.cond.cleanup, label %vector.body1585 1586for.cond.cleanup: ; preds = %vector.body, %entry1587 ret void1588}1589 1590 1591declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)1592declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)1593declare <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i32>)1594declare <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i32>)1595declare <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x float>)1596declare <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x float>)1597declare <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x float>)1598declare <2 x i16> @llvm.masked.gather.v2i16.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i16>)1599declare <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i16>)1600declare <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i16>)1601declare <16 x i16> @llvm.masked.gather.v16i16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i16>)1602declare <4 x half> @llvm.masked.gather.v4f16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x half>)1603declare <8 x half> @llvm.masked.gather.v8f16.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x half>)1604declare <16 x half> @llvm.masked.gather.v16f16.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x half>)1605declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i8>)1606declare <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr>, i32, <8 x i1>, <8 x i8>)1607declare <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i8>)1608declare <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x i8>)1609declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)1610declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)1611