1114 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2 3; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp --arm-memtransfer-tploop=allow %s -o - | FileCheck %s4 5!0 = !{i32 1, !"wchar_size", i32 4}6!1 = !{i32 1, !"min_enum_size", i32 4}7!2 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 26f04d01a39a33d73fd23165c208b215bf5c350d)"}8!3 = !{!4, !4, i64 0}9!4 = !{!"int", !5, i64 0}10!5 = !{!"omnipotent char", !6, i64 0}11!6 = !{!"Simple C/C++ TBAA"}12!7 = distinct !{!7, !8}13!8 = !{!"llvm.loop.isvectorized", i32 1}14!9 = distinct !{!9, !10, !8}15!10 = !{!"llvm.loop.unroll.runtime.disable"}16 17 18 19define arm_aapcs_vfpcc void @push_out_mul_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {20; CHECK-LABEL: push_out_mul_gather:21; CHECK: @ %bb.0: @ %vector.ph22; CHECK-NEXT: adr r3, .LCPI0_023; CHECK-NEXT: vldrw.u32 q0, [r3]24; CHECK-NEXT: vadd.i32 q0, q0, r025; CHECK-NEXT: .LBB0_1: @ %vector.body26; CHECK-NEXT: @ =>This Inner Loop Header: Depth=127; CHECK-NEXT: vldrw.u32 q1, [q0, #96]!28; CHECK-NEXT: subs r2, #429; CHECK-NEXT: vstrb.8 q1, [r1], #1630; CHECK-NEXT: bne .LBB0_131; CHECK-NEXT: @ %bb.2: @ %end32; CHECK-NEXT: bx lr33; CHECK-NEXT: .p2align 434; CHECK-NEXT: @ %bb.3:35; CHECK-NEXT: .LCPI0_0:36; CHECK-NEXT: .long 4294967200 @ 0xffffffa037; CHECK-NEXT: .long 4294967224 @ 0xffffffb838; CHECK-NEXT: .long 4294967248 @ 0xffffffd039; CHECK-NEXT: .long 4294967272 @ 0xffffffe840 41vector.ph: ; preds = %for.body.preheader42 br label %vector.body43 44vector.body: ; preds = %vector.body, %vector.ph45 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]46 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]47 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>48 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %049 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)50 %2 = getelementptr inbounds i32, ptr %dst, i32 %index51 %3 = bitcast ptr %2 to ptr52 store <4 x i32> %wide.masked.gather, ptr %3, align 453 %index.next = add i32 %index, 454 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>55 %4 = icmp eq i32 %index.next, %n.vec56 br i1 %4, label %end, label %vector.body57 58end:59 ret void;60}61 62define arm_aapcs_vfpcc void @push_out_add_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {63; CHECK-LABEL: push_out_add_gather:64; CHECK: @ %bb.0: @ %vector.ph65; CHECK-NEXT: adr r3, .LCPI1_066; CHECK-NEXT: vldrw.u32 q0, [r3]67; CHECK-NEXT: vadd.i32 q0, q0, r068; CHECK-NEXT: .LBB1_1: @ %vector.body69; CHECK-NEXT: @ =>This Inner Loop Header: Depth=170; CHECK-NEXT: vldrw.u32 q1, [q0, #32]!71; CHECK-NEXT: subs r2, #472; CHECK-NEXT: vstrb.8 q1, [r1], #1673; CHECK-NEXT: bne .LBB1_174; CHECK-NEXT: @ %bb.2: @ %end75; CHECK-NEXT: bx lr76; CHECK-NEXT: .p2align 477; CHECK-NEXT: @ %bb.3:78; CHECK-NEXT: .LCPI1_0:79; CHECK-NEXT: .long 4294967288 @ 0xfffffff880; CHECK-NEXT: .long 0 @ 0x081; CHECK-NEXT: .long 8 @ 0x882; CHECK-NEXT: .long 16 @ 0x1083 84vector.ph: ; preds = %for.body.preheader85 br label %vector.body86 87vector.body: ; preds = %vector.body, %vector.ph88 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]89 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]90 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>91 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %092 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)93 %2 = getelementptr inbounds i32, ptr %dst, i32 %index94 %3 = bitcast ptr %2 to ptr95 store <4 x i32> %wide.masked.gather, ptr %3, align 496 %index.next = add i32 %index, 497 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>98 %4 = icmp eq i32 %index.next, %n.vec99 br i1 %4, label %end, label %vector.body100 101end:102 ret void;103}104 105define arm_aapcs_vfpcc void @push_out_mul_add_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {106; CHECK-LABEL: push_out_mul_add_gather:107; CHECK: @ %bb.0: @ %vector.ph108; CHECK-NEXT: adr r3, .LCPI2_0109; CHECK-NEXT: vldrw.u32 q0, [r3]110; CHECK-NEXT: vadd.i32 q0, q0, r0111; CHECK-NEXT: .LBB2_1: @ %vector.body112; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1113; CHECK-NEXT: vldrw.u32 q1, [q0, #96]!114; CHECK-NEXT: subs r2, #4115; CHECK-NEXT: vstrb.8 q1, [r1], #16116; CHECK-NEXT: bne .LBB2_1117; CHECK-NEXT: @ %bb.2: @ %end118; CHECK-NEXT: bx lr119; CHECK-NEXT: .p2align 4120; CHECK-NEXT: @ %bb.3:121; CHECK-NEXT: .LCPI2_0:122; CHECK-NEXT: .long 4294967224 @ 0xffffffb8123; CHECK-NEXT: .long 4294967248 @ 0xffffffd0124; CHECK-NEXT: .long 4294967272 @ 0xffffffe8125; CHECK-NEXT: .long 0 @ 0x0126 127vector.ph: ; preds = %for.body.preheader128 br label %vector.body129 130vector.body: ; preds = %vector.body, %vector.ph131 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]132 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]133 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>134 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>135 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1136 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)137 %3 = getelementptr inbounds i32, ptr %dst, i32 %index138 %4 = bitcast ptr %3 to ptr139 store <4 x i32> %wide.masked.gather, ptr %4, align 4140 %index.next = add i32 %index, 4141 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>142 %5 = icmp eq i32 %index.next, %n.vec143 br i1 %5, label %end, label %vector.body144 145end:146 ret void;147}148 149define arm_aapcs_vfpcc void @push_out_mul_scatter(ptr noalias nocapture readonly %data,150; CHECK-LABEL: push_out_mul_scatter:151; CHECK: @ %bb.0: @ %vector.ph152; CHECK-NEXT: adr r1, .LCPI3_0153; CHECK-NEXT: vldrw.u32 q1, [r1]154; CHECK-NEXT: vadd.i32 q1, q1, r0155; CHECK-NEXT: .LBB3_1: @ %vector.body156; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1157; CHECK-NEXT: subs r2, #4158; CHECK-NEXT: vstrw.32 q0, [q1, #96]!159; CHECK-NEXT: bne .LBB3_1160; CHECK-NEXT: @ %bb.2: @ %end161; CHECK-NEXT: bx lr162; CHECK-NEXT: .p2align 4163; CHECK-NEXT: @ %bb.3:164; CHECK-NEXT: .LCPI3_0:165; CHECK-NEXT: .long 4294967200 @ 0xffffffa0166; CHECK-NEXT: .long 4294967224 @ 0xffffffb8167; CHECK-NEXT: .long 4294967248 @ 0xffffffd0168; CHECK-NEXT: .long 4294967272 @ 0xffffffe8169 ptr noalias nocapture %dst, i32 %n.vec,170 <4 x i32> %to.store) {171 172vector.ph: ; preds = %for.body.preheader173 br label %vector.body174 175vector.body: ; preds = %vector.body, %vector.ph176 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]177 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]178 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>179 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0180 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %to.store, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)181 %index.next = add i32 %index, 4182 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>183 %2 = icmp eq i32 %index.next, %n.vec184 br i1 %2, label %end, label %vector.body185 186end:187 ret void;188}189 190define arm_aapcs_vfpcc void @push_out_add_scatter(ptr noalias nocapture readonly %data,191; CHECK-LABEL: push_out_add_scatter:192; CHECK: @ %bb.0: @ %vector.ph193; CHECK-NEXT: adr r1, .LCPI4_0194; CHECK-NEXT: vldrw.u32 q1, [r1]195; CHECK-NEXT: vadd.i32 q1, q1, r0196; CHECK-NEXT: .LBB4_1: @ %vector.body197; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1198; CHECK-NEXT: subs r2, #4199; CHECK-NEXT: vstrw.32 q0, [q1, #32]!200; CHECK-NEXT: bne .LBB4_1201; CHECK-NEXT: @ %bb.2: @ %end202; CHECK-NEXT: bx lr203; CHECK-NEXT: .p2align 4204; CHECK-NEXT: @ %bb.3:205; CHECK-NEXT: .LCPI4_0:206; CHECK-NEXT: .long 4294967288 @ 0xfffffff8207; CHECK-NEXT: .long 0 @ 0x0208; CHECK-NEXT: .long 8 @ 0x8209; CHECK-NEXT: .long 16 @ 0x10210 ptr noalias nocapture %dst, i32 %n.vec,211 <4 x i32> %to.store) {212 213vector.ph: ; preds = %for.body.preheader214 br label %vector.body215 216vector.body: ; preds = %vector.body, %vector.ph217 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]218 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]219 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>220 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0221 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %to.store, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)222 %index.next = add i32 %index, 4223 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>224 %2 = icmp eq i32 %index.next, %n.vec225 br i1 %2, label %end, label %vector.body226 227end:228 ret void;229}230 231define arm_aapcs_vfpcc void @push_out_mul_gather_scatter(ptr noalias nocapture readonly %data,232; CHECK-LABEL: push_out_mul_gather_scatter:233; CHECK: @ %bb.0: @ %vector.ph234; CHECK-NEXT: adr r1, .LCPI5_0235; CHECK-NEXT: vmov.i32 q0, #0x18236; CHECK-NEXT: vldrw.u32 q1, [r1]237; CHECK-NEXT: .LBB5_1: @ %vector.body238; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1239; CHECK-NEXT: vldrw.u32 q2, [r0, q1, uxtw #2]240; CHECK-NEXT: vadd.i32 q3, q1, q0241; CHECK-NEXT: subs r2, #4242; CHECK-NEXT: vstrw.32 q2, [r0, q1, uxtw #2]243; CHECK-NEXT: vmov q1, q3244; CHECK-NEXT: bne .LBB5_1245; CHECK-NEXT: @ %bb.2: @ %end246; CHECK-NEXT: bx lr247; CHECK-NEXT: .p2align 4248; CHECK-NEXT: @ %bb.3:249; CHECK-NEXT: .LCPI5_0:250; CHECK-NEXT: .long 0 @ 0x0251; CHECK-NEXT: .long 6 @ 0x6252; CHECK-NEXT: .long 12 @ 0xc253; CHECK-NEXT: .long 18 @ 0x12254 ptr noalias nocapture %dst, i32 %n.vec) {255 256vector.ph: ; preds = %for.body.preheader257 br label %vector.body258 259vector.body: ; preds = %vector.body, %vector.ph260 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]261 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]262 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>263 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0264 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)265 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %wide.masked.gather, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)266 %index.next = add i32 %index, 4267 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>268 %2 = icmp eq i32 %index.next, %n.vec269 br i1 %2, label %end, label %vector.body270 271end:272 ret void;273}274 275define arm_aapcs_vfpcc void @push_out_add_sub_block(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {276; CHECK-LABEL: push_out_add_sub_block:277; CHECK: @ %bb.0: @ %vector.ph278; CHECK-NEXT: adr r3, .LCPI6_0279; CHECK-NEXT: vldrw.u32 q0, [r3]280; CHECK-NEXT: vadd.i32 q0, q0, r0281; CHECK-NEXT: .LBB6_1: @ %vector.body282; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1283; CHECK-NEXT: vldrw.u32 q1, [q0, #32]!284; CHECK-NEXT: subs r2, #4285; CHECK-NEXT: vstrb.8 q1, [r1], #16286; CHECK-NEXT: bne .LBB6_1287; CHECK-NEXT: @ %bb.2: @ %end288; CHECK-NEXT: bx lr289; CHECK-NEXT: .p2align 4290; CHECK-NEXT: @ %bb.3:291; CHECK-NEXT: .LCPI6_0:292; CHECK-NEXT: .long 4294967288 @ 0xfffffff8293; CHECK-NEXT: .long 0 @ 0x0294; CHECK-NEXT: .long 8 @ 0x8295; CHECK-NEXT: .long 16 @ 0x10296 297vector.ph: ; preds = %for.body.preheader298 br label %vector.body299 300vector.body: ; preds = %vector.body, %vector.ph301 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]302 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]303 br label %lower.block;304 305lower.block: ; preds = %vector.body306 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>307 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0308 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)309 %2 = getelementptr inbounds i32, ptr %dst, i32 %index310 %3 = bitcast ptr %2 to ptr311 store <4 x i32> %wide.masked.gather, ptr %3, align 4312 %index.next = add i32 %index, 4313 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>314 br label %vector.body.end315 316vector.body.end: ; preds = %lower.block317 %4 = icmp eq i32 %index.next, %n.vec318 br i1 %4, label %end, label %vector.body319 320end:321 ret void;322}323 324define arm_aapcs_vfpcc void @non_gatscat_use1(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec, ptr %x) {325; CHECK-LABEL: non_gatscat_use1:326; CHECK: @ %bb.0: @ %vector.ph327; CHECK-NEXT: .save {r4, lr}328; CHECK-NEXT: push {r4, lr}329; CHECK-NEXT: .vsave {d8, d9}330; CHECK-NEXT: vpush {d8, d9}331; CHECK-NEXT: adr r4, .LCPI7_0332; CHECK-NEXT: mov.w r12, #9333; CHECK-NEXT: vldrw.u32 q1, [r4]334; CHECK-NEXT: mov.w lr, #12335; CHECK-NEXT: movs r4, #8336; CHECK-NEXT: vdup.32 q0, r0337; CHECK-NEXT: .LBB7_1: @ %vector.body338; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1339; CHECK-NEXT: vmov q3, q0340; CHECK-NEXT: vadd.i32 q2, q1, r4341; CHECK-NEXT: vmla.i32 q3, q1, lr342; CHECK-NEXT: vmul.i32 q1, q1, r12343; CHECK-NEXT: vldrw.u32 q4, [q3, #24]344; CHECK-NEXT: subs r2, #4345; CHECK-NEXT: vstrw.32 q1, [r3]346; CHECK-NEXT: vmov q1, q2347; CHECK-NEXT: vstrb.8 q4, [r1], #16348; CHECK-NEXT: bne .LBB7_1349; CHECK-NEXT: @ %bb.2: @ %end350; CHECK-NEXT: vpop {d8, d9}351; CHECK-NEXT: pop {r4, pc}352; CHECK-NEXT: .p2align 4353; CHECK-NEXT: @ %bb.3:354; CHECK-NEXT: .LCPI7_0:355; CHECK-NEXT: .long 0 @ 0x0356; CHECK-NEXT: .long 2 @ 0x2357; CHECK-NEXT: .long 4 @ 0x4358; CHECK-NEXT: .long 6 @ 0x6359 360vector.ph: ; preds = %for.body.preheader361 br label %vector.body362 363vector.body: ; preds = %vector.body, %vector.ph364 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]365 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]366 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>367 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>368 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1369 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)370 %3 = getelementptr inbounds i32, ptr %dst, i32 %index371 %4 = bitcast ptr %3 to ptr372 store <4 x i32> %wide.masked.gather, ptr %4, align 4373 %non_gatscat_use = mul <4 x i32> %0, <i32 3, i32 3, i32 3, i32 3>374 store <4 x i32> %non_gatscat_use, ptr %x, align 4375 %index.next = add i32 %index, 4376 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>377 %5 = icmp eq i32 %index.next, %n.vec378 br i1 %5, label %end, label %vector.body379 380end:381 ret void;382}383 384define arm_aapcs_vfpcc void @non_gatscat_use2(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec, ptr %x) {385; CHECK-LABEL: non_gatscat_use2:386; CHECK: @ %bb.0: @ %vector.ph387; CHECK-NEXT: .save {r4, r5, r7, lr}388; CHECK-NEXT: push {r4, r5, r7, lr}389; CHECK-NEXT: .vsave {d8, d9, d10, d11}390; CHECK-NEXT: vpush {d8, d9, d10, d11}391; CHECK-NEXT: adr r4, .LCPI8_0392; CHECK-NEXT: movs r5, #18393; CHECK-NEXT: vldrw.u32 q2, [r4]394; CHECK-NEXT: mov.w r12, #9395; CHECK-NEXT: mov.w lr, #12396; CHECK-NEXT: movs r4, #8397; CHECK-NEXT: vdup.32 q0, r0398; CHECK-NEXT: vdup.32 q1, r5399; CHECK-NEXT: .LBB8_1: @ %vector.body400; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1401; CHECK-NEXT: vmov q4, q0402; CHECK-NEXT: vadd.i32 q3, q2, r4403; CHECK-NEXT: vmla.i32 q4, q2, lr404; CHECK-NEXT: subs r2, #4405; CHECK-NEXT: vldrw.u32 q5, [q4, #24]406; CHECK-NEXT: vmov q4, q1407; CHECK-NEXT: vmla.i32 q4, q2, r12408; CHECK-NEXT: vmov q2, q3409; CHECK-NEXT: vstrb.8 q5, [r1], #16410; CHECK-NEXT: vstrw.32 q4, [r3]411; CHECK-NEXT: bne .LBB8_1412; CHECK-NEXT: @ %bb.2: @ %end413; CHECK-NEXT: vpop {d8, d9, d10, d11}414; CHECK-NEXT: pop {r4, r5, r7, pc}415; CHECK-NEXT: .p2align 4416; CHECK-NEXT: @ %bb.3:417; CHECK-NEXT: .LCPI8_0:418; CHECK-NEXT: .long 0 @ 0x0419; CHECK-NEXT: .long 2 @ 0x2420; CHECK-NEXT: .long 4 @ 0x4421; CHECK-NEXT: .long 6 @ 0x6422 423vector.ph: ; preds = %for.body.preheader424 br label %vector.body425 426vector.body: ; preds = %vector.body, %vector.ph427 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]428 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]429 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>430 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>431 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1432 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)433 %3 = getelementptr inbounds i32, ptr %dst, i32 %index434 %4 = bitcast ptr %3 to ptr435 store <4 x i32> %wide.masked.gather, ptr %4, align 4436 %non_gatscat_use = mul <4 x i32> %1, <i32 3, i32 3, i32 3, i32 3>437 store <4 x i32> %non_gatscat_use, ptr %x, align 4438 %index.next = add i32 %index, 4439 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>440 %5 = icmp eq i32 %index.next, %n.vec441 br i1 %5, label %end, label %vector.body442 443end:444 ret void;445}446 447define dso_local void @arm_mat_mult_q31(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %n, i32 %m, i32 %l) local_unnamed_addr #0 {448; CHECK-LABEL: arm_mat_mult_q31:449; CHECK: @ %bb.0: @ %for.cond8.preheader.us.us.preheader.preheader450; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr}451; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, lr}452; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}453; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}454; CHECK-NEXT: .pad #32455; CHECK-NEXT: sub sp, #32456; CHECK-NEXT: ldrd r9, r12, [sp, #128]457; CHECK-NEXT: sub.w r7, r12, #1458; CHECK-NEXT: movs r6, #1459; CHECK-NEXT: mov.w r8, #0460; CHECK-NEXT: add.w r7, r6, r7, lsr #1461; CHECK-NEXT: bic r7, r7, #3462; CHECK-NEXT: subs r7, #4463; CHECK-NEXT: add.w r10, r6, r7, lsr #2464; CHECK-NEXT: adr r6, .LCPI9_1465; CHECK-NEXT: vldrw.u32 q0, [r6]466; CHECK-NEXT: adr r7, .LCPI9_0467; CHECK-NEXT: vldrw.u32 q1, [r7]468; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill469; CHECK-NEXT: vdup.32 q0, r9470; CHECK-NEXT: vmov q2, q0471; CHECK-NEXT: vshl.i32 q3, q0, #3472; CHECK-NEXT: vstrw.32 q1, [sp, #16] @ 16-byte Spill473; CHECK-NEXT: .LBB9_1: @ %for.cond8.preheader.us.us.preheader474; CHECK-NEXT: @ =>This Loop Header: Depth=1475; CHECK-NEXT: @ Child Loop BB9_2 Depth 2476; CHECK-NEXT: @ Child Loop BB9_3 Depth 3477; CHECK-NEXT: mul lr, r8, r12478; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload479; CHECK-NEXT: movs r7, #0480; CHECK-NEXT: mul r6, r8, r9481; CHECK-NEXT: vdup.32 q4, lr482; CHECK-NEXT: vshl.i32 q4, q4, #2483; CHECK-NEXT: vadd.i32 q4, q4, r0484; CHECK-NEXT: vadd.i32 q4, q4, q0485; CHECK-NEXT: .LBB9_2: @ %vector.ph486; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1487; CHECK-NEXT: @ => This Loop Header: Depth=2488; CHECK-NEXT: @ Child Loop BB9_3 Depth 3489; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload490; CHECK-NEXT: vmov q7, q2491; CHECK-NEXT: dls lr, r10492; CHECK-NEXT: vmov.i32 q5, #0x0493; CHECK-NEXT: vmlas.i32 q7, q0, r7494; CHECK-NEXT: vmov q6, q4495; CHECK-NEXT: .LBB9_3: @ %vector.body496; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1497; CHECK-NEXT: @ Parent Loop BB9_2 Depth=2498; CHECK-NEXT: @ => This Inner Loop Header: Depth=3499; CHECK-NEXT: vadd.i32 q0, q7, q3500; CHECK-NEXT: vldrw.u32 q1, [r1, q7, uxtw #2]501; CHECK-NEXT: vldrw.u32 q7, [q6, #32]!502; CHECK-NEXT: vmul.i32 q1, q1, q7503; CHECK-NEXT: vmov q7, q0504; CHECK-NEXT: vadd.i32 q5, q1, q5505; CHECK-NEXT: le lr, .LBB9_3506; CHECK-NEXT: @ %bb.4: @ %middle.block507; CHECK-NEXT: @ in Loop: Header=BB9_2 Depth=2508; CHECK-NEXT: adds r5, r7, r6509; CHECK-NEXT: adds r7, #1510; CHECK-NEXT: vaddv.u32 r4, q5511; CHECK-NEXT: cmp r7, r9512; CHECK-NEXT: str.w r4, [r2, r5, lsl #2]513; CHECK-NEXT: bne .LBB9_2514; CHECK-NEXT: @ %bb.5: @ %for.cond4.for.cond.cleanup6_crit_edge.us515; CHECK-NEXT: @ in Loop: Header=BB9_1 Depth=1516; CHECK-NEXT: add.w r8, r8, #1517; CHECK-NEXT: cmp r8, r3518; CHECK-NEXT: bne .LBB9_1519; CHECK-NEXT: @ %bb.6: @ %for.end25520; CHECK-NEXT: add sp, #32521; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}522; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, pc}523; CHECK-NEXT: .p2align 4524; CHECK-NEXT: @ %bb.7:525; CHECK-NEXT: .LCPI9_0:526; CHECK-NEXT: .long 0 @ 0x0527; CHECK-NEXT: .long 2 @ 0x2528; CHECK-NEXT: .long 4 @ 0x4529; CHECK-NEXT: .long 6 @ 0x6530; CHECK-NEXT: .LCPI9_1:531; CHECK-NEXT: .long 4294967264 @ 0xffffffe0532; CHECK-NEXT: .long 4294967272 @ 0xffffffe8533; CHECK-NEXT: .long 4294967280 @ 0xfffffff0534; CHECK-NEXT: .long 4294967288 @ 0xfffffff8535 536for.cond8.preheader.us.us.preheader.preheader: ; preds = %entry537 %0 = add i32 %l, -1538 %1 = lshr i32 %0, 1539 %2 = add nuw i32 %1, 1540 %min.iters.check = icmp ult i32 %0, 6541 %n.vec = and i32 %2, -4542 %broadcast.splatinsert86 = insertelement <4 x i32> undef, i32 %m, i32 0543 %broadcast.splat87 = shufflevector <4 x i32> %broadcast.splatinsert86, <4 x i32> undef, <4 x i32> zeroinitializer544 %cmp.n = icmp eq i32 %2, %n.vec545 br label %for.cond8.preheader.us.us.preheader546 547for.cond8.preheader.us.us.preheader: ; preds = %for.cond8.preheader.us.us.preheader.preheader, %for.cond4.for.cond.cleanup6_crit_edge.us548 %i.054.us = phi i32 [ %inc24.us, %for.cond4.for.cond.cleanup6_crit_edge.us ], [ 0, %for.cond8.preheader.us.us.preheader.preheader ]549 %mul.us = mul i32 %i.054.us, %l550 %mul18.us = mul i32 %i.054.us, %m551 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %mul.us, i32 0552 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer553 br label %vector.ph554 555for.cond4.for.cond.cleanup6_crit_edge.us: ; preds = %for.cond8.for.cond.cleanup10_crit_edge.us.us556 %inc24.us = add nuw nsw i32 %i.054.us, 1557 %exitcond85 = icmp eq i32 %inc24.us, %n558 br i1 %exitcond85, label %for.end25, label %for.cond8.preheader.us.us.preheader559 560vector.ph: ; preds = %middle.block, %for.cond8.preheader.us.us.preheader561 %j.051.us.us = phi i32 [ %inc.us.us, %middle.block ], [ 0, %for.cond8.preheader.us.us.preheader ]562 %broadcast.splatinsert88 = insertelement <4 x i32> undef, i32 %j.051.us.us, i32 0563 %broadcast.splat89 = shufflevector <4 x i32> %broadcast.splatinsert88, <4 x i32> undef, <4 x i32> zeroinitializer564 br label %vector.body565 566vector.body: ; preds = %vector.body, %vector.ph567 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]568 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]569 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %9, %vector.body ]570 %3 = add <4 x i32> %vec.ind, %broadcast.splat571 %4 = getelementptr inbounds i32, ptr %A, <4 x i32> %3572 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %4, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), !tbaa !3573 %5 = mul <4 x i32> %vec.ind, %broadcast.splat87574 %6 = add <4 x i32> %5, %broadcast.splat89575 %7 = getelementptr inbounds i32, ptr %B, <4 x i32> %6576 %wide.masked.gather90 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %7, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), !tbaa !3577 %8 = mul nsw <4 x i32> %wide.masked.gather90, %wide.masked.gather578 %9 = add <4 x i32> %8, %vec.phi579 %index.next = add i32 %index, 4580 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>581 %10 = icmp eq i32 %index.next, %n.vec582 br i1 %10, label %middle.block, label %vector.body, !llvm.loop !7583 584middle.block: ; preds = %vector.body585 %11 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %9)586;for.cond8.for.cond.cleanup10_crit_edge.us.us: ; preds = %for.body11.us.us, %middle.block587 %add19.us.us = add i32 %j.051.us.us, %mul18.us588 %arrayidx20.us.us = getelementptr inbounds i32, ptr %C, i32 %add19.us.us589 store i32 %11, ptr %arrayidx20.us.us, align 4, !tbaa !3590 %inc.us.us = add nuw nsw i32 %j.051.us.us, 1591 %exitcond = icmp eq i32 %inc.us.us, %m592 br i1 %exitcond, label %for.cond4.for.cond.cleanup6_crit_edge.us, label %vector.ph593 594for.end25: ; preds = %for.cond4.for.cond.cleanup6_crit_edge.us, %entry595 ret void596}597 598define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %n, i32 %m, i32 %l) local_unnamed_addr #0 {599; CHECK-LABEL: arm_mat_mult_q15:600; CHECK: @ %bb.0: @ %entry601; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}602; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}603; CHECK-NEXT: .pad #4604; CHECK-NEXT: sub sp, #4605; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}606; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}607; CHECK-NEXT: .pad #32608; CHECK-NEXT: sub sp, #32609; CHECK-NEXT: strd r0, r2, [sp, #24] @ 8-byte Folded Spill610; CHECK-NEXT: cmp r3, #0611; CHECK-NEXT: str r3, [sp, #8] @ 4-byte Spill612; CHECK-NEXT: itt ne613; CHECK-NEXT: ldrne r0, [sp, #136]614; CHECK-NEXT: cmpne r0, #0615; CHECK-NEXT: bne .LBB10_2616; CHECK-NEXT: .LBB10_1: @ %for.cond.cleanup617; CHECK-NEXT: add sp, #32618; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}619; CHECK-NEXT: add sp, #4620; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}621; CHECK-NEXT: .LBB10_2: @ %for.cond1.preheader.us.preheader622; CHECK-NEXT: ldr.w r12, [sp, #140]623; CHECK-NEXT: movs r7, #1624; CHECK-NEXT: mov.w r11, #0625; CHECK-NEXT: vmov.i32 q0, #0x0626; CHECK-NEXT: bic r2, r12, #3627; CHECK-NEXT: subs r3, r2, #4628; CHECK-NEXT: add.w r0, r7, r3, lsr #2629; CHECK-NEXT: ldr r7, [sp, #136]630; CHECK-NEXT: adr r3, .LCPI10_0631; CHECK-NEXT: str r0, [sp, #16] @ 4-byte Spill632; CHECK-NEXT: lsl.w r0, r12, #1633; CHECK-NEXT: vdup.32 q1, r7634; CHECK-NEXT: vldrw.u32 q2, [r3]635; CHECK-NEXT: str r0, [sp, #4] @ 4-byte Spill636; CHECK-NEXT: ldr r0, [sp, #24] @ 4-byte Reload637; CHECK-NEXT: lsls r6, r7, #1638; CHECK-NEXT: vshl.i32 q3, q1, #2639; CHECK-NEXT: movs r3, #0640; CHECK-NEXT: str r0, [sp, #20] @ 4-byte Spill641; CHECK-NEXT: b .LBB10_5642; CHECK-NEXT: .LBB10_3: @ %for.cond5.preheader.us73.preheader643; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1644; CHECK-NEXT: ldr r0, [sp, #28] @ 4-byte Reload645; CHECK-NEXT: add.w r3, r0, r5, lsl #1646; CHECK-NEXT: wlstp.8 lr, r6, .LBB10_4647; CHECK-NEXT: b .LBB10_15648; CHECK-NEXT: .LBB10_4: @ %for.cond1.for.cond.cleanup3_crit_edge.us649; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1650; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload651; CHECK-NEXT: add r11, r12652; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload653; CHECK-NEXT: add r3, r0654; CHECK-NEXT: str r3, [sp, #20] @ 4-byte Spill655; CHECK-NEXT: ldr r3, [sp, #12] @ 4-byte Reload656; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload657; CHECK-NEXT: adds r3, #1658; CHECK-NEXT: cmp r3, r0659; CHECK-NEXT: beq .LBB10_1660; CHECK-NEXT: .LBB10_5: @ %for.cond1.preheader.us661; CHECK-NEXT: @ =>This Loop Header: Depth=1662; CHECK-NEXT: @ Child Loop BB10_8 Depth 2663; CHECK-NEXT: @ Child Loop BB10_11 Depth 3664; CHECK-NEXT: @ Child Loop BB10_14 Depth 3665; CHECK-NEXT: @ Child Loop BB10_15 Depth 2666; CHECK-NEXT: mul r5, r3, r7667; CHECK-NEXT: cmp.w r12, #0668; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill669; CHECK-NEXT: beq .LBB10_3670; CHECK-NEXT: @ %bb.6: @ %for.cond5.preheader.us.us.preheader671; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1672; CHECK-NEXT: mov.w r8, #0673; CHECK-NEXT: b .LBB10_8674; CHECK-NEXT: .LBB10_7: @ %for.cond5.for.cond.cleanup7_crit_edge.us.us675; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2676; CHECK-NEXT: ldr r3, [sp, #28] @ 4-byte Reload677; CHECK-NEXT: add.w r0, r8, r5678; CHECK-NEXT: add.w r8, r8, #1679; CHECK-NEXT: cmp r8, r7680; CHECK-NEXT: strh.w r10, [r3, r0, lsl #1]681; CHECK-NEXT: beq .LBB10_4682; CHECK-NEXT: .LBB10_8: @ %for.cond5.preheader.us.us683; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1684; CHECK-NEXT: @ => This Loop Header: Depth=2685; CHECK-NEXT: @ Child Loop BB10_11 Depth 3686; CHECK-NEXT: @ Child Loop BB10_14 Depth 3687; CHECK-NEXT: cmp.w r12, #3688; CHECK-NEXT: bhi .LBB10_10689; CHECK-NEXT: @ %bb.9: @ in Loop: Header=BB10_8 Depth=2690; CHECK-NEXT: movs r4, #0691; CHECK-NEXT: mov.w r10, #0692; CHECK-NEXT: b .LBB10_13693; CHECK-NEXT: .LBB10_10: @ %vector.ph694; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2695; CHECK-NEXT: ldr r0, [sp, #16] @ 4-byte Reload696; CHECK-NEXT: vmov q5, q1697; CHECK-NEXT: vmov.i32 q4, #0x0698; CHECK-NEXT: vmlas.i32 q5, q2, r8699; CHECK-NEXT: dls lr, r0700; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload701; CHECK-NEXT: .LBB10_11: @ %vector.body702; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1703; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2704; CHECK-NEXT: @ => This Inner Loop Header: Depth=3705; CHECK-NEXT: vadd.i32 q6, q5, q3706; CHECK-NEXT: vldrh.s32 q7, [r1, q5, uxtw #1]707; CHECK-NEXT: vldrh.s32 q5, [r3], #8708; CHECK-NEXT: vmul.i32 q5, q7, q5709; CHECK-NEXT: vadd.i32 q4, q5, q4710; CHECK-NEXT: vmov q5, q6711; CHECK-NEXT: le lr, .LBB10_11712; CHECK-NEXT: @ %bb.12: @ %middle.block713; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2714; CHECK-NEXT: vaddv.u32 r10, q4715; CHECK-NEXT: mov r4, r2716; CHECK-NEXT: cmp r2, r12717; CHECK-NEXT: beq .LBB10_7718; CHECK-NEXT: .LBB10_13: @ %for.body8.us.us.preheader719; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2720; CHECK-NEXT: mla r3, r7, r4, r8721; CHECK-NEXT: add.w r0, r11, r4722; CHECK-NEXT: ldr r7, [sp, #24] @ 4-byte Reload723; CHECK-NEXT: sub.w lr, r12, r4724; CHECK-NEXT: add.w r9, r7, r0, lsl #1725; CHECK-NEXT: ldr r7, [sp, #136]726; CHECK-NEXT: add.w r3, r1, r3, lsl #1727; CHECK-NEXT: .LBB10_14: @ %for.body8.us.us728; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1729; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2730; CHECK-NEXT: @ => This Inner Loop Header: Depth=3731; CHECK-NEXT: ldrsh.w r4, [r3]732; CHECK-NEXT: add r3, r6733; CHECK-NEXT: ldrsh r0, [r9], #2734; CHECK-NEXT: smlabb r10, r4, r0, r10735; CHECK-NEXT: le lr, .LBB10_14736; CHECK-NEXT: b .LBB10_7737; CHECK-NEXT: .LBB10_15: @ Parent Loop BB10_5 Depth=1738; CHECK-NEXT: @ => This Inner Loop Header: Depth=2739; CHECK-NEXT: vstrb.8 q0, [r3], #16740; CHECK-NEXT: letp lr, .LBB10_15741; CHECK-NEXT: b .LBB10_4742; CHECK-NEXT: .p2align 4743; CHECK-NEXT: @ %bb.16:744; CHECK-NEXT: .LCPI10_0:745; CHECK-NEXT: .long 0 @ 0x0746; CHECK-NEXT: .long 1 @ 0x1747; CHECK-NEXT: .long 2 @ 0x2748; CHECK-NEXT: .long 3 @ 0x3749entry:750 %cmp48 = icmp eq i32 %n, 0751 br i1 %cmp48, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph752 753for.cond1.preheader.lr.ph: ; preds = %entry754 %cmp245 = icmp eq i32 %m, 0755 %cmp642 = icmp eq i32 %l, 0756 br i1 %cmp245, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader757 758for.cond1.preheader.us.preheader: ; preds = %for.cond1.preheader.lr.ph759 %0 = shl nuw i32 %m, 1760 %min.iters.check = icmp ult i32 %l, 4761 %n.vec = and i32 %l, -4762 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %m, i32 0763 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer764 %cmp.n = icmp eq i32 %n.vec, %l765 br label %for.cond1.preheader.us766 767for.cond1.preheader.us: ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader768 %i.049.us = phi i32 [ %inc23.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]769 %1 = mul i32 %i.049.us, %m770 %mul.us = mul i32 %i.049.us, %l771 br i1 %cmp642, label %for.cond5.preheader.us73.preheader, label %for.cond5.preheader.us.us772 773for.cond5.preheader.us73.preheader: ; preds = %for.cond1.preheader.us774 %scevgep = getelementptr i16, ptr %C, i32 %1775 %scevgep82 = bitcast ptr %scevgep to ptr776 call void @llvm.memset.p0.i32(ptr align 2 %scevgep82, i8 0, i32 %0, i1 false)777 br label %for.cond1.for.cond.cleanup3_crit_edge.us778 779for.cond1.for.cond.cleanup3_crit_edge.us: ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us.us, %for.cond5.preheader.us73.preheader780 %inc23.us = add nuw nsw i32 %i.049.us, 1781 %exitcond84 = icmp eq i32 %inc23.us, %n782 br i1 %exitcond84, label %for.cond.cleanup, label %for.cond1.preheader.us783 784for.cond5.preheader.us.us: ; preds = %for.cond1.preheader.us, %for.cond5.for.cond.cleanup7_crit_edge.us.us785 %j.046.us.us = phi i32 [ %inc20.us.us, %for.cond5.for.cond.cleanup7_crit_edge.us.us ], [ 0, %for.cond1.preheader.us ]786 br i1 %min.iters.check, label %for.body8.us.us.preheader, label %vector.ph787 788for.body8.us.us.preheader: ; preds = %middle.block, %for.cond5.preheader.us.us789 %k.044.us.us.ph = phi i32 [ 0, %for.cond5.preheader.us.us ], [ %n.vec, %middle.block ]790 %sum.043.us.us.ph = phi i32 [ 0, %for.cond5.preheader.us.us ], [ %13, %middle.block ]791 br label %for.body8.us.us792 793vector.ph: ; preds = %for.cond5.preheader.us.us794 %broadcast.splatinsert85 = insertelement <4 x i32> undef, i32 %j.046.us.us, i32 0795 %broadcast.splat86 = shufflevector <4 x i32> %broadcast.splatinsert85, <4 x i32> undef, <4 x i32> zeroinitializer796 br label %vector.body797 798vector.body: ; preds = %vector.body, %vector.ph799 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]800 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ]801 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %11, %vector.body ]802 %2 = add i32 %index, %mul.us803 %3 = getelementptr inbounds i16, ptr %A, i32 %2804 %4 = bitcast ptr %3 to ptr805 %wide.load = load <4 x i16>, ptr %4, align 2, !tbaa !3806 %5 = sext <4 x i16> %wide.load to <4 x i32>807 %6 = mul <4 x i32> %vec.ind, %broadcast.splat808 %7 = add <4 x i32> %6, %broadcast.splat86809 %8 = getelementptr inbounds i16, ptr %B, <4 x i32> %7810 %wide.masked.gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> %8, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef), !tbaa !3811 %9 = sext <4 x i16> %wide.masked.gather to <4 x i32>812 %10 = mul nsw <4 x i32> %9, %5813 %11 = add <4 x i32> %10, %vec.phi814 %index.next = add i32 %index, 4815 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>816 %12 = icmp eq i32 %index.next, %n.vec817 br i1 %12, label %middle.block, label %vector.body, !llvm.loop !7818 819middle.block: ; preds = %vector.body820 %13 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %11)821 br i1 %cmp.n, label %for.cond5.for.cond.cleanup7_crit_edge.us.us, label %for.body8.us.us.preheader822 823for.cond5.for.cond.cleanup7_crit_edge.us.us: ; preds = %for.body8.us.us, %middle.block824 %add14.us.us.lcssa = phi i32 [ %13, %middle.block ], [ %add14.us.us, %for.body8.us.us ]825 %conv15.us.us = trunc i32 %add14.us.us.lcssa to i16826 %add17.us.us = add i32 %j.046.us.us, %1827 %arrayidx18.us.us = getelementptr inbounds i16, ptr %C, i32 %add17.us.us828 store i16 %conv15.us.us, ptr %arrayidx18.us.us, align 2, !tbaa !3829 %inc20.us.us = add nuw nsw i32 %j.046.us.us, 1830 %exitcond83 = icmp eq i32 %inc20.us.us, %m831 br i1 %exitcond83, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.cond5.preheader.us.us832 833for.body8.us.us: ; preds = %for.body8.us.us.preheader, %for.body8.us.us834 %k.044.us.us = phi i32 [ %inc.us.us, %for.body8.us.us ], [ %k.044.us.us.ph, %for.body8.us.us.preheader ]835 %sum.043.us.us = phi i32 [ %add14.us.us, %for.body8.us.us ], [ %sum.043.us.us.ph, %for.body8.us.us.preheader ]836 %add.us.us = add i32 %k.044.us.us, %mul.us837 %arrayidx.us.us = getelementptr inbounds i16, ptr %A, i32 %add.us.us838 %14 = load i16, ptr %arrayidx.us.us, align 2, !tbaa !3839 %conv.us.us = sext i16 %14 to i32840 %mul9.us.us = mul i32 %k.044.us.us, %m841 %add10.us.us = add i32 %mul9.us.us, %j.046.us.us842 %arrayidx11.us.us = getelementptr inbounds i16, ptr %B, i32 %add10.us.us843 %15 = load i16, ptr %arrayidx11.us.us, align 2, !tbaa !3844 %conv12.us.us = sext i16 %15 to i32845 %mul13.us.us = mul nsw i32 %conv12.us.us, %conv.us.us846 %add14.us.us = add nsw i32 %mul13.us.us, %sum.043.us.us847 %inc.us.us = add nuw nsw i32 %k.044.us.us, 1848 %exitcond = icmp eq i32 %inc.us.us, %l849 br i1 %exitcond, label %for.cond5.for.cond.cleanup7_crit_edge.us.us, label %for.body8.us.us, !llvm.loop !9850 851for.cond.cleanup: ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.lr.ph, %entry852 ret void853}854 855define hidden arm_aapcs_vfpcc i32 @arm_depthwise_conv_s8(ptr nocapture readonly %input, i16 zeroext %input_x, i16 zeroext %input_y, i16 zeroext %input_ch, ptr nocapture readonly %kernel, i16 zeroext %output_ch, i16 zeroext %ch_mult, i16 zeroext %kernel_x, i16 zeroext %kernel_y, i16 zeroext %pad_x, i16 zeroext %pad_y, i16 zeroext %stride_x, i16 zeroext %stride_y, ptr nocapture readonly %bias, ptr nocapture %output, ptr nocapture readonly %output_shift, ptr nocapture readonly %output_mult, i16 zeroext %output_x, i16 zeroext %output_y, i32 %output_offset, i32 %input_offset, i32 %output_activation_min, i32 %output_activation_max, i16 zeroext %dilation_x, i16 zeroext %dilation_y, ptr nocapture readnone %buffer_a) local_unnamed_addr #0 {856; CHECK-LABEL: arm_depthwise_conv_s8:857; CHECK: @ %bb.0: @ %entry858; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}859; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}860; CHECK-NEXT: .pad #4861; CHECK-NEXT: sub sp, #4862; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}863; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}864; CHECK-NEXT: .pad #24865; CHECK-NEXT: sub sp, #24866; CHECK-NEXT: ldrd r2, r7, [sp, #136]867; CHECK-NEXT: add.w r8, r7, #10868; CHECK-NEXT: adr r7, .LCPI11_0869; CHECK-NEXT: ldr r1, [sp, #128]870; CHECK-NEXT: vdup.32 q0, r2871; CHECK-NEXT: vldrw.u32 q1, [r7]872; CHECK-NEXT: movs r4, #0873; CHECK-NEXT: mov.w r10, #6874; CHECK-NEXT: movs r6, #11875; CHECK-NEXT: vshl.i32 q0, q0, #2876; CHECK-NEXT: movs r5, #0877; CHECK-NEXT: .LBB11_1: @ %for.body10.i878; CHECK-NEXT: @ =>This Loop Header: Depth=1879; CHECK-NEXT: @ Child Loop BB11_2 Depth 2880; CHECK-NEXT: @ Child Loop BB11_3 Depth 3881; CHECK-NEXT: @ Child Loop BB11_4 Depth 4882; CHECK-NEXT: @ Child Loop BB11_5 Depth 5883; CHECK-NEXT: mov.w r9, #0884; CHECK-NEXT: str r5, [sp, #4] @ 4-byte Spill885; CHECK-NEXT: .LBB11_2: @ %for.cond22.preheader.i886; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1887; CHECK-NEXT: @ => This Loop Header: Depth=2888; CHECK-NEXT: @ Child Loop BB11_3 Depth 3889; CHECK-NEXT: @ Child Loop BB11_4 Depth 4890; CHECK-NEXT: @ Child Loop BB11_5 Depth 5891; CHECK-NEXT: movs r7, #0892; CHECK-NEXT: vdup.32 q2, r9893; CHECK-NEXT: vstrw.32 q2, [sp, #8] @ 16-byte Spill894; CHECK-NEXT: .LBB11_3: @ %for.body27.i895; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1896; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2897; CHECK-NEXT: @ => This Loop Header: Depth=3898; CHECK-NEXT: @ Child Loop BB11_4 Depth 4899; CHECK-NEXT: @ Child Loop BB11_5 Depth 5900; CHECK-NEXT: dls lr, r10901; CHECK-NEXT: mov.w r12, #0902; CHECK-NEXT: mov.w r11, #4903; CHECK-NEXT: vdup.32 q3, r7904; CHECK-NEXT: .LBB11_4: @ %for.body78.us.i905; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1906; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2907; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3908; CHECK-NEXT: @ => This Loop Header: Depth=4909; CHECK-NEXT: @ Child Loop BB11_5 Depth 5910; CHECK-NEXT: mul r5, r11, r6911; CHECK-NEXT: vmov q4, q3912; CHECK-NEXT: vadd.i32 q5, q1, r5913; CHECK-NEXT: vmla.i32 q4, q5, r2914; CHECK-NEXT: vldrw.u32 q5, [sp, #8] @ 16-byte Reload915; CHECK-NEXT: adds r5, #113916; CHECK-NEXT: vadd.i32 q6, q1, r5917; CHECK-NEXT: mov r5, r8918; CHECK-NEXT: vmla.i32 q5, q6, r2919; CHECK-NEXT: .LBB11_5: @ %vector.body920; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1921; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2922; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3923; CHECK-NEXT: @ Parent Loop BB11_4 Depth=4924; CHECK-NEXT: @ => This Inner Loop Header: Depth=5925; CHECK-NEXT: vldrb.s32 q2, [r0, q5]926; CHECK-NEXT: vadd.i32 q7, q5, q0927; CHECK-NEXT: vldrb.s32 q5, [r1, q4]928; CHECK-NEXT: vadd.i32 q6, q4, q0929; CHECK-NEXT: vadd.i32 q2, q2, r2930; CHECK-NEXT: subs r5, #4931; CHECK-NEXT: vmlava.u32 r12, q2, q5932; CHECK-NEXT: vmov q5, q7933; CHECK-NEXT: vmov q4, q6934; CHECK-NEXT: bne .LBB11_5935; CHECK-NEXT: @ %bb.6: @ %middle.block936; CHECK-NEXT: @ in Loop: Header=BB11_4 Depth=4937; CHECK-NEXT: add.w r11, r11, #1938; CHECK-NEXT: le lr, .LBB11_4939; CHECK-NEXT: @ %bb.7: @ %for.cond.cleanup77.i940; CHECK-NEXT: @ in Loop: Header=BB11_3 Depth=3941; CHECK-NEXT: adds r7, #1942; CHECK-NEXT: adds r4, #1943; CHECK-NEXT: cmp r7, r2944; CHECK-NEXT: bne .LBB11_3945; CHECK-NEXT: @ %bb.8: @ %for.cond.cleanup26.i946; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=2947; CHECK-NEXT: add.w r9, r9, #1948; CHECK-NEXT: cmp r9, r3949; CHECK-NEXT: bne .LBB11_2950; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup20.i951; CHECK-NEXT: @ in Loop: Header=BB11_1 Depth=1952; CHECK-NEXT: ldr r5, [sp, #4] @ 4-byte Reload953; CHECK-NEXT: ldr r7, [sp, #180]954; CHECK-NEXT: adds r5, #1955; CHECK-NEXT: cmp r5, r7956; CHECK-NEXT: it eq957; CHECK-NEXT: moveq r5, #0958; CHECK-NEXT: b .LBB11_1959; CHECK-NEXT: .p2align 4960; CHECK-NEXT: @ %bb.10:961; CHECK-NEXT: .LCPI11_0:962; CHECK-NEXT: .long 0 @ 0x0963; CHECK-NEXT: .long 1 @ 0x1964; CHECK-NEXT: .long 2 @ 0x2965; CHECK-NEXT: .long 3 @ 0x3966entry:967 %conv = zext i16 %ch_mult to i32968 %conv6.i = zext i16 %output_x to i32969 %conv17.i = zext i16 %input_ch to i32970 %conv60.i = zext i16 %kernel_x to i32971 %broadcast.splatinsert63 = insertelement <4 x i32> undef, i32 %conv, i32 0972 %broadcast.splat64 = shufflevector <4 x i32> %broadcast.splatinsert63, <4 x i32> undef, <4 x i32> zeroinitializer973 %broadcast.splatinsert69 = insertelement <4 x i32> undef, i32 %conv, i32 0974 %broadcast.splat70 = shufflevector <4 x i32> %broadcast.splatinsert69, <4 x i32> undef, <4 x i32> zeroinitializer975 %broadcast.splatinsert73 = insertelement <4 x i32> undef, i32 %conv, i32 0976 %broadcast.splat74 = shufflevector <4 x i32> %broadcast.splatinsert73, <4 x i32> undef, <4 x i32> zeroinitializer977 %unroll_iter = and i32 %conv, 65534978 br label %for.body.i38979 980for.body.i38: ; preds = %for.cond.cleanup9.i, %entry981 %i_out.024.i = phi i32 [ 0, %entry ], [ %i_out.1.lcssa.i, %for.cond.cleanup9.i ]982 %i_out_y.023.i = phi i32 [ 0, %entry ], [ %inc140.i, %for.cond.cleanup9.i ]983 br label %for.body10.i984 985for.cond.cleanup9.i: ; preds = %for.cond.cleanup20.i, %for.body.i38986 %i_out.1.lcssa.i = phi i32 [ %i_out.2.lcssa.i, %for.cond.cleanup20.i ]987 %inc140.i = add nuw nsw i32 %i_out_y.023.i, 1988 br i1 0, label %if.end, label %for.body.i38989 990for.body10.i: ; preds = %for.cond.cleanup20.i, %for.body.i38991 %i_out.120.i = phi i32 [ %i_out.024.i, %for.body.i38 ], [ %i_out.2.lcssa.i, %for.cond.cleanup20.i ]992 %i_out_x.019.i = phi i32 [ 0, %for.body.i38 ], [ %inc137.i, %for.cond.cleanup20.i ]993 %n.vec = add nsw i32 %conv60.i, 10994 br i1 0, label %for.cond.cleanup20.i, label %for.cond22.preheader.lr.ph.i995 996for.cond22.preheader.lr.ph.i: ; preds = %for.body10.i997 %.splatinsert = insertelement <4 x i32> undef, i32 0, i32 0998 %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer999 %induction = add <4 x i32> %.splat, <i32 0, i32 1, i32 2, i32 3>1000 %cmp.n = icmp eq i32 10, %n.vec1001 br label %for.cond22.preheader.i1002 1003for.cond22.preheader.i: ; preds = %for.cond.cleanup26.i, %for.cond22.preheader.lr.ph.i1004 %i_out.216.i = phi i32 [ %i_out.120.i, %for.cond22.preheader.lr.ph.i ], [ %i_out.3.lcssa.i, %for.cond.cleanup26.i ]1005 %i_input_ch.014.i = phi i32 [ 0, %for.cond22.preheader.lr.ph.i ], [ %inc134.i, %for.cond.cleanup26.i ]1006 br i1 0, label %for.cond.cleanup26.i, label %for.body27.lr.ph.i1007 1008for.body27.lr.ph.i: ; preds = %for.cond22.preheader.i1009 br i1 0, label %for.body27.i.us.preheader, label %for.body27.i.preheader1010 1011for.body27.i.preheader: ; preds = %for.body27.lr.ph.i1012 %broadcast.splatinsert65 = insertelement <4 x i32> undef, i32 %i_input_ch.014.i, i32 01013 %broadcast.splat66 = shufflevector <4 x i32> %broadcast.splatinsert65, <4 x i32> undef, <4 x i32> zeroinitializer1014 br label %for.body27.i1015 1016for.body27.i.us.preheader: ; preds = %for.body27.lr.ph.i1017 br i1 0, label %for.cond.cleanup26.i.loopexit.unr-lcssa, label %for.body27.i.us1018 1019for.body27.i.us: ; preds = %for.body27.i.us, %for.body27.i.us.preheader1020 %i_out.311.i.us = phi i32 [ %inc128.i.us.1, %for.body27.i.us ], [ %i_out.216.i, %for.body27.i.us.preheader ]1021 %i_ch_mult.010.i.us = phi i32 [ %inc131.i.us.1, %for.body27.i.us ], [ 0, %for.body27.i.us.preheader ]1022 %niter = phi i32 [ 0, %for.body27.i.us ], [ %unroll_iter, %for.body27.i.us.preheader ]1023 %inc128.i.us.1 = add nsw i32 %i_out.311.i.us, 21024 %inc131.i.us.1 = add nuw nsw i32 %i_ch_mult.010.i.us, 21025 br i1 0, label %for.cond.cleanup26.i.loopexit.unr-lcssa, label %for.body27.i.us1026 1027for.cond.cleanup20.i: ; preds = %for.cond.cleanup26.i, %for.body10.i1028 %i_out.2.lcssa.i = phi i32 [ %i_out.120.i, %for.body10.i ], [ %i_out.3.lcssa.i, %for.cond.cleanup26.i ]1029 %inc137.i = add nuw nsw i32 %i_out_x.019.i, 11030 %exitcond27.i = icmp eq i32 %inc137.i, %conv6.i1031 br i1 %exitcond27.i, label %for.cond.cleanup9.i, label %for.body10.i1032 1033for.cond.cleanup26.i.loopexit.unr-lcssa: ; preds = %for.body27.i.us, %for.body27.i.us.preheader1034 %inc128.i.us.lcssa.ph = phi i32 [ undef, %for.body27.i.us.preheader ], [ %inc128.i.us.1, %for.body27.i.us ]1035 br label %for.cond.cleanup26.i1036 1037for.cond.cleanup26.i: ; preds = %for.cond.cleanup77.i, %for.cond.cleanup26.i.loopexit.unr-lcssa, %for.cond22.preheader.i1038 %i_out.3.lcssa.i = phi i32 [ %i_out.216.i, %for.cond22.preheader.i ], [ %inc128.i.us.lcssa.ph, %for.cond.cleanup26.i.loopexit.unr-lcssa ], [ %inc128.i, %for.cond.cleanup77.i ]1039 %inc134.i = add nuw nsw i32 %i_input_ch.014.i, 11040 %exitcond26.i = icmp eq i32 %inc134.i, %conv17.i1041 br i1 %exitcond26.i, label %for.cond.cleanup20.i, label %for.cond22.preheader.i1042 1043for.body27.i: ; preds = %for.cond.cleanup77.i, %for.body27.i.preheader1044 %i_out.311.i = phi i32 [ %inc128.i, %for.cond.cleanup77.i ], [ %i_out.216.i, %for.body27.i.preheader ]1045 %i_ch_mult.010.i = phi i32 [ %inc131.i, %for.cond.cleanup77.i ], [ 0, %for.body27.i.preheader ]1046 %broadcast.splatinsert71 = insertelement <4 x i32> undef, i32 %i_ch_mult.010.i, i32 01047 %broadcast.splat72 = shufflevector <4 x i32> %broadcast.splatinsert71, <4 x i32> undef, <4 x i32> zeroinitializer1048 br label %for.body78.us.i1049 1050for.body78.us.i: ; preds = %middle.block, %for.body27.i1051 %i_ker_y.06.us.i = phi i32 [ %inc110.us.i, %middle.block ], [ 4, %for.body27.i ]1052 %acc_0.05.us.i = phi i32 [ %tmp89, %middle.block ], [ 0, %for.body27.i ]1053 %add80.us.i43 = add nsw i32 %i_ker_y.06.us.i, 101054 %mul89.us.i = mul nsw i32 %add80.us.i43, 111055 %add87.us.i44 = add i32 %mul89.us.i, 31056 %mul95.us.i = mul nsw i32 %i_ker_y.06.us.i, 111057 br label %vector.ph1058 1059vector.ph: ; preds = %for.body78.us.i1060 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %add87.us.i44, i32 01061 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer1062 %broadcast.splatinsert67 = insertelement <4 x i32> undef, i32 %mul95.us.i, i32 01063 %broadcast.splat68 = shufflevector <4 x i32> %broadcast.splatinsert67, <4 x i32> undef, <4 x i32> zeroinitializer1064 br label %vector.body1065 1066vector.body: ; preds = %vector.body, %vector.ph1067 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]1068 %vec.ind = phi <4 x i32> [ %induction, %vector.ph ], [ %vec.ind.next, %vector.body ]1069 %vec.phi = phi i32 [ %acc_0.05.us.i, %vector.ph ], [ %tmp89, %vector.body ]1070 %tmp76 = add <4 x i32> %broadcast.splat, %vec.ind1071 %tmp77 = mul nsw <4 x i32> %tmp76, %broadcast.splat641072 %tmp78 = add nsw <4 x i32> %tmp77, %broadcast.splat661073 %tmp79 = add nsw <4 x i32> %vec.ind, %broadcast.splat681074 %tmp80 = mul nsw <4 x i32> %broadcast.splat70, %tmp791075 %tmp81 = add nsw <4 x i32> %tmp80, %broadcast.splat721076 %tmp82 = getelementptr inbounds i8, ptr %input, <4 x i32> %tmp781077 %wide.masked.gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %tmp82, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)1078 %tmp83 = sext <4 x i8> %wide.masked.gather to <4 x i32>1079 %tmp84 = add nsw <4 x i32> %broadcast.splat74, %tmp831080 %tmp85 = getelementptr inbounds i8, ptr %kernel, <4 x i32> %tmp811081 %wide.masked.gather75 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %tmp85, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)1082 %tmp86 = sext <4 x i8> %wide.masked.gather75 to <4 x i32>1083 %tmp87 = mul nsw <4 x i32> %tmp84, %tmp861084 %tmp88 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %tmp87)1085 %tmp89 = add i32 %tmp88, %vec.phi1086 %index.next = add i32 %index, 41087 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>1088 %tmp90 = icmp eq i32 %index.next, %n.vec1089 br i1 %tmp90, label %middle.block, label %vector.body1090 1091middle.block: ; preds = %vector.body1092 %inc110.us.i = add nsw i32 %i_ker_y.06.us.i, 11093 %cmp75.us.i = icmp slt i32 %inc110.us.i, 101094 br i1 %cmp75.us.i, label %for.body78.us.i, label %for.cond.cleanup77.i1095 1096for.cond.cleanup77.i: ; preds = %middle.block1097 %inc128.i = add nsw i32 %i_out.311.i, 11098 %inc131.i = add nuw nsw i32 %i_ch_mult.010.i, 11099 %exitcond.i50 = icmp eq i32 %inc131.i, %conv1100 br i1 %exitcond.i50, label %for.cond.cleanup26.i, label %for.body27.i1101 1102if.end: ; preds = %for.cond.cleanup9.i, %entry, %for.cond.cleanup7.i, %if.then1103 ret i32 01104}1105 1106declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)1107declare <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i16>)1108declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i8>) #31109 1110declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)1111declare void @llvm.memset.p0.i32(ptr align 2, i8, i32, i1)1112 1113declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32, <4 x i1>)1114