440 lines · plain
1; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT2; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER3; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER4; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-ENABLE-TP5 6target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"7target triple = "thumbv8.1m.main-arm-unknown-eabihf"8 9; This IR corresponds to this type of C-code:10;11; void f(char *a, char *b, char *c, int N) {12; while (N-- > 0)13; *c++ = *a++ + *b++;14; }15;16define dso_local void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {17; COMMON-LABEL: @sgt_loopguard(18; COMMON: vector.body:19 20; CHECK-TF: %[[VIVELEM0:.*]] = extractelement <16 x i32> %vec.iv, i32 021; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[VIVELEM0]], i32 %N)22; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask23; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask24; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)25entry:26 %cmp5 = icmp sgt i32 %N, 027 br i1 %cmp5, label %while.body.preheader, label %while.end28 29while.body.preheader:30 br label %while.body31 32while.body:33 %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]34 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]35 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]36 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]37 %dec = add nsw i32 %N.addr.09, -138 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 139 %0 = load i8, ptr %a.addr.06, align 140 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 141 %1 = load i8, ptr %b.addr.07, align 142 %add = add i8 %1, %043 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 144 store i8 %add, ptr %c.addr.08, align 145 %cmp = icmp sgt i32 %N.addr.09, 146 br i1 %cmp, label %while.body, label %while.end.loopexit47 48while.end.loopexit:49 br label %while.end50 51while.end:52 ret void53}54 55; No loop-guard: we need one for this to be valid.56;57define dso_local void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {58; COMMON-LABEL: @sgt_no_loopguard(59; COMMON: vector.body:60; CHECK-TF: masked.load61; CHECK-TF: masked.load62; CHECK-TF: masked.store63entry:64 br label %while.body65 66while.body:67 %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]68 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]69 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]70 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]71 %dec = add nsw i32 %N.addr.09, -172 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 173 %0 = load i8, ptr %a.addr.06, align 174 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 175 %1 = load i8, ptr %b.addr.07, align 176 %add = add i8 %1, %077 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 178 store i8 %add, ptr %c.addr.08, align 179 %cmp = icmp sgt i32 %N.addr.09, 180 br i1 %cmp, label %while.body, label %while.end.loopexit81 82while.end.loopexit:83 br label %while.end84 85while.end:86 ret void87}88 89define dso_local void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {90; COMMON-LABEL: @sgt_extra_use_cmp(91; COMMON: vector.body:92; CHECK-TF: masked.load93; CHECK-TF: masked.load94; CHECK-TF: masked.store95entry:96 br label %while.body97 98while.body:99 %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ]100 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]101 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]102 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry ]103 %dec = add nsw i32 %N.addr.09, -1104 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1105 %0 = load i8, ptr %a.addr.06, align 1106 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1107 %1 = load i8, ptr %b.addr.07, align 1108 %add = add i8 %1, %0109 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1110 %cmp = icmp sgt i32 %N.addr.09, 1111 %select = select i1 %cmp, i8 %0, i8 %1112 %add2 = add i8 %add, %select113 store i8 %add2, ptr %c.addr.08, align 1114 br i1 %cmp, label %while.body, label %while.end.loopexit115 116while.end.loopexit:117 br label %while.end118 119while.end:120 ret void121}122 123define dso_local void @sgt_const_tripcount(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {124; COMMON-LABEL: @sgt_const_tripcount(125; COMMON: vector.body:126; CHECK-TF: masked.load127; CHECK-TF: masked.load128; CHECK-TF: masked.store129entry:130 %cmp5 = icmp sgt i32 %N, 0131 br i1 %cmp5, label %while.body.preheader, label %while.end132 133while.body.preheader:134 br label %while.body135 136while.body:137 %N.addr.09 = phi i32 [ %dec, %while.body ], [ 2049, %while.body.preheader ]138 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]139 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]140 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]141 %dec = add nsw i32 %N.addr.09, -1142 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1143 %0 = load i8, ptr %a.addr.06, align 1144 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1145 %1 = load i8, ptr %b.addr.07, align 1146 %add = add i8 %1, %0147 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1148 store i8 %add, ptr %c.addr.08, align 1149 %cmp = icmp sgt i32 %N.addr.09, 1150 br i1 %cmp, label %while.body, label %while.end.loopexit151 152while.end.loopexit:153 br label %while.end154 155while.end:156 ret void157}158 159define dso_local void @sgt_no_guard_0_startval(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {160; COMMON-LABEL: @sgt_no_guard_0_startval(161; COMMON-NOT: vector.body:162entry:163 br label %while.body164 165while.body:166 %N.addr.09 = phi i32 [ %dec, %while.body ], [ 0, %entry ]167 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %entry ]168 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %entry ]169 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %entry]170 %dec = add nsw i32 %N.addr.09, -1171 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1172 %0 = load i8, ptr %a.addr.06, align 1173 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1174 %1 = load i8, ptr %b.addr.07, align 1175 %add = add i8 %1, %0176 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1177 store i8 %add, ptr %c.addr.08, align 1178 %cmp = icmp sgt i32 %N.addr.09, 1179 br i1 %cmp, label %while.body, label %while.end.loopexit180 181while.end.loopexit:182 br label %while.end183 184while.end:185 ret void186}187 188define dso_local void @sgt_step_minus_two(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {189; COMMON-LABEL: @sgt_step_minus_two(190; COMMON: vector.body:191; CHECK-TF: masked.load192; CHECK-TF: masked.load193; CHECK-TF: masked.store194entry:195 %cmp5 = icmp sgt i32 %N, 0196 br i1 %cmp5, label %while.body.preheader, label %while.end197 198while.body.preheader:199 br label %while.body200 201while.body:202 %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]203 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]204 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]205 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]206 %dec = add nsw i32 %N.addr.09, -2207 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1208 %0 = load i8, ptr %a.addr.06, align 1209 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1210 %1 = load i8, ptr %b.addr.07, align 1211 %add = add i8 %1, %0212 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1213 store i8 %add, ptr %c.addr.08, align 1214 %cmp = icmp sgt i32 %N.addr.09, 1215 br i1 %cmp, label %while.body, label %while.end.loopexit216 217while.end.loopexit:218 br label %while.end219 220while.end:221 ret void222}223 224define dso_local void @sgt_step_not_constant(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N, i32 %S) local_unnamed_addr #0 {225; COMMON-LABEL: @sgt_step_not_constant(226; COMMON-NOT: vector.body:227entry:228 %cmp5 = icmp sgt i32 %N, 0229 br i1 %cmp5, label %while.body.preheader, label %while.end230 231while.body.preheader:232 br label %while.body233 234while.body:235 %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]236 %c.addr.08 = phi ptr [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ]237 %b.addr.07 = phi ptr [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ]238 %a.addr.06 = phi ptr [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ]239 %dec = add nsw i32 %N.addr.09, %S240 %incdec.ptr = getelementptr inbounds i8, ptr %a.addr.06, i32 1241 %0 = load i8, ptr %a.addr.06, align 1242 %incdec.ptr1 = getelementptr inbounds i8, ptr %b.addr.07, i32 1243 %1 = load i8, ptr %b.addr.07, align 1244 %add = add i8 %1, %0245 %incdec.ptr4 = getelementptr inbounds i8, ptr %c.addr.08, i32 1246 store i8 %add, ptr %c.addr.08, align 1247 %cmp = icmp sgt i32 %N.addr.09, 1248 br i1 %cmp, label %while.body, label %while.end.loopexit249 250while.end.loopexit:251 br label %while.end252 253while.end:254 ret void255}256 257define dso_local void @icmp_eq(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %N) #0 {258; COMMON-LABEL: @icmp_eq259; COMMON: vector.body:260entry:261 %cmp6 = icmp eq i32 %N, 0262 br i1 %cmp6, label %while.end, label %while.body.preheader263 264while.body.preheader:265 br label %while.body266 267while.body:268 %N.addr.010 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ]269 %C.addr.09 = phi ptr [ %incdec.ptr4, %while.body ], [ %C, %while.body.preheader ]270 %B.addr.08 = phi ptr [ %incdec.ptr1, %while.body ], [ %B, %while.body.preheader ]271 %A.addr.07 = phi ptr [ %incdec.ptr, %while.body ], [ %A, %while.body.preheader ]272 %incdec.ptr = getelementptr inbounds i8, ptr %A.addr.07, i32 1273 %0 = load i8, ptr %A.addr.07, align 1274 %incdec.ptr1 = getelementptr inbounds i8, ptr %B.addr.08, i32 1275 %1 = load i8, ptr %B.addr.08, align 1276 %add = add i8 %1, %0277 %incdec.ptr4 = getelementptr inbounds i8, ptr %C.addr.09, i32 1278 store i8 %add, ptr %C.addr.09, align 1279 %dec = add i32 %N.addr.010, -1280 %cmp = icmp eq i32 %dec, 0281 br i1 %cmp, label %while.end.loopexit, label %while.body282 283while.end.loopexit:284 br label %while.end285 286while.end:287 ret void288}289 290; This IR corresponds to this type of C-code:291;292; void f(char *a, char *b, char * __restrict c, int N) {293; #pragma clang loop vectorize_width(16)294; for (int i = N; i>0; i--)295; c[i] = a[i] + b[i];296; }297;298define dso_local void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {299; COMMON-LABEL: @sgt_for_loop(300; COMMON: vector.body:301; CHECK-PREFER: masked.load302; CHECK-PREFER: masked.load303; CHECK-PREFER: masked.store304;305; TODO: if tail-predication is requested, tail-folding isn't triggered because306; the profitability check returns "Different strides found, can't tail-predicate",307; investigate this.308;309; CHECK-ENABLE-TP-NOT: masked.load310; CHECK-ENABLE-TP-NOT: masked.load311; CHECK-ENABLE-TP-NOT: masked.store312;313entry:314 %cmp5 = icmp sgt i32 %N, 0315 br i1 %cmp5, label %for.body.preheader, label %for.end316 317for.body.preheader:318 br label %for.body319 320for.body:321 %i.011 = phi i32 [ %dec, %for.body ], [ %N, %for.body.preheader ]322 %arrayidx = getelementptr inbounds i8, ptr %a, i32 %i.011323 %0 = load i8, ptr %arrayidx, align 1324 %arrayidx1 = getelementptr inbounds i8, ptr %b, i32 %i.011325 %1 = load i8, ptr %arrayidx1, align 1326 %add = add i8 %1, %0327 %arrayidx4 = getelementptr inbounds i8, ptr %c, i32 %i.011328 store i8 %add, ptr %arrayidx4, align 1329 %dec = add nsw i32 %i.011, -1330 %cmp = icmp sgt i32 %i.011, 1331 br i1 %cmp, label %for.body, label %for.end, !llvm.loop !1332 333for.end:334 ret void335}336 337define dso_local void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {338; COMMON-LABEL: @sgt_for_loop_i64(339; COMMON: vector.body:340;341; CHECK-PREFER: masked.load342; CHECK-PREFER: masked.load343; CHECK-PREFER: masked.store344;345; With -disable-mve-tail-predication=false, the target hook returns346; "preferPredicateOverEpilogue: hardware-loop is not profitable."347; so here we don't expect the tail-folding. TODO: look into this.348;349; CHECK-ENABLE-TP-NOT: masked.load350; CHECK-ENABLE-TP-NOT: masked.load351; CHECK-ENABLE-TP-NOT: masked.store352;353entry:354 %cmp14 = icmp sgt i32 %N, 0355 br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup356 357for.body.preheader:358 %conv16 = zext i32 %N to i64359 br label %for.body360 361for.cond.cleanup.loopexit:362 br label %for.cond.cleanup363 364for.cond.cleanup:365 ret void366 367for.body:368 %i.015 = phi i64 [ %dec, %for.body ], [ %conv16, %for.body.preheader ]369 %idxprom = trunc i64 %i.015 to i32370 %arrayidx = getelementptr inbounds i8, ptr %a, i32 %idxprom371 %0 = load i8, ptr %arrayidx, align 1372 %arrayidx4 = getelementptr inbounds i8, ptr %b, i32 %idxprom373 %1 = load i8, ptr %arrayidx4, align 1374 %add = add i8 %1, %0375 %arrayidx8 = getelementptr inbounds i8, ptr %c, i32 %idxprom376 store i8 %add, ptr %arrayidx8, align 1377 %dec = add nsw i64 %i.015, -1378 %cmp = icmp sgt i64 %i.015, 1379 br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !1380}381 382; This IR corresponds to this nested-loop:383;384; for (int i = 0; i<N; i++)385; for (int j = i+1; j>0; j--)386; c[j] = a[j] + b[j];387;388; while the inner-loop looks similar to previous examples, we can't389; transform this because the inner loop because isGuarded returns390; false for the inner-loop.391;392define dso_local void @sgt_nested_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) local_unnamed_addr #0 {393; COMMON-LABEL: @sgt_nested_loop(394; DEFAULT-NOT: vector.body:395; CHECK-TF-NOT: masked.load396; CHECK-TF-NOT: masked.load397; CHECK-TF-NOT: masked.store398; COMMON: }399;400entry:401 %cmp21 = icmp sgt i32 %N, 0402 br i1 %cmp21, label %for.body.preheader, label %for.cond.cleanup403 404for.body.preheader:405 br label %for.body406 407for.cond.loopexit:408 %exitcond = icmp eq i32 %add, %N409 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body410 411for.cond.cleanup.loopexit:412 br label %for.cond.cleanup413 414for.cond.cleanup:415 ret void416 417for.body:418 %i.022 = phi i32 [ %add, %for.cond.loopexit ], [ 0, %for.body.preheader ]419 %add = add nuw nsw i32 %i.022, 1420 br label %for.body4421 422for.body4: ; preds = %for.body, %for.body4423 %j.020 = phi i32 [ %add, %for.body ], [ %dec, %for.body4 ]424 %arrayidx = getelementptr inbounds i8, ptr %a, i32 %j.020425 %0 = load i8, ptr %arrayidx, align 1426 %arrayidx5 = getelementptr inbounds i8, ptr %b, i32 %j.020427 %1 = load i8, ptr %arrayidx5, align 1428 %add7 = add i8 %1, %0429 %arrayidx9 = getelementptr inbounds i8, ptr %c, i32 %j.020430 store i8 %add7, ptr %arrayidx9, align 1431 %dec = add nsw i32 %j.020, -1432 %cmp2 = icmp sgt i32 %j.020, 1433 br i1 %cmp2, label %for.body4, label %for.cond.loopexit434}435 436attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }437 438!1 = distinct !{!1, !2}439!2 = !{!"llvm.loop.vectorize.width", i32 16}440