432 lines · plain
1; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf \2; RUN: -enable-arm-maskedgatscat=false \3; RUN: -tail-predication=enabled -passes=loop-vectorize -S < %s | \4; RUN: FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING5 6; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=-mve \7; RUN: -tail-predication=enabled -passes=loop-vectorize \8; RUN: -enable-arm-maskedgatscat=false \9; RUN: -enable-arm-maskedldst=true -S < %s | \10; RUN: FileCheck %s -check-prefixes=CHECK,NO-FOLDING11 12; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \13; RUN: -tail-predication=enabled -passes=loop-vectorize \14; RUN: -enable-arm-maskedgatscat=false \15; RUN: -enable-arm-maskedldst=false -S < %s | \16; RUN: FileCheck %s -check-prefixes=CHECK,NO-FOLDING17 18; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \19; RUN: -tail-predication=disabled -passes=loop-vectorize \20; RUN: -enable-arm-maskedgatscat=false \21; RUN: -enable-arm-maskedldst=true -S < %s | \22; RUN: FileCheck %s -check-prefixes=CHECK,NO-FOLDING23 24; Disabling the low-overhead branch extension will make25; 'isHardwareLoopProfitable' return false, so that we test avoiding folding for26; these cases.27; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve,-lob \28; RUN: -tail-predication=enabled -passes=loop-vectorize \29; RUN: -enable-arm-maskedgatscat=false \30; RUN: -enable-arm-maskedldst=true -S < %s | \31; RUN: FileCheck %s -check-prefixes=CHECK,NO-FOLDING32 33; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \34; RUN: -tail-predication=enabled -passes=loop-vectorize \35; RUN: -enable-arm-maskedgatscat=false \36; RUN: -enable-arm-maskedldst=true -S < %s | \37; RUN: FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING38 39; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \40; RUN: -prefer-predicate-over-epilogue=scalar-epilogue \41; RUN: -tail-predication=enabled -passes=loop-vectorize \42; RUN: -enable-arm-maskedgatscat=false \43; RUN: -enable-arm-maskedldst=true -S < %s | \44; RUN: FileCheck %s -check-prefixes=CHECK,NO-FOLDING45 46; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \47; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \48; RUN: -tail-predication=enabled -passes=loop-vectorize \49; RUN: -enable-arm-maskedgatscat=false \50; RUN: -enable-arm-maskedldst=true -S < %s | \51; RUN: FileCheck %s -check-prefixes=CHECK52 53target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"54 55define void @prefer_folding(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {56; CHECK-LABEL: prefer_folding(57; PREFER-FOLDING: vector.body:58; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]59; PREFER-FOLDING: %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 431)60; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %active.lane.mask,61; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0({{.*}}, <4 x i1> %active.lane.mask,62; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0({{.*}}, <4 x i1> %active.lane.mask63; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body64;65; NO-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0(66; NO-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0(67; NO-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %for.body68entry:69 br label %for.body70 71for.cond.cleanup:72 ret void73 74for.body:75 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]76 %arrayidx = getelementptr inbounds i32, ptr %B, i32 %i.0977 %0 = load i32, ptr %arrayidx, align 478 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.0979 %1 = load i32, ptr %arrayidx1, align 480 %add = add nsw i32 %1, %081 %arrayidx2 = getelementptr inbounds i32, ptr %A, i32 %i.0982 store i32 %add, ptr %arrayidx2, align 483 %add3 = add nuw nsw i32 %i.09, 184 %exitcond = icmp eq i32 %add3, 43185 br i1 %exitcond, label %for.cond.cleanup, label %for.body86}87 88define void @mixed_types(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, ptr noalias nocapture %D, ptr noalias nocapture readonly %E, ptr noalias nocapture readonly %F) #0 {89; CHECK-LABEL: mixed_types(90; PREFER-FOLDING: vector.body:91; PREFER-FOLDING: call <4 x i16> @llvm.masked.load.v4i16.p092; PREFER-FOLDING: call <4 x i16> @llvm.masked.load.v4i16.p093; PREFER-FOLDING: call void @llvm.masked.store.v4i16.p094; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p095; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p096; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p097; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body98entry:99 br label %for.body100 101for.cond.cleanup:102 ret void103 104for.body:105 %i.018 = phi i32 [ 0, %entry ], [ %add9, %for.body ]106 %arrayidx = getelementptr inbounds i16, ptr %B, i32 %i.018107 %0 = load i16, ptr %arrayidx, align 2108 %arrayidx1 = getelementptr inbounds i16, ptr %C, i32 %i.018109 %1 = load i16, ptr %arrayidx1, align 2110 %add = add i16 %1, %0111 %arrayidx4 = getelementptr inbounds i16, ptr %A, i32 %i.018112 store i16 %add, ptr %arrayidx4, align 2113 %arrayidx5 = getelementptr inbounds i32, ptr %E, i32 %i.018114 %2 = load i32, ptr %arrayidx5, align 4115 %arrayidx6 = getelementptr inbounds i32, ptr %F, i32 %i.018116 %3 = load i32, ptr %arrayidx6, align 4117 %add7 = add nsw i32 %3, %2118 %arrayidx8 = getelementptr inbounds i32, ptr %D, i32 %i.018119 store i32 %add7, ptr %arrayidx8, align 4120 %add9 = add nuw nsw i32 %i.018, 1121 %exitcond = icmp eq i32 %add9, 431122 br i1 %exitcond, label %for.cond.cleanup, label %for.body123}124 125define void @zero_extending_load_allowed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {126; CHECK-LABEL: zero_extending_load_allowed(127; PREFER-FOLDING: vector.body:128; PREFER-FOLDING: call <4 x i8> @llvm.masked.load.v4i8.p0129; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0130; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0131; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body132entry:133 br label %for.body134 135for.cond.cleanup:136 ret void137 138for.body:139 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]140 %arrayidx = getelementptr inbounds i8, ptr %B, i32 %i.09141 %0 = load i8, ptr %arrayidx, align 1142 %conv = zext i8 %0 to i32143 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.09144 %1 = load i32, ptr %arrayidx1, align 4145 %add = add nsw i32 %1, %conv146 %arrayidx2 = getelementptr inbounds i32, ptr %A, i32 %i.09147 store i32 %add, ptr %arrayidx2, align 4148 %add3 = add nuw nsw i32 %i.09, 1149 %exitcond = icmp eq i32 %add3, 431150 br i1 %exitcond, label %for.cond.cleanup, label %for.body151}152 153define void @sign_extending_load_allowed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {154; CHECK-LABEL: sign_extending_load_allowed(155; PREFER-FOLDING: vector.body:156; PREFER-FOLDING: call <4 x i8> @llvm.masked.load.v4i8.p0157; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0158; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0159; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body160entry:161 br label %for.body162 163for.cond.cleanup:164 ret void165 166for.body:167 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]168 %arrayidx = getelementptr inbounds i8, ptr %B, i32 %i.09169 %0 = load i8, ptr %arrayidx, align 1170 %conv = sext i8 %0 to i32171 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.09172 %1 = load i32, ptr %arrayidx1, align 4173 %add = add nsw i32 %1, %conv174 %arrayidx2 = getelementptr inbounds i32, ptr %A, i32 %i.09175 store i32 %add, ptr %arrayidx2, align 4176 %add3 = add nuw nsw i32 %i.09, 1177 %exitcond = icmp eq i32 %add3, 431178 br i1 %exitcond, label %for.cond.cleanup, label %for.body179}180 181define void @narrowing_store_allowed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {182; CHECK-LABEL: narrowing_store_allowed(183; PREFER-FOLDING: call void @llvm.masked.store.v4i8.p0184; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body185entry:186 br label %for.body187 188for.cond.cleanup:189 ret void190 191for.body:192 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]193 %arrayidx = getelementptr inbounds i32, ptr %B, i32 %i.09194 %0 = load i32, ptr %arrayidx, align 4195 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.09196 %1 = load i32, ptr %arrayidx1, align 4197 %add = add nsw i32 %1, %0198 %conv = trunc i32 %add to i8199 %arrayidx2 = getelementptr inbounds i8, ptr %A, i32 %i.09200 store i8 %conv, ptr %arrayidx2, align 1201 %add3 = add nuw nsw i32 %i.09, 1202 %exitcond = icmp eq i32 %add3, 431203 br i1 %exitcond, label %for.cond.cleanup, label %for.body204}205 206@tab = common global [32 x i8] zeroinitializer, align 1207 208define i32 @icmp_not_allowed() #0 {209; CHECK-LABEL: icmp_not_allowed(210; PREFER-FOLDING: vector.body:211; PREFER-FOLDING-NOT: llvm.masked.load212; PREFER-FOLDING-NOT: llvm.masked.store213; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body214entry:215 br label %for.body216 217for.body:218 %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]219 %arrayidx = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 %i.08220 %0 = load i8, ptr %arrayidx, align 1221 %cmp1 = icmp eq i8 %0, 0222 %. = select i1 %cmp1, i8 2, i8 1223 store i8 %., ptr %arrayidx, align 1224 %inc = add nsw i32 %i.08, 1225 %exitcond = icmp slt i32 %inc, 1000226 br i1 %exitcond, label %for.body, label %for.end227 228for.end:229 ret i32 0230}231 232@ftab = common global [32 x float] zeroinitializer, align 1233 234define float @fcmp_not_allowed() #0 {235; CHECK-LABEL: fcmp_not_allowed(236; PREFER-FOLDING: vector.body:237; PREFER-FOLDING-NOT: llvm.masked.load238; PREFER-FOLDING-NOT: llvm.masked.store239; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body240entry:241 br label %for.body242 243for.body:244 %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]245 %arrayidx = getelementptr inbounds [32 x float], ptr @ftab, i32 0, i32 %i.08246 %0 = load float, ptr %arrayidx, align 4247 %cmp1 = fcmp oeq float %0, 0.000000e+00248 %. = select i1 %cmp1, float 2.000000e+00, float 1.000000e+00249 store float %., ptr %arrayidx, align 4250 %inc = add nsw i32 %i.08, 1251 %exitcond = icmp slt i32 %inc, 999252 br i1 %exitcond, label %for.body, label %for.end253 254for.end:255 ret float 0.000000e+00256}257 258define void @pragma_vect_predicate_disable(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {259; CHECK-LABEL: pragma_vect_predicate_disable(260; PREFER-FOLDING: vector.body:261; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0262; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0263; PREFER-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0264; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body265entry:266 br label %for.body267 268for.cond.cleanup:269 ret void270 271for.body:272 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]273 %arrayidx = getelementptr inbounds i32, ptr %B, i32 %i.09274 %0 = load i32, ptr %arrayidx, align 4275 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.09276 %1 = load i32, ptr %arrayidx1, align 4277 %add = add nsw i32 %1, %0278 %arrayidx2 = getelementptr inbounds i32, ptr %A, i32 %i.09279 store i32 %add, ptr %arrayidx2, align 4280 %add3 = add nuw nsw i32 %i.09, 1281 %exitcond = icmp eq i32 %add3, 431282 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !7283}284 285define void @stride_4(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {286; CHECK-LABEL: stride_4(287; PREFER-FOLDING: vector.body:288; PREFER-FOLDING-NOT: llvm.masked.load289; PREFER-FOLDING-NOT: llvm.masked.store290; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body291entry:292 br label %for.body293 294for.cond.cleanup:295 ret void296 297for.body:298 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]299 %arrayidx = getelementptr inbounds i32, ptr %B, i32 %i.09300 %0 = load i32, ptr %arrayidx, align 4301 %arrayidx1 = getelementptr inbounds i32, ptr %C, i32 %i.09302 %1 = load i32, ptr %arrayidx1, align 4303 %add = add nsw i32 %1, %0304 %arrayidx2 = getelementptr inbounds i32, ptr %A, i32 %i.09305 store i32 %add, ptr %arrayidx2, align 4306 %add3 = add nuw nsw i32 %i.09, 4307 %cmp = icmp ult i32 %add3, 731308 br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !5309}310 311define dso_local void @half(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {312; CHECK-LABEL: half(313; PREFER-FOLDING: vector.body:314; PREFER-FOLDING: call <8 x half> @llvm.masked.load.v8f16.p0315; PREFER-FOLDING: call <8 x half> @llvm.masked.load.v8f16.p0316; PREFER-FOLDING: call void @llvm.masked.store.v8f16.p0317; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body318entry:319 br label %for.body320 321for.cond.cleanup:322 ret void323 324for.body:325 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]326 %arrayidx = getelementptr inbounds half, ptr %B, i32 %i.09327 %0 = load half, ptr %arrayidx, align 2328 %arrayidx1 = getelementptr inbounds half, ptr %C, i32 %i.09329 %1 = load half, ptr %arrayidx1, align 2330 %add = fadd fast half %1, %0331 %arrayidx2 = getelementptr inbounds half, ptr %A, i32 %i.09332 store half %add, ptr %arrayidx2, align 2333 %add3 = add nuw nsw i32 %i.09, 1334 %exitcond = icmp eq i32 %add3, 431335 br i1 %exitcond, label %for.cond.cleanup, label %for.body336}337 338define void @float(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {339; CHECK-LABEL: float(340; PREFER-FOLDING: vector.body:341; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]342; PREFER-FOLDING: %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 431)343; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0({{.*}}%active.lane.mask344; PREFER-FOLDING: call <4 x float> @llvm.masked.load.v4f32.p0({{.*}}%active.lane.mask345; PREFER-FOLDING: call void @llvm.masked.store.v4f32.p0({{.*}}%active.lane.mask346; PREFER-FOLDING: %index.next = add nuw i32 %index, 4347; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body348entry:349 br label %for.body350 351for.cond.cleanup:352 ret void353 354for.body:355 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]356 %arrayidx = getelementptr inbounds float, ptr %B, i32 %i.09357 %0 = load float, ptr %arrayidx, align 4358 %arrayidx1 = getelementptr inbounds float, ptr %C, i32 %i.09359 %1 = load float, ptr %arrayidx1, align 4360 %add = fadd fast float %1, %0361 %arrayidx2 = getelementptr inbounds float, ptr %A, i32 %i.09362 store float %add, ptr %arrayidx2, align 4363 %add3 = add nuw nsw i32 %i.09, 1364 %exitcond = icmp eq i32 %add3, 431365 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10366}367 368define void @fpext_allowed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {369; CHECK-LABEL: fpext_allowed(370; PREFER-FOLDING: vector.body:371; PREFER-FOLDING-NOT: llvm.masked.load372; PREFER-FOLDING-NOT: llvm.masked.store373; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body374entry:375 br label %for.body376 377for.cond.cleanup:378 ret void379 380for.body:381 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]382 %arrayidx = getelementptr inbounds half, ptr %B, i32 %i.09383 %0 = load half, ptr %arrayidx, align 2384 %conv = fpext half %0 to float385 %arrayidx1 = getelementptr inbounds float, ptr %C, i32 %i.09386 %1 = load float, ptr %arrayidx1, align 4387 %add = fadd fast float %1, %conv388 %arrayidx2 = getelementptr inbounds float, ptr %A, i32 %i.09389 store float %add, ptr %arrayidx2, align 4390 %add3 = add nuw nsw i32 %i.09, 1391 %exitcond = icmp eq i32 %add3, 431392 br i1 %exitcond, label %for.cond.cleanup, label %for.body393}394 395define void @fptrunc_allowed(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) #0 {396; CHECK-LABEL: fptrunc_allowed(397; PREFER-FOLDING: vector.body:398; PREFER-FOLDING-NOT: llvm.masked.load399; PREFER-FOLDING-NOT: llvm.masked.store400; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body401entry:402 br label %for.body403 404for.cond.cleanup:405 ret void406 407for.body:408 %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]409 %arrayidx = getelementptr inbounds float, ptr %B, i32 %i.09410 %0 = load float, ptr %arrayidx, align 4411 %arrayidx1 = getelementptr inbounds float, ptr %C, i32 %i.09412 %1 = load float, ptr %arrayidx1, align 4413 %add = fadd fast float %1, %0414 %conv = fptrunc float %add to half415 %arrayidx2 = getelementptr inbounds half, ptr %A, i32 %i.09416 store half %conv, ptr %arrayidx2, align 2417 %add3 = add nuw nsw i32 %i.09, 1418 %exitcond = icmp eq i32 %add3, 431419 br i1 %exitcond, label %for.cond.cleanup, label %for.body420}421 422attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }423 424!5 = distinct !{!5, !6}425!6 = !{!"llvm.loop.vectorize.enable", i1 true}426 427!7 = distinct !{!7, !8}428!8 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}429 430!10 = distinct !{!10, !11}431!11 = !{!"llvm.loop.vectorize.width", i32 4}432