182 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s3; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s4 5target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"6target triple = "x86_64-unknown-linux-gnu"7 8define dso_local void @tail_folding_enabled(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) local_unnamed_addr #0 {9; CHECK-LABEL: @tail_folding_enabled(10; CHECK-NEXT: entry:11; CHECK-NEXT: br label [[VECTOR_PH:%.*]]12; CHECK: vector.ph:13; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]14; CHECK: vector.body:15; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]16; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 017; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer18; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>19; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], splat (i64 429)20; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]21; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP2]], <8 x i1> [[TMP1]], <8 x i32> poison)22; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[INDEX]]23; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[TMP1]], <8 x i32> poison)24; CHECK-NEXT: [[TMP6:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]25; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]26; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP6]], ptr align 4 [[TMP7]], <8 x i1> [[TMP1]])27; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 828; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 43229; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]30; CHECK: middle.block:31; CHECK-NEXT: br label [[FOR_BODY:%.*]]32; CHECK: for.cond.cleanup:33; CHECK-NEXT: ret void34;35entry:36 br label %for.body37 38for.cond.cleanup:39 ret void40 41for.body:42 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]43 %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv44 %0 = load i32, ptr %arrayidx, align 445 %arrayidx2 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv46 %1 = load i32, ptr %arrayidx2, align 447 %add = add nsw i32 %1, %048 %arrayidx4 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv49 store i32 %add, ptr %arrayidx4, align 450 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 151 %exitcond = icmp eq i64 %indvars.iv.next, 43052 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !653}54 55; Marking function as optsize turns tail folding on, as if explicit tail folding56; flag was enabled.57define dso_local void @tail_folding_disabled(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C) local_unnamed_addr #0 {58; CHECK-LABEL: @tail_folding_disabled(59; CHECK-NEXT: entry:60; CHECK-NEXT: br label [[VECTOR_PH:%.*]]61; CHECK: vector.ph:62; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]63; CHECK: vector.body:64; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]65; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 066; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer67; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>68; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], splat (i64 429)69; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]70; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP2]], <8 x i1> [[TMP1]], <8 x i32> poison)71; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[INDEX]]72; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP4]], <8 x i1> [[TMP1]], <8 x i32> poison)73; CHECK-NEXT: [[TMP6:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]]74; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]75; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP6]], ptr align 4 [[TMP7]], <8 x i1> [[TMP1]])76; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 877; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 43278; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]79; CHECK: middle.block:80; CHECK-NEXT: br label [[FOR_BODY:%.*]]81; CHECK: for.cond.cleanup:82; CHECK-NEXT: ret void83;84entry:85 br label %for.body86 87for.cond.cleanup:88 ret void89 90for.body:91 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]92 %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv93 %0 = load i32, ptr %arrayidx, align 494 %arrayidx2 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv95 %1 = load i32, ptr %arrayidx2, align 496 %add = add nsw i32 %1, %097 %arrayidx4 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv98 store i32 %add, ptr %arrayidx4, align 499 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1100 %exitcond = icmp eq i64 %indvars.iv.next, 430101 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10102}103 104; Check that fold tail under optsize passes the reduction live-out value105; through a select.106; int reduction_i32(int *A, int *B, int N) {107; int sum = 0;108; for (int i = 0; i < N; ++i)109; sum += (A[i] + B[i]);110; return sum;111; }112;113 114define i32 @reduction_i32(ptr nocapture readonly %A, ptr nocapture readonly %B, i32 %N) #0 {115; CHECK-LABEL: @reduction_i32(116; CHECK-NEXT: entry:117; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1118; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64119; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1120; CHECK-NEXT: br label [[VECTOR_PH:%.*]]121; CHECK: vector.ph:122; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP2]], 7123; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8124; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]125; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP2]], 1126; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0127; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer128; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]129; CHECK: vector.body:130; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]131; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]132; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0133; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer134; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>135; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <8 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]136; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]137; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP5]], <8 x i1> [[TMP4]], <8 x i32> poison)138; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]139; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP7]], <8 x i1> [[TMP4]], <8 x i32> poison)140; CHECK-NEXT: [[TMP9:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]141; CHECK-NEXT: [[TMP10]] = add <8 x i32> [[TMP9]], [[VEC_PHI]]142; CHECK-NEXT: [[TMP11:%.*]] = select <8 x i1> [[TMP4]], <8 x i32> [[TMP10]], <8 x i32> [[VEC_PHI]]143; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8144; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]145; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]146; CHECK: middle.block:147; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])148; CHECK-NEXT: br label [[FOR_BODY:%.*]]149; CHECK: for.cond.cleanup:150; CHECK-NEXT: ret i32 [[TMP13]]151;152entry:153 br label %for.body154 155for.body:156 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]157 %sum.0 = phi i32 [ %sum.1, %for.body ], [ 0, %entry ]158 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1159 %arrayidxA = getelementptr inbounds i32, ptr %A, i64 %indvars.iv160 %0 = load i32, ptr %arrayidxA, align 4161 %arrayidxB = getelementptr inbounds i32, ptr %B, i64 %indvars.iv162 %1 = load i32, ptr %arrayidxB, align 4163 %add = add nsw i32 %1, %0164 %sum.1 = add nuw nsw i32 %add, %sum.0165 %lftr.wideiv = trunc i64 %indvars.iv.next to i32166 %exitcond = icmp eq i32 %lftr.wideiv, %N167 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6168 169for.cond.cleanup:170 ret i32 %sum.1171}172 173 174attributes #0 = { nounwind optsize uwtable "target-cpu"="core-avx2" "target-features"="+avx,+avx2" }175 176!6 = distinct !{!6, !7, !8}177!7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}178!8 = !{!"llvm.loop.vectorize.enable", i1 true}179 180!10 = distinct !{!10, !11}181!11 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}182