217 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize -mcpu=corei7-avx -S -vectorizer-min-trip-count=21 | FileCheck %s3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"5target triple = "x86_64-unknown-linux"6 7;8; The source code for the test:9;10; void foo(ptr restrict A, ptr restrict B)11; {12; for (int i = 0; i < 20; ++i) A[i] += B[i];13; }14;15 16;17; This loop will be vectorized, although the trip count is below the threshold, but18; vectorization is explicitly forced in metadata. The trip count of 4 is chosen as19; it more nicely divides the loop count of 20, produce a lower total cost.20;21define void @vectorized(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {22; CHECK-LABEL: @vectorized(23; CHECK-NEXT: iter.check:24; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]25; CHECK: vector.main.loop.iter.check:26; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]27; CHECK: vector.ph:28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]29; CHECK: vector.body:30; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]32; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 433; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 834; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 1235; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4, !llvm.access.group [[ACC_GRP0:![0-9]+]]36; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !llvm.access.group [[ACC_GRP0]]37; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4, !llvm.access.group [[ACC_GRP0]]38; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP3]], align 4, !llvm.access.group [[ACC_GRP0]]39; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]40; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 441; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 842; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 1243; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !llvm.access.group [[ACC_GRP0]]44; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP5]], align 4, !llvm.access.group [[ACC_GRP0]]45; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP8]], align 4, !llvm.access.group [[ACC_GRP0]]46; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP7]], align 4, !llvm.access.group [[ACC_GRP0]]47; CHECK-NEXT: [[TMP11:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]48; CHECK-NEXT: [[TMP12:%.*]] = fadd fast <4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]49; CHECK-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]50; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]51; CHECK-NEXT: store <4 x float> [[TMP11]], ptr [[TMP6]], align 4, !llvm.access.group [[ACC_GRP0]]52; CHECK-NEXT: store <4 x float> [[TMP12]], ptr [[TMP5]], align 4, !llvm.access.group [[ACC_GRP0]]53; CHECK-NEXT: store <4 x float> [[TMP13]], ptr [[TMP8]], align 4, !llvm.access.group [[ACC_GRP0]]54; CHECK-NEXT: store <4 x float> [[TMP14]], ptr [[TMP7]], align 4, !llvm.access.group [[ACC_GRP0]]55; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1656; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1657; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]58; CHECK: middle.block:59; CHECK-NEXT: br i1 false, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]60; CHECK: vec.epilog.iter.check:61; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]62; CHECK: vec.epilog.ph:63; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]64; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]65; CHECK: vec.epilog.vector.body:66; CHECK-NEXT: [[INDEX8:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]67; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX8]]68; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP17]], align 4, !llvm.access.group [[ACC_GRP0]]69; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX8]]70; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x float>, ptr [[TMP19]], align 4, !llvm.access.group [[ACC_GRP0]]71; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x float> [[WIDE_LOAD9]], [[WIDE_LOAD10]]72; CHECK-NEXT: store <4 x float> [[TMP21]], ptr [[TMP19]], align 4, !llvm.access.group [[ACC_GRP0]]73; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX8]], 474; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT11]], 2075; CHECK-NEXT: br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]76; CHECK: vec.epilog.middle.block:77; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]78; CHECK: vec.epilog.scalar.ph:79; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 16, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]80; CHECK-NEXT: br label [[FOR_BODY:%.*]]81; CHECK: for.body:82; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]83; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]84; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !llvm.access.group [[ACC_GRP0]]85; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]86; CHECK-NEXT: [[TMP24:%.*]] = load float, ptr [[ARRAYIDX2]], align 4, !llvm.access.group [[ACC_GRP0]]87; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP23]], [[TMP24]]88; CHECK-NEXT: store float [[ADD]], ptr [[ARRAYIDX2]], align 4, !llvm.access.group [[ACC_GRP0]]89; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 190; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 2091; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]92; CHECK: for.end:93; CHECK-NEXT: ret void94;95entry:96 br label %for.body97 98for.body:99 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]100 %arrayidx = getelementptr inbounds float, ptr %B, i64 %indvars.iv101 %0 = load float, ptr %arrayidx, align 4, !llvm.access.group !11102 %arrayidx2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv103 %1 = load float, ptr %arrayidx2, align 4, !llvm.access.group !11104 %add = fadd fast float %0, %1105 store float %add, ptr %arrayidx2, align 4, !llvm.access.group !11106 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1107 %exitcond = icmp eq i64 %indvars.iv.next, 20108 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1109 110for.end:111 ret void112}113 114!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}115!2 = !{!"llvm.loop.vectorize.enable", i1 true}116!11 = distinct !{}117 118;119; This loop will be vectorized as the trip count is below the threshold but no120; scalar iterations are needed thanks to folding its tail.121;122define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {123; CHECK-LABEL: @vectorized1(124; CHECK-NEXT: entry:125; CHECK-NEXT: br label [[VECTOR_PH:%.*]]126; CHECK: vector.ph:127; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]128; CHECK: vector.body:129; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]130; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[INDEX]], i64 0131; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer132; CHECK-NEXT: [[VEC_IV:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>133; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IV]], splat (i64 19)134; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]135; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP2]], <8 x i1> [[TMP1]], <8 x float> poison), !llvm.access.group [[ACC_GRP8:![0-9]+]]136; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]137; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 [[TMP4]], <8 x i1> [[TMP1]], <8 x float> poison), !llvm.access.group [[ACC_GRP8]]138; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]139; CHECK-NEXT: call void @llvm.masked.store.v8f32.p0(<8 x float> [[TMP6]], ptr align 4 [[TMP4]], <8 x i1> [[TMP1]]), !llvm.access.group [[ACC_GRP8]]140; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8141; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24142; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]143; CHECK: middle.block:144; CHECK-NEXT: br label [[FOR_BODY:%.*]]145; CHECK: for.end:146; CHECK-NEXT: ret void147;148entry:149 br label %for.body150 151for.body:152 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]153 %arrayidx = getelementptr inbounds float, ptr %B, i64 %indvars.iv154 %0 = load float, ptr %arrayidx, align 4, !llvm.access.group !13155 %arrayidx2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv156 %1 = load float, ptr %arrayidx2, align 4, !llvm.access.group !13157 %add = fadd fast float %0, %1158 store float %add, ptr %arrayidx2, align 4, !llvm.access.group !13159 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1160 %exitcond = icmp eq i64 %indvars.iv.next, 20161 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3162 163for.end:164 ret void165}166 167!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}168!13 = distinct !{}169 170;171; This loop will be vectorized as the trip count is below the threshold but no172; scalar iterations are needed.173;174define void @vectorized2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {175; CHECK-LABEL: @vectorized2(176; CHECK-NEXT: entry:177; CHECK-NEXT: br label [[VECTOR_PH:%.*]]178; CHECK: vector.ph:179; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]180; CHECK: vector.body:181; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]182; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]183; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP1]], align 4, !llvm.access.group [[ACC_GRP8]]184; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]185; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x float>, ptr [[TMP3]], align 4, !llvm.access.group [[ACC_GRP8]]186; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]187; CHECK-NEXT: store <8 x float> [[TMP5]], ptr [[TMP3]], align 4, !llvm.access.group [[ACC_GRP8]]188; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8189; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16190; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]191; CHECK: middle.block:192; CHECK-NEXT: br label [[FOR_BODY:%.*]]193; CHECK: for.end:194; CHECK-NEXT: ret void195;196entry:197 br label %for.body198 199for.body:200 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]201 %arrayidx = getelementptr inbounds float, ptr %B, i64 %indvars.iv202 %0 = load float, ptr %arrayidx, align 4, !llvm.access.group !13203 %arrayidx2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv204 %1 = load float, ptr %arrayidx2, align 4, !llvm.access.group !13205 %add = fadd fast float %0, %1206 store float %add, ptr %arrayidx2, align 4, !llvm.access.group !13207 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1208 %exitcond = icmp eq i64 %indvars.iv.next, 16209 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4210 211for.end:212 ret void213}214 215!4 = !{!4}216 217