350 lines · plain
1; RUN: opt < %s -passes=loop-vectorize,dce -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -force-vector-interleave=0 -S \2; RUN: | FileCheck %s --check-prefix=CHECK-VECTOR3; RUN: opt < %s -passes=loop-vectorize,dce -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=1 -force-vector-interleave=0 -S \4; RUN: | FileCheck %s --check-prefix=CHECK-SCALAR5 6target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"7target triple = "x86_64-apple-macosx10.8.0"8 9; We don't unroll this loop because it has a small constant trip count 10; that is not profitable for generating a scalar epilogue11;12; CHECK-VECTOR-LABEL: @foo_trip_count_8(13; CHECK-VECTOR: load <4 x i32>14; CHECK-VECTOR-NOT: load <4 x i32>15; CHECK-VECTOR: store <4 x i32>16; CHECK-VECTOR-NOT: store <4 x i32>17; CHECK-VECTOR: ret18;19; CHECK-SCALAR-LABEL: @foo_trip_count_8(20; CHECK-SCALAR: load i32, ptr21; CHECK-SCALAR-NOT: load i32, ptr22; CHECK-SCALAR: store i3223; CHECK-SCALAR-NOT: store i3224; CHECK-SCALAR: ret25define void @foo_trip_count_8(ptr nocapture %A) nounwind uwtable ssp {26entry:27 br label %for.body28 29for.body: ; preds = %for.body, %entry30 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]31 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv32 %1 = load i32, ptr %0, align 433 %2 = add nsw i32 %1, 634 store i32 %2, ptr %0, align 435 %indvars.iv.next = add i64 %indvars.iv, 136 %lftr.wideiv = trunc i64 %indvars.iv.next to i3237 %exitcond = icmp eq i32 %lftr.wideiv, 838 br i1 %exitcond, label %for.end, label %for.body39 40for.end: ; preds = %for.body41 ret void42}43 44; We should unroll this loop 4 times since TC being a multiple of VF means45; that the epilogue loop may not need to run, making it profitable for46; the vector loop to run even once47;48; CHECK-VECTOR-LABEL: @foo_trip_count_16(49; CHECK-VECTOR: load <4 x i32>50; CHECK-VECTOR: load <4 x i32>51; CHECK-VECTOR: load <4 x i32>52; CHECK-VECTOR: load <4 x i32>53; CHECK-VECTOR-NOT: load <4 x i32>54; CHECK-VECTOR: store <4 x i32>55; CHECK-VECTOR: store <4 x i32>56; CHECK-VECTOR: store <4 x i32>57; CHECK-VECTOR: store <4 x i32>58; CHECK-VECTOR-NOT: store <4 x i32>59; CHECK-VECTOR: ret60;61; CHECK-SCALAR-LABEL: @foo_trip_count_16(62; CHECK-SCALAR: load i32, ptr63; CHECK-SCALAR-NOT: load i32, ptr64; CHECK-SCALAR: store i3265; CHECK-SCALAR-NOT: store i3266; CHECK-SCALAR: ret67define void @foo_trip_count_16(ptr nocapture %A) nounwind uwtable ssp {68entry:69 br label %for.body70 71for.body: ; preds = %for.body, %entry72 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]73 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv74 %1 = load i32, ptr %0, align 475 %2 = add nsw i32 %1, 676 store i32 %2, ptr %0, align 477 %indvars.iv.next = add i64 %indvars.iv, 178 %lftr.wideiv = trunc i64 %indvars.iv.next to i3279 %exitcond = icmp eq i32 %lftr.wideiv, 1680 br i1 %exitcond, label %for.end, label %for.body81 82for.end: ; preds = %for.body83 ret void84}85 86; We should unroll this loop four times since unrolling it twice87; will produce the same epilogue TC of 1, making larger unroll count88; more profitable89;90; CHECK-VECTOR-LABEL: @foo_trip_count_17(91; CHECK-VECTOR: load <4 x i32>92; CHECK-VECTOR: load <4 x i32>93; CHECK-VECTOR: load <4 x i32>94; CHECK-VECTOR: load <4 x i32>95; CHECK-VECTOR-NOT: load <4 x i32>96; CHECK-VECTOR: store <4 x i32>97; CHECK-VECTOR: store <4 x i32>98; CHECK-VECTOR: store <4 x i32>99; CHECK-VECTOR: store <4 x i32>100; CHECK-VECTOR-NOT: store <4 x i32>101; CHECK-VECTOR: ret102;103; CHECK-SCALAR-LABEL: @foo_trip_count_17(104; CHECK-SCALAR: load i32, ptr105; CHECK-SCALAR-NOT: load i32, ptr106; CHECK-SCALAR: store i32107; CHECK-SCALAR-NOT: store i32108; CHECK-SCALAR: ret109define void @foo_trip_count_17(ptr nocapture %A) nounwind uwtable ssp {110entry:111 br label %for.body112 113for.body: ; preds = %for.body, %entry114 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]115 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv116 %1 = load i32, ptr %0, align 4117 %2 = add nsw i32 %1, 6118 store i32 %2, ptr %0, align 4119 %indvars.iv.next = add i64 %indvars.iv, 1120 %lftr.wideiv = trunc i64 %indvars.iv.next to i32121 %exitcond = icmp eq i32 %lftr.wideiv, 17122 br i1 %exitcond, label %for.end, label %for.body123 124for.end: ; preds = %for.body125 ret void126}127 128; We should unroll this loop twice since unrolling four times will 129; create an epilogue loop of TC 8, while unrolling it twice will 130; eliminate the epologue loop altogether131;132; CHECK-VECTOR-LABEL: @foo_trip_count_24(133; CHECK-VECTOR: load <4 x i32>134; CHECK-VECTOR: load <4 x i32>135; CHECK-VECTOR-NOT: load <4 x i32>136; CHECK-VECTOR: store <4 x i32>137; CHECK-VECTOR: store <4 x i32>138; CHECK-VECTOR-NOT: store <4 x i32>139; CHECK-VECTOR: ret140;141; CHECK-SCALAR-LABEL: @foo_trip_count_24(142; CHECK-SCALAR: load i32, ptr143; CHECK-SCALAR-NOT: load i32, ptr144; CHECK-SCALAR: store i32145; CHECK-SCALAR-NOT: store i32146; CHECK-SCALAR: ret147define void @foo_trip_count_24(ptr nocapture %A) nounwind uwtable ssp {148entry:149 br label %for.body150 151for.body: ; preds = %for.body, %entry152 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]153 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv154 %1 = load i32, ptr %0, align 4155 %2 = add nsw i32 %1, 6156 store i32 %2, ptr %0, align 4157 %indvars.iv.next = add i64 %indvars.iv, 1158 %lftr.wideiv = trunc i64 %indvars.iv.next to i32159 %exitcond = icmp eq i32 %lftr.wideiv, 24160 br i1 %exitcond, label %for.end, label %for.body161 162for.end: ; preds = %for.body163 ret void164}165 166; We should unroll this loop twice since TC not being a multiple of VF may require167; the epilogue loop to run, making it profitable when the vector loop runs168; at least twice.169;170; CHECK-VECTOR-LABEL: @foo_trip_count_25(171; CHECK-VECTOR: load <4 x i32>172; CHECK-VECTOR: load <4 x i32>173; CHECK-VECTOR-NOT: load <4 x i32>174; CHECK-VECTOR: store <4 x i32>175; CHECK-VECTOR: store <4 x i32>176; CHECK-VECTOR-NOT: store <4 x i32>177; CHECK-VECTOR: ret178;179; CHECK-SCALAR-LABEL: @foo_trip_count_25(180; CHECK-SCALAR: load i32, ptr181; CHECK-SCALAR-NOT: load i32, ptr182; CHECK-SCALAR: store i32183; CHECK-SCALAR-NOT: store i32184; CHECK-SCALAR: ret185define void @foo_trip_count_25(ptr nocapture %A) nounwind uwtable ssp {186entry:187 br label %for.body188 189for.body: ; preds = %for.body, %entry190 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]191 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv192 %1 = load i32, ptr %0, align 4193 %2 = add nsw i32 %1, 6194 store i32 %2, ptr %0, align 4195 %indvars.iv.next = add i64 %indvars.iv, 1196 %lftr.wideiv = trunc i64 %indvars.iv.next to i32197 %exitcond = icmp eq i32 %lftr.wideiv, 25198 br i1 %exitcond, label %for.end, label %for.body199 200for.end: ; preds = %for.body201 ret void202}203 204; We should unroll this loop 4 times since TC not being a multiple of VF may require205; the epilogue loop to run, making it profitable when the vector loop runs206; at least twice.207;208; CHECK-VECTOR-LABEL: @foo_trip_count_33(209; CHECK-VECTOR: load <4 x i32>210; CHECK-VECTOR: load <4 x i32>211; CHECK-VECTOR: load <4 x i32>212; CHECK-VECTOR: load <4 x i32>213; CHECK-VECTOR-NOT: load <4 x i32>214; CHECK-VECTOR: store <4 x i32>215; CHECK-VECTOR: store <4 x i32>216; CHECK-VECTOR: store <4 x i32>217; CHECK-VECTOR: store <4 x i32>218; CHECK-VECTOR-NOT: store <4 x i32>219; CHECK-VECTOR: ret220;221; CHECK-SCALAR-LABEL: @foo_trip_count_33(222; CHECK-SCALAR: load i32, ptr223; CHECK-SCALAR-NOT: load i32, ptr224; CHECK-SCALAR: store i32225; CHECK-SCALAR-NOT: store i32226; CHECK-SCALAR: ret227define void @foo_trip_count_33(ptr nocapture %A) nounwind uwtable ssp {228entry:229 br label %for.body230 231for.body: ; preds = %for.body, %entry232 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]233 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv234 %1 = load i32, ptr %0, align 4235 %2 = add nsw i32 %1, 6236 store i32 %2, ptr %0, align 4237 %indvars.iv.next = add i64 %indvars.iv, 1238 %lftr.wideiv = trunc i64 %indvars.iv.next to i32239 %exitcond = icmp eq i32 %lftr.wideiv, 33240 br i1 %exitcond, label %for.end, label %for.body241 242for.end: ; preds = %for.body243 ret void244}245 246; We should unroll this loop 4 times since TC not being a multiple of VF may require247; the epilogue loop to run, making it profitable when the vector loop runs248; at least twice. The IC is restricted to 4 since that is the maximum supported 249; for the target.250;251; CHECK-VECTOR-LABEL: @foo_trip_count_101(252; CHECK-VECTOR: load <4 x i32>253; CHECK-VECTOR: load <4 x i32>254; CHECK-VECTOR: load <4 x i32>255; CHECK-VECTOR: load <4 x i32>256; CHECK-VECTOR-NOT: load <4 x i32>257; CHECK-VECTOR: store <4 x i32>258; CHECK-VECTOR: store <4 x i32>259; CHECK-VECTOR: store <4 x i32>260; CHECK-VECTOR: store <4 x i32>261; CHECK-VECTOR-NOT: store <4 x i32>262; CHECK-VECTOR: ret263;264; CHECK-SCALAR-LABEL: @foo_trip_count_101(265; CHECK-SCALAR: load i32, ptr266; CHECK-SCALAR-NOT: load i32, ptr267; CHECK-SCALAR: store i32268; CHECK-SCALAR-NOT: store i32269; CHECK-SCALAR: ret270define void @foo_trip_count_101(ptr nocapture %A) nounwind uwtable ssp {271entry:272 br label %for.body273 274for.body: ; preds = %for.body, %entry275 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]276 %0 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv277 %1 = load i32, ptr %0, align 4278 %2 = add nsw i32 %1, 6279 store i32 %2, ptr %0, align 4280 %indvars.iv.next = add i64 %indvars.iv, 1281 %lftr.wideiv = trunc i64 %indvars.iv.next to i32282 %exitcond = icmp eq i32 %lftr.wideiv, 101283 br i1 %exitcond, label %for.end, label %for.body284 285for.end: ; preds = %for.body286 ret void287}288 289; But this is a good small loop to unroll as we don't know of a bound on its290; trip count.291;292; CHECK-VECTOR-LABEL: @bar(293; CHECK-VECTOR: store <4 x i32>294; CHECK-VECTOR: store <4 x i32>295; CHECK-VECTOR: ret296;297; For x86, loop unroll in loop vectorizer is disabled when VF==1.298;299; CHECK-SCALAR-LABEL: @bar(300; CHECK-SCALAR: store i32301; CHECK-SCALAR-NOT: store i32302; CHECK-SCALAR: ret303define void @bar(ptr nocapture %A, i32 %n) nounwind uwtable ssp {304 %1 = icmp sgt i32 %n, 0305 br i1 %1, label %.lr.ph, label %._crit_edge306 307.lr.ph: ; preds = %0, %.lr.ph308 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]309 %2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv310 %3 = load i32, ptr %2, align 4311 %4 = add nsw i32 %3, 6312 store i32 %4, ptr %2, align 4313 %indvars.iv.next = add i64 %indvars.iv, 1314 %lftr.wideiv = trunc i64 %indvars.iv.next to i32315 %exitcond = icmp eq i32 %lftr.wideiv, %n316 br i1 %exitcond, label %._crit_edge, label %.lr.ph317 318._crit_edge: ; preds = %.lr.ph, %0319 ret void320}321 322; Also unroll if we need a runtime check but it was going to be added for323; vectorization anyways.324; CHECK-VECTOR-LABEL: @runtime_chk(325; CHECK-VECTOR: store <4 x float>326; CHECK-VECTOR: store <4 x float>327;328; But not if the unrolling would introduce the runtime check.329; CHECK-SCALAR-LABEL: @runtime_chk(330; CHECK-SCALAR: store float331; CHECK-SCALAR-NOT: store float332define void @runtime_chk(ptr %A, ptr %B, float %N) {333entry:334 br label %for.body335 336for.body:337 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]338 %arrayidx = getelementptr inbounds float, ptr %B, i64 %indvars.iv339 %0 = load float, ptr %arrayidx, align 4340 %mul = fmul float %0, %N341 %arrayidx2 = getelementptr inbounds float, ptr %A, i64 %indvars.iv342 store float %mul, ptr %arrayidx2, align 4343 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1344 %exitcond = icmp eq i64 %indvars.iv.next, 256345 br i1 %exitcond, label %for.end, label %for.body346 347for.end:348 ret void349}350