427 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:" --version 52; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v1 -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefix=CHECK-V1-IC13; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v1 -force-vector-interleave=1 \4; RUN: -epilogue-vectorization-force-VF=4 -S < %s | FileCheck %s -check-prefix=CHECK-V1-IC1-FORCE-EPI45; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=1 -S < %s | FileCheck %s -check-prefix=CHECK-V2-IC16; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=4 -S < %s | FileCheck %s -check-prefix=CHECK-V2-IC47 8target triple = "aarch64-unknown-linux-gnu"9 10@a = global [1024 x i32] zeroinitializer, align 1611@b = global [1024 x i32] zeroinitializer, align 1612 13; We expect the branch weight computations after vectorisation to use14; vscale=2 for neoverse-v1 and vscale=1 for neoverse-v2.15define void @foo_i32(i64 %n) {16; CHECK-V1-IC1-LABEL: define void @foo_i32(17; CHECK-V1-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {18; CHECK-V1-IC1: [[ENTRY:.*:]]19; CHECK-V1-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]20; CHECK-V1-IC1: [[VECTOR_PH]]:21; CHECK-V1-IC1: br label %[[VECTOR_BODY:.*]]22; CHECK-V1-IC1: [[VECTOR_BODY]]:23; CHECK-V1-IC1: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]24; CHECK-V1-IC1: [[MIDDLE_BLOCK]]:25; CHECK-V1-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF5:![0-9]+]]26; CHECK-V1-IC1: [[SCALAR_PH]]:27; CHECK-V1-IC1: br label %[[FOR_BODY:.*]]28; CHECK-V1-IC1: [[FOR_BODY]]:29; CHECK-V1-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]30; CHECK-V1-IC1: [[FOR_COND_CLEANUP]]:31;32; CHECK-V1-IC1-FORCE-EPI4-LABEL: define void @foo_i32(33; CHECK-V1-IC1-FORCE-EPI4-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {34; CHECK-V1-IC1-FORCE-EPI4: [[ITER_CHECK:.*:]]35; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0:![0-9]+]]36; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:37; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]38; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_PH]]:39; CHECK-V1-IC1-FORCE-EPI4: br label %[[VECTOR_BODY:.*]]40; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_BODY]]:41; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]42; CHECK-V1-IC1-FORCE-EPI4: [[MIDDLE_BLOCK]]:43; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF5:![0-9]+]]44; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_ITER_CHECK]]:45; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6:![0-9]+]]46; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_PH]]:47; CHECK-V1-IC1-FORCE-EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]48; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_VECTOR_BODY]]:49; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]50; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:51; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF9:![0-9]+]]52; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_SCALAR_PH]]:53; CHECK-V1-IC1-FORCE-EPI4: br label %[[FOR_BODY:.*]]54; CHECK-V1-IC1-FORCE-EPI4: [[FOR_BODY]]:55; CHECK-V1-IC1-FORCE-EPI4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]56; CHECK-V1-IC1-FORCE-EPI4: [[FOR_COND_CLEANUP]]:57;58; CHECK-V2-IC1-LABEL: define void @foo_i32(59; CHECK-V2-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {60; CHECK-V2-IC1: [[ENTRY:.*:]]61; CHECK-V2-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]62; CHECK-V2-IC1: [[VECTOR_PH]]:63; CHECK-V2-IC1: br label %[[VECTOR_BODY:.*]]64; CHECK-V2-IC1: [[VECTOR_BODY]]:65; CHECK-V2-IC1: br i1 [[TMP2:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]66; CHECK-V2-IC1: [[MIDDLE_BLOCK]]:67; CHECK-V2-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF6:![0-9]+]]68; CHECK-V2-IC1: [[SCALAR_PH]]:69; CHECK-V2-IC1: br label %[[FOR_BODY:.*]]70; CHECK-V2-IC1: [[FOR_BODY]]:71; CHECK-V2-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF7:![0-9]+]], !llvm.loop [[LOOP8:![0-9]+]]72; CHECK-V2-IC1: [[FOR_COND_CLEANUP]]:73;74; CHECK-V2-IC4-LABEL: define void @foo_i32(75; CHECK-V2-IC4-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {76; CHECK-V2-IC4: [[ITER_CHECK:.*:]]77; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0:![0-9]+]]78; CHECK-V2-IC4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:79; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]80; CHECK-V2-IC4: [[VECTOR_PH]]:81; CHECK-V2-IC4: br label %[[VECTOR_BODY:.*]]82; CHECK-V2-IC4: [[VECTOR_BODY]]:83; CHECK-V2-IC4: br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]84; CHECK-V2-IC4: [[MIDDLE_BLOCK]]:85; CHECK-V2-IC4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF6:![0-9]+]]86; CHECK-V2-IC4: [[VEC_EPILOG_ITER_CHECK]]:87; CHECK-V2-IC4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]88; CHECK-V2-IC4: [[VEC_EPILOG_PH]]:89; CHECK-V2-IC4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]90; CHECK-V2-IC4: [[VEC_EPILOG_VECTOR_BODY]]:91; CHECK-V2-IC4: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]92; CHECK-V2-IC4: [[VEC_EPILOG_MIDDLE_BLOCK]]:93; CHECK-V2-IC4: br i1 [[CMP_N10:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF10:![0-9]+]]94; CHECK-V2-IC4: [[VEC_EPILOG_SCALAR_PH]]:95; CHECK-V2-IC4: br label %[[FOR_BODY:.*]]96; CHECK-V2-IC4: [[FOR_BODY]]:97; CHECK-V2-IC4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF11:![0-9]+]], !llvm.loop [[LOOP12:![0-9]+]]98; CHECK-V2-IC4: [[FOR_COND_CLEANUP]]:99;100entry:101 br label %for.body102 103for.body: ; preds = %for.body, %entry104 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]105 %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv106 %load = load i32, ptr %arrayidx, align 4107 %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv108 store i32 %load, ptr %arrayidx2, align 4109 %iv.next = add nuw nsw i64 %iv, 1110 %exitcond = icmp eq i64 %iv.next, %n111 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0112 113for.cond.cleanup: ; preds = %for.body114 ret void115}116 117define void @foo_i8(i64 %n) {118; CHECK-V1-IC1-LABEL: define void @foo_i8(119; CHECK-V1-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {120; CHECK-V1-IC1: [[ITER_CHECK:.*:]]121; CHECK-V1-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]122; CHECK-V1-IC1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:123; CHECK-V1-IC1: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]124; CHECK-V1-IC1: [[VECTOR_PH]]:125; CHECK-V1-IC1: br label %[[VECTOR_BODY:.*]]126; CHECK-V1-IC1: [[VECTOR_BODY]]:127; CHECK-V1-IC1: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]128; CHECK-V1-IC1: [[MIDDLE_BLOCK]]:129; CHECK-V1-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF9]]130; CHECK-V1-IC1: [[VEC_EPILOG_ITER_CHECK]]:131; CHECK-V1-IC1: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF12:![0-9]+]]132; CHECK-V1-IC1: [[VEC_EPILOG_PH]]:133; CHECK-V1-IC1: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]134; CHECK-V1-IC1: [[VEC_EPILOG_VECTOR_BODY]]:135; CHECK-V1-IC1: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]136; CHECK-V1-IC1: [[VEC_EPILOG_MIDDLE_BLOCK]]:137; CHECK-V1-IC1: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF14:![0-9]+]]138; CHECK-V1-IC1: [[VEC_EPILOG_SCALAR_PH]]:139; CHECK-V1-IC1: br label %[[FOR_BODY:.*]]140; CHECK-V1-IC1: [[FOR_BODY]]:141; CHECK-V1-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6]], !llvm.loop [[LOOP15:![0-9]+]]142; CHECK-V1-IC1: [[FOR_COND_CLEANUP]]:143;144; CHECK-V1-IC1-FORCE-EPI4-LABEL: define void @foo_i8(145; CHECK-V1-IC1-FORCE-EPI4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {146; CHECK-V1-IC1-FORCE-EPI4: [[ITER_CHECK:.*:]]147; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]148; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:149; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]150; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_PH]]:151; CHECK-V1-IC1-FORCE-EPI4: br label %[[VECTOR_BODY:.*]]152; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_BODY]]:153; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF12:![0-9]+]], !llvm.loop [[LOOP13:![0-9]+]]154; CHECK-V1-IC1-FORCE-EPI4: [[MIDDLE_BLOCK]]:155; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF12]]156; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_ITER_CHECK]]:157; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF15:![0-9]+]]158; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_PH]]:159; CHECK-V1-IC1-FORCE-EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]160; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_VECTOR_BODY]]:161; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]162; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:163; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF9]]164; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_SCALAR_PH]]:165; CHECK-V1-IC1-FORCE-EPI4: br label %[[FOR_BODY:.*]]166; CHECK-V1-IC1-FORCE-EPI4: [[FOR_BODY]]:167; CHECK-V1-IC1-FORCE-EPI4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF10]], !llvm.loop [[LOOP17:![0-9]+]]168; CHECK-V1-IC1-FORCE-EPI4: [[FOR_COND_CLEANUP]]:169;170; CHECK-V2-IC1-LABEL: define void @foo_i8(171; CHECK-V2-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {172; CHECK-V2-IC1: [[ITER_CHECK:.*:]]173; CHECK-V2-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]174; CHECK-V2-IC1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:175; CHECK-V2-IC1: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]176; CHECK-V2-IC1: [[VECTOR_PH]]:177; CHECK-V2-IC1: br label %[[VECTOR_BODY:.*]]178; CHECK-V2-IC1: [[VECTOR_BODY]]:179; CHECK-V2-IC1: br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]180; CHECK-V2-IC1: [[MIDDLE_BLOCK]]:181; CHECK-V2-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF13:![0-9]+]]182; CHECK-V2-IC1: [[VEC_EPILOG_ITER_CHECK]]:183; CHECK-V2-IC1: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF14:![0-9]+]]184; CHECK-V2-IC1: [[VEC_EPILOG_PH]]:185; CHECK-V2-IC1: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]186; CHECK-V2-IC1: [[VEC_EPILOG_VECTOR_BODY]]:187; CHECK-V2-IC1: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]188; CHECK-V2-IC1: [[VEC_EPILOG_MIDDLE_BLOCK]]:189; CHECK-V2-IC1: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF6]]190; CHECK-V2-IC1: [[VEC_EPILOG_SCALAR_PH]]:191; CHECK-V2-IC1: br label %[[FOR_BODY:.*]]192; CHECK-V2-IC1: [[FOR_BODY]]:193; CHECK-V2-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF7]], !llvm.loop [[LOOP16:![0-9]+]]194; CHECK-V2-IC1: [[FOR_COND_CLEANUP]]:195;196; CHECK-V2-IC4-LABEL: define void @foo_i8(197; CHECK-V2-IC4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {198; CHECK-V2-IC4: [[ITER_CHECK:.*:]]199; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]200; CHECK-V2-IC4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:201; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]202; CHECK-V2-IC4: [[VECTOR_PH]]:203; CHECK-V2-IC4: br label %[[VECTOR_BODY:.*]]204; CHECK-V2-IC4: [[VECTOR_BODY]]:205; CHECK-V2-IC4: br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF6]], !llvm.loop [[LOOP13:![0-9]+]]206; CHECK-V2-IC4: [[MIDDLE_BLOCK]]:207; CHECK-V2-IC4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF1]]208; CHECK-V2-IC4: [[VEC_EPILOG_ITER_CHECK]]:209; CHECK-V2-IC4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF15:![0-9]+]]210; CHECK-V2-IC4: [[VEC_EPILOG_PH]]:211; CHECK-V2-IC4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]212; CHECK-V2-IC4: [[VEC_EPILOG_VECTOR_BODY]]:213; CHECK-V2-IC4: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]214; CHECK-V2-IC4: [[VEC_EPILOG_MIDDLE_BLOCK]]:215; CHECK-V2-IC4: br i1 [[CMP_N10:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF17:![0-9]+]]216; CHECK-V2-IC4: [[VEC_EPILOG_SCALAR_PH]]:217; CHECK-V2-IC4: br label %[[FOR_BODY:.*]]218; CHECK-V2-IC4: [[FOR_BODY]]:219; CHECK-V2-IC4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF11]], !llvm.loop [[LOOP18:![0-9]+]]220; CHECK-V2-IC4: [[FOR_COND_CLEANUP]]:221;222entry:223 br label %for.body224 225for.body:226 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]227 %arrayidx = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %iv228 %load = load i8, ptr %arrayidx, align 1229 %arrayidx2 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %iv230 store i8 %load, ptr %arrayidx2, align 1231 %iv.next = add nuw nsw i64 %iv, 1232 %exitcond = icmp eq i64 %iv.next, %n233 br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0234 235for.cond.cleanup:236 ret void237}238 239define void @foo_i32_no_bw(i64 %n) {240; CHECK-V1-IC1-LABEL: define void @foo_i32_no_bw(241; CHECK-V1-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {242; CHECK-V1-IC1: [[ENTRY:.*:]]243; CHECK-V1-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]244; CHECK-V1-IC1: [[VECTOR_PH]]:245; CHECK-V1-IC1: br label %[[VECTOR_BODY:.*]]246; CHECK-V1-IC1: [[VECTOR_BODY]]:247; CHECK-V1-IC1: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]248; CHECK-V1-IC1: [[MIDDLE_BLOCK]]:249; CHECK-V1-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]250; CHECK-V1-IC1: [[SCALAR_PH]]:251; CHECK-V1-IC1: br label %[[FOR_BODY:.*]]252; CHECK-V1-IC1: [[FOR_BODY]]:253; CHECK-V1-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]254; CHECK-V1-IC1: [[FOR_COND_CLEANUP]]:255;256; CHECK-V1-IC1-FORCE-EPI4-LABEL: define void @foo_i32_no_bw(257; CHECK-V1-IC1-FORCE-EPI4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {258; CHECK-V1-IC1-FORCE-EPI4: [[ITER_CHECK:.*:]]259; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]260; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:261; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]262; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_PH]]:263; CHECK-V1-IC1-FORCE-EPI4: br label %[[VECTOR_BODY:.*]]264; CHECK-V1-IC1-FORCE-EPI4: [[VECTOR_BODY]]:265; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]266; CHECK-V1-IC1-FORCE-EPI4: [[MIDDLE_BLOCK]]:267; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]268; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_ITER_CHECK]]:269; CHECK-V1-IC1-FORCE-EPI4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF6]]270; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_PH]]:271; CHECK-V1-IC1-FORCE-EPI4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]272; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_VECTOR_BODY]]:273; CHECK-V1-IC1-FORCE-EPI4: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]274; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_MIDDLE_BLOCK]]:275; CHECK-V1-IC1-FORCE-EPI4: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]]276; CHECK-V1-IC1-FORCE-EPI4: [[VEC_EPILOG_SCALAR_PH]]:277; CHECK-V1-IC1-FORCE-EPI4: br label %[[FOR_BODY:.*]]278; CHECK-V1-IC1-FORCE-EPI4: [[FOR_BODY]]:279; CHECK-V1-IC1-FORCE-EPI4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]280; CHECK-V1-IC1-FORCE-EPI4: [[FOR_COND_CLEANUP]]:281;282; CHECK-V2-IC1-LABEL: define void @foo_i32_no_bw(283; CHECK-V2-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {284; CHECK-V2-IC1: [[ENTRY:.*:]]285; CHECK-V2-IC1: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]286; CHECK-V2-IC1: [[VECTOR_PH]]:287; CHECK-V2-IC1: br label %[[VECTOR_BODY:.*]]288; CHECK-V2-IC1: [[VECTOR_BODY]]:289; CHECK-V2-IC1: br i1 [[TMP2:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]290; CHECK-V2-IC1: [[MIDDLE_BLOCK]]:291; CHECK-V2-IC1: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]]292; CHECK-V2-IC1: [[SCALAR_PH]]:293; CHECK-V2-IC1: br label %[[FOR_BODY:.*]]294; CHECK-V2-IC1: [[FOR_BODY]]:295; CHECK-V2-IC1: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]296; CHECK-V2-IC1: [[FOR_COND_CLEANUP]]:297;298; CHECK-V2-IC4-LABEL: define void @foo_i32_no_bw(299; CHECK-V2-IC4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {300; CHECK-V2-IC4: [[ITER_CHECK:.*:]]301; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]302; CHECK-V2-IC4: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:303; CHECK-V2-IC4: br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]304; CHECK-V2-IC4: [[VECTOR_PH]]:305; CHECK-V2-IC4: br label %[[VECTOR_BODY:.*]]306; CHECK-V2-IC4: [[VECTOR_BODY]]:307; CHECK-V2-IC4: br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]308; CHECK-V2-IC4: [[MIDDLE_BLOCK]]:309; CHECK-V2-IC4: br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]310; CHECK-V2-IC4: [[VEC_EPILOG_ITER_CHECK]]:311; CHECK-V2-IC4: br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF7]]312; CHECK-V2-IC4: [[VEC_EPILOG_PH]]:313; CHECK-V2-IC4: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]314; CHECK-V2-IC4: [[VEC_EPILOG_VECTOR_BODY]]:315; CHECK-V2-IC4: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]316; CHECK-V2-IC4: [[VEC_EPILOG_MIDDLE_BLOCK]]:317; CHECK-V2-IC4: br i1 [[CMP_N10:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]]318; CHECK-V2-IC4: [[VEC_EPILOG_SCALAR_PH]]:319; CHECK-V2-IC4: br label %[[FOR_BODY:.*]]320; CHECK-V2-IC4: [[FOR_BODY]]:321; CHECK-V2-IC4: br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]322; CHECK-V2-IC4: [[FOR_COND_CLEANUP]]:323;324entry:325 br label %for.body326 327for.body: ; preds = %for.body, %entry328 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]329 %arrayidx = getelementptr inbounds [1024 x i32], ptr @b, i64 0, i64 %iv330 %load = load i32, ptr %arrayidx, align 4331 %arrayidx2 = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 %iv332 store i32 %load, ptr %arrayidx2, align 4333 %iv.next = add nuw nsw i64 %iv, 1334 %exitcond = icmp eq i64 %iv.next, %n335 br i1 %exitcond, label %for.cond.cleanup, label %for.body336 337for.cond.cleanup: ; preds = %for.body338 ret void339}340 341!0 = !{!"branch_weights", i32 1, i32 1023}342;.343; CHECK-V1-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}344; CHECK-V1-IC1: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}345; CHECK-V1-IC1: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}346; CHECK-V1-IC1: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}347; CHECK-V1-IC1: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 128}348; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 1, i32 7}349; CHECK-V1-IC1: [[PROF6]] = !{!"branch_weights", i32 0, i32 0}350; CHECK-V1-IC1: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META2]], [[META8:![0-9]+]]}351; CHECK-V1-IC1: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 0}352; CHECK-V1-IC1: [[PROF9]] = !{!"branch_weights", i32 1, i32 31}353; CHECK-V1-IC1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META3]], [[META11:![0-9]+]]}354; CHECK-V1-IC1: [[META11]] = !{!"llvm.loop.estimated_trip_count", i32 32}355; CHECK-V1-IC1: [[PROF12]] = !{!"branch_weights", i32 16, i32 16}356; CHECK-V1-IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META8]], [[META3]]}357; CHECK-V1-IC1: [[PROF14]] = !{!"branch_weights", i32 1, i32 15}358; CHECK-V1-IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]], [[META2]], [[META8]]}359; CHECK-V1-IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META3]]}360; CHECK-V1-IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]], [[META2]]}361;.362; CHECK-V1-IC1-FORCE-EPI4: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}363; CHECK-V1-IC1-FORCE-EPI4: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]], [[META4:![0-9]+]]}364; CHECK-V1-IC1-FORCE-EPI4: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}365; CHECK-V1-IC1-FORCE-EPI4: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}366; CHECK-V1-IC1-FORCE-EPI4: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 128}367; CHECK-V1-IC1-FORCE-EPI4: [[PROF5]] = !{!"branch_weights", i32 1, i32 7}368; CHECK-V1-IC1-FORCE-EPI4: [[PROF6]] = !{!"branch_weights", i32 4, i32 4}369; CHECK-V1-IC1-FORCE-EPI4: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META8:![0-9]+]], [[META3]]}370; CHECK-V1-IC1-FORCE-EPI4: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 0}371; CHECK-V1-IC1-FORCE-EPI4: [[PROF9]] = !{!"branch_weights", i32 1, i32 3}372; CHECK-V1-IC1-FORCE-EPI4: [[PROF10]] = !{!"branch_weights", i32 0, i32 0}373; CHECK-V1-IC1-FORCE-EPI4: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META2]], [[META8]]}374; CHECK-V1-IC1-FORCE-EPI4: [[PROF12]] = !{!"branch_weights", i32 1, i32 31}375; CHECK-V1-IC1-FORCE-EPI4: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META3]], [[META14:![0-9]+]]}376; CHECK-V1-IC1-FORCE-EPI4: [[META14]] = !{!"llvm.loop.estimated_trip_count", i32 32}377; CHECK-V1-IC1-FORCE-EPI4: [[PROF15]] = !{!"branch_weights", i32 4, i32 28}378; CHECK-V1-IC1-FORCE-EPI4: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META8]], [[META3]]}379; CHECK-V1-IC1-FORCE-EPI4: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]], [[META2]], [[META8]]}380; CHECK-V1-IC1-FORCE-EPI4: [[LOOP18]] = distinct !{[[LOOP18]], [[META2]], [[META3]]}381; CHECK-V1-IC1-FORCE-EPI4: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META3]]}382; CHECK-V1-IC1-FORCE-EPI4: [[LOOP20]] = distinct !{[[LOOP20]], [[META3]], [[META2]]}383;.384; CHECK-V2-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}385; CHECK-V2-IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 255}386; CHECK-V2-IC1: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}387; CHECK-V2-IC1: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}388; CHECK-V2-IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}389; CHECK-V2-IC1: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 256}390; CHECK-V2-IC1: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}391; CHECK-V2-IC1: [[PROF7]] = !{!"branch_weights", i32 0, i32 0}392; CHECK-V2-IC1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META3]], [[META9:![0-9]+]]}393; CHECK-V2-IC1: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}394; CHECK-V2-IC1: [[PROF10]] = !{!"branch_weights", i32 1, i32 63}395; CHECK-V2-IC1: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]], [[META12:![0-9]+]]}396; CHECK-V2-IC1: [[META12]] = !{!"llvm.loop.estimated_trip_count", i32 64}397; CHECK-V2-IC1: [[PROF13]] = !{!"branch_weights", i32 1, i32 15}398; CHECK-V2-IC1: [[PROF14]] = !{!"branch_weights", i32 4, i32 12}399; CHECK-V2-IC1: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]], [[META9]], [[META4]]}400; CHECK-V2-IC1: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]], [[META3]], [[META9]]}401; CHECK-V2-IC1: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]], [[META4]]}402; CHECK-V2-IC1: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]], [[META3]]}403;.404; CHECK-V2-IC4: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}405; CHECK-V2-IC4: [[PROF1]] = !{!"branch_weights", i32 1, i32 63}406; CHECK-V2-IC4: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]], [[META5:![0-9]+]]}407; CHECK-V2-IC4: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}408; CHECK-V2-IC4: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}409; CHECK-V2-IC4: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 64}410; CHECK-V2-IC4: [[PROF6]] = !{!"branch_weights", i32 1, i32 15}411; CHECK-V2-IC4: [[PROF7]] = !{!"branch_weights", i32 4, i32 12}412; CHECK-V2-IC4: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META9:![0-9]+]], [[META4]]}413; CHECK-V2-IC4: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}414; CHECK-V2-IC4: [[PROF10]] = !{!"branch_weights", i32 1, i32 3}415; CHECK-V2-IC4: [[PROF11]] = !{!"branch_weights", i32 0, i32 0}416; CHECK-V2-IC4: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META3]], [[META9]]}417; CHECK-V2-IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META4]], [[META14:![0-9]+]]}418; CHECK-V2-IC4: [[META14]] = !{!"llvm.loop.estimated_trip_count", i32 16}419; CHECK-V2-IC4: [[PROF15]] = !{!"branch_weights", i32 8, i32 56}420; CHECK-V2-IC4: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META9]], [[META4]]}421; CHECK-V2-IC4: [[PROF17]] = !{!"branch_weights", i32 1, i32 7}422; CHECK-V2-IC4: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]], [[META3]], [[META9]]}423; CHECK-V2-IC4: [[LOOP19]] = distinct !{[[LOOP19]], [[META3]], [[META4]]}424; CHECK-V2-IC4: [[LOOP20]] = distinct !{[[LOOP20]], [[META3]], [[META4]]}425; CHECK-V2-IC4: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]], [[META3]]}426;.427