693 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^loop:"2; REQUIRES: asserts3; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -epilogue-vectorization-minimum-VF=0 -prefer-predicate-over-epilogue=scalar-epilogue \4; RUN: -debug-only=loop-vectorize -force-target-instruction-cost=1 -S 2>%t | FileCheck %s --check-prefix=CHECK5; RUN: cat %t | FileCheck %s --check-prefix=DEBUG6; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -epilogue-vectorization-force-VF=8 \7; RUN: -debug-only=loop-vectorize -S 2>%t | FileCheck %s --check-prefix=CHECK-VF88; RUN: cat %t | FileCheck %s --check-prefix=DEBUG-FORCED9 10target triple = "aarch64-linux-gnu"11 12; DEBUG: LV: Checking a loop in 'main_vf_vscale_x_16'13; DEBUG-NOT: Create Skeleton for epilogue vectorized loop (first pass)14; DEBUG: Executing best plan with VF=vscale x 16, UF=215 16; DEBUG-FORCED: LV: Checking a loop in 'main_vf_vscale_x_16'17; DEBUG-FORCED: LEV: Epilogue vectorization factor is forced.18; DEBUG-FORCED: Create Skeleton for epilogue vectorized loop (first pass)19; DEBUG-FORCED: Main Loop VF:vscale x 16, Main Loop UF:2, Epilogue Loop VF:8, Epilogue Loop UF:120 21define void @main_vf_vscale_x_16(ptr %A) #0 {22; CHECK-LABEL: @main_vf_vscale_x_16(23; CHECK-NEXT: entry:24; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()25; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 526; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]27; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]28; CHECK: vector.ph:29; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()30; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 3231; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]32; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]33; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]34; CHECK: vector.body:35; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]36; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]37; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()38; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 439; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 [[TMP6]]40; CHECK-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP4]], align 141; CHECK-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP7]], align 142; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]43; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]44; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_ITER_CHECK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]45; CHECK: middle.block:46; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]47; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]48; CHECK: scalar.ph:49; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]] ]50; CHECK-NEXT: br label [[FOR_BODY:%.*]]51; CHECK: for.body:52;53; CHECK-VF8-LABEL: @main_vf_vscale_x_16(54; CHECK-VF8-NEXT: iter.check:55; CHECK-VF8-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]56; CHECK-VF8: vector.main.loop.iter.check:57; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()58; CHECK-VF8-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 559; CHECK-VF8-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]60; CHECK-VF8-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]61; CHECK-VF8: vector.ph:62; CHECK-VF8-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()63; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 3264; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]65; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]66; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]67; CHECK-VF8: vector.body:68; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]69; CHECK-VF8-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]70; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()71; CHECK-VF8-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 472; CHECK-VF8-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 [[TMP6]]73; CHECK-VF8-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP4]], align 174; CHECK-VF8-NEXT: store <vscale x 16 x i8> splat (i8 1), ptr [[TMP7]], align 175; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]76; CHECK-VF8-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]77; CHECK-VF8-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]78; CHECK-VF8: middle.block:79; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]80; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]81; CHECK-VF8: vec.epilog.iter.check:82; CHECK-VF8-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 883; CHECK-VF8-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]84; CHECK-VF8: vec.epilog.ph:85; CHECK-VF8-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]86; CHECK-VF8-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]87; CHECK-VF8: vec.epilog.vector.body:88; CHECK-VF8-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]89; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX1]]90; CHECK-VF8-NEXT: store <8 x i8> splat (i8 1), ptr [[TMP9]], align 191; CHECK-VF8-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 892; CHECK-VF8-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 102493; CHECK-VF8-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]94; CHECK-VF8: vec.epilog.middle.block:95; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]96; CHECK-VF8: vec.epilog.scalar.ph:97; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]98; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]99; CHECK-VF8: for.body:100;101entry:102 br label %for.body103 104for.body:105 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]106 %arrayidx = getelementptr inbounds i8, ptr %A, i64 %iv107 store i8 1, ptr %arrayidx, align 1108 %iv.next = add nuw nsw i64 %iv, 1109 %exitcond = icmp ne i64 %iv.next, 1024110 br i1 %exitcond, label %for.body, label %exit111 112exit:113 ret void114}115 116define void @main_vf_vscale_x_2_no_epi_iteration(ptr %A) #0 vscale_range(8, 8) {117; CHECK-LABEL: @main_vf_vscale_x_2_no_epi_iteration(118; CHECK-NEXT: entry:119; CHECK-NEXT: br label [[VECTOR_PH:%.*]]120; CHECK: vector.ph:121; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()122; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4123; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]124; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]125; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]126; CHECK: vector.body:127; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]128; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]129; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()130; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1131; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 [[TMP4]]132; CHECK-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP2]], align 1133; CHECK-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP5]], align 1134; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]135; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]136; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]137; CHECK: middle.block:138; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]139; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]140; CHECK: scalar.ph:141; CHECK-NEXT: br label [[FOR_BODY:%.*]]142; CHECK: for.body:143;144; CHECK-VF8-LABEL: @main_vf_vscale_x_2_no_epi_iteration(145; CHECK-VF8-NEXT: iter.check:146; CHECK-VF8-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]147; CHECK-VF8: vector.main.loop.iter.check:148; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()149; CHECK-VF8-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2150; CHECK-VF8-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]151; CHECK-VF8-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]152; CHECK-VF8: vector.ph:153; CHECK-VF8-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()154; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4155; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]156; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]157; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]158; CHECK-VF8: vector.body:159; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]160; CHECK-VF8-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]161; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()162; CHECK-VF8-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1163; CHECK-VF8-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP6]]164; CHECK-VF8-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP4]], align 1165; CHECK-VF8-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP7]], align 1166; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]167; CHECK-VF8-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]168; CHECK-VF8-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]169; CHECK-VF8: middle.block:170; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]171; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]172; CHECK-VF8: vec.epilog.iter.check:173; CHECK-VF8-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8174; CHECK-VF8-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]175; CHECK-VF8: vec.epilog.ph:176; CHECK-VF8-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]177; CHECK-VF8-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]178; CHECK-VF8: vec.epilog.vector.body:179; CHECK-VF8-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]180; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX1]]181; CHECK-VF8-NEXT: store <8 x i64> splat (i64 1), ptr [[TMP9]], align 1182; CHECK-VF8-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 8183; CHECK-VF8-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 1024184; CHECK-VF8-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]185; CHECK-VF8: vec.epilog.middle.block:186; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]187; CHECK-VF8: vec.epilog.scalar.ph:188; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]189; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]190; CHECK-VF8: for.body:191;192entry:193 br label %for.body194 195for.body:196 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]197 %arrayidx = getelementptr inbounds i64, ptr %A, i64 %iv198 store i64 1, ptr %arrayidx, align 1199 %iv.next = add nuw nsw i64 %iv, 1200 %exitcond = icmp ne i64 %iv.next, 1024201 br i1 %exitcond, label %for.body, label %exit202 203exit:204 ret void205}206 207; DEBUG: LV: Checking a loop in 'main_vf_vscale_x_2'208; DEBUG: Create Skeleton for epilogue vectorized loop (first pass)209; DEBUG: Main Loop VF:vscale x 2, Main Loop UF:2, Epilogue Loop VF:8, Epilogue Loop UF:1210 211; DEBUG-FORCED: LV: Checking a loop in 'main_vf_vscale_x_2'212; DEBUG-FORCED: LEV: Epilogue vectorization factor is forced.213; DEBUG-FORCED: Create Skeleton for epilogue vectorized loop (first pass)214; DEBUG-FORCED: Main Loop VF:vscale x 2, Main Loop UF:2, Epilogue Loop VF:8, Epilogue Loop UF:1215 216; When the vector.body uses VF=vscale x 1 (or VF=vscale x 2 because217; that's the minimum supported VF by SVE), we could still use a wide218; fixed-width VF=8 for the epilogue if the vectors are known to be219; sufficiently wide. This information can be deduced from vscale_range or220; VScaleForTuning (set by mcpu/mtune).221define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {222; CHECK-LABEL: @main_vf_vscale_x_2(223; CHECK-NEXT: iter.check:224; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8225; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]226; CHECK: vector.main.loop.iter.check:227; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()228; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2229; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]230; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]231; CHECK: vector.ph:232; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()233; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4234; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]235; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]236; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]237; CHECK: vector.body:238; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]239; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]240; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()241; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1242; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP6]]243; CHECK-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP4]], align 1244; CHECK-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP7]], align 1245; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]246; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]247; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]248; CHECK: middle.block:249; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]250; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]251; CHECK: vec.epilog.iter.check:252; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8253; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF7:![0-9]+]]254; CHECK: vec.epilog.ph:255; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]256; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 8257; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]258; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]259; CHECK: vec.epilog.vector.body:260; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]261; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX4]]262; CHECK-NEXT: store <8 x i64> splat (i64 1), ptr [[TMP9]], align 1263; CHECK-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8264; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]265; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]266; CHECK: vec.epilog.middle.block:267; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]268; CHECK-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]269; CHECK: vec.epilog.scalar.ph:270; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]271; CHECK-NEXT: br label [[FOR_BODY:%.*]]272; CHECK: for.body:273;274; CHECK-VF8-LABEL: @main_vf_vscale_x_2(275; CHECK-VF8-NEXT: iter.check:276; CHECK-VF8-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8277; CHECK-VF8-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]278; CHECK-VF8: vector.main.loop.iter.check:279; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()280; CHECK-VF8-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2281; CHECK-VF8-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]282; CHECK-VF8-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]283; CHECK-VF8: vector.ph:284; CHECK-VF8-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()285; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4286; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]287; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]288; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]289; CHECK-VF8: vector.body:290; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]291; CHECK-VF8-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]292; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()293; CHECK-VF8-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1294; CHECK-VF8-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i64 [[TMP6]]295; CHECK-VF8-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP4]], align 1296; CHECK-VF8-NEXT: store <vscale x 2 x i64> splat (i64 1), ptr [[TMP7]], align 1297; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]298; CHECK-VF8-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]299; CHECK-VF8-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]300; CHECK-VF8: middle.block:301; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]302; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]303; CHECK-VF8: vec.epilog.iter.check:304; CHECK-VF8-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8305; CHECK-VF8-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]306; CHECK-VF8: vec.epilog.ph:307; CHECK-VF8-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]308; CHECK-VF8-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 8309; CHECK-VF8-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]310; CHECK-VF8-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]311; CHECK-VF8: vec.epilog.vector.body:312; CHECK-VF8-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]313; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX4]]314; CHECK-VF8-NEXT: store <8 x i64> splat (i64 1), ptr [[TMP9]], align 1315; CHECK-VF8-NEXT: [[INDEX_NEXT5]] = add nuw i64 [[INDEX4]], 8316; CHECK-VF8-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]317; CHECK-VF8-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]318; CHECK-VF8: vec.epilog.middle.block:319; CHECK-VF8-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]320; CHECK-VF8-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]321; CHECK-VF8: vec.epilog.scalar.ph:322; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]323; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]324; CHECK-VF8: for.body:325;326entry:327 br label %for.body328 329for.body:330 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]331 %arrayidx = getelementptr inbounds i64, ptr %A, i64 %iv332 store i64 1, ptr %arrayidx, align 1333 %iv.next = add nuw nsw i64 %iv, 1334 %exitcond = icmp ne i64 %iv.next, %n335 br i1 %exitcond, label %for.body, label %exit336 337exit:338 ret void339}340 341; FIXME: The epilogue loop is currently miscompiled: the pointer induction342; uses an incorrect resume value.343define void @test_pr57912_pointer_induction(ptr %start) #0 {344; CHECK-LABEL: @test_pr57912_pointer_induction(345; CHECK-NEXT: iter.check:346; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]347; CHECK: vector.main.loop.iter.check:348; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()349; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5350; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]351; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]352; CHECK: vector.ph:353; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()354; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32355; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]]356; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]357; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]358; CHECK: vector.body:359; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]360; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]361; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()362; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4363; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP5]]364; CHECK-NEXT: store <vscale x 16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1365; CHECK-NEXT: store <vscale x 16 x i8> zeroinitializer, ptr [[TMP6]], align 1366; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]367; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]368; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]369; CHECK: middle.block:370; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 10000, [[N_VEC]]371; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]372; CHECK: vec.epilog.iter.check:373; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]374; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8375; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF7]]376; CHECK: vec.epilog.ph:377; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]378; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[START]], i64 10000379; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]380; CHECK: vec.epilog.vector.body:381; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]382; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX1]]383; CHECK-NEXT: store <8 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1384; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 8385; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 10000386; CHECK-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]387; CHECK: vec.epilog.middle.block:388; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]389; CHECK: vec.epilog.scalar.ph:390; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]391; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]392; CHECK-NEXT: br label [[LOOP:%.*]]393; CHECK: loop:394;395; CHECK-VF8-LABEL: @test_pr57912_pointer_induction(396; CHECK-VF8-NEXT: iter.check:397; CHECK-VF8-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]398; CHECK-VF8: vector.main.loop.iter.check:399; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()400; CHECK-VF8-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5401; CHECK-VF8-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]402; CHECK-VF8-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]403; CHECK-VF8: vector.ph:404; CHECK-VF8-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()405; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32406; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]]407; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]408; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]409; CHECK-VF8: vector.body:410; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]411; CHECK-VF8-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]412; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()413; CHECK-VF8-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4414; CHECK-VF8-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP5]]415; CHECK-VF8-NEXT: store <vscale x 16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1416; CHECK-VF8-NEXT: store <vscale x 16 x i8> zeroinitializer, ptr [[TMP6]], align 1417; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]418; CHECK-VF8-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]419; CHECK-VF8-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]420; CHECK-VF8: middle.block:421; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 10000, [[N_VEC]]422; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]423; CHECK-VF8: vec.epilog.iter.check:424; CHECK-VF8-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]425; CHECK-VF8-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8426; CHECK-VF8-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]427; CHECK-VF8: vec.epilog.ph:428; CHECK-VF8-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]429; CHECK-VF8-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[START]], i64 10000430; CHECK-VF8-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]431; CHECK-VF8: vec.epilog.vector.body:432; CHECK-VF8-NEXT: [[INDEX1:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]433; CHECK-VF8-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX1]]434; CHECK-VF8-NEXT: store <8 x i8> zeroinitializer, ptr [[NEXT_GEP2]], align 1435; CHECK-VF8-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 8436; CHECK-VF8-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 10000437; CHECK-VF8-NEXT: br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]438; CHECK-VF8: vec.epilog.middle.block:439; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]440; CHECK-VF8: vec.epilog.scalar.ph:441; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]442; CHECK-VF8-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP8]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]443; CHECK-VF8-NEXT: br label [[LOOP:%.*]]444; CHECK-VF8: loop:445;446entry:447 br label %loop448 449loop:450 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]451 %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ]452 store i8 0, ptr %ptr.iv, align 1453 %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 1454 %iv.next = add nuw nsw i64 %iv, 1455 %cmp = icmp eq i64 %iv.next, 10000456 br i1 %cmp, label %exit, label %loop457 458exit:459 ret void460}461 462; Loop with vscale-based trip count vscale x 1033.463define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 16) #0 {464; CHECK-LABEL: @trip_count_vscale(465; CHECK-NEXT: iter.check:466; CHECK-NEXT: [[V:%.*]] = tail call i64 @llvm.vscale.i64()467; CHECK-NEXT: [[N:%.*]] = mul nuw nsw i64 [[V]], 1033468; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]469; CHECK: vector.main.loop.iter.check:470; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()471; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3472; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]473; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]474; CHECK: vector.ph:475; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()476; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8477; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]478; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]479; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]480; CHECK: vector.body:481; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]482; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]483; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()484; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2485; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[TMP4]], i64 [[TMP6]]486; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4487; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4488; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]489; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()490; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2491; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP10]]492; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4493; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4494; CHECK-NEXT: [[TMP12:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]495; CHECK-NEXT: [[TMP13:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]496; CHECK-NEXT: store <vscale x 4 x float> [[TMP12]], ptr [[TMP8]], align 4497; CHECK-NEXT: store <vscale x 4 x float> [[TMP13]], ptr [[TMP11]], align 4498; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]499; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]500; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]501; CHECK: middle.block:502; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]503; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]504; CHECK: vec.epilog.iter.check:505; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 2506; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF14:![0-9]+]]507; CHECK: vec.epilog.ph:508; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]509; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[N]], 2510; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[N]], [[N_MOD_VF4]]511; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]512; CHECK: vec.epilog.vector.body:513; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]514; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX6]]515; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <2 x float>, ptr [[TMP15]], align 4516; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX6]]517; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <2 x float>, ptr [[TMP16]], align 4518; CHECK-NEXT: [[TMP17:%.*]] = fmul <2 x float> [[WIDE_LOAD7]], [[WIDE_LOAD8]]519; CHECK-NEXT: store <2 x float> [[TMP17]], ptr [[TMP16]], align 4520; CHECK-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX6]], 2521; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC5]]522; CHECK-NEXT: br i1 [[TMP18]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]523; CHECK: vec.epilog.middle.block:524; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]525; CHECK-NEXT: br i1 [[CMP_N10]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]526; CHECK: vec.epilog.scalar.ph:527; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]528; CHECK-NEXT: br label [[FOR_BODY:%.*]]529; CHECK: for.body:530;531; CHECK-VF8-LABEL: @trip_count_vscale(532; CHECK-VF8-NEXT: entry:533; CHECK-VF8-NEXT: [[V:%.*]] = tail call i64 @llvm.vscale.i64()534; CHECK-VF8-NEXT: [[N:%.*]] = mul nuw nsw i64 [[V]], 1033535; CHECK-VF8-NEXT: br label [[VECTOR_PH:%.*]]536; CHECK-VF8: vector.ph:537; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()538; CHECK-VF8-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8539; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]540; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]541; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]542; CHECK-VF8: vector.body:543; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]544; CHECK-VF8-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]545; CHECK-VF8-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()546; CHECK-VF8-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2547; CHECK-VF8-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP2]], i64 [[TMP4]]548; CHECK-VF8-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 4549; CHECK-VF8-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4550; CHECK-VF8-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]551; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()552; CHECK-VF8-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2553; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP8]]554; CHECK-VF8-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4555; CHECK-VF8-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4556; CHECK-VF8-NEXT: [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]557; CHECK-VF8-NEXT: [[TMP11:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]558; CHECK-VF8-NEXT: store <vscale x 4 x float> [[TMP10]], ptr [[TMP6]], align 4559; CHECK-VF8-NEXT: store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4560; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]561; CHECK-VF8-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]562; CHECK-VF8-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]563; CHECK-VF8: middle.block:564; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]565; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]566; CHECK-VF8: scalar.ph:567; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]568; CHECK-VF8: for.body:569;570entry:571 %v = tail call i64 @llvm.vscale.i64()572 %n = mul nuw nsw i64 %v, 1033573 br label %for.body574 575for.body:576 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]577 %arrayidx = getelementptr inbounds nuw float, ptr %a, i64 %iv578 %l.a = load float, ptr %arrayidx, align 4579 %arrayidx3 = getelementptr inbounds nuw float, ptr %b, i64 %iv580 %l.b = load float, ptr %arrayidx3, align 4581 %mul4 = fmul float %l.a, %l.b582 store float %mul4, ptr %arrayidx3, align 4583 %iv.next = add nuw nsw i64 %iv, 1584 %ec = icmp eq i64 %iv.next, %n585 br i1 %ec, label %exit, label %for.body586 587exit:588 ret void589}590 591; Loop with vscale-based trip count vscale x 1024.592define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalias %b) vscale_range(1, 16) #0 {593; CHECK-LABEL: @trip_count_vscale_no_epilogue_iterations(594; CHECK-NEXT: entry:595; CHECK-NEXT: [[V:%.*]] = tail call i64 @llvm.vscale.i64()596; CHECK-NEXT: [[N:%.*]] = mul nuw nsw i64 [[V]], 1024597; CHECK-NEXT: br label [[VECTOR_PH:%.*]]598; CHECK: vector.ph:599; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()600; CHECK-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8601; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]602; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]603; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]604; CHECK: vector.body:605; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]606; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]607; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()608; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2609; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP2]], i64 [[TMP4]]610; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 4611; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4612; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]613; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()614; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2615; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP8]]616; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4617; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4618; CHECK-NEXT: [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]619; CHECK-NEXT: [[TMP11:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]620; CHECK-NEXT: store <vscale x 4 x float> [[TMP10]], ptr [[TMP6]], align 4621; CHECK-NEXT: store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4622; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]623; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]624; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]625; CHECK: middle.block:626; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]627; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]628; CHECK: scalar.ph:629; CHECK-NEXT: br label [[FOR_BODY:%.*]]630; CHECK: for.body:631;632; CHECK-VF8-LABEL: @trip_count_vscale_no_epilogue_iterations(633; CHECK-VF8-NEXT: entry:634; CHECK-VF8-NEXT: [[V:%.*]] = tail call i64 @llvm.vscale.i64()635; CHECK-VF8-NEXT: [[N:%.*]] = mul nuw nsw i64 [[V]], 1024636; CHECK-VF8-NEXT: br label [[VECTOR_PH:%.*]]637; CHECK-VF8: vector.ph:638; CHECK-VF8-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()639; CHECK-VF8-NEXT: [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8640; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]641; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]642; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]]643; CHECK-VF8: vector.body:644; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]645; CHECK-VF8-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]646; CHECK-VF8-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()647; CHECK-VF8-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2648; CHECK-VF8-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP2]], i64 [[TMP4]]649; CHECK-VF8-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP2]], align 4650; CHECK-VF8-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4651; CHECK-VF8-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]652; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()653; CHECK-VF8-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2654; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP8]]655; CHECK-VF8-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4656; CHECK-VF8-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4657; CHECK-VF8-NEXT: [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]658; CHECK-VF8-NEXT: [[TMP11:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]659; CHECK-VF8-NEXT: store <vscale x 4 x float> [[TMP10]], ptr [[TMP6]], align 4660; CHECK-VF8-NEXT: store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4661; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]662; CHECK-VF8-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]663; CHECK-VF8-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]664; CHECK-VF8: middle.block:665; CHECK-VF8-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]666; CHECK-VF8-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]667; CHECK-VF8: scalar.ph:668; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]669; CHECK-VF8: for.body:670;671entry:672 %v = tail call i64 @llvm.vscale.i64()673 %n = mul nuw nsw i64 %v, 1024674 br label %for.body675 676for.body:677 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]678 %arrayidx = getelementptr inbounds nuw float, ptr %a, i64 %iv679 %l.a = load float, ptr %arrayidx, align 4680 %arrayidx3 = getelementptr inbounds nuw float, ptr %b, i64 %iv681 %l.b = load float, ptr %arrayidx3, align 4682 %mul4 = fmul float %l.a, %l.b683 store float %mul4, ptr %arrayidx3, align 4684 %iv.next = add nuw nsw i64 %iv, 1685 %ec = icmp eq i64 %iv.next, %n686 br i1 %ec, label %exit, label %for.body687 688exit:689 ret void690}691 692attributes #0 = { "target-features"="+sve" }693