440 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 62; RUN: opt -p loop-vectorize -force-target-instruction-cost=1 -S %s | FileCheck --check-prefixes=COMMON,COST1 %s3; RUN: opt -p loop-vectorize -force-target-instruction-cost=10 -S %s | FileCheck --check-prefixes=COMMON,COST10 %s4 5target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"6target triple = "arm64-apple-macosx14.0.0"7 8define double @test_reduction_costs() {9; COMMON-LABEL: define double @test_reduction_costs() {10; COMMON-NEXT: [[ENTRY:.*:]]11; COMMON-NEXT: br label %[[VECTOR_PH:.*]]12; COMMON: [[VECTOR_PH]]:13; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]14; COMMON: [[VECTOR_BODY]]:15; COMMON-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]16; COMMON-NEXT: [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]17; COMMON-NEXT: [[TMP0]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> splat (double 3.000000e+00))18; COMMON-NEXT: [[TMP1]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI1]], <2 x double> splat (double 9.000000e+00))19; COMMON-NEXT: br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]20; COMMON: [[MIDDLE_BLOCK]]:21; COMMON-NEXT: br label %[[EXIT:.*]]22; COMMON: [[EXIT]]:23; COMMON-NEXT: [[DIV:%.*]] = fmul double [[TMP0]], [[TMP1]]24; COMMON-NEXT: ret double [[DIV]]25;26entry:27 br label %loop.128 29loop.1:30 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.1 ]31 %r.1 = phi double [ 0.000000e+00, %entry ], [ %r.1.next, %loop.1 ]32 %r.2 = phi double [ 0.000000e+00, %entry ], [ %r.2.next, %loop.1 ]33 %r.1.next = fadd double %r.1, 3.000000e+0034 %r.2.next = fadd double %r.2, 9.000000e+0035 %iv.next = add i64 %iv, 136 %ec = icmp eq i64 %iv, 137 br i1 %ec, label %exit, label %loop.138 39exit:40 %div = fmul double %r.1.next, %r.2.next41 ret double %div42}43 44define void @test_iv_cost(ptr %ptr.start, i8 %a, i64 %b) {45; COST1-LABEL: define void @test_iv_cost(46; COST1-SAME: ptr [[PTR_START:%.*]], i8 [[A:%.*]], i64 [[B:%.*]]) {47; COST1-NEXT: [[ENTRY:.*:]]48; COST1-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i6449; COST1-NEXT: [[START:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 [[A_EXT]])50; COST1-NEXT: [[C:%.*]] = icmp eq i64 [[START]], 051; COST1-NEXT: br i1 [[C]], [[EXIT:label %.*]], label %[[ITER_CHECK:.*]]52; COST1: [[ITER_CHECK]]:53; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[START]], 454; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]55; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:56; COST1-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[START]], 3257; COST1-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]58; COST1: [[VECTOR_PH]]:59; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 3260; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]]61; COST1-NEXT: br label %[[VECTOR_BODY:.*]]62; COST1: [[VECTOR_BODY]]:63; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]64; COST1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]]65; COST1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 1666; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 167; COST1-NEXT: store <16 x i8> zeroinitializer, ptr [[TMP0]], align 168; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 3269; COST1-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]70; COST1-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]71; COST1: [[MIDDLE_BLOCK]]:72; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[START]], [[N_VEC]]73; COST1-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]74; COST1: [[VEC_EPILOG_ITER_CHECK]]:75; COST1-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]]76; COST1-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]]77; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 478; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]]79; COST1: [[VEC_EPILOG_PH]]:80; COST1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]81; COST1-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[START]], 482; COST1-NEXT: [[N_VEC3:%.*]] = sub i64 [[START]], [[N_MOD_VF2]]83; COST1-NEXT: [[TMP2:%.*]] = sub i64 [[START]], [[N_VEC3]]84; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC3]]85; COST1-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]86; COST1: [[VEC_EPILOG_VECTOR_BODY]]:87; COST1-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]88; COST1-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX4]]89; COST1-NEXT: store <4 x i8> zeroinitializer, ptr [[NEXT_GEP5]], align 190; COST1-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 491; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]92; COST1-NEXT: br i1 [[TMP4]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]93; COST1: [[VEC_EPILOG_MIDDLE_BLOCK]]:94; COST1-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[START]], [[N_VEC3]]95; COST1-NEXT: br i1 [[CMP_N7]], [[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]96; COST1: [[VEC_EPILOG_SCALAR_PH]]:97;98; COST10-LABEL: define void @test_iv_cost(99; COST10-SAME: ptr [[PTR_START:%.*]], i8 [[A:%.*]], i64 [[B:%.*]]) {100; COST10-NEXT: [[ENTRY:.*:]]101; COST10-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i64102; COST10-NEXT: [[START:%.*]] = call i64 @llvm.umin.i64(i64 [[B]], i64 [[A_EXT]])103; COST10-NEXT: [[C:%.*]] = icmp eq i64 [[START]], 0104; COST10-NEXT: br i1 [[C]], [[EXIT:label %.*]], label %[[ITER_CHECK:.*]]105; COST10: [[ITER_CHECK]]:106; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[START]], 4107; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]108; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:109; COST10-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[START]], 16110; COST10-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]111; COST10: [[VECTOR_PH]]:112; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[START]], 16113; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[START]], [[N_MOD_VF]]114; COST10-NEXT: br label %[[VECTOR_BODY:.*]]115; COST10: [[VECTOR_BODY]]:116; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]117; COST10-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX]]118; COST10-NEXT: store <16 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1119; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16120; COST10-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]121; COST10-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]122; COST10: [[MIDDLE_BLOCK]]:123; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[START]], [[N_VEC]]124; COST10-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]125; COST10: [[VEC_EPILOG_ITER_CHECK]]:126; COST10-NEXT: [[IND_END:%.*]] = sub i64 [[START]], [[N_VEC]]127; COST10-NEXT: [[IND_END9:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC]]128; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4129; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF4:![0-9]+]]130; COST10: [[VEC_EPILOG_PH]]:131; COST10-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]132; COST10-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[START]], 4133; COST10-NEXT: [[N_VEC3:%.*]] = sub i64 [[START]], [[N_MOD_VF2]]134; COST10-NEXT: [[TMP1:%.*]] = sub i64 [[START]], [[N_VEC3]]135; COST10-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[N_VEC3]]136; COST10-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]137; COST10: [[VEC_EPILOG_VECTOR_BODY]]:138; COST10-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]139; COST10-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[INDEX4]]140; COST10-NEXT: store <4 x i8> zeroinitializer, ptr [[NEXT_GEP5]], align 1141; COST10-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4142; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]143; COST10-NEXT: br i1 [[TMP3]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]144; COST10: [[VEC_EPILOG_MIDDLE_BLOCK]]:145; COST10-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[START]], [[N_VEC3]]146; COST10-NEXT: br i1 [[CMP_N7]], [[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]]147; COST10: [[VEC_EPILOG_SCALAR_PH]]:148;149entry:150 %a.ext = zext i8 %a to i64151 %start = call i64 @llvm.umin.i64(i64 %b, i64 %a.ext)152 %c = icmp eq i64 %start, 0153 br i1 %c, label %exit, label %loop154 155loop:156 %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ]157 %ptr.iv = phi ptr [ %ptr.start, %entry ], [ %ptr.iv.next, %loop ]158 %iv.next = add i64 %iv, -1159 %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 1160 store i8 0, ptr %ptr.iv, align 1161 %ec = icmp eq i64 %iv.next, 0162 br i1 %ec, label %exit, label %loop163 164exit:165 ret void166}167 168define void @test_exit_branch_cost(ptr %dst, ptr noalias %x.ptr, ptr noalias %y.ptr, ptr noalias %dst.1, i1 %c.4, ptr %src, ptr noalias %dst.3, i1 %c.3, ptr noalias %dst.2) {169; COMMON-LABEL: define void @test_exit_branch_cost(170; COMMON-SAME: ptr [[DST:%.*]], ptr noalias [[X_PTR:%.*]], ptr noalias [[Y_PTR:%.*]], ptr noalias [[DST_1:%.*]], i1 [[C_4:%.*]], ptr [[SRC:%.*]], ptr noalias [[DST_3:%.*]], i1 [[C_3:%.*]], ptr noalias [[DST_2:%.*]]) {171; COMMON-NEXT: [[ENTRY:.*:]]172; COMMON-NEXT: br label %[[VECTOR_MEMCHECK:.*]]173; COMMON: [[VECTOR_MEMCHECK]]:174; COMMON-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8175; COMMON-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 8176; COMMON-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]177; COMMON-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]178; COMMON-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]179; COMMON-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]180; COMMON: [[VECTOR_PH]]:181; COMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C_3]], i64 0182; COMMON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer183; COMMON-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x i1> poison, i1 [[C_4]], i64 0184; COMMON-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT2]], <2 x i1> poison, <2 x i32> zeroinitializer185; COMMON-NEXT: [[TMP0:%.*]] = select i1 [[C_4]], <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> zeroinitializer186; COMMON-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP0]], splat (i1 true)187; COMMON-NEXT: [[TMP2:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> [[BROADCAST_SPLAT3]], <2 x i1> zeroinitializer188; COMMON-NEXT: [[TMP3:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT3]], splat (i1 true)189; COMMON-NEXT: [[TMP4:%.*]] = select <2 x i1> [[BROADCAST_SPLAT]], <2 x i1> [[TMP3]], <2 x i1> zeroinitializer190; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]191; COMMON: [[VECTOR_BODY]]:192; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE17:.*]] ]193; COMMON-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[X_PTR]], i64 [[INDEX]]194; COMMON-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 8195; COMMON-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[WIDE_LOAD]], zeroinitializer196; COMMON-NEXT: [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true)197; COMMON-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0198; COMMON-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]199; COMMON: [[PRED_STORE_IF]]:200; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8201; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE]]202; COMMON: [[PRED_STORE_CONTINUE]]:203; COMMON-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1204; COMMON-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]205; COMMON: [[PRED_STORE_IF4]]:206; COMMON-NEXT: store i64 0, ptr [[DST_1]], align 8207; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE5]]208; COMMON: [[PRED_STORE_CONTINUE5]]:209; COMMON-NEXT: [[TMP10:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP1]], <2 x i1> zeroinitializer210; COMMON-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0211; COMMON-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7:.*]]212; COMMON: [[PRED_STORE_IF6]]:213; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8214; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE7]]215; COMMON: [[PRED_STORE_CONTINUE7]]:216; COMMON-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP10]], i32 1217; COMMON-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF8:.*]], label %[[PRED_STORE_CONTINUE9:.*]]218; COMMON: [[PRED_STORE_IF8]]:219; COMMON-NEXT: store i64 0, ptr [[DST_3]], align 8220; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE9]]221; COMMON: [[PRED_STORE_CONTINUE9]]:222; COMMON-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP2]], <2 x i1> zeroinitializer223; COMMON-NEXT: [[TMP14:%.*]] = or <2 x i1> [[TMP6]], [[TMP13]]224; COMMON-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP13]], <2 x i64> zeroinitializer, <2 x i64> splat (i64 1)225; COMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[TMP14]], i32 0226; COMMON-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF10:.*]], label %[[PRED_STORE_CONTINUE11:.*]]227; COMMON: [[PRED_STORE_IF10]]:228; COMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i64> [[PREDPHI]], i32 0229; COMMON-NEXT: store i64 [[TMP16]], ptr [[DST_2]], align 8230; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE11]]231; COMMON: [[PRED_STORE_CONTINUE11]]:232; COMMON-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP14]], i32 1233; COMMON-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF12:.*]], label %[[PRED_STORE_CONTINUE13:.*]]234; COMMON: [[PRED_STORE_IF12]]:235; COMMON-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[PREDPHI]], i32 1236; COMMON-NEXT: store i64 [[TMP18]], ptr [[DST_2]], align 8237; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE13]]238; COMMON: [[PRED_STORE_CONTINUE13]]:239; COMMON-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP4]], <2 x i1> zeroinitializer240; COMMON-NEXT: [[TMP20:%.*]] = or <2 x i1> [[TMP14]], [[TMP19]]241; COMMON-NEXT: [[TMP21:%.*]] = extractelement <2 x i1> [[TMP20]], i32 0242; COMMON-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF14:.*]], label %[[PRED_STORE_CONTINUE15:.*]]243; COMMON: [[PRED_STORE_IF14]]:244; COMMON-NEXT: [[TMP22:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META7:![0-9]+]]245; COMMON-NEXT: store i64 [[TMP22]], ptr [[DST]], align 8, !alias.scope [[META10:![0-9]+]], !noalias [[META7]]246; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE15]]247; COMMON: [[PRED_STORE_CONTINUE15]]:248; COMMON-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP20]], i32 1249; COMMON-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF16:.*]], label %[[PRED_STORE_CONTINUE17]]250; COMMON: [[PRED_STORE_IF16]]:251; COMMON-NEXT: [[TMP24:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META7]]252; COMMON-NEXT: store i64 [[TMP24]], ptr [[DST]], align 8, !alias.scope [[META10]], !noalias [[META7]]253; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE17]]254; COMMON: [[PRED_STORE_CONTINUE17]]:255; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2256; COMMON-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64257; COMMON-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]258; COMMON: [[MIDDLE_BLOCK]]:259; COMMON-NEXT: br label %[[SCALAR_PH]]260; COMMON: [[SCALAR_PH]]:261;262entry:263 br label %loop.header264 265loop.header:266 %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]267 %x.gep = getelementptr i64, ptr %x.ptr, i64 %iv268 %x = load i64, ptr %x.gep269 %y.gep = getelementptr i32, ptr %y.ptr, i64 %iv270 %y = load i32, ptr %y.gep271 %c1 = icmp eq i64 %x, 0272 br i1 %c1, label %then.4, label %then.1273 274then.1:275 %and32831 = and i32 %y, 1276 store i64 0, ptr %dst.1, align 8277 %c.2 = icmp eq i32 %y, 0278 %or.cond = select i1 %c.4, i1 %c.3, i1 false279 br i1 %or.cond, label %then.2, label %else.1280 281else.1: ; preds = %then.1282 store i64 0, ptr %dst.3, align 8283 br label %then.2284 285then.2:286 br i1 %c.3, label %then.3, label %loop.latch287 288then.3:289 br i1 %c.4, label %then.5, label %else.2290 291then.4:292 call void @llvm.assume(i1 %c.4)293 br label %then.5294 295then.5:296 %1 = phi i64 [ 1, %then.4 ], [ 0, %then.3 ]297 store i64 %1, ptr %dst.2, align 8298 br label %else.2299 300else.2:301 %l = load i64, ptr %src, align 8302 store i64 %l, ptr %dst, align 8303 br label %loop.latch304 305loop.latch:306 %iv.next = add i64 %iv, 1307 %ec = icmp eq i64 %iv, 64308 br i1 %ec, label %exit, label %loop.header309 310exit:311 ret void312}313 314define void @invalid_legacy_cost(i64 %N, ptr %x) #0 {315; COST1-LABEL: define void @invalid_legacy_cost(316; COST1-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {317; COST1-NEXT: [[ENTRY:.*:]]318; COST1-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1319; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4320; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]321; COST1: [[VECTOR_PH]]:322; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4323; COST1-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]324; COST1-NEXT: br label %[[VECTOR_BODY:.*]]325; COST1: [[VECTOR_BODY]]:326; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]327; COST1-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16328; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0329; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer330; COST1-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]]331; COST1-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[TMP2]], i64 2332; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8333; COST1-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 8334; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4335; COST1-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]336; COST1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]337; COST1: [[MIDDLE_BLOCK]]:338; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]339; COST1-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]340; COST1: [[SCALAR_PH]]:341;342; COST10-LABEL: define void @invalid_legacy_cost(343; COST10-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {344; COST10-NEXT: [[ENTRY:.*:]]345; COST10-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1346; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2347; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]348; COST10: [[VECTOR_PH]]:349; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2350; COST10-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]351; COST10-NEXT: br label %[[VECTOR_BODY:.*]]352; COST10: [[VECTOR_BODY]]:353; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]354; COST10-NEXT: [[TMP1:%.*]] = alloca i8, i64 0, align 16355; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP1]], i64 0356; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer357; COST10-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[X]], i64 [[INDEX]]358; COST10-NEXT: store <2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP2]], align 8359; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2360; COST10-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]361; COST10-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]362; COST10: [[MIDDLE_BLOCK]]:363; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]364; COST10-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]]365; COST10: [[SCALAR_PH]]:366;367entry:368 br label %for.body369 370for.body:371 %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]372 %0 = alloca i8, i64 0, align 16373 %arrayidx = getelementptr ptr, ptr %x, i64 %iv374 store ptr %0, ptr %arrayidx, align 8375 %iv.next = add i64 %iv, 1376 %exitcond.not = icmp eq i64 %iv, %N377 br i1 %exitcond.not, label %for.end, label %for.body378 379for.end:380 ret void381}382 383define void @loop_with_freeze_and_conditional_srem(ptr %dst, ptr %keyinfo, ptr %invariant.ptr, i32 %divisor) #1 {384; COMMON-LABEL: define void @loop_with_freeze_and_conditional_srem(385; COMMON-SAME: ptr [[DST:%.*]], ptr [[KEYINFO:%.*]], ptr [[INVARIANT_PTR:%.*]], i32 [[DIVISOR:%.*]]) {386; COMMON-NEXT: [[ENTRY:.*]]:387; COMMON-NEXT: br label %[[LOOP:.*]]388; COMMON: [[LOOP]]:389; COMMON-NEXT: [[INDEX_NEXT:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]390; COMMON-NEXT: [[LOADED:%.*]] = load i32, ptr [[INVARIANT_PTR]], align 4391; COMMON-NEXT: [[FROZEN:%.*]] = freeze i32 [[LOADED]]392; COMMON-NEXT: [[CMP:%.*]] = icmp eq i32 [[FROZEN]], 0393; COMMON-NEXT: br i1 [[CMP]], label %[[IF_ZERO:.*]], label %[[IF_NONZERO:.*]]394; COMMON: [[IF_ZERO]]:395; COMMON-NEXT: store i32 0, ptr [[KEYINFO]], align 4396; COMMON-NEXT: br label %[[LOOP_LATCH]]397; COMMON: [[IF_NONZERO]]:398; COMMON-NEXT: [[TMP11:%.*]] = srem i32 1, [[DIVISOR]]399; COMMON-NEXT: store i32 [[TMP11]], ptr [[DST]], align 4400; COMMON-NEXT: br label %[[LOOP_LATCH]]401; COMMON: [[LOOP_LATCH]]:402; COMMON-NEXT: [[IV_NEXT]] = add i64 [[INDEX_NEXT]], 1403; COMMON-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32404; COMMON-NEXT: br i1 [[TMP16]], label %[[EXIT:.*]], label %[[LOOP]]405; COMMON: [[EXIT]]:406; COMMON-NEXT: ret void407;408entry:409 br label %loop410 411loop: ; preds = %loop.latch, %entry412 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]413 %loaded = load i32, ptr %invariant.ptr, align 4414 %frozen = freeze i32 %loaded415 %cmp = icmp eq i32 %frozen, 0416 br i1 %cmp, label %if.zero, label %if.nonzero417 418if.zero: ; preds = %loop419 store i32 0, ptr %keyinfo, align 4420 br label %loop.latch421 422if.nonzero: ; preds = %loop423 %rem = srem i32 1, %divisor424 store i32 %rem, ptr %dst, align 4425 br label %loop.latch426 427loop.latch: ; preds = %if.nonzero, %if.zero428 %iv.next = add i64 %iv, 1429 %exitcond = icmp eq i64 %iv, 32430 br i1 %exitcond, label %exit, label %loop431 432exit: ; preds = %loop.latch433 ret void434}435 436attributes #0 = { "target-features"="+neon,+sve" vscale_range(1,16) }437 438declare void @llvm.assume(i1 noundef)439declare i64 @llvm.umin.i64(i64, i64)440