194 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize,instsimplify -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=TFNONE3; RUN: opt < %s -passes=loop-vectorize,instsimplify,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=TFCOMMON,TFALWAYS4; RUN: opt < %s -passes=loop-vectorize,instsimplify,simplifycfg -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck %s --check-prefixes=TFCOMMON,TFFALLBACK5; RUN: opt < %s -passes=loop-vectorize,instsimplify,simplifycfg -force-vector-interleave=2 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s --check-prefixes=TFA_INTERLEAVE6 7target triple = "aarch64-unknown-linux-gnu"8 9define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {10; TFNONE-LABEL: @test_widen_exp_v2(11; TFNONE-NEXT: entry:12; TFNONE-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 113; TFNONE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 214; TFNONE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]15; TFNONE: vector.ph:16; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 217; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]18; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]]19; TFNONE: vector.body:20; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]21; TFNONE-NEXT: [[TMP7:%.*]] = load double, ptr [[P2:%.*]], align 822; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 023; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer24; TFNONE-NEXT: [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])25; TFNONE-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer26; TFNONE-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)27; TFNONE-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 128; TFNONE-NEXT: store double [[TMP14]], ptr [[P:%.*]], align 829; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 230; TFNONE-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]31; TFNONE-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]32; TFNONE: middle.block:33; TFNONE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]34; TFNONE-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]35; TFNONE: scalar.ph:36; TFNONE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]37; TFNONE-NEXT: br label [[LOOP:%.*]]38; TFNONE: loop:39; TFNONE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_END:%.*]] ]40; TFNONE-NEXT: [[LD:%.*]] = load double, ptr [[P2]], align 841; TFNONE-NEXT: [[EXP:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR2:[0-9]+]]42; TFNONE-NEXT: [[COND1:%.*]] = fcmp ogt double [[EXP]], 0.000000e+0043; TFNONE-NEXT: br i1 [[COND1]], label [[LOOP_MIDDLE:%.*]], label [[LOOP_END]]44; TFNONE: loop.middle:45; TFNONE-NEXT: br label [[LOOP_END]]46; TFNONE: loop.end:47; TFNONE-NEXT: [[SINK:%.*]] = phi double [ 0.000000e+00, [[LOOP_MIDDLE]] ], [ 1.000000e+00, [[LOOP]] ]48; TFNONE-NEXT: store double [[SINK]], ptr [[P]], align 849; TFNONE-NEXT: [[IV_NEXT]] = add i64 [[IV]], 150; TFNONE-NEXT: [[COND2:%.*]] = icmp eq i64 [[IV]], [[N]]51; TFNONE-NEXT: br i1 [[COND2]], label [[END]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]52; TFNONE: end:53; TFNONE-NEXT: ret void54;55; TFCOMMON-LABEL: @test_widen_exp_v2(56; TFCOMMON-NEXT: entry:57; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0:%.*]], 158; TFCOMMON-NEXT: [[TMP1:%.*]] = sub i64 [[N_RND_UP]], 259; TFCOMMON-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[N_RND_UP]], 260; TFCOMMON-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 061; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N_RND_UP]])62; TFCOMMON-NEXT: br label [[LOOP:%.*]]63; TFCOMMON: vector.body:64; TFCOMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]65; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]66; TFCOMMON-NEXT: [[LD:%.*]] = load double, ptr [[P2:%.*]], align 867; TFCOMMON-NEXT: [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[LD]]) #[[ATTR2:[0-9]+]]68; TFCOMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 069; TFCOMMON-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer70; TFCOMMON-NEXT: [[TMP9:%.*]] = fcmp ogt <2 x double> [[TMP8]], zeroinitializer71; TFCOMMON-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP9]], i32 072; TFCOMMON-NEXT: [[PREDPHI:%.*]] = select i1 [[TMP7]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)73; TFCOMMON-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 074; TFCOMMON-NEXT: br i1 [[TMP16]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]75; TFCOMMON: pred.store.if:76; TFCOMMON-NEXT: [[SINK:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 077; TFCOMMON-NEXT: store double [[SINK]], ptr [[P:%.*]], align 878; TFCOMMON-NEXT: br label [[PRED_STORE_CONTINUE]]79; TFCOMMON: pred.store.continue:80; TFCOMMON-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 181; TFCOMMON-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE6]]82; TFCOMMON: pred.store.if1:83; TFCOMMON-NEXT: [[TMP19:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 084; TFCOMMON-NEXT: store double [[TMP19]], ptr [[P]], align 885; TFCOMMON-NEXT: br label [[PRED_STORE_CONTINUE6]]86; TFCOMMON: pred.store.continue2:87; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 288; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])89; TFCOMMON-NEXT: [[TMP15:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 090; TFCOMMON-NEXT: [[TMP17:%.*]] = xor i1 [[TMP15]], true91; TFCOMMON-NEXT: br i1 [[TMP17]], label [[END:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]92; TFCOMMON: end:93; TFCOMMON-NEXT: ret void94;95; TFA_INTERLEAVE-LABEL: @test_widen_exp_v2(96; TFA_INTERLEAVE-NEXT: entry:97; TFA_INTERLEAVE-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0:%.*]], 198; TFA_INTERLEAVE-NEXT: [[TMP1:%.*]] = sub i64 [[N_RND_UP]], 499; TFA_INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[N_RND_UP]], 4100; TFA_INTERLEAVE-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP1]], i64 0101; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 0, i64 [[N_RND_UP]])102; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 2, i64 [[N_RND_UP]])103; TFA_INTERLEAVE-NEXT: br label [[VECTOR_BODY:%.*]]104; TFA_INTERLEAVE: vector.body:105; TFA_INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE9:%.*]] ]106; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_STORE_CONTINUE9]] ]107; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT10:%.*]], [[PRED_STORE_CONTINUE9]] ]108; TFA_INTERLEAVE-NEXT: [[TMP4:%.*]] = load double, ptr [[P2:%.*]], align 8109; TFA_INTERLEAVE-NEXT: [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR2:[0-9]+]]110; TFA_INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0111; TFA_INTERLEAVE-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer112; TFA_INTERLEAVE-NEXT: [[TMP14:%.*]] = fcmp ogt <2 x double> [[TMP12]], zeroinitializer113; TFA_INTERLEAVE-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP14]], i32 0114; TFA_INTERLEAVE-NEXT: [[PREDPHI3:%.*]] = select i1 [[TMP7]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)115; TFA_INTERLEAVE-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 0116; TFA_INTERLEAVE-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]117; TFA_INTERLEAVE: pred.store.if:118; TFA_INTERLEAVE-NEXT: [[TMP20:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 0119; TFA_INTERLEAVE-NEXT: store double [[TMP20]], ptr [[P:%.*]], align 8120; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE]]121; TFA_INTERLEAVE: pred.store.continue:122; TFA_INTERLEAVE-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i32 1123; TFA_INTERLEAVE-NEXT: br i1 [[TMP29]], label [[PRED_STORE_IF4:%.*]], label [[PRED_STORE_CONTINUE5:%.*]]124; TFA_INTERLEAVE: pred.store.if3:125; TFA_INTERLEAVE-NEXT: [[TMP22:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 0126; TFA_INTERLEAVE-NEXT: store double [[TMP22]], ptr [[P]], align 8127; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE5]]128; TFA_INTERLEAVE: pred.store.continue4:129; TFA_INTERLEAVE-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i32 0130; TFA_INTERLEAVE-NEXT: br i1 [[TMP31]], label [[PRED_STORE_IF6:%.*]], label [[PRED_STORE_CONTINUE7:%.*]]131; TFA_INTERLEAVE: pred.store.if5:132; TFA_INTERLEAVE-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 0133; TFA_INTERLEAVE-NEXT: store double [[TMP32]], ptr [[P]], align 8134; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE7]]135; TFA_INTERLEAVE: pred.store.continue6:136; TFA_INTERLEAVE-NEXT: [[TMP25:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK2]], i32 1137; TFA_INTERLEAVE-NEXT: br i1 [[TMP25]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9]]138; TFA_INTERLEAVE: pred.store.if7:139; TFA_INTERLEAVE-NEXT: [[TMP34:%.*]] = extractelement <2 x double> [[PREDPHI3]], i32 0140; TFA_INTERLEAVE-NEXT: store double [[TMP34]], ptr [[P]], align 8141; TFA_INTERLEAVE-NEXT: br label [[PRED_STORE_CONTINUE9]]142; TFA_INTERLEAVE: pred.store.continue8:143; TFA_INTERLEAVE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4144; TFA_INTERLEAVE-NEXT: [[TMP27:%.*]] = add i64 [[INDEX]], 2145; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])146; TFA_INTERLEAVE-NEXT: [[ACTIVE_LANE_MASK_NEXT10]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP27]], i64 [[TMP3]])147; TFA_INTERLEAVE-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0148; TFA_INTERLEAVE-NEXT: [[TMP28:%.*]] = xor i1 [[TMP26]], true149; TFA_INTERLEAVE-NEXT: br i1 [[TMP28]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]150; TFA_INTERLEAVE: end:151; TFA_INTERLEAVE-NEXT: ret void152;153entry:154 br label %loop155 156loop:157 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.end ]158 %ld = load double, ptr %p2, align 8159 %exp = tail call double @llvm.exp.f64(double %ld) #6160 %cond1 = fcmp ogt double %exp, 0.000000e+00161 br i1 %cond1, label %loop.middle, label %loop.end162 163loop.middle:164 br label %loop.end165 166loop.end:167 %sink = phi double [ 0.000000e+00, %loop.middle ], [ 1.000000e+00, %loop ]168 store double %sink, ptr %p, align 8169 %iv.next = add i64 %iv, 1170 %cond2 = icmp eq i64 %iv, %n171 br i1 %cond2, label %end, label %loop, !llvm.loop !0172 173end:174 ret void175}176 177 178declare double @llvm.exp.f64(double)179 180; fixed-width variant of exp181declare <2 x double> @exp_fixed(<2 x double>)182 183;; scalable vector variant of exp184declare <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double>, <vscale x 2 x i1>)185 186attributes #5 = { "target-cpu"="neoverse-v2" vscale_range(1,16) }187attributes #6 = { "vector-function-abi-variant"="_ZGV_LLVM_N2v_llvm.exp.f64(exp_fixed),_ZGVsMxv_llvm.exp.f64(exp_masked_scalable)" }188 189!0 = distinct !{!0, !1}190!1 = !{!"llvm.loop.vectorize.width", i32 2}191;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:192; TFALWAYS: {{.*}}193; TFFALLBACK: {{.*}}194