124 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"2; RUN: opt -passes='loop-vectorize,instcombine' -sve-tail-folding-insn-threshold=0 -sve-tail-folding=all -S < %s | FileCheck %s3 4target triple = "aarch64"5 6; Test that this uses the 'DataAndControlFlow' style of tail folding7; where it performs the loop-indvar increment before the active.lane.mask8; and uses %N directly for the tripcount.9define void @cannot_overflow_i32_induction_var(ptr noalias %dst, ptr readonly %src, i32 %N) #0 {10; CHECK-LABEL: @cannot_overflow_i32_induction_var(11; CHECK-NEXT: entry:12; CHECK-NEXT: [[CMP6_NOT:%.*]] = icmp eq i32 [[N:%.*]], 013; CHECK-NEXT: br i1 [[CMP6_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]14; CHECK: for.body.preheader:15; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i6416; CHECK-NEXT: br label [[VECTOR_PH:%.*]]17; CHECK: vector.ph:18; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()19; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 220; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])21; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]22; CHECK: vector.body:23; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]24; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]25; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]26; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP0]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)27; CHECK-NEXT: [[TMP1:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)28; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]29; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP1]], ptr align 4 [[TMP2]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])30; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]31; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])32; CHECK-NEXT: [[TMP5:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 033; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]34; CHECK: middle.block:35; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]36; CHECK: for.cond.cleanup.loopexit:37; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]38; CHECK: for.cond.cleanup:39; CHECK-NEXT: ret void40;41entry:42 %cmp6.not = icmp eq i32 %N, 043 br i1 %cmp6.not, label %for.cond.cleanup, label %for.body.preheader44 45for.body.preheader:46 %wide.trip.count = zext i32 %N to i6447 br label %for.body48 49for.body:50 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]51 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %indvars.iv52 %0 = load i32, ptr %arrayidx, align 453 %add = add nsw i32 %0, 4254 %arrayidx2 = getelementptr inbounds i32, ptr %dst, i64 %indvars.iv55 store i32 %add, ptr %arrayidx2, align 456 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 157 %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count58 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body59 60for.cond.cleanup:61 ret void62}63 64; Test that this uses the 'DataAndControlFlowWithoutRuntimeCheck' style of65; tail folding where it uses an updated trip count and do the loop-indvar66; increment after the active.lane.mask.67define void @can_overflow_i64_induction_var(ptr noalias %dst, ptr readonly %src, i64 %N) #0 {68; CHECK-LABEL: @can_overflow_i64_induction_var(69; CHECK-NEXT: entry:70; CHECK-NEXT: [[CMP6_NOT:%.*]] = icmp eq i64 [[N:%.*]], 071; CHECK-NEXT: br i1 [[CMP6_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]]72; CHECK: for.body.preheader:73; CHECK-NEXT: br label [[VECTOR_PH:%.*]]74; CHECK: vector.ph:75; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()76; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 277; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()78; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 279; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]])80; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])81; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]82; CHECK: vector.body:83; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]84; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]85; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]86; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP3]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)87; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 42)88; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]89; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP4]], ptr align 4 [[TMP5]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])90; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]91; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP2]])92; CHECK-NEXT: [[TMP8:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 093; CHECK-NEXT: br i1 [[TMP8]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP3:![0-9]+]]94; CHECK: middle.block:95; CHECK-NEXT: br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]96; CHECK: for.cond.cleanup.loopexit:97; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]98; CHECK: for.cond.cleanup:99; CHECK-NEXT: ret void100;101entry:102 %cmp6.not = icmp eq i64 %N, 0103 br i1 %cmp6.not, label %for.cond.cleanup, label %for.body.preheader104 105for.body.preheader:106 br label %for.body107 108for.body:109 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]110 %arrayidx = getelementptr inbounds i32, ptr %src, i64 %indvars.iv111 %0 = load i32, ptr %arrayidx, align 4112 %add = add nsw i32 %0, 42113 %arrayidx2 = getelementptr inbounds i32, ptr %dst, i64 %indvars.iv114 store i32 %add, ptr %arrayidx2, align 4115 %indvars.iv.next = add nuw i64 %indvars.iv, 1116 %exitcond.not = icmp eq i64 %indvars.iv.next, %N117 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body118 119for.cond.cleanup:120 ret void121}122 123attributes #0 = { vscale_range(1,16) "target-features"="+sve" }124