172 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; REQUIRES: asserts3; RUN: opt -passes=loop-vectorize -S -mtriple=aarch64 -mattr=+sve -debug-only=loop-vectorize \4; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s 2>&1 | FileCheck %s5 6target triple = "aarch64-unknown-linux-gnu"7 8; CHECK-LABEL: LV: Checking a loop in 'pointer_induction_used_as_vector'9; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 110;11; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {12 13; In the test below the pointer phi %ptr.iv.2 is used as14; 1. As a uniform address for the load, and15; 2. Non-uniform use by the getelementptr which is stored. This requires the16; vector value.17define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias %start.2, i64 %N) {18; CHECK-LABEL: @pointer_induction_used_as_vector(19; CHECK-NEXT: entry:20; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()21; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 122; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]23; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]24; CHECK: vector.ph:25; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()26; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 227; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]28; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]29; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 830; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]]31; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]]32; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]33; CHECK: vector.body:34; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]35; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]36; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()37; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP7]]38; CHECK-NEXT: [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[VECTOR_GEP]], i32 039; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 840; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[OFFSET_IDX]]41; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 142; CHECK-NEXT: store <vscale x 2 x ptr> [[TMP9]], ptr [[NEXT_GEP]], align 843; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP10]], align 144; CHECK-NEXT: [[TMP12:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)45; CHECK-NEXT: store <vscale x 2 x i8> [[TMP12]], ptr [[TMP10]], align 146; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]47; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP3]]48; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]49; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]50; CHECK: middle.block:51; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]52; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]53; CHECK: scalar.ph:54; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]55; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY]] ]56; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]57; CHECK-NEXT: br label [[LOOP_BODY:%.*]]58; CHECK: loop.body:59; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY]] ]60; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_1_NEXT:%.*]], [[LOOP_BODY]] ]61; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], [[LOOP_BODY]] ]62; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr inbounds ptr, ptr [[PTR_IV_1]], i64 163; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV_2]], i64 164; CHECK-NEXT: store ptr [[PTR_IV_2_NEXT]], ptr [[PTR_IV_1]], align 865; CHECK-NEXT: [[LV:%.*]] = load i8, ptr [[PTR_IV_2]], align 166; CHECK-NEXT: [[ADD:%.*]] = add i8 [[LV]], 167; CHECK-NEXT: store i8 [[ADD]], ptr [[PTR_IV_2]], align 168; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 169; CHECK-NEXT: [[C:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]70; CHECK-NEXT: br i1 [[C]], label [[LOOP_BODY]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]71; CHECK: exit:72; CHECK-NEXT: ret void73;74 75 76 77entry:78 br label %loop.body79 80loop.body: ; preds = %loop.body, %entry81 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.body ]82 %ptr.iv.1 = phi ptr [ %start.1, %entry ], [ %ptr.iv.1.next, %loop.body ]83 %ptr.iv.2 = phi ptr [ %start.2, %entry ], [ %ptr.iv.2.next, %loop.body ]84 %ptr.iv.1.next = getelementptr inbounds ptr, ptr %ptr.iv.1, i64 185 %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 186 store ptr %ptr.iv.2.next, ptr %ptr.iv.1, align 887 %lv = load i8, ptr %ptr.iv.2, align 188 %add = add i8 %lv, 189 store i8 %add, ptr %ptr.iv.2, align 190 %iv.next = add nuw i64 %iv, 191 %c = icmp ne i64 %iv.next, %N92 br i1 %c, label %loop.body, label %exit, !llvm.loop !093 94exit: ; preds = %loop.body95 ret void96}97 98define void @pointer_induction(ptr noalias %start, i64 %N) {99; CHECK-LABEL: @pointer_induction(100; CHECK-NEXT: entry:101; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1102; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()103; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1104; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]105; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]106; CHECK: vector.ph:107; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()108; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2109; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]110; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]111; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]]112; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]113; CHECK: vector.body:114; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]115; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX2]]116; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP15]], align 1117; CHECK-NEXT: [[TMP17:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)118; CHECK-NEXT: store <vscale x 2 x i8> [[TMP17]], ptr [[TMP15]], align 1119; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX2]], [[TMP4]]120; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]121; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]122; CHECK: middle.block:123; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]124; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]125; CHECK: scalar.ph:126; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ]127; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]128; CHECK-NEXT: br label [[FOR_BODY:%.*]]129; CHECK: for.body:130; CHECK-NEXT: [[PTR_PHI:%.*]] = phi ptr [ [[PTR_PHI_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]131; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]132; CHECK-NEXT: [[INDEX_NXT]] = add i64 [[INDEX]], 1133; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[PTR_PHI]], align 1134; CHECK-NEXT: [[ADD:%.*]] = add i8 [[TMP19]], 1135; CHECK-NEXT: store i8 [[ADD]], ptr [[PTR_PHI]], align 1136; CHECK-NEXT: [[PTR_PHI_NEXT]] = getelementptr inbounds i8, ptr [[PTR_PHI]], i64 1137; CHECK-NEXT: [[CMP_I_NOT:%.*]] = icmp eq ptr [[PTR_PHI_NEXT]], [[START]]138; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[INDEX]], [[N]]139; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[END]], !llvm.loop [[LOOP5:![0-9]+]]140; CHECK: end:141; CHECK-NEXT: ret void142;143entry:144 br label %for.body145 146for.body:147 %ptr.phi = phi ptr [ %ptr.phi.next, %for.body ], [ %start, %entry ]148 %index = phi i64 [ %index_nxt, %for.body ], [ 0, %entry ]149 %index_nxt = add i64 %index, 1150 %0 = load i8, ptr %ptr.phi, align 1151 %add = add i8 %0, 1152 store i8 %add, ptr %ptr.phi153 %ptr.phi.next = getelementptr inbounds i8, ptr %ptr.phi, i64 1154 %cmp.i.not = icmp eq ptr %ptr.phi.next, %start155 %cmp = icmp ult i64 %index, %N156 br i1 %cmp, label %for.body, label %end, !llvm.loop !0157 158end:159 ret void160}161 162attributes #0 = {"target-features"="+sve"}163 164!0 = distinct !{!0, !1, !2, !3}165!1 = !{!"llvm.loop.interleave.count", i32 1}166!2 = !{!"llvm.loop.vectorize.width", i32 2}167!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}168!4 = !{ !5 }169!5 = distinct !{ !5, !6 }170!6 = distinct !{ !7 }171!7 = distinct !{ !7, !6 }172