254 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 52; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"5 6; Check that the interleaved-mem-access analysis currently does not create an7; interleave group for access 'a' due to the possible pointer wrap-around.8;9; To begin with, in this test the candidate interleave group can be created10; only when getPtrStride is called with Assume=true. Next, because11; the interleave-group of the loads is not full (has gaps), we also need to check12; for possible pointer wrapping. Here we currently use Assume=false and as a13; result cannot prove the transformation is safe and therefore invalidate the14; candidate interleave group.15;16 17; void func(unsigned * __restrict a, unsigned * __restrict b, unsigned char x, unsigned char y) {18; int i = 0;19; for (unsigned char index = x; i < y; index +=2, ++i)20; b[i] = aptr 2;21;22; }23 24define void @wrap_around_scev_check(ptr noalias %a, ptr noalias %b, i8 %x, i8 %y) {25; CHECK-LABEL: define void @wrap_around_scev_check(26; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {27; CHECK-NEXT: [[ENTRY:.*:]]28; CHECK-NEXT: [[CMP9:%.*]] = icmp eq i8 [[Y]], 029; CHECK-NEXT: br i1 [[CMP9]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]30; CHECK: [[LOOP_PREHEADER]]:31; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i8 [[Y]] to i6432; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[WIDE_TRIP_COUNT]], 433; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]34; CHECK: [[VECTOR_SCEVCHECK]]:35; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -136; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i837; CHECK-NEXT: [[MUL1:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 2, i8 [[TMP1]])38; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL1]], 039; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i8, i1 } [[MUL1]], 140; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[X]], [[MUL_RESULT]]41; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i8 [[TMP2]], [[X]]42; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]43; CHECK-NEXT: [[TMP17:%.*]] = icmp ugt i64 [[TMP0]], 25544; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP5]], [[TMP17]]45; CHECK-NEXT: br i1 [[TMP18]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]46; CHECK: [[VECTOR_PH]]:47; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 448; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[N_MOD_VF]], 049; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], i64 4, i64 [[N_MOD_VF]]50; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[TMP7]]51; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i852; CHECK-NEXT: [[TMP8:%.*]] = mul i8 [[DOTCAST]], 253; CHECK-NEXT: [[TMP9:%.*]] = add i8 [[X]], [[TMP8]]54; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]55; CHECK: [[VECTOR_BODY]]:56; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]57; CHECK-NEXT: [[DOTCAST2:%.*]] = trunc i64 [[INDEX]] to i858; CHECK-NEXT: [[TMP10:%.*]] = mul i8 [[DOTCAST2]], 259; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i8 [[X]], [[TMP10]]60; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[OFFSET_IDX]] to i6461; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]62; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP12]], align 463; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>64; CHECK-NEXT: [[TMP13:%.*]] = shl <4 x i32> [[STRIDED_VEC]], splat (i32 1)65; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]66; CHECK-NEXT: store <4 x i32> [[TMP13]], ptr [[TMP14]], align 467; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 468; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]69; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]70; CHECK: [[MIDDLE_BLOCK]]:71; CHECK-NEXT: br label %[[SCALAR_PH]]72; CHECK: [[SCALAR_PH]]:73; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_PREHEADER]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]74; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i8 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[X]], %[[LOOP_PREHEADER]] ], [ [[X]], %[[VECTOR_SCEVCHECK]] ]75; CHECK-NEXT: br label %[[LOOP:.*]]76; CHECK: [[LOOP]]:77; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]78; CHECK-NEXT: [[INDEX_011:%.*]] = phi i8 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ]79; CHECK-NEXT: [[IDXPROM:%.*]] = zext i8 [[INDEX_011]] to i6480; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]]81; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 482; CHECK-NEXT: [[MUL:%.*]] = shl i32 [[TMP16]], 183; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]84; CHECK-NEXT: store i32 [[MUL]], ptr [[ARRAYIDX2]], align 485; CHECK-NEXT: [[ADD]] = add i8 [[INDEX_011]], 286; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 187; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]88; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]89; CHECK: [[EXIT_LOOPEXIT]]:90; CHECK-NEXT: br label %[[EXIT]]91; CHECK: [[EXIT]]:92; CHECK-NEXT: ret void93;94entry:95 %cmp9 = icmp eq i8 %y, 096 br i1 %cmp9, label %exit, label %loop.preheader97 98loop.preheader:99 %wide.trip.count = zext i8 %y to i64100 br label %loop101 102loop:103 %iv = phi i64 [ 0, %loop.preheader ], [ %iv.next, %loop ]104 %index.011 = phi i8 [ %x, %loop.preheader ], [ %add, %loop ]105 %idxprom = zext i8 %index.011 to i64106 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %idxprom107 %0 = load i32, ptr %arrayidx, align 4108 %mul = shl i32 %0, 1109 %arrayidx2 = getelementptr inbounds i32, ptr %b, i64 %iv110 store i32 %mul, ptr %arrayidx2, align 4111 %add = add i8 %index.011, 2112 %iv.next = add nuw nsw i64 %iv, 1113 %exitcond = icmp eq i64 %iv.next, %wide.trip.count114 br i1 %exitcond, label %exit, label %loop115 116exit:117 ret void118}119 120; For %gep, we have the following SCEV: ((4 * (zext i4 {0,+,5}<%loop> to i64))<nuw><nsw> + %x).121; Note the i4 bit wide AddRec {0,+,5}. It is known to wrap in the loop with trip count 16.122define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(ptr noalias %x, ptr noalias %out) {123; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_wraps_for_known_trip_count(124; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]]) {125; CHECK-NEXT: [[START:.*]]:126; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]127; CHECK: [[VECTOR_SCEVCHECK]]:128; CHECK-NEXT: [[MUL:%.*]] = call { i4, i1 } @llvm.umul.with.overflow.i4(i4 5, i4 -1)129; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i4, i1 } [[MUL]], 1130; CHECK-NEXT: br i1 [[MUL_OVERFLOW]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]131; CHECK: [[VECTOR_PH]]:132; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]133; CHECK: [[VECTOR_BODY]]:134; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]135; CHECK-NEXT: [[TMP0:%.*]] = mul nuw nsw i64 [[INDEX]], 5136; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 15137; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP1]]138; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <20 x i32>, ptr [[TMP2]], align 4139; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <20 x i32> [[WIDE_VEC]], <20 x i32> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>140; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]141; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP3]], align 4142; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4143; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12144; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]145; CHECK: [[MIDDLE_BLOCK]]:146; CHECK-NEXT: br label %[[SCALAR_PH]]147; CHECK: [[SCALAR_PH]]:148; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 12, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]149; CHECK-NEXT: br label %[[LOOP:.*]]150; CHECK: [[LOOP]]:151; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]152; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1153; CHECK-NEXT: [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 5154; CHECK-NEXT: [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15155; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV_MUL5_MASKED]]156; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4157; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]158; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4159; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 16160; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]161; CHECK: [[EXIT]]:162; CHECK-NEXT: ret void163;164start:165 br label %loop166 167loop:168 %iv = phi i64 [ 0, %start ], [ %iv.next, %loop ]169 %iv.next = add nuw nsw i64 %iv, 1170 %iv.mul5 = mul nuw nsw i64 %iv, 5171 %iv.mul5.masked = and i64 %iv.mul5, 15172 %gep = getelementptr inbounds nuw i32, ptr %x, i64 %iv.mul5.masked173 %v = load i32, ptr %gep, align 4174 %out.i = getelementptr inbounds nuw i32, ptr %out, i64 %iv175 store i32 %v, ptr %out.i, align 4176 %exitcond.not = icmp eq i64 %iv.next, 16177 br i1 %exitcond.not, label %exit, label %loop178 179exit:180 ret void181}182 183; For %gep, we have the following SCEV: ((4 * (zext i4 {0,+,3}<%loop> to i64))<nuw><nsw> + %x).184; Note the i4 bit wide AddRec {0,+,3}. It may wrap, depending on the trip count.185define void @wrap_predicate_for_interleave_group_unknown_trip_count(ptr noalias %x, ptr noalias %out, i64 %n) {186; CHECK-LABEL: define void @wrap_predicate_for_interleave_group_unknown_trip_count(187; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) {188; CHECK-NEXT: [[START:.*]]:189; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N]], 4190; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]191; CHECK: [[VECTOR_SCEVCHECK]]:192; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1193; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP0]] to i4194; CHECK-NEXT: [[MUL:%.*]] = call { i4, i1 } @llvm.umul.with.overflow.i4(i4 3, i4 [[TMP9]])195; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i4, i1 } [[MUL]], 1196; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[TMP0]], 15197; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[MUL_OVERFLOW]], [[TMP1]]198; CHECK-NEXT: br i1 [[TMP10]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]199; CHECK: [[VECTOR_PH]]:200; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4201; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[N_MOD_VF]], 0202; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i64 4, i64 [[N_MOD_VF]]203; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP7]]204; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]205; CHECK: [[VECTOR_BODY]]:206; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]207; CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[INDEX]], 3208; CHECK-NEXT: [[TMP3:%.*]] = and i64 [[TMP8]], 15209; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[TMP3]]210; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i32>, ptr [[TMP4]], align 4211; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i32> [[WIDE_VEC]], <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>212; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[INDEX]]213; CHECK-NEXT: store <4 x i32> [[STRIDED_VEC]], ptr [[TMP5]], align 4214; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4215; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]216; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]217; CHECK: [[MIDDLE_BLOCK]]:218; CHECK-NEXT: br label %[[SCALAR_PH]]219; CHECK: [[SCALAR_PH]]:220; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[START]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]221; CHECK-NEXT: br label %[[LOOP:.*]]222; CHECK: [[LOOP]]:223; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]224; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1225; CHECK-NEXT: [[IV_MUL5:%.*]] = mul nuw nsw i64 [[IV]], 3226; CHECK-NEXT: [[IV_MUL5_MASKED:%.*]] = and i64 [[IV_MUL5]], 15227; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[IV_MUL5_MASKED]]228; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4229; CHECK-NEXT: [[OUT_I:%.*]] = getelementptr inbounds nuw i32, ptr [[OUT]], i64 [[IV]]230; CHECK-NEXT: store i32 [[V]], ptr [[OUT_I]], align 4231; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]232; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]233; CHECK: [[EXIT]]:234; CHECK-NEXT: ret void235;236start:237 br label %loop238 239loop:240 %iv = phi i64 [ 0, %start ], [ %iv.next, %loop ]241 %iv.next = add nuw nsw i64 %iv, 1242 %iv.mul3 = mul nuw nsw i64 %iv, 3243 %iv.mul3.masked = and i64 %iv.mul3, 15244 %gep = getelementptr inbounds nuw i32, ptr %x, i64 %iv.mul3.masked245 %v = load i32, ptr %gep, align 4246 %out.i = getelementptr inbounds nuw i32, ptr %out, i64 %iv247 store i32 %v, ptr %out.i, align 4248 %exitcond.not = icmp eq i64 %iv.next, %n249 br i1 %exitcond.not, label %exit, label %loop250 251exit:252 ret void253}254