226 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 52; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 < %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"5 6; Check the scenario where we have an unknown Stride, which happens to also be7; the loop iteration count, so if we specialize the loop for the Stride==1 case,8; this also implies that the loop will iterate no more than a single iteration,9; as in the following example:10;11; unsigned int N;12; int tmp = 0;13; for(unsigned int k=0;k<N;k++) {14; tmp+=(int)B[k*N+j];15; }16;17; We check here that the following runtime scev guard for Stride==1 is NOT generated:18; vector.scevcheck:19; %ident.check = icmp ne i32 %N, 120; %0 = or i1 false, %ident.check21; br i1 %0, label %scalar.ph, label %vector.ph22; Instead the loop is vectorized with an unknown stride.23 24 25 26define i32 @foo1(i32 %N, ptr nocapture readnone %A, ptr nocapture readonly %B, i32 %i, i32 %j) {27; CHECK-LABEL: define i32 @foo1(28; CHECK-SAME: i32 [[N:%.*]], ptr readnone captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i32 [[I:%.*]], i32 [[J:%.*]]) {29; CHECK-NEXT: [[ENTRY:.*:]]30; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 031; CHECK-NEXT: br i1 [[CMP8]], [[FOR_END:label %.*]], label %[[FOR_BODY_LR_PH:.*]]32; CHECK: [[FOR_BODY_LR_PH]]:33; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 434; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]35; CHECK: [[VECTOR_SCEVCHECK]]:36; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -137; CHECK-NEXT: [[TMP1:%.*]] = sub i32 0, [[N]]38; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i32 [[N]], 039; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 [[N]]40; CHECK-NEXT: [[MUL1:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 [[TMP0]])41; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL1]], 042; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL1]], 143; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[J]], [[MUL_RESULT]]44; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[J]], [[MUL_RESULT]]45; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i32 [[TMP4]], [[J]]46; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[J]]47; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[TMP2]], i1 [[TMP7]], i1 [[TMP6]]48; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP8]], [[MUL_OVERFLOW]]49; CHECK-NEXT: br i1 [[TMP9]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]50; CHECK: [[VECTOR_PH]]:51; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 452; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]53; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 054; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer55; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[J]], i64 056; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer57; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]58; CHECK: [[VECTOR_BODY]]:59; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]60; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]61; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]62; CHECK-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]63; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i32> [[TMP10]], [[BROADCAST_SPLAT3]]64; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP11]], i32 065; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP11]], i32 166; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP11]], i32 267; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[TMP11]], i32 368; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP12]]69; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP14]]70; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP16]]71; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP18]]72; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[TMP13]], align 273; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[TMP15]], align 274; CHECK-NEXT: [[TMP22:%.*]] = load i16, ptr [[TMP17]], align 275; CHECK-NEXT: [[TMP23:%.*]] = load i16, ptr [[TMP19]], align 276; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> poison, i16 [[TMP20]], i32 077; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i16> [[TMP24]], i16 [[TMP21]], i32 178; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x i16> [[TMP25]], i16 [[TMP22]], i32 279; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i16> [[TMP26]], i16 [[TMP23]], i32 380; CHECK-NEXT: [[TMP28:%.*]] = sext <4 x i16> [[TMP27]] to <4 x i32>81; CHECK-NEXT: [[TMP29]] = add <4 x i32> [[VEC_PHI]], [[TMP28]]82; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 483; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i32> [[VEC_IND]], splat (i32 4)84; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]85; CHECK-NEXT: br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]86; CHECK: [[MIDDLE_BLOCK]]:87; CHECK-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP29]])88; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]89; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]90; CHECK: [[SCALAR_PH]]:91;92entry:93 %cmp8 = icmp eq i32 %N, 094 br i1 %cmp8, label %for.end, label %for.body.lr.ph95 96for.body.lr.ph:97 br label %for.body98 99for.body:100 %tmp.010 = phi i32 [ 0, %for.body.lr.ph ], [ %add1, %for.body ]101 %k.09 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]102 %mul = mul i32 %k.09, %N103 %add = add i32 %mul, %j104 %arrayidx = getelementptr inbounds i16, ptr %B, i32 %add105 %0 = load i16, ptr %arrayidx, align 2106 %conv = sext i16 %0 to i32107 %add1 = add nsw i32 %tmp.010, %conv108 %inc = add nuw i32 %k.09, 1109 %exitcond = icmp eq i32 %inc, %N110 br i1 %exitcond, label %for.end.loopexit, label %for.body111 112for.end.loopexit:113 %add1.lcssa = phi i32 [ %add1, %for.body ]114 br label %for.end115 116for.end:117 %tmp.0.lcssa = phi i32 [ 0, %entry ], [ %add1.lcssa, %for.end.loopexit ]118 ret i32 %tmp.0.lcssa119}120 121 122; Check the same, but also where the Stride and the loop iteration count123; are not of the same data type.124;125; unsigned short N;126; int tmp = 0;127; for(unsigned int k=0;k<N;k++) {128; tmp+=(int)B[k*N+j];129; }130;131; We check here that the following runtime scev guard for Stride==1 is NOT generated:132; vector.scevcheck:133; %ident.check = icmp ne i16 %N, 1134; %0 = or i1 false, %ident.check135; br i1 %0, label %scalar.ph, label %vector.ph136define i32 @foo2(i16 zeroext %N, ptr nocapture readnone %A, ptr nocapture readonly %B, i32 %i, i32 %j) {137; CHECK-LABEL: define i32 @foo2(138; CHECK-SAME: i16 zeroext [[N:%.*]], ptr readnone captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i32 [[I:%.*]], i32 [[J:%.*]]) {139; CHECK-NEXT: [[ENTRY:.*:]]140; CHECK-NEXT: [[CONV:%.*]] = zext i16 [[N]] to i32141; CHECK-NEXT: [[CMP11:%.*]] = icmp eq i16 [[N]], 0142; CHECK-NEXT: br i1 [[CMP11]], [[FOR_END:label %.*]], label %[[FOR_BODY_LR_PH:.*]]143; CHECK: [[FOR_BODY_LR_PH]]:144; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[CONV]], 4145; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]146; CHECK: [[VECTOR_SCEVCHECK]]:147; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[CONV]], -1148; CHECK-NEXT: [[MUL1:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[CONV]], i32 [[TMP0]])149; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL1]], 0150; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL1]], 1151; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[J]], [[MUL_RESULT]]152; CHECK-NEXT: [[TMP3:%.*]] = icmp slt i32 [[TMP1]], [[J]]153; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]154; CHECK-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]155; CHECK: [[VECTOR_PH]]:156; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[CONV]], 4157; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[CONV]], [[N_MOD_VF]]158; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[CONV]], i64 0159; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer160; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[J]], i64 0161; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer162; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]163; CHECK: [[VECTOR_BODY]]:164; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]165; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]166; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]167; CHECK-NEXT: [[TMP7:%.*]] = mul nuw <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]168; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], [[BROADCAST_SPLAT3]]169; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 0170; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP8]], i32 1171; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP8]], i32 2172; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3173; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP9]]174; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP11]]175; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP13]]176; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP15]]177; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[TMP10]], align 2178; CHECK-NEXT: [[TMP18:%.*]] = load i16, ptr [[TMP12]], align 2179; CHECK-NEXT: [[TMP19:%.*]] = load i16, ptr [[TMP14]], align 2180; CHECK-NEXT: [[TMP20:%.*]] = load i16, ptr [[TMP16]], align 2181; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i16> poison, i16 [[TMP17]], i32 0182; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[TMP18]], i32 1183; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[TMP19]], i32 2184; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> [[TMP23]], i16 [[TMP20]], i32 3185; CHECK-NEXT: [[TMP25:%.*]] = sext <4 x i16> [[TMP24]] to <4 x i32>186; CHECK-NEXT: [[TMP26]] = add <4 x i32> [[VEC_PHI]], [[TMP25]]187; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4188; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)189; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]190; CHECK-NEXT: br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]191; CHECK: [[MIDDLE_BLOCK]]:192; CHECK-NEXT: [[TMP28:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP26]])193; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[CONV]], [[N_VEC]]194; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]195; CHECK: [[SCALAR_PH]]:196;197entry:198 %conv = zext i16 %N to i32199 %cmp11 = icmp eq i16 %N, 0200 br i1 %cmp11, label %for.end, label %for.body.lr.ph201 202for.body.lr.ph:203 br label %for.body204 205for.body:206 %tmp.013 = phi i32 [ 0, %for.body.lr.ph ], [ %add4, %for.body ]207 %k.012 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]208 %mul = mul nuw i32 %k.012, %conv209 %add = add i32 %mul, %j210 %arrayidx = getelementptr inbounds i16, ptr %B, i32 %add211 %0 = load i16, ptr %arrayidx, align 2212 %conv3 = sext i16 %0 to i32213 %add4 = add nsw i32 %tmp.013, %conv3214 %inc = add nuw nsw i32 %k.012, 1215 %exitcond = icmp eq i32 %inc, %conv216 br i1 %exitcond, label %for.end.loopexit, label %for.body217 218for.end.loopexit:219 %add4.lcssa = phi i32 [ %add4, %for.body ]220 br label %for.end221 222for.end:223 %tmp.0.lcssa = phi i32 [ 0, %entry ], [ %add4.lcssa, %for.end.loopexit ]224 ret i32 %tmp.0.lcssa225}226