274 lines · plain
1; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S | FileCheck %s2; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s -check-prefix=WIDTH3; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 < %s | FileCheck %s -check-prefix=RIGHTVF4; RUN: opt -S -passes=loop-vectorize -force-vector-width=8 < %s | FileCheck %s -check-prefix=WRONGVF5 6target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"7 8; Vectorization with dependence checks.9 10; No plausible dependence - can be vectorized.11; for (i = 0; i < 1024; ++i)12; A[i] = A[i + 1] + 1;13 14; CHECK-LABEL: @f1_vec(15; CHECK: <2 x i32>16 17define void @f1_vec(ptr %A) {18entry:19 br label %for.body20 21for.body:22 %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]23 %indvars.iv.next = add i32 %indvars.iv, 124 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv.next25 %0 = load i32, ptr %arrayidx, align 426 %add1 = add nsw i32 %0, 127 %arrayidx3 = getelementptr inbounds i32, ptr %A, i32 %indvars.iv28 store i32 %add1, ptr %arrayidx3, align 429 %exitcond = icmp ne i32 %indvars.iv.next, 102430 br i1 %exitcond, label %for.body, label %for.end31 32for.end:33 ret void34}35 36; Plausible dependence of distance 1 - can't be vectorized.37; for (i = 0; i < 1024; ++i)38; A[i+1] = A[i] + 1;39 40; CHECK-LABEL: @f2_novec(41; CHECK-NOT: <2 x i32>42 43define void @f2_novec(ptr %A) {44entry:45 br label %for.body46 47for.body:48 %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ]49 %arrayidx = getelementptr inbounds i32, ptr %A, i32 %indvars.iv50 %0 = load i32, ptr %arrayidx, align 451 %add = add nsw i32 %0, 152 %indvars.iv.next = add i32 %indvars.iv, 153 %arrayidx3 = getelementptr inbounds i32, ptr %A, i32 %indvars.iv.next54 store i32 %add, ptr %arrayidx3, align 455 %exitcond = icmp ne i32 %indvars.iv.next, 102456 br i1 %exitcond, label %for.body, label %for.end57 58for.end:59 ret void60}61 62; Plausible dependence of distance 2 - can be vectorized with a width of 2.63; for (i = 0; i < 1024; ++i)64; A[i+2] = A[i] + 1;65 66; CHECK-LABEL: @f3_vec_len(67; CHECK: <2 x i32>68 69; WIDTH: f3_vec_len70; WIDTH-NOT: <4 x i32>71 72define void @f3_vec_len(ptr %A) {73entry:74 br label %for.body75 76for.body:77 %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ]78 %idxprom = sext i32 %i.01 to i6479 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %idxprom80 %0 = load i32, ptr %arrayidx, align 481 %add = add nsw i32 %0, 182 %add1 = add nsw i32 %i.01, 283 %idxprom2 = sext i32 %add1 to i6484 %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom285 store i32 %add, ptr %arrayidx3, align 486 %inc = add nsw i32 %i.01, 187 %cmp = icmp slt i32 %inc, 102488 br i1 %cmp, label %for.body, label %for.end89 90for.end:91 ret void92}93 94; Plausible dependence of distance 1 - cannot be vectorized (without reordering95; accesses).96; for (i = 0; i < 1024; ++i) {97; B[i] = A[i];98; A[i] = B[i + 1];99; }100 101; CHECK-LABEL: @f5(102; CHECK-NOT: <2 x i32>103 104define void @f5(ptr %A, ptr %B) {105entry:106 br label %for.body107 108for.body:109 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]110 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv111 %0 = load i32, ptr %arrayidx, align 4112 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv113 store i32 %0, ptr %arrayidx2, align 4114 %indvars.iv.next = add nsw i64 %indvars.iv, 1115 %arrayidx4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv.next116 %1 = load i32, ptr %arrayidx4, align 4117 store i32 %1, ptr %arrayidx, align 4118 %lftr.wideiv = trunc i64 %indvars.iv.next to i32119 %exitcond = icmp ne i32 %lftr.wideiv, 1024120 br i1 %exitcond, label %for.body, label %for.end121 122for.end:123 ret void124}125 126; Dependence through a phi node - must not vectorize.127; for (i = 0; i < 1024; ++i) {128; a[i+1] = tmp;129; tmp = a[i];130; }131 132; CHECK-LABEL: @f6133; CHECK-NOT: <2 x i32>134 135define void @f6(ptr %a, i32 %tmp) {136entry:137 br label %for.body138 139for.body:140 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]141 %tmp.addr.08 = phi i32 [ %tmp, %entry ], [ %0, %for.body ]142 %indvars.iv.next = add nsw i64 %indvars.iv, 1143 %arrayidx = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.next144 store i32 %tmp.addr.08, ptr %arrayidx, align 4145 %arrayidx3 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv146 %0 = load i32, ptr %arrayidx3, align 4147 %lftr.wideiv = trunc i64 %indvars.iv.next to i32148 %exitcond = icmp ne i32 %lftr.wideiv, 1024149 br i1 %exitcond, label %for.body, label %for.end150 151for.end:152 ret void153}154 155; Don't vectorize true loop carried dependencies that are not a multiple of the156; vector width.157; Example:158; for (int i = ...; ++i) {159; a[i] = a[i-3] + ...;160; It is a bad idea to vectorize this loop because store-load forwarding will not161; happen.162;163 164; CHECK-LABEL: @nostoreloadforward(165; CHECK-NOT: <2 x i32>166 167define void @nostoreloadforward(ptr %A) {168entry:169 br label %for.body170 171for.body:172 %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ]173 %0 = add nsw i64 %indvars.iv, -3174 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %0175 %1 = load i32, ptr %arrayidx, align 4176 %2 = add nsw i64 %indvars.iv, 4177 %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %2178 %3 = load i32, ptr %arrayidx2, align 4179 %add3 = add nsw i32 %3, %1180 %arrayidx5 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv181 store i32 %add3, ptr %arrayidx5, align 4182 %indvars.iv.next = add i64 %indvars.iv, 1183 %lftr.wideiv = trunc i64 %indvars.iv.next to i32184 %exitcond = icmp ne i32 %lftr.wideiv, 128185 br i1 %exitcond, label %for.body, label %for.end186 187for.end:188 ret void189}190 191; Example:192; for (int i = ...; ++i) {193; a[i] = b[i];194; c[i] = a[i-3] + ...;195; It is a bad idea to vectorize this loop because store-load forwarding will not196; happen.197;198 199; CHECK-LABEL: @nostoreloadforward2(200; CHECK-NOT: <2 x i32>201 202define void @nostoreloadforward2(ptr noalias %A, ptr noalias %B, ptr noalias %C) {203entry:204 br label %for.body205 206for.body:207 %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ]208 %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv209 %0 = load i32, ptr %arrayidx, align 4210 %arrayidx2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv211 store i32 %0, ptr %arrayidx2, align 4212 %1 = add nsw i64 %indvars.iv, -3213 %arrayidx4 = getelementptr inbounds i32, ptr %A, i64 %1214 %2 = load i32, ptr %arrayidx4, align 4215 %arrayidx6 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv216 store i32 %2, ptr %arrayidx6, align 4217 %indvars.iv.next = add i64 %indvars.iv, 1218 %lftr.wideiv = trunc i64 %indvars.iv.next to i32219 %exitcond = icmp ne i32 %lftr.wideiv, 128220 br i1 %exitcond, label %for.body, label %for.end221 222for.end:223 ret void224}225 226 227;Check the new calculation of the maximum safe distance in bits which can be vectorized.228;The previous behavior did not take account that the stride was 2.229;Therefore the maxVF was computed as 8 instead of 4, as the dependence distance here is 6 iterations, given by |N-(N-12)|/2.230 231;#define M 32232;#define N 2 * M233;unsigned int a [N];234;void pr34283(){235; unsigned int j=0;236; for (j = 0; j < M - 6; ++j)237; {238; a[N - 2 * j] = 69;239; a[N - 12 - 2 * j] = 7;240; }241;242;}243 244; RIGHTVF-LABEL: @pr34283245; RIGHTVF: <4 x i64>246 247; WRONGVF-LABLE: @pr34283248; WRONGVF-NOT: <8 x i64>249 250@a = common local_unnamed_addr global [64 x i32] zeroinitializer, align 16251 252; Function Attrs: norecurse nounwind uwtable253define void @pr34283() local_unnamed_addr {254entry:255 br label %for.body256 257for.body:258 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]259 %0 = shl i64 %indvars.iv, 1260 %1 = sub nuw nsw i64 64, %0261 %arrayidx = getelementptr inbounds [64 x i32], ptr @a, i64 0, i64 %1262 store i32 69, ptr %arrayidx, align 8263 %2 = sub nuw nsw i64 52, %0264 %arrayidx4 = getelementptr inbounds [64 x i32], ptr @a, i64 0, i64 %2265 store i32 7, ptr %arrayidx4, align 8266 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1267 %exitcond = icmp eq i64 %indvars.iv.next, 26268 br i1 %exitcond, label %for.end, label %for.body269 270for.end:271 ret void272}273 274