253 lines · plain
1; RUN: opt -S -passes=loop-vectorize -force-vector-width=8 -force-vector-interleave=1 < %s | FileCheck %s -check-prefix=VF82; RUN: opt -S -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 < %s | FileCheck %s -check-prefix=VF13 4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"5 6; Given a loop with an induction variable which is being7; truncated/extended using casts that had been proven to8; be redundant under a runtime test, we want to make sure9; that these casts, do not get vectorized/scalarized/widened. 10; This is the case for inductions whose SCEV expression is11; of the form "ExtTrunc(%phi) + %step", where "ExtTrunc"12; can be a result of the IR sequences we check below.13; 14; See also pr30654.15;16 17; Case1: Check the following induction pattern:18;19; %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]20; %sext = shl i32 %p.09, 2421; %conv = ashr exact i32 %sext, 2422; %add = add nsw i32 %conv, %step23; 24; This is the case in the following code:25;26; void doit1(int n, int step) {27; int i;28; char p = 0;29; for (i = 0; i < n; i++) {30; a[i] = p;31; p = p + step;32; }33; }34;35; The "ExtTrunc" IR sequence here is:36; "%sext = shl i32 %p.09, 24"37; "%conv = ashr exact i32 %sext, 24"38; We check that it does not appear in the vector loop body, whether39; we vectorize or scalarize the induction.40; In the case of widened induction, this means that the induction phi41; is directly used, without shl/ashr on the way.42 43; VF8-LABEL: @doit144; VF8: vector.body:45; VF8: %vec.ind = phi <8 x i32>46; VF8: store <8 x i32> %vec.ind47; VF8: middle.block: 48 49; VF1-LABEL: @doit150; VF1: vector.body:51; VF1-NOT: %{{.*}} = shl i3252; VF1: middle.block: 53 54@a = common local_unnamed_addr global [250 x i32] zeroinitializer, align 1655 56define void @doit1(i32 %n, i32 %step) {57entry:58 %cmp7 = icmp sgt i32 %n, 059 br i1 %cmp7, label %for.body.lr.ph, label %for.end60 61for.body.lr.ph:62 %wide.trip.count = zext i32 %n to i6463 br label %for.body64 65for.body:66 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]67 %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]68 %sext = shl i32 %p.09, 2469 %conv = ashr exact i32 %sext, 2470 %arrayidx = getelementptr inbounds [250 x i32], ptr @a, i64 0, i64 %indvars.iv71 store i32 %conv, ptr %arrayidx, align 472 %add = add nsw i32 %conv, %step73 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 174 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count75 br i1 %exitcond, label %for.end.loopexit, label %for.body76 77for.end.loopexit:78 br label %for.end79 80for.end:81 ret void82}83 84 85; Case2: Another variant of the above pattern is where the induction variable86; is used only for address compuation (i.e. it is a GEP index) and therefore87; the induction is not vectorized but rather only the step is widened. 88;89; This is the case in the following code, where the induction variable 'w_ix' 90; is only used to access the array 'in':91;92; void doit2(int *in, int *out, size_t size, size_t step)93; {94; int w_ix = 0;95; for (size_t offset = 0; offset < size; ++offset)96; {97; int w = in[w_ix];98; out[offset] = w;99; w_ix += step;100; }101; }102;103; The "ExtTrunc" IR sequence here is similar to the previous case:104; "%sext = shl i64 %w_ix.012, 32105; %idxprom = ashr exact i64 %sext, 32"106; We check that it does not appear in the vector loop body, whether107; we widen or scalarize the induction.108; In the case of widened induction, this means that the induction phi109; is directly used, without shl/ashr on the way.110 111; VF8-LABEL: @doit2112; VF8: vector.body:113; VF8-NEXT: [[INDEX:%.+]] = phi i64 [ 0, %vector.ph ]114; VF8-NEXT: [[OFFSET_IDX:%.+]] = mul i64 [[INDEX]], %step115; VF8-NEXT: [[MUL0:%.+]] = mul i64 0, %step116; VF8-NEXT: [[ADD:%.+]] = add i64 [[OFFSET_IDX]], [[MUL0]]117; VF8: getelementptr inbounds i32, ptr %in, i64 [[ADD]]118; VF8: middle.block:119 120; VF1-LABEL: @doit2121; VF1: vector.body:122; VF1-NOT: %{{.*}} = shl i64123; VF1: middle.block:124;125 126define void @doit2(ptr nocapture readonly %in, ptr nocapture %out, i64 %size, i64 %step) {127entry:128 %cmp9 = icmp eq i64 %size, 0129 br i1 %cmp9, label %for.cond.cleanup, label %for.body.lr.ph130 131for.body.lr.ph:132 br label %for.body133 134for.cond.cleanup.loopexit:135 br label %for.cond.cleanup136 137for.cond.cleanup:138 ret void139 140for.body:141 %w_ix.011 = phi i64 [ 0, %for.body.lr.ph ], [ %add, %for.body ]142 %offset.010 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]143 %sext = shl i64 %w_ix.011, 32144 %idxprom = ashr exact i64 %sext, 32145 %arrayidx = getelementptr inbounds i32, ptr %in, i64 %idxprom146 %0 = load i32, ptr %arrayidx, align 4147 %arrayidx1 = getelementptr inbounds i32, ptr %out, i64 %offset.010148 store i32 %0, ptr %arrayidx1, align 4149 %add = add i64 %idxprom, %step150 %inc = add nuw i64 %offset.010, 1151 %exitcond = icmp eq i64 %inc, %size152 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body153}154 155; Case3: Lastly, check also the following induction pattern:156; 157; %p.09 = phi i32 [ %val0, %scalar.ph ], [ %add, %for.body ]158; %conv = and i32 %p.09, 255159; %add = add nsw i32 %conv, %step160; 161; This is the case in the following code:162;163; int a[N];164; void doit3(int n, int step) {165; int i;166; unsigned char p = 0;167; for (i = 0; i < n; i++) {168; a[i] = p;169; p = p + step;170; }171; }172; 173; The "ExtTrunc" IR sequence here is:174; "%conv = and i32 %p.09, 255".175; We check that it does not appear in the vector loop body, whether176; we vectorize or scalarize the induction.177 178; VF8-LABEL: @doit3179; VF8: vector.body:180; VF8: %vec.ind = phi <8 x i32>181; VF8: store <8 x i32> %vec.ind182; VF8: middle.block: 183 184; VF1-LABEL: @doit3185; VF1: vector.body:186; VF1-NOT: %{{.*}} = and i32 187; VF1: middle.block: 188 189define void @doit3(i32 %n, i32 %step) {190entry:191 %cmp7 = icmp sgt i32 %n, 0192 br i1 %cmp7, label %for.body.lr.ph, label %for.end193 194for.body.lr.ph:195 %wide.trip.count = zext i32 %n to i64196 br label %for.body197 198for.body:199 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]200 %p.09 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ]201 %conv = and i32 %p.09, 255202 %arrayidx = getelementptr inbounds [250 x i32], ptr @a, i64 0, i64 %indvars.iv203 store i32 %conv, ptr %arrayidx, align 4204 %add = add nsw i32 %conv, %step205 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1206 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count207 br i1 %exitcond, label %for.end.loopexit, label %for.body208 209for.end.loopexit:210 br label %for.end211 212for.end:213 ret void214}215 216; VF8-LABEL: @test_conv_in_latch_block217; VF8: vector.body:218; VF8-NEXT: %index = phi i64219; VF8-NEXT: %vec.ind = phi <8 x i32>220; VF8: store <8 x i32> %vec.ind221; VF8: middle.block:222;223define void @test_conv_in_latch_block(i32 %n, i32 %step, ptr noalias %A, ptr noalias %B) {224entry:225 %wide.trip.count = zext i32 %n to i64226 br label %loop227 228loop:229 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]230 %p.09 = phi i32 [ 0, %entry ], [ %add, %latch ]231 %B.gep = getelementptr inbounds i32, ptr %B, i64 %iv232 %l = load i32, ptr %B.gep233 %c = icmp eq i32 %l, 0234 br i1 %c, label %then, label %latch235 236then:237 %A.gep = getelementptr inbounds i32, ptr %A, i64 %iv238 store i32 0, ptr %A.gep239 br label %latch240 241latch:242 %sext = shl i32 %p.09, 24243 %conv = ashr exact i32 %sext, 24244 %add = add nsw i32 %conv, %step245 store i32 %conv, ptr %B.gep, align 4246 %iv.next = add nuw nsw i64 %iv, 1247 %exitcond = icmp eq i64 %iv.next, %wide.trip.count248 br i1 %exitcond, label %exit, label %loop249 250exit:251 ret void252}253