229 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -enable-if-conversion -S | FileCheck %s3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"5 6define void @foo(ptr nocapture %A, ptr nocapture %B, i32 %n) {7; CHECK-LABEL: @foo(8; CHECK-NEXT: entry:9; CHECK-NEXT: [[CMP26:%.*]] = icmp sgt i32 [[N:%.*]], 010; CHECK-NEXT: br i1 [[CMP26]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]11; CHECK: for.body.preheader:12; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i6413; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 414; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]15; CHECK: vector.memcheck:16; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[N]], -117; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i6418; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 219; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 420; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP4]]21; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]]22; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]23; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]24; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]25; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]26; CHECK: vector.ph:27; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 214748364428; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]29; CHECK: vector.body:30; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]31; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]32; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]33; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]34; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !alias.scope [[META3]]35; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]36; CHECK-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 19)37; CHECK-NEXT: [[TMP9:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD2]], splat (i32 4)38; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> splat (i32 4), <4 x i32> splat (i32 5)39; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> splat (i32 3), <4 x i32> [[TMP10]]40; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> [[PREDPHI]], <4 x i32> splat (i32 9)41; CHECK-NEXT: store <4 x i32> [[PREDPHI3]], ptr [[TMP5]], align 4, !alias.scope [[META0]], !noalias [[META3]]42; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 443; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]44; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]45; CHECK: middle.block:46; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]47; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]48; CHECK: scalar.ph:49; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]50; CHECK-NEXT: br label [[FOR_BODY:%.*]]51; CHECK: for.body:52; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]53; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]54; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 455; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]56; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 457; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[TMP12]], [[TMP13]]58; CHECK-NEXT: br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[IF_END14]]59; CHECK: if.then:60; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP12]], 1961; CHECK-NEXT: br i1 [[CMP6]], label [[IF_END14]], label [[IF_ELSE:%.*]]62; CHECK: if.else:63; CHECK-NEXT: [[CMP10:%.*]] = icmp slt i32 [[TMP13]], 464; CHECK-NEXT: [[DOT:%.*]] = select i1 [[CMP10]], i32 4, i32 565; CHECK-NEXT: br label [[IF_END14]]66; CHECK: if.end14:67; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 9, [[FOR_BODY]] ], [ 3, [[IF_THEN]] ], [ [[DOT]], [[IF_ELSE]] ]68; CHECK-NEXT: store i32 [[X_0]], ptr [[ARRAYIDX]], align 469; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 170; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i3271; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]72; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]73; CHECK: for.end.loopexit:74; CHECK-NEXT: br label [[FOR_END]]75; CHECK: for.end:76; CHECK-NEXT: ret void77;78entry:79 %cmp26 = icmp sgt i32 %n, 080 br i1 %cmp26, label %for.body, label %for.end81 82for.body:83 %indvars.iv = phi i64 [ %indvars.iv.next, %if.end14 ], [ 0, %entry ]84 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv85 %0 = load i32, ptr %arrayidx, align 486 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv87 %1 = load i32, ptr %arrayidx2, align 488 %cmp3 = icmp sgt i32 %0, %189 br i1 %cmp3, label %if.then, label %if.end1490 91if.then:92 %cmp6 = icmp sgt i32 %0, 1993 br i1 %cmp6, label %if.end14, label %if.else94 95if.else:96 %cmp10 = icmp slt i32 %1, 497 %. = select i1 %cmp10, i32 4, i32 598 br label %if.end1499 100if.end14:101 %x.0 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %., %if.else ] ; <------------- A PHI with 3 entries that we can still vectorize.102 store i32 %x.0, ptr %arrayidx, align 4103 %indvars.iv.next = add i64 %indvars.iv, 1104 %lftr.wideiv = trunc i64 %indvars.iv.next to i32105 %exitcond = icmp eq i32 %lftr.wideiv, %n106 br i1 %exitcond, label %for.end, label %for.body107 108for.end:109 ret void110}111 112; As above but with multiple variables set per block.113define void @multi_variable_if_nest(ptr nocapture %A, ptr nocapture %B, i32 %n) {114; CHECK-LABEL: @multi_variable_if_nest(115; CHECK-NEXT: entry:116; CHECK-NEXT: [[CMP26:%.*]] = icmp sgt i32 [[N:%.*]], 0117; CHECK-NEXT: br i1 [[CMP26]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]118; CHECK: for.body.preheader:119; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[N]] to i64120; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4121; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]122; CHECK: vector.memcheck:123; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[N]], -1124; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64125; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2126; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 4127; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP4]]128; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]]129; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP1]]130; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[B]], [[SCEVGEP]]131; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]132; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]133; CHECK: vector.ph:134; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644135; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]136; CHECK: vector.body:137; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]138; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]139; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]]140; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]141; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4, !alias.scope [[META12]]142; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]143; CHECK-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], splat (i32 19)144; CHECK-NEXT: [[TMP11:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD2]], splat (i32 4)145; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> splat (i32 4), <4 x i32> splat (i32 5)146; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> splat (i32 6), <4 x i32> splat (i32 11)147; CHECK-NEXT: [[TMP14:%.*]] = and <4 x i1> [[TMP7]], [[TMP8]]148; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> splat (i32 3), <4 x i32> [[TMP12]]149; CHECK-NEXT: [[PREDPHI3:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> [[PREDPHI]], <4 x i32> splat (i32 9)150; CHECK-NEXT: [[PREDPHI4:%.*]] = select <4 x i1> [[TMP14]], <4 x i32> splat (i32 7), <4 x i32> [[TMP13]]151; CHECK-NEXT: [[PREDPHI5:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> [[PREDPHI4]], <4 x i32> splat (i32 18)152; CHECK-NEXT: store <4 x i32> [[PREDPHI3]], ptr [[TMP5]], align 4, !alias.scope [[META9]], !noalias [[META12]]153; CHECK-NEXT: store <4 x i32> [[PREDPHI5]], ptr [[TMP6]], align 4, !alias.scope [[META12]]154; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4155; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]156; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]157; CHECK: middle.block:158; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]159; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]160; CHECK: scalar.ph:161; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ], [ 0, [[VECTOR_MEMCHECK]] ]162; CHECK-NEXT: br label [[FOR_BODY:%.*]]163; CHECK: for.body:164; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[IF_END14:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]165; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]166; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4167; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]168; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4169; CHECK-NEXT: [[CMP3:%.*]] = icmp sgt i32 [[TMP16]], [[TMP17]]170; CHECK-NEXT: br i1 [[CMP3]], label [[IF_THEN:%.*]], label [[IF_END14]]171; CHECK: if.then:172; CHECK-NEXT: [[CMP6:%.*]] = icmp sgt i32 [[TMP16]], 19173; CHECK-NEXT: br i1 [[CMP6]], label [[IF_END14]], label [[IF_ELSE:%.*]]174; CHECK: if.else:175; CHECK-NEXT: [[CMP10:%.*]] = icmp slt i32 [[TMP17]], 4176; CHECK-NEXT: [[X_ELSE:%.*]] = select i1 [[CMP10]], i32 4, i32 5177; CHECK-NEXT: [[Y_ELSE:%.*]] = select i1 [[CMP10]], i32 6, i32 11178; CHECK-NEXT: br label [[IF_END14]]179; CHECK: if.end14:180; CHECK-NEXT: [[X_0:%.*]] = phi i32 [ 9, [[FOR_BODY]] ], [ 3, [[IF_THEN]] ], [ [[X_ELSE]], [[IF_ELSE]] ]181; CHECK-NEXT: [[Y_0:%.*]] = phi i32 [ 18, [[FOR_BODY]] ], [ 7, [[IF_THEN]] ], [ [[Y_ELSE]], [[IF_ELSE]] ]182; CHECK-NEXT: store i32 [[X_0]], ptr [[ARRAYIDX]], align 4183; CHECK-NEXT: store i32 [[Y_0]], ptr [[ARRAYIDX2]], align 4184; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1185; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32186; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[N]], [[LFTR_WIDEIV]]187; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]188; CHECK: for.end.loopexit:189; CHECK-NEXT: br label [[FOR_END]]190; CHECK: for.end:191; CHECK-NEXT: ret void192;193entry:194 %cmp26 = icmp sgt i32 %n, 0195 br i1 %cmp26, label %for.body, label %for.end196 197for.body:198 %indvars.iv = phi i64 [ %indvars.iv.next, %if.end14 ], [ 0, %entry ]199 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv200 %0 = load i32, ptr %arrayidx, align 4201 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv202 %1 = load i32, ptr %arrayidx2, align 4203 %cmp3 = icmp sgt i32 %0, %1204 br i1 %cmp3, label %if.then, label %if.end14205 206if.then:207 %cmp6 = icmp sgt i32 %0, 19208 br i1 %cmp6, label %if.end14, label %if.else209 210if.else:211 %cmp10 = icmp slt i32 %1, 4212 %x.else = select i1 %cmp10, i32 4, i32 5213 %y.else = select i1 %cmp10, i32 6, i32 11214 br label %if.end14215 216if.end14:217 %x.0 = phi i32 [ 9, %for.body ], [ 3, %if.then ], [ %x.else, %if.else ] ; <------------- A PHI with 3 entries that we can still vectorize.218 %y.0 = phi i32 [ 18, %for.body ], [ 7, %if.then ], [ %y.else, %if.else ] ; <------------- A PHI with 3 entries that we can still vectorize.219 store i32 %x.0, ptr %arrayidx, align 4220 store i32 %y.0, ptr %arrayidx2, align 4221 %indvars.iv.next = add i64 %indvars.iv, 1222 %lftr.wideiv = trunc i64 %indvars.iv.next to i32223 %exitcond = icmp eq i32 %lftr.wideiv, %n224 br i1 %exitcond, label %for.end, label %for.body225 226for.end:227 ret void228}229