619 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -loop-vectorize-with-block-frequency -S | FileCheck %s3 4target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"5target triple = "x86_64-apple-macosx10.8.0"6 7@b = common global [2048 x i32] zeroinitializer, align 168@c = common global [2048 x i32] zeroinitializer, align 169@a = common global [2048 x i32] zeroinitializer, align 1610@G = common global [32 x [1024 x i32]] zeroinitializer, align 1611@ub = common global [1024 x i32] zeroinitializer, align 1612@uc = common global [1024 x i32] zeroinitializer, align 1613@d = common global [2048 x i32] zeroinitializer, align 1614@fa = common global [1024 x float] zeroinitializer, align 1615@fb = common global [1024 x float] zeroinitializer, align 1616@ic = common global [1024 x i32] zeroinitializer, align 1617@da = common global [1024 x float] zeroinitializer, align 1618@db = common global [1024 x float] zeroinitializer, align 1619@dc = common global [1024 x float] zeroinitializer, align 1620@dd = common global [1024 x float] zeroinitializer, align 1621@dj = common global [1024 x i32] zeroinitializer, align 1622 23; We can optimize this test without a tail.24define void @example1() optsize {25; CHECK-LABEL: @example1(26; CHECK-NEXT: br label [[VECTOR_PH:%.*]]27; CHECK: vector.ph:28; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]29; CHECK: vector.body:30; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[INDEX]]32; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 433; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr @c, i64 [[INDEX]]34; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 435; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]36; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr @a, i64 [[INDEX]]37; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP4]], align 438; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 439; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 25640; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]41; CHECK: middle.block:42; CHECK-NEXT: br label [[TMP6:%.*]]43; CHECK: 6:44; CHECK-NEXT: ret void45;46 br label %147 48; <label>:1 ; preds = %1, %049 %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ]50 %2 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv51 %3 = load i32, ptr %2, align 452 %4 = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %indvars.iv53 %5 = load i32, ptr %4, align 454 %6 = add nsw i32 %5, %355 %7 = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %indvars.iv56 store i32 %6, ptr %7, align 457 %indvars.iv.next = add i64 %indvars.iv, 158 %lftr.wideiv = trunc i64 %indvars.iv.next to i3259 %exitcond = icmp eq i32 %lftr.wideiv, 25660 br i1 %exitcond, label %8, label %161 62; <label>:8 ; preds = %163 ret void64}65 66; Can vectorize in 'optsize' mode by masking the needed tail.67define void @example2(i32 %n, i32 %x) optsize {68; CHECK-LABEL: @example2(69; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt i32 [[N:%.*]], 070; CHECK-NEXT: br i1 [[TMP1]], label [[DOTLR_PH5_PREHEADER:%.*]], label [[DOTPREHEADER:%.*]]71; CHECK: .lr.ph5.preheader:72; CHECK-NEXT: br label [[VECTOR_PH:%.*]]73; CHECK: vector.ph:74; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[N]] to i6475; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP2]], 376; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 429496729277; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[TMP2]], -178; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 079; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer80; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]81; CHECK: vector.body:82; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]83; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]84; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]85; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 086; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]87; CHECK: pred.store.if:88; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[INDEX]]89; CHECK-NEXT: store i32 [[X:%.*]], ptr [[TMP5]], align 490; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]91; CHECK: pred.store.continue:92; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 193; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2:%.*]]94; CHECK: pred.store.if1:95; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr @b, i64 [[INDEX]]96; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i64 497; CHECK-NEXT: store i32 [[X]], ptr [[TMP8]], align 498; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]99; CHECK: pred.store.continue2:100; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2101; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]102; CHECK: pred.store.if3:103; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr @b, i64 [[INDEX]]104; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8105; CHECK-NEXT: store i32 [[X]], ptr [[TMP11]], align 4106; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]107; CHECK: pred.store.continue4:108; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3109; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6]]110; CHECK: pred.store.if5:111; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr @b, i64 [[INDEX]]112; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i64 12113; CHECK-NEXT: store i32 [[X]], ptr [[TMP14]], align 4114; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]115; CHECK: pred.store.continue6:116; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4117; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)118; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]119; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]120; CHECK: middle.block:121; CHECK-NEXT: br label [[DOT_PREHEADER_CRIT_EDGE:%.*]]122; CHECK: ..preheader_crit_edge:123; CHECK-NEXT: [[PHITMP:%.*]] = zext nneg i32 [[N]] to i64124; CHECK-NEXT: br label [[DOTPREHEADER]]125; CHECK: .preheader:126; CHECK-NEXT: [[I_0_LCSSA:%.*]] = phi i64 [ [[PHITMP]], [[DOT_PREHEADER_CRIT_EDGE]] ], [ 0, [[TMP0:%.*]] ]127; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[N]], 0128; CHECK-NEXT: br i1 [[TMP16]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]]129; CHECK: .lr.ph.preheader:130; CHECK-NEXT: br label [[VECTOR_PH8:%.*]]131; CHECK: vector.ph7:132; CHECK-NEXT: [[TMP17:%.*]] = zext i32 [[N]] to i64133; CHECK-NEXT: [[N_RND_UP10:%.*]] = add nuw nsw i64 [[TMP17]], 3134; CHECK-NEXT: [[N_VEC12:%.*]] = and i64 [[N_RND_UP10]], 8589934588135; CHECK-NEXT: [[TRIP_COUNT_MINUS_114:%.*]] = add nsw i64 [[TMP17]], -1136; CHECK-NEXT: [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_114]], i64 0137; CHECK-NEXT: [[BROADCAST_SPLAT20:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT19]], <4 x i64> poison, <4 x i32> zeroinitializer138; CHECK-NEXT: br label [[VECTOR_BODY13:%.*]]139; CHECK: vector.body14:140; CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ 0, [[VECTOR_PH8]] ], [ [[INDEX_NEXT29:%.*]], [[PRED_STORE_CONTINUE26:%.*]] ]141; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX16]]142; CHECK-NEXT: [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX16]], i64 0143; CHECK-NEXT: [[BROADCAST_SPLAT18:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT17]], <4 x i64> poison, <4 x i32> zeroinitializer144; CHECK-NEXT: [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT18]], <i64 0, i64 1, i64 2, i64 3>145; CHECK-NEXT: [[TMP18:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT20]]146; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0147; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]148; CHECK: pred.store.if18:149; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[OFFSET_IDX]]150; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4151; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr @c, i64 [[OFFSET_IDX]]152; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4153; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr @a, i64 [[OFFSET_IDX]]154; CHECK-NEXT: [[TMP25:%.*]] = and i32 [[TMP23]], [[TMP21]]155; CHECK-NEXT: store i32 [[TMP25]], ptr [[TMP24]], align 4156; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]]157; CHECK: pred.store.continue19:158; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP18]], i64 1159; CHECK-NEXT: br i1 [[TMP26]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]160; CHECK: pred.store.if20:161; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[OFFSET_IDX]], 1162; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[TMP27]]163; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4164; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr @c, i64 [[TMP27]]165; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4166; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr @a, i64 [[TMP27]]167; CHECK-NEXT: [[TMP33:%.*]] = and i32 [[TMP31]], [[TMP29]]168; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4169; CHECK-NEXT: br label [[PRED_STORE_CONTINUE22]]170; CHECK: pred.store.continue21:171; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP18]], i64 2172; CHECK-NEXT: br i1 [[TMP34]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]173; CHECK: pred.store.if22:174; CHECK-NEXT: [[TMP35:%.*]] = add i64 [[OFFSET_IDX]], 2175; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[TMP35]]176; CHECK-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4177; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr @c, i64 [[TMP35]]178; CHECK-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4179; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i32, ptr @a, i64 [[TMP35]]180; CHECK-NEXT: [[TMP41:%.*]] = and i32 [[TMP39]], [[TMP37]]181; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP40]], align 4182; CHECK-NEXT: br label [[PRED_STORE_CONTINUE24]]183; CHECK: pred.store.continue23:184; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP18]], i64 3185; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26]]186; CHECK: pred.store.if24:187; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[OFFSET_IDX]], 3188; CHECK-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr @b, i64 [[TMP43]]189; CHECK-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4190; CHECK-NEXT: [[TMP46:%.*]] = getelementptr inbounds i32, ptr @c, i64 [[TMP43]]191; CHECK-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP46]], align 4192; CHECK-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, ptr @a, i64 [[TMP43]]193; CHECK-NEXT: [[TMP49:%.*]] = and i32 [[TMP47]], [[TMP45]]194; CHECK-NEXT: store i32 [[TMP49]], ptr [[TMP48]], align 4195; CHECK-NEXT: br label [[PRED_STORE_CONTINUE26]]196; CHECK: pred.store.continue25:197; CHECK-NEXT: [[INDEX_NEXT29]] = add nuw i64 [[INDEX16]], 4198; CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT29]], [[N_VEC12]]199; CHECK-NEXT: br i1 [[TMP50]], label [[MIDDLE_BLOCK27:%.*]], label [[VECTOR_BODY13]], !llvm.loop [[LOOP4:![0-9]+]]200; CHECK: middle.block27:201; CHECK-NEXT: br label [[DOTLR_PH1:%.*]]202; CHECK: ._crit_edge.loopexit:203; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]204; CHECK: ._crit_edge:205; CHECK-NEXT: ret void206;207 %1 = icmp sgt i32 %n, 0208 br i1 %1, label %.lr.ph5, label %.preheader209 210..preheader_crit_edge: ; preds = %.lr.ph5211 %phitmp = sext i32 %n to i64212 br label %.preheader213 214.preheader: ; preds = %..preheader_crit_edge, %0215 %i.0.lcssa = phi i64 [ %phitmp, %..preheader_crit_edge ], [ 0, %0 ]216 %2 = icmp eq i32 %n, 0217 br i1 %2, label %._crit_edge, label %.lr.ph218 219.lr.ph5: ; preds = %0, %.lr.ph5220 %indvars.iv6 = phi i64 [ %indvars.iv.next7, %.lr.ph5 ], [ 0, %0 ]221 %3 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv6222 store i32 %x, ptr %3, align 4223 %indvars.iv.next7 = add i64 %indvars.iv6, 1224 %lftr.wideiv = trunc i64 %indvars.iv.next7 to i32225 %exitcond = icmp eq i32 %lftr.wideiv, %n226 br i1 %exitcond, label %..preheader_crit_edge, label %.lr.ph5227 228.lr.ph: ; preds = %.preheader, %.lr.ph229 %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ %i.0.lcssa, %.preheader ]230 %.02 = phi i32 [ %4, %.lr.ph ], [ %n, %.preheader ]231 %4 = add nsw i32 %.02, -1232 %5 = getelementptr inbounds [2048 x i32], ptr @b, i64 0, i64 %indvars.iv233 %6 = load i32, ptr %5, align 4234 %7 = getelementptr inbounds [2048 x i32], ptr @c, i64 0, i64 %indvars.iv235 %8 = load i32, ptr %7, align 4236 %9 = and i32 %8, %6237 %10 = getelementptr inbounds [2048 x i32], ptr @a, i64 0, i64 %indvars.iv238 store i32 %9, ptr %10, align 4239 %indvars.iv.next = add i64 %indvars.iv, 1240 %11 = icmp eq i32 %4, 0241 br i1 %11, label %._crit_edge, label %.lr.ph242 243._crit_edge: ; preds = %.lr.ph, %.preheader244 ret void245}246 247; Loop has no primary induction as its integer IV has step -1 starting at248; unknown N, but can still be vectorized.249; Note: Most scalar pointer induction GEPs could be sunk into the conditional250; blocks.251define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q) optsize {252; CHECK-LABEL: @example3(253; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N:%.*]], 0254; CHECK-NEXT: br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH_PREHEADER:%.*]]255; CHECK: .lr.ph.preheader:256; CHECK-NEXT: br label [[VECTOR_PH:%.*]]257; CHECK: vector.ph:258; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[N]] to i64259; CHECK-NEXT: [[N_RND_UP:%.*]] = add nuw nsw i64 [[TMP2]], 3260; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N_RND_UP]], 8589934588261; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add nsw i64 [[TMP2]], -1262; CHECK-NEXT: [[BROADCAST_SPLATINSERT11:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0263; CHECK-NEXT: [[BROADCAST_SPLAT12:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT11]], <4 x i64> poison, <4 x i32> zeroinitializer264; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]265; CHECK: vector.body:266; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ]267; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2268; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[OFFSET_IDX]]269; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[TMP11]], i64 4270; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFFSET_IDX]]271; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[TMP12]], i64 8272; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFFSET_IDX]]273; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[TMP15]], i64 12274; CHECK-NEXT: [[OFFSET_IDX6:%.*]] = shl i64 [[INDEX]], 2275; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[Q:%.*]], i64 [[OFFSET_IDX6]]276; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TMP16]], i64 4277; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[Q]], i64 [[OFFSET_IDX6]]278; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[TMP7]], i64 8279; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[Q]], i64 [[OFFSET_IDX6]]280; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[TMP8]], i64 12281; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0282; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer283; CHECK-NEXT: [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>284; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT12]]285; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0286; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]287; CHECK: pred.store.if:288; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[Q]], i64 [[OFFSET_IDX6]]289; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[OFFSET_IDX]]290; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[NEXT_GEP7]], align 16291; CHECK-NEXT: store i32 [[TMP5]], ptr [[NEXT_GEP]], align 16292; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]293; CHECK: pred.store.continue:294; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1295; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]296; CHECK: pred.store.if11:297; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP8]], align 16298; CHECK-NEXT: store i32 [[TMP9]], ptr [[NEXT_GEP3]], align 16299; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]300; CHECK: pred.store.continue12:301; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2302; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]303; CHECK: pred.store.if13:304; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP9]], align 16305; CHECK-NEXT: store i32 [[TMP13]], ptr [[NEXT_GEP4]], align 16306; CHECK-NEXT: br label [[PRED_STORE_CONTINUE14]]307; CHECK: pred.store.continue14:308; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3309; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16]]310; CHECK: pred.store.if15:311; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16312; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP5]], align 16313; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]]314; CHECK: pred.store.continue16:315; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4316; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]317; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]318; CHECK: middle.block:319; CHECK-NEXT: br label [[DOTLR_PH:%.*]]320; CHECK: ._crit_edge.loopexit:321; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]322; CHECK: ._crit_edge:323; CHECK-NEXT: ret void324;325 %1 = icmp eq i32 %n, 0326 br i1 %1, label %._crit_edge, label %.lr.ph327 328.lr.ph: ; preds = %0, %.lr.ph329 %.05 = phi i32 [ %2, %.lr.ph ], [ %n, %0 ]330 %.014 = phi ptr [ %5, %.lr.ph ], [ %p, %0 ]331 %.023 = phi ptr [ %3, %.lr.ph ], [ %q, %0 ]332 %2 = add nsw i32 %.05, -1333 %3 = getelementptr inbounds i32, ptr %.023, i64 1334 %4 = load i32, ptr %.023, align 16335 %5 = getelementptr inbounds i32, ptr %.014, i64 1336 store i32 %4, ptr %.014, align 16337 %6 = icmp eq i32 %2, 0338 br i1 %6, label %._crit_edge, label %.lr.ph339 340._crit_edge: ; preds = %.lr.ph, %0341 ret void342}343 344; We can't vectorize this one because we need a runtime ptr check.345define void @example23(ptr nocapture %src, ptr nocapture %dst) optsize {346; CHECK-LABEL: @example23(347; CHECK-NEXT: br label [[TMP1:%.*]]348; CHECK: 1:349; CHECK-NEXT: [[DOT04:%.*]] = phi ptr [ [[SRC:%.*]], [[TMP0:%.*]] ], [ [[TMP2:%.*]], [[TMP1]] ]350; CHECK-NEXT: [[DOT013:%.*]] = phi ptr [ [[DST:%.*]], [[TMP0]] ], [ [[TMP6:%.*]], [[TMP1]] ]351; CHECK-NEXT: [[I_02:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]352; CHECK-NEXT: [[TMP2]] = getelementptr inbounds nuw i8, ptr [[DOT04]], i64 2353; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[DOT04]], align 2354; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32355; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7356; CHECK-NEXT: [[TMP6]] = getelementptr inbounds nuw i8, ptr [[DOT013]], i64 4357; CHECK-NEXT: store i32 [[TMP5]], ptr [[DOT013]], align 4358; CHECK-NEXT: [[TMP7]] = add nuw nsw i32 [[I_02]], 1359; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[TMP7]], 256360; CHECK-NEXT: br i1 [[EXITCOND]], label [[TMP8:%.*]], label [[TMP1]]361; CHECK: 8:362; CHECK-NEXT: ret void363;364 br label %1365 366; <label>:1 ; preds = %1, %0367 %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]368 %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]369 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]370 %2 = getelementptr inbounds i16, ptr %.04, i64 1371 %3 = load i16, ptr %.04, align 2372 %4 = zext i16 %3 to i32373 %5 = shl nuw nsw i32 %4, 7374 %6 = getelementptr inbounds i32, ptr %.013, i64 1375 store i32 %5, ptr %.013, align 4376 %7 = add nsw i32 %i.02, 1377 %exitcond = icmp eq i32 %7, 256378 br i1 %exitcond, label %8, label %1379 380; <label>:8 ; preds = %1381 ret void382}383 384 385; We CAN vectorize this example because the pointers are marked as noalias.386define void @example23b(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {387; CHECK-LABEL: @example23b(388; CHECK-NEXT: br label [[VECTOR_PH:%.*]]389; CHECK: vector.ph:390; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]391; CHECK: vector.body:392; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]393; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1394; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]395; CHECK-NEXT: [[OFFSET_IDX2:%.*]] = shl i64 [[INDEX]], 2396; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX2]]397; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[NEXT_GEP]], align 2398; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i16> [[WIDE_LOAD]] to <4 x i32>399; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw <4 x i32> [[TMP1]], splat (i32 7)400; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[NEXT_GEP3]], align 4401; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4402; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256403; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]404; CHECK: middle.block:405; CHECK-NEXT: br label [[TMP4:%.*]]406; CHECK: 4:407; CHECK-NEXT: ret void408;409 br label %1410 411; <label>:1 ; preds = %1, %0412 %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]413 %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]414 %i.02 = phi i32 [ 0, %0 ], [ %7, %1 ]415 %2 = getelementptr inbounds i16, ptr %.04, i64 1416 %3 = load i16, ptr %.04, align 2417 %4 = zext i16 %3 to i32418 %5 = shl nuw nsw i32 %4, 7419 %6 = getelementptr inbounds i32, ptr %.013, i64 1420 store i32 %5, ptr %.013, align 4421 %7 = add nsw i32 %i.02, 1422 %exitcond = icmp eq i32 %7, 256423 br i1 %exitcond, label %8, label %1424 425; <label>:8 ; preds = %1426 ret void427}428 429; We CAN vectorize this example by folding the tail it entails.430; Note: Most scalar pointer induction GEPs could be sunk into the conditional431; blocks.432define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {433; CHECK-LABEL: @example23c(434; CHECK-NEXT: br label [[VECTOR_PH:%.*]]435; CHECK: vector.ph:436; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]437; CHECK: vector.body:438; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ]439; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1440; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]441; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[TMP7]], i64 2442; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]443; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[TMP8]], i64 4444; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]445; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[TMP13]], i64 6446; CHECK-NEXT: [[OFFSET_IDX5:%.*]] = shl i64 [[INDEX]], 2447; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX5]]448; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[TMP14]], i64 4449; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX5]]450; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TMP19]], i64 8451; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX5]]452; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[TMP20]], i64 12453; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0454; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer455; CHECK-NEXT: [[VEC_IV:%.*]] = or disjoint <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>456; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <4 x i64> [[VEC_IV]], splat (i64 257)457; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0458; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]459; CHECK: pred.store.if:460; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX5]]461; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]462; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[NEXT_GEP]], align 2463; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32464; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7465; CHECK-NEXT: store i32 [[TMP5]], ptr [[NEXT_GEP6]], align 4466; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]467; CHECK: pred.store.continue:468; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1469; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]470; CHECK: pred.store.if9:471; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[NEXT_GEP2]], align 2472; CHECK-NEXT: [[TMP10:%.*]] = zext i16 [[TMP9]] to i32473; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7474; CHECK-NEXT: store i32 [[TMP11]], ptr [[NEXT_GEP7]], align 4475; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]476; CHECK: pred.store.continue10:477; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2478; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]479; CHECK: pred.store.if11:480; CHECK-NEXT: [[TMP15:%.*]] = load i16, ptr [[NEXT_GEP3]], align 2481; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP15]] to i32482; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7483; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP8]], align 4484; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]485; CHECK: pred.store.continue12:486; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3487; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE15]]488; CHECK: pred.store.if13:489; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP4]], align 2490; CHECK-NEXT: [[TMP22:%.*]] = zext i16 [[TMP21]] to i32491; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7492; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP9]], align 4493; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]]494; CHECK: pred.store.continue14:495; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4496; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260497; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]498; CHECK: middle.block:499; CHECK-NEXT: br label [[TMP25:%.*]]500; CHECK: 25:501; CHECK-NEXT: ret void502;503 br label %1504 505; <label>:1 ; preds = %1, %0506 %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]507 %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]508 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]509 %2 = getelementptr inbounds i16, ptr %.04, i64 1510 %3 = load i16, ptr %.04, align 2511 %4 = zext i16 %3 to i32512 %5 = shl nuw nsw i32 %4, 7513 %6 = getelementptr inbounds i32, ptr %.013, i64 1514 store i32 %5, ptr %.013, align 4515 %7 = add nsw i64 %i.02, 1516 %exitcond = icmp eq i64 %7, 257517 br i1 %exitcond, label %8, label %1518 519; <label>:8 ; preds = %1520 ret void521}522 523; We CAN'T vectorize this example because it would entail a tail and an524; induction is used outside the loop.525define i64 @example23d(ptr noalias nocapture %src, ptr noalias nocapture %dst) optsize {526; CHECK-LABEL: @example23d(527; CHECK-NEXT: br label [[VECTOR_PH:%.*]]528; CHECK: vector.ph:529; CHECK-NEXT: br label [[TMP1:%.*]]530; CHECK: vector.body:531; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE14:%.*]] ]532; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]533; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1534; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFFSET_IDX]]535; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[TMP9]], i64 2536; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]537; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 4538; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]539; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[TMP10]], i64 6540; CHECK-NEXT: [[OFFSET_IDX4:%.*]] = shl i64 [[INDEX]], 2541; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[OFFSET_IDX4]]542; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[TMP11]], i64 4543; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX4]]544; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[TMP32]], i64 8545; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX4]]546; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[TMP6]], i64 12547; CHECK-NEXT: [[TMP33:%.*]] = icmp ult <4 x i64> [[VEC_IND]], splat (i64 257)548; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP33]], i64 0549; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]550; CHECK: pred.store.if:551; CHECK-NEXT: [[DOT013:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET_IDX4]]552; CHECK-NEXT: [[DOT04:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]553; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[DOT04]], align 2554; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32555; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 7556; CHECK-NEXT: store i32 [[TMP5]], ptr [[DOT013]], align 4557; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]558; CHECK: pred.store.continue:559; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP33]], i64 1560; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]561; CHECK: pred.store.if9:562; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[NEXT_GEP1]], align 2563; CHECK-NEXT: [[TMP14:%.*]] = zext i16 [[TMP13]] to i32564; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i32 [[TMP14]], 7565; CHECK-NEXT: store i32 [[TMP15]], ptr [[NEXT_GEP6]], align 4566; CHECK-NEXT: br label [[PRED_STORE_CONTINUE10]]567; CHECK: pred.store.continue10:568; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP33]], i64 2569; CHECK-NEXT: br i1 [[TMP16]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]570; CHECK: pred.store.if11:571; CHECK-NEXT: [[TMP17:%.*]] = load i16, ptr [[NEXT_GEP2]], align 2572; CHECK-NEXT: [[TMP18:%.*]] = zext i16 [[TMP17]] to i32573; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i32 [[TMP18]], 7574; CHECK-NEXT: store i32 [[TMP19]], ptr [[NEXT_GEP7]], align 4575; CHECK-NEXT: br label [[PRED_STORE_CONTINUE12]]576; CHECK: pred.store.continue12:577; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP33]], i64 3578; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14]]579; CHECK: pred.store.if13:580; CHECK-NEXT: [[TMP21:%.*]] = load i16, ptr [[NEXT_GEP3]], align 2581; CHECK-NEXT: [[TMP22:%.*]] = zext i16 [[TMP21]] to i32582; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7583; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP8]], align 4584; CHECK-NEXT: br label [[PRED_STORE_CONTINUE14]]585; CHECK: pred.store.continue14:586; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4587; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)588; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260589; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[TMP1]], !llvm.loop [[LOOP8:![0-9]+]]590; CHECK: middle.block:591; CHECK-NEXT: br label [[TMP30:%.*]]592; CHECK: 25:593; CHECK-NEXT: [[TMP25:%.*]] = xor <4 x i1> [[TMP33]], splat (i1 true)594; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP25]], i1 false)595; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], -1596; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP27]]597; CHECK-NEXT: [[TMP29:%.*]] = add nsw i64 [[TMP28]], 1598; CHECK-NEXT: ret i64 [[TMP29]]599;600 br label %1601 602; <label>:1 ; preds = %1, %0603 %.04 = phi ptr [ %src, %0 ], [ %2, %1 ]604 %.013 = phi ptr [ %dst, %0 ], [ %6, %1 ]605 %i.02 = phi i64 [ 0, %0 ], [ %7, %1 ]606 %2 = getelementptr inbounds i16, ptr %.04, i64 1607 %3 = load i16, ptr %.04, align 2608 %4 = zext i16 %3 to i32609 %5 = shl nuw nsw i32 %4, 7610 %6 = getelementptr inbounds i32, ptr %.013, i64 1611 store i32 %5, ptr %.013, align 4612 %7 = add nsw i64 %i.02, 1613 %exitcond = icmp eq i64 %7, 257614 br i1 %exitcond, label %8, label %1615 616; <label>:8 ; preds = %1617 ret i64 %7618}619