1123 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; This test verifies that the loop vectorizer will NOT produce a tail3; loop with the optimize for size or the minimize size attributes.4; REQUIRES: asserts5; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -S | FileCheck %s6; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -pgso -S | FileCheck %s -check-prefix=PGSO7; RUN: opt < %s -passes='require<profile-summary>,loop-vectorize' -pgso=false -S | FileCheck %s -check-prefix=NPGSO8 9target datalayout = "E-m:e-p:32:32-i64:32-f64:32:64-a:0:32-n32-S128"10 11@tab = common global [32 x i8] zeroinitializer, align 112 13define i32 @foo_optsize() #0 {14; CHECK-LABEL: define i32 @foo_optsize(15; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {16; CHECK-NEXT: [[ENTRY:.*]]:17; CHECK-NEXT: br label %[[FOR_BODY:.*]]18; CHECK: [[FOR_BODY]]:19; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]20; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]21; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 122; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 023; CHECK-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 124; CHECK-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 125; CHECK-NEXT: [[INC]] = add nsw i32 [[I_08]], 126; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 20227; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]28; CHECK: [[FOR_END]]:29; CHECK-NEXT: ret i32 030;31; PGSO-LABEL: define i32 @foo_optsize(32; PGSO-SAME: ) #[[ATTR0:[0-9]+]] {33; PGSO-NEXT: [[ENTRY:.*]]:34; PGSO-NEXT: br label %[[FOR_BODY:.*]]35; PGSO: [[FOR_BODY]]:36; PGSO-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]37; PGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]38; PGSO-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 139; PGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 040; PGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 141; PGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 142; PGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 143; PGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 20244; PGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]45; PGSO: [[FOR_END]]:46; PGSO-NEXT: ret i32 047;48; NPGSO-LABEL: define i32 @foo_optsize(49; NPGSO-SAME: ) #[[ATTR0:[0-9]+]] {50; NPGSO-NEXT: [[ENTRY:.*]]:51; NPGSO-NEXT: br label %[[FOR_BODY:.*]]52; NPGSO: [[FOR_BODY]]:53; NPGSO-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]54; NPGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]55; NPGSO-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 156; NPGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 057; NPGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 158; NPGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 159; NPGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 160; NPGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 20261; NPGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]62; NPGSO: [[FOR_END]]:63; NPGSO-NEXT: ret i32 064;65 66entry:67 br label %for.body68 69for.body: ; preds = %for.body, %entry70 %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]71 %arrayidx = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 %i.0872 %0 = load i8, ptr %arrayidx, align 173 %cmp1 = icmp eq i8 %0, 074 %. = select i1 %cmp1, i8 2, i8 175 store i8 %., ptr %arrayidx, align 176 %inc = add nsw i32 %i.08, 177 %exitcond = icmp eq i32 %i.08, 20278 br i1 %exitcond, label %for.end, label %for.body79 80for.end: ; preds = %for.body81 ret i32 082}83 84attributes #0 = { optsize }85 86define i32 @foo_minsize() #1 {87; CHECK-LABEL: define i32 @foo_minsize(88; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {89; CHECK-NEXT: [[ENTRY:.*]]:90; CHECK-NEXT: br label %[[FOR_BODY:.*]]91; CHECK: [[FOR_BODY]]:92; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]93; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]94; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 195; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 096; CHECK-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 197; CHECK-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 198; CHECK-NEXT: [[INC]] = add nsw i32 [[I_08]], 199; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202100; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]101; CHECK: [[FOR_END]]:102; CHECK-NEXT: ret i32 0103;104; PGSO-LABEL: define i32 @foo_minsize(105; PGSO-SAME: ) #[[ATTR1:[0-9]+]] {106; PGSO-NEXT: [[ENTRY:.*]]:107; PGSO-NEXT: br label %[[FOR_BODY:.*]]108; PGSO: [[FOR_BODY]]:109; PGSO-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]110; PGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]111; PGSO-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1112; PGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0113; PGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1114; PGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 1115; PGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 1116; PGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202117; PGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]118; PGSO: [[FOR_END]]:119; PGSO-NEXT: ret i32 0120;121; NPGSO-LABEL: define i32 @foo_minsize(122; NPGSO-SAME: ) #[[ATTR1:[0-9]+]] {123; NPGSO-NEXT: [[ENTRY:.*]]:124; NPGSO-NEXT: br label %[[FOR_BODY:.*]]125; NPGSO: [[FOR_BODY]]:126; NPGSO-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]127; NPGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]128; NPGSO-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1129; NPGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0130; NPGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1131; NPGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 1132; NPGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 1133; NPGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202134; NPGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]135; NPGSO: [[FOR_END]]:136; NPGSO-NEXT: ret i32 0137;138 139entry:140 br label %for.body141 142for.body: ; preds = %for.body, %entry143 %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]144 %arrayidx = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 %i.08145 %0 = load i8, ptr %arrayidx, align 1146 %cmp1 = icmp eq i8 %0, 0147 %. = select i1 %cmp1, i8 2, i8 1148 store i8 %., ptr %arrayidx, align 1149 %inc = add nsw i32 %i.08, 1150 %exitcond = icmp eq i32 %i.08, 202151 br i1 %exitcond, label %for.end, label %for.body152 153for.end: ; preds = %for.body154 ret i32 0155}156 157attributes #1 = { minsize }158 159define i32 @foo_pgso() !prof !14 {160; CHECK-LABEL: define i32 @foo_pgso(161; CHECK-SAME: ) !prof [[PROF14:![0-9]+]] {162; CHECK-NEXT: [[ENTRY:.*]]:163; CHECK-NEXT: br label %[[FOR_BODY:.*]]164; CHECK: [[FOR_BODY]]:165; CHECK-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]166; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]167; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1168; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0169; CHECK-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1170; CHECK-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 1171; CHECK-NEXT: [[INC]] = add nsw i32 [[I_08]], 1172; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202173; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]174; CHECK: [[FOR_END]]:175; CHECK-NEXT: ret i32 0176;177; PGSO-LABEL: define i32 @foo_pgso(178; PGSO-SAME: ) !prof [[PROF14:![0-9]+]] {179; PGSO-NEXT: [[ENTRY:.*]]:180; PGSO-NEXT: br label %[[FOR_BODY:.*]]181; PGSO: [[FOR_BODY]]:182; PGSO-NEXT: [[I_08:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]183; PGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]184; PGSO-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1185; PGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP0]], 0186; PGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1187; PGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 1188; PGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 1189; PGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202190; PGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]]191; PGSO: [[FOR_END]]:192; PGSO-NEXT: ret i32 0193;194; NPGSO-LABEL: define i32 @foo_pgso(195; NPGSO-SAME: ) !prof [[PROF14:![0-9]+]] {196; NPGSO-NEXT: [[ENTRY:.*:]]197; NPGSO-NEXT: br label %[[VECTOR_PH:.*]]198; NPGSO: [[VECTOR_PH]]:199; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]200; NPGSO: [[VECTOR_BODY]]:201; NPGSO-NEXT: [[TMP0:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]202; NPGSO-NEXT: [[TMP1:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[TMP0]]203; NPGSO-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1204; NPGSO-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer205; NPGSO-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i8> splat (i8 2), <4 x i8> splat (i8 1)206; NPGSO-NEXT: store <4 x i8> [[TMP4]], ptr [[TMP1]], align 1207; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP0]], 4208; NPGSO-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 200209; NPGSO-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]210; NPGSO: [[MIDDLE_BLOCK]]:211; NPGSO-NEXT: br label %[[SCALAR_PH:.*]]212; NPGSO: [[SCALAR_PH]]:213; NPGSO-NEXT: br label %[[FOR_BODY:.*]]214; NPGSO: [[FOR_BODY]]:215; NPGSO-NEXT: [[I_08:%.*]] = phi i32 [ 200, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]216; NPGSO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]217; NPGSO-NEXT: [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX]], align 1218; NPGSO-NEXT: [[CMP1:%.*]] = icmp eq i8 [[TMP6]], 0219; NPGSO-NEXT: [[DOT:%.*]] = select i1 [[CMP1]], i8 2, i8 1220; NPGSO-NEXT: store i8 [[DOT]], ptr [[ARRAYIDX]], align 1221; NPGSO-NEXT: [[INC]] = add nsw i32 [[I_08]], 1222; NPGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[I_08]], 202223; NPGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]224; NPGSO: [[FOR_END]]:225; NPGSO-NEXT: ret i32 0226;227 228entry:229 br label %for.body230 231for.body: ; preds = %for.body, %entry232 %i.08 = phi i32 [ 0, %entry ], [ %inc, %for.body ]233 %arrayidx = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 %i.08234 %0 = load i8, ptr %arrayidx, align 1235 %cmp1 = icmp eq i8 %0, 0236 %. = select i1 %cmp1, i8 2, i8 1237 store i8 %., ptr %arrayidx, align 1238 %inc = add nsw i32 %i.08, 1239 %exitcond = icmp eq i32 %i.08, 202240 br i1 %exitcond, label %for.end, label %for.body241 242for.end: ; preds = %for.body243 ret i32 0244}245 246; PR43371: don't run into an assert due to emitting SCEV runtime checks247; with OptForSize.248;249@cm_array = external global [2592 x i16], align 1250 251define void @pr43371(i16 %val) optsize {252;253; CHECK-LABEL: define void @pr43371(254; CHECK-SAME: i16 [[VAL:%.*]]) #[[ATTR0]] {255; CHECK-NEXT: [[ENTRY:.*:]]256; CHECK-NEXT: br label %[[VECTOR_PH:.*]]257; CHECK: [[VECTOR_PH]]:258; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[VAL]], i64 0259; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer260; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]261; CHECK: [[VECTOR_BODY]]:262; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]263; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]264; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], [[VEC_IND]]265; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>266; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0267; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1268; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]269; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP3]]270; CHECK-NEXT: store i16 0, ptr [[TMP5]], align 1271; CHECK-NEXT: store i16 0, ptr [[TMP7]], align 1272; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2273; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)274; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756275; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]276; CHECK: [[MIDDLE_BLOCK]]:277; CHECK-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]278; CHECK: [[FOR_COND_CLEANUP28]]:279; CHECK-NEXT: unreachable280;281; PGSO-LABEL: define void @pr43371(282; PGSO-SAME: i16 [[VAL:%.*]]) #[[ATTR0]] {283; PGSO-NEXT: [[ENTRY:.*:]]284; PGSO-NEXT: br label %[[VECTOR_PH:.*]]285; PGSO: [[VECTOR_PH]]:286; PGSO-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[VAL]], i64 0287; PGSO-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer288; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]289; PGSO: [[VECTOR_BODY]]:290; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]291; PGSO-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]292; PGSO-NEXT: [[TMP0:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], [[VEC_IND]]293; PGSO-NEXT: [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>294; PGSO-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0295; PGSO-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1296; PGSO-NEXT: [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]297; PGSO-NEXT: [[TMP7:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP3]]298; PGSO-NEXT: store i16 0, ptr [[TMP5]], align 1299; PGSO-NEXT: store i16 0, ptr [[TMP7]], align 1300; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2301; PGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)302; PGSO-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756303; PGSO-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]304; PGSO: [[MIDDLE_BLOCK]]:305; PGSO-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]306; PGSO: [[FOR_COND_CLEANUP28]]:307; PGSO-NEXT: unreachable308;309; NPGSO-LABEL: define void @pr43371(310; NPGSO-SAME: i16 [[VAL:%.*]]) #[[ATTR0]] {311; NPGSO-NEXT: [[ENTRY:.*:]]312; NPGSO-NEXT: br label %[[VECTOR_PH:.*]]313; NPGSO: [[VECTOR_PH]]:314; NPGSO-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[VAL]], i64 0315; NPGSO-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer316; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]317; NPGSO: [[VECTOR_BODY]]:318; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]319; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]320; NPGSO-NEXT: [[TMP0:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], [[VEC_IND]]321; NPGSO-NEXT: [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>322; NPGSO-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0323; NPGSO-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1324; NPGSO-NEXT: [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]325; NPGSO-NEXT: [[TMP7:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP3]]326; NPGSO-NEXT: store i16 0, ptr [[TMP5]], align 1327; NPGSO-NEXT: store i16 0, ptr [[TMP7]], align 1328; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2329; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)330; NPGSO-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756331; NPGSO-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]332; NPGSO: [[MIDDLE_BLOCK]]:333; NPGSO-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]334; NPGSO: [[FOR_COND_CLEANUP28]]:335; NPGSO-NEXT: unreachable336;337; We do not want to generate SCEV predicates when optimising for size, because338; that will lead to extra code generation such as the SCEV overflow runtime339; checks. Not generating SCEV predicates can still result in vectorisation as340; the non-consecutive loads/stores can be scalarized:341entry:342 br label %for.body29343 344for.cond.cleanup28:345 unreachable346 347for.body29:348 %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]349 %add33 = add i16 %val, %i24.0170350 %idxprom34 = zext i16 %add33 to i32351 %arrayidx35 = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 %idxprom34352 store i16 0, ptr %arrayidx35, align 1353 %inc37 = add i16 %i24.0170, 1354 %cmp26 = icmp ult i16 %inc37, 756355 br i1 %cmp26, label %for.body29, label %for.cond.cleanup28356}357 358define void @pr43371_pgso(i16 %val) !prof !14 {359;360; CHECK-LABEL: define void @pr43371_pgso(361; CHECK-SAME: i16 [[VAL:%.*]]) !prof [[PROF14]] {362; CHECK-NEXT: [[ENTRY:.*:]]363; CHECK-NEXT: br label %[[VECTOR_PH:.*]]364; CHECK: [[VECTOR_PH]]:365; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[VAL]], i64 0366; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer367; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]368; CHECK: [[VECTOR_BODY]]:369; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]370; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]371; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], [[VEC_IND]]372; CHECK-NEXT: [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>373; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0374; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1375; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]376; CHECK-NEXT: [[TMP7:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP3]]377; CHECK-NEXT: store i16 0, ptr [[TMP5]], align 1378; CHECK-NEXT: store i16 0, ptr [[TMP7]], align 1379; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2380; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)381; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756382; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]383; CHECK: [[MIDDLE_BLOCK]]:384; CHECK-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]385; CHECK: [[FOR_COND_CLEANUP28]]:386; CHECK-NEXT: unreachable387;388; PGSO-LABEL: define void @pr43371_pgso(389; PGSO-SAME: i16 [[VAL:%.*]]) !prof [[PROF14]] {390; PGSO-NEXT: [[ENTRY:.*:]]391; PGSO-NEXT: br label %[[VECTOR_PH:.*]]392; PGSO: [[VECTOR_PH]]:393; PGSO-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[VAL]], i64 0394; PGSO-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer395; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]396; PGSO: [[VECTOR_BODY]]:397; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]398; PGSO-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]399; PGSO-NEXT: [[TMP0:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], [[VEC_IND]]400; PGSO-NEXT: [[TMP1:%.*]] = zext <2 x i16> [[TMP0]] to <2 x i32>401; PGSO-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0402; PGSO-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1403; PGSO-NEXT: [[TMP5:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP4]]404; PGSO-NEXT: [[TMP7:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP3]]405; PGSO-NEXT: store i16 0, ptr [[TMP5]], align 1406; PGSO-NEXT: store i16 0, ptr [[TMP7]], align 1407; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2408; PGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2)409; PGSO-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756410; PGSO-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]411; PGSO: [[MIDDLE_BLOCK]]:412; PGSO-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]413; PGSO: [[FOR_COND_CLEANUP28]]:414; PGSO-NEXT: unreachable415;416; NPGSO-LABEL: define void @pr43371_pgso(417; NPGSO-SAME: i16 [[VAL:%.*]]) !prof [[PROF14]] {418; NPGSO-NEXT: [[ENTRY:.*:]]419; NPGSO-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]420; NPGSO: [[VECTOR_SCEVCHECK]]:421; NPGSO-NEXT: [[TMP0:%.*]] = add i16 [[VAL]], 755422; NPGSO-NEXT: [[TMP4:%.*]] = icmp ult i16 [[TMP0]], [[VAL]]423; NPGSO-NEXT: br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]424; NPGSO: [[VECTOR_PH]]:425; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]426; NPGSO: [[VECTOR_BODY]]:427; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]428; NPGSO-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16429; NPGSO-NEXT: [[TMP1:%.*]] = add i16 [[VAL]], [[OFFSET_IDX]]430; NPGSO-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i32431; NPGSO-NEXT: [[TMP3:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[TMP2]]432; NPGSO-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP3]], align 1433; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2434; NPGSO-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 756435; NPGSO-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]436; NPGSO: [[MIDDLE_BLOCK]]:437; NPGSO-NEXT: br label %[[FOR_COND_CLEANUP28:.*]]438; NPGSO: [[SCALAR_PH]]:439; NPGSO-NEXT: br label %[[FOR_BODY29:.*]]440; NPGSO: [[FOR_COND_CLEANUP28]]:441; NPGSO-NEXT: unreachable442; NPGSO: [[FOR_BODY29]]:443; NPGSO-NEXT: [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]444; NPGSO-NEXT: [[ADD33:%.*]] = add i16 [[VAL]], [[I24_0170]]445; NPGSO-NEXT: [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32446; NPGSO-NEXT: [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]447; NPGSO-NEXT: store i16 0, ptr [[ARRAYIDX35]], align 1448; NPGSO-NEXT: [[INC37]] = add i16 [[I24_0170]], 1449; NPGSO-NEXT: [[CMP26:%.*]] = icmp ult i16 [[INC37]], 756450; NPGSO-NEXT: br i1 [[CMP26]], label %[[FOR_BODY29]], label %[[FOR_COND_CLEANUP28]], !llvm.loop [[LOOP21:![0-9]+]]451;452; We do not want to generate SCEV predicates when optimising for size, because453; that will lead to extra code generation such as the SCEV overflow runtime454; checks. Not generating SCEV predicates can still result in vectorisation as455; the non-consecutive loads/stores can be scalarized:456entry:457 br label %for.body29458 459for.cond.cleanup28:460 unreachable461 462for.body29:463 %i24.0170 = phi i16 [ 0, %entry], [ %inc37, %for.body29]464 %add33 = add i16 %val, %i24.0170465 %idxprom34 = zext i16 %add33 to i32466 %arrayidx35 = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 %idxprom34467 store i16 0, ptr %arrayidx35, align 1468 %inc37 = add i16 %i24.0170, 1469 %cmp26 = icmp ult i16 %inc37, 756470 br i1 %cmp26, label %for.body29, label %for.cond.cleanup28471}472 473; PR45526: don't vectorize with fold-tail if first-order-recurrence is live-out.474;475define i32 @pr45526() optsize {476;477; CHECK-LABEL: define i32 @pr45526(478; CHECK-SAME: ) #[[ATTR0]] {479; CHECK-NEXT: [[ENTRY:.*:]]480; CHECK-NEXT: br label %[[VECTOR_PH:.*]]481; CHECK: [[VECTOR_PH]]:482; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]483; CHECK: [[VECTOR_BODY]]:484; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]485; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]486; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]487; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)488; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)489; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4490; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)491; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512492; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]493; CHECK: [[MIDDLE_BLOCK]]:494; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)495; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1496; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1497; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 [[TMP6]]498; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3499; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 0500; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]501; CHECK-NEXT: br label %[[EXIT:.*]]502; CHECK: [[EXIT]]:503; CHECK-NEXT: ret i32 [[TMP10]]504;505; PGSO-LABEL: define i32 @pr45526(506; PGSO-SAME: ) #[[ATTR0]] {507; PGSO-NEXT: [[ENTRY:.*:]]508; PGSO-NEXT: br label %[[VECTOR_PH:.*]]509; PGSO: [[VECTOR_PH]]:510; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]511; PGSO: [[VECTOR_BODY]]:512; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]513; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]514; PGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]515; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)516; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)517; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4518; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)519; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512520; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]521; PGSO: [[MIDDLE_BLOCK]]:522; PGSO-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)523; PGSO-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1524; PGSO-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1525; PGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 [[TMP6]]526; PGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3527; PGSO-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 0528; PGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]529; PGSO-NEXT: br label %[[EXIT:.*]]530; PGSO: [[EXIT]]:531; PGSO-NEXT: ret i32 [[TMP10]]532;533; NPGSO-LABEL: define i32 @pr45526(534; NPGSO-SAME: ) #[[ATTR0]] {535; NPGSO-NEXT: [[ENTRY:.*:]]536; NPGSO-NEXT: br label %[[VECTOR_PH:.*]]537; NPGSO: [[VECTOR_PH]]:538; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]539; NPGSO: [[VECTOR_BODY]]:540; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]541; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]542; NPGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]543; NPGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)544; NPGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)545; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4546; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)547; NPGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512548; NPGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]549; NPGSO: [[MIDDLE_BLOCK]]:550; NPGSO-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)551; NPGSO-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1552; NPGSO-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1553; NPGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 [[TMP6]]554; NPGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3555; NPGSO-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 0556; NPGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]557; NPGSO-NEXT: br label %[[EXIT:.*]]558; NPGSO: [[EXIT]]:559; NPGSO-NEXT: ret i32 [[TMP10]]560;561entry:562 br label %loop563 564loop:565 %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]566 %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]567 %pivPlus1 = add nuw nsw i32 %piv, 1568 %cond = icmp ult i32 %piv, 510569 br i1 %cond, label %loop, label %exit570 571exit:572 ret i32 %for573}574 575define i32 @pr45526_pgso() !prof !14 {576;577; CHECK-LABEL: define i32 @pr45526_pgso(578; CHECK-SAME: ) !prof [[PROF14]] {579; CHECK-NEXT: [[ENTRY:.*:]]580; CHECK-NEXT: br label %[[VECTOR_PH:.*]]581; CHECK: [[VECTOR_PH]]:582; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]583; CHECK: [[VECTOR_BODY]]:584; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]585; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]586; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]587; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)588; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)589; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4590; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)591; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512592; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]593; CHECK: [[MIDDLE_BLOCK]]:594; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)595; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1596; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1597; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 [[TMP6]]598; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3599; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 0600; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]601; CHECK-NEXT: br label %[[EXIT:.*]]602; CHECK: [[EXIT]]:603; CHECK-NEXT: ret i32 [[TMP10]]604;605; PGSO-LABEL: define i32 @pr45526_pgso(606; PGSO-SAME: ) !prof [[PROF14]] {607; PGSO-NEXT: [[ENTRY:.*:]]608; PGSO-NEXT: br label %[[VECTOR_PH:.*]]609; PGSO: [[VECTOR_PH]]:610; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]611; PGSO: [[VECTOR_BODY]]:612; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]613; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]614; PGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]615; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)616; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)617; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4618; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)619; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512620; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]621; PGSO: [[MIDDLE_BLOCK]]:622; PGSO-NEXT: [[TMP4:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP0]], i1 false)623; PGSO-NEXT: [[TMP5:%.*]] = sub i64 [[TMP4]], 1624; PGSO-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], 1625; PGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i64 [[TMP6]]626; PGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3627; PGSO-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP5]], 0628; PGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]629; PGSO-NEXT: br label %[[EXIT:.*]]630; PGSO: [[EXIT]]:631; PGSO-NEXT: ret i32 [[TMP10]]632;633; NPGSO-LABEL: define i32 @pr45526_pgso(634; NPGSO-SAME: ) !prof [[PROF14]] {635; NPGSO-NEXT: [[ENTRY:.*:]]636; NPGSO-NEXT: br label %[[VECTOR_PH:.*]]637; NPGSO: [[VECTOR_PH]]:638; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]639; NPGSO: [[VECTOR_BODY]]:640; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]641; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]642; NPGSO-NEXT: [[TMP0:%.*]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)643; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4644; NPGSO-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)645; NPGSO-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 508646; NPGSO-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]647; NPGSO: [[MIDDLE_BLOCK]]:648; NPGSO-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3649; NPGSO-NEXT: br label %[[SCALAR_PH:.*]]650; NPGSO: [[SCALAR_PH]]:651; NPGSO-NEXT: br label %[[LOOP:.*]]652; NPGSO: [[LOOP]]:653; NPGSO-NEXT: [[PIV:%.*]] = phi i32 [ 508, %[[SCALAR_PH]] ], [ [[PIVPLUS1:%.*]], %[[LOOP]] ]654; NPGSO-NEXT: [[FOR:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[SCALAR_PH]] ], [ [[PIVPLUS1]], %[[LOOP]] ]655; NPGSO-NEXT: [[PIVPLUS1]] = add nuw nsw i32 [[PIV]], 1656; NPGSO-NEXT: [[COND:%.*]] = icmp ult i32 [[PIV]], 510657; NPGSO-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP24:![0-9]+]]658; NPGSO: [[EXIT]]:659; NPGSO-NEXT: [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ]660; NPGSO-NEXT: ret i32 [[FOR_LCSSA]]661;662entry:663 br label %loop664 665loop:666 %piv = phi i32 [ 0, %entry ], [ %pivPlus1, %loop ]667 %for = phi i32 [ 5, %entry ], [ %pivPlus1, %loop ]668 %pivPlus1 = add nuw nsw i32 %piv, 1669 %cond = icmp ult i32 %piv, 510670 br i1 %cond, label %loop, label %exit671 672exit:673 ret i32 %for674}675 676; PR46228: Vectorize w/o versioning for unit stride under optsize and enabled677; vectorization.678 679; NOTE: Some assertions have been autogenerated by utils/update_test_checks.py680define void @stride1(ptr noalias %B, i32 %BStride) optsize {681; CHECK-LABEL: define void @stride1(682; CHECK-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {683; CHECK-NEXT: [[ENTRY:.*:]]684; CHECK-NEXT: br label %[[VECTOR_PH:.*]]685; CHECK: [[VECTOR_PH]]:686; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0687; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer688; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]689; CHECK: [[VECTOR_BODY]]:690; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]691; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]692; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)693; CHECK-NEXT: [[TMP0:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]694; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0695; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]696; CHECK: [[PRED_STORE_IF]]:697; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP0]], i32 0698; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]699; CHECK-NEXT: store i16 42, ptr [[TMP4]], align 4700; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]701; CHECK: [[PRED_STORE_CONTINUE]]:702; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1703; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]704; CHECK: [[PRED_STORE_IF1]]:705; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1706; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]707; CHECK-NEXT: store i16 42, ptr [[TMP7]], align 4708; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]709; CHECK: [[PRED_STORE_CONTINUE2]]:710; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2711; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)712; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026713; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]714; CHECK: [[MIDDLE_BLOCK]]:715; CHECK-NEXT: br label %[[FOR_END:.*]]716; CHECK: [[FOR_END]]:717; CHECK-NEXT: ret void718;719; PGSO-LABEL: define void @stride1(720; PGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {721; PGSO-NEXT: [[ENTRY:.*:]]722; PGSO-NEXT: br label %[[VECTOR_PH:.*]]723; PGSO: [[VECTOR_PH]]:724; PGSO-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0725; PGSO-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer726; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]727; PGSO: [[VECTOR_BODY]]:728; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]729; PGSO-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]730; PGSO-NEXT: [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)731; PGSO-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]732; PGSO-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0733; PGSO-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]734; PGSO: [[PRED_STORE_IF]]:735; PGSO-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0736; PGSO-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]737; PGSO-NEXT: store i16 42, ptr [[TMP4]], align 4738; PGSO-NEXT: br label %[[PRED_STORE_CONTINUE]]739; PGSO: [[PRED_STORE_CONTINUE]]:740; PGSO-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1741; PGSO-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]742; PGSO: [[PRED_STORE_IF1]]:743; PGSO-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1744; PGSO-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]745; PGSO-NEXT: store i16 42, ptr [[TMP7]], align 4746; PGSO-NEXT: br label %[[PRED_STORE_CONTINUE2]]747; PGSO: [[PRED_STORE_CONTINUE2]]:748; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2749; PGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)750; PGSO-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026751; PGSO-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]752; PGSO: [[MIDDLE_BLOCK]]:753; PGSO-NEXT: br label %[[FOR_END:.*]]754; PGSO: [[FOR_END]]:755; PGSO-NEXT: ret void756;757; NPGSO-LABEL: define void @stride1(758; NPGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {759; NPGSO-NEXT: [[ENTRY:.*:]]760; NPGSO-NEXT: br label %[[VECTOR_PH:.*]]761; NPGSO: [[VECTOR_PH]]:762; NPGSO-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0763; NPGSO-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer764; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]765; NPGSO: [[VECTOR_BODY]]:766; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]767; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]768; NPGSO-NEXT: [[TMP0:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 1024)769; NPGSO-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]770; NPGSO-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0771; NPGSO-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]772; NPGSO: [[PRED_STORE_IF]]:773; NPGSO-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0774; NPGSO-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP3]]775; NPGSO-NEXT: store i16 42, ptr [[TMP4]], align 4776; NPGSO-NEXT: br label %[[PRED_STORE_CONTINUE]]777; NPGSO: [[PRED_STORE_CONTINUE]]:778; NPGSO-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1779; NPGSO-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]780; NPGSO: [[PRED_STORE_IF1]]:781; NPGSO-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1782; NPGSO-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP6]]783; NPGSO-NEXT: store i16 42, ptr [[TMP7]], align 4784; NPGSO-NEXT: br label %[[PRED_STORE_CONTINUE2]]785; NPGSO: [[PRED_STORE_CONTINUE2]]:786; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2787; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)788; NPGSO-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1026789; NPGSO-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]790; NPGSO: [[MIDDLE_BLOCK]]:791; NPGSO-NEXT: br label %[[FOR_END:.*]]792; NPGSO: [[FOR_END]]:793; NPGSO-NEXT: ret void794;795 796entry:797 br label %for.body798 799for.body:800 %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]801 %mulB = mul nsw i32 %iv, %BStride802 %gepOfB = getelementptr inbounds i16, ptr %B, i32 %mulB803 store i16 42, ptr %gepOfB, align 4804 %iv.next = add nuw nsw i32 %iv, 1805 %exitcond = icmp eq i32 %iv.next, 1025806 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15807 808for.end:809 ret void810}811 812; Vectorize with versioning for unit stride for PGSO and enabled vectorization.813;814define void @stride1_pgso(ptr noalias %B, i32 %BStride) !prof !14 {815; CHECK-LABEL: define void @stride1_pgso(816; CHECK-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {817; CHECK-NEXT: [[ENTRY:.*:]]818; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]819; CHECK: [[VECTOR_SCEVCHECK]]:820; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1821; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]822; CHECK: [[VECTOR_PH]]:823; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]824; CHECK: [[VECTOR_BODY]]:825; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]826; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]827; CHECK-NEXT: store <2 x i16> splat (i16 42), ptr [[TMP1]], align 4828; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP0]], 2829; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024830; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]831; CHECK: [[MIDDLE_BLOCK]]:832; CHECK-NEXT: br label %[[SCALAR_PH]]833; CHECK: [[SCALAR_PH]]:834; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]835; CHECK-NEXT: br label %[[FOR_BODY:.*]]836; CHECK: [[FOR_BODY]]:837; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]838; CHECK-NEXT: [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]839; CHECK-NEXT: [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]840; CHECK-NEXT: store i16 42, ptr [[GEPOFB]], align 4841; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1842; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025843; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]844; CHECK: [[FOR_END]]:845; CHECK-NEXT: ret void846;847; PGSO-LABEL: define void @stride1_pgso(848; PGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {849; PGSO-NEXT: [[ENTRY:.*:]]850; PGSO-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]851; PGSO: [[VECTOR_SCEVCHECK]]:852; PGSO-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1853; PGSO-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]854; PGSO: [[VECTOR_PH]]:855; PGSO-NEXT: br label %[[VECTOR_BODY:.*]]856; PGSO: [[VECTOR_BODY]]:857; PGSO-NEXT: [[TMP0:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]858; PGSO-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]859; PGSO-NEXT: store <2 x i16> splat (i16 42), ptr [[TMP1]], align 4860; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP0]], 2861; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024862; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]863; PGSO: [[MIDDLE_BLOCK]]:864; PGSO-NEXT: br label %[[SCALAR_PH]]865; PGSO: [[SCALAR_PH]]:866; PGSO-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]867; PGSO-NEXT: br label %[[FOR_BODY:.*]]868; PGSO: [[FOR_BODY]]:869; PGSO-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]870; PGSO-NEXT: [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]871; PGSO-NEXT: [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]872; PGSO-NEXT: store i16 42, ptr [[GEPOFB]], align 4873; PGSO-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1874; PGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025875; PGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]876; PGSO: [[FOR_END]]:877; PGSO-NEXT: ret void878;879; NPGSO-LABEL: define void @stride1_pgso(880; NPGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) !prof [[PROF14]] {881; NPGSO-NEXT: [[ENTRY:.*:]]882; NPGSO-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]883; NPGSO: [[VECTOR_SCEVCHECK]]:884; NPGSO-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[BSTRIDE]], 1885; NPGSO-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]886; NPGSO: [[VECTOR_PH]]:887; NPGSO-NEXT: br label %[[VECTOR_BODY:.*]]888; NPGSO: [[VECTOR_BODY]]:889; NPGSO-NEXT: [[TMP0:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]890; NPGSO-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[TMP0]]891; NPGSO-NEXT: store <2 x i16> splat (i16 42), ptr [[TMP1]], align 4892; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP0]], 2893; NPGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024894; NPGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]895; NPGSO: [[MIDDLE_BLOCK]]:896; NPGSO-NEXT: br label %[[SCALAR_PH]]897; NPGSO: [[SCALAR_PH]]:898; NPGSO-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]899; NPGSO-NEXT: br label %[[FOR_BODY:.*]]900; NPGSO: [[FOR_BODY]]:901; NPGSO-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]902; NPGSO-NEXT: [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]903; NPGSO-NEXT: [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]904; NPGSO-NEXT: store i16 42, ptr [[GEPOFB]], align 4905; NPGSO-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1906; NPGSO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1025907; NPGSO-NEXT: br i1 [[EXITCOND]], label %[[FOR_END:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]908; NPGSO: [[FOR_END]]:909; NPGSO-NEXT: ret void910;911 912entry:913 br label %for.body914 915for.body:916 %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]917 %mulB = mul nsw i32 %iv, %BStride918 %gepOfB = getelementptr inbounds i16, ptr %B, i32 %mulB919 store i16 42, ptr %gepOfB, align 4920 %iv.next = add nuw nsw i32 %iv, 1921 %exitcond = icmp eq i32 %iv.next, 1025922 br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !15923 924for.end:925 ret void926}927 928; PR46652: Check that the need for stride==1 check prevents vectorizing a loop929; having tiny trip count, when compiling w/o -Os/-Oz.930 931@g = external global [1 x i16], align 1932 933define void @pr46652(i16 %stride) {934; CHECK-LABEL: define void @pr46652(935; CHECK-SAME: i16 [[STRIDE:%.*]]) {936; CHECK-NEXT: [[ENTRY:.*]]:937; CHECK-NEXT: br label %[[FOR_BODY:.*]]938; CHECK: [[FOR_BODY]]:939; CHECK-NEXT: [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]940; CHECK-NEXT: [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]941; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]942; CHECK-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1943; CHECK-NEXT: [[INC9]] = add nuw nsw i16 [[L1_02]], 1944; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16945; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]946; CHECK: [[FOR_END]]:947; CHECK-NEXT: ret void948;949; PGSO-LABEL: define void @pr46652(950; PGSO-SAME: i16 [[STRIDE:%.*]]) {951; PGSO-NEXT: [[ENTRY:.*]]:952; PGSO-NEXT: br label %[[FOR_BODY:.*]]953; PGSO: [[FOR_BODY]]:954; PGSO-NEXT: [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]955; PGSO-NEXT: [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]956; PGSO-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]957; PGSO-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1958; PGSO-NEXT: [[INC9]] = add nuw nsw i16 [[L1_02]], 1959; PGSO-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16960; PGSO-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]961; PGSO: [[FOR_END]]:962; PGSO-NEXT: ret void963;964; NPGSO-LABEL: define void @pr46652(965; NPGSO-SAME: i16 [[STRIDE:%.*]]) {966; NPGSO-NEXT: [[ENTRY:.*]]:967; NPGSO-NEXT: br label %[[FOR_BODY:.*]]968; NPGSO: [[FOR_BODY]]:969; NPGSO-NEXT: [[L1_02:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[INC9:%.*]], %[[FOR_BODY]] ]970; NPGSO-NEXT: [[MUL:%.*]] = mul nsw i16 [[L1_02]], [[STRIDE]]971; NPGSO-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 [[MUL]]972; NPGSO-NEXT: [[TMP0:%.*]] = load i16, ptr [[ARRAYIDX6]], align 1973; NPGSO-NEXT: [[INC9]] = add nuw nsw i16 [[L1_02]], 1974; NPGSO-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i16 [[INC9]], 16975; NPGSO-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]976; NPGSO: [[FOR_END]]:977; NPGSO-NEXT: ret void978;979entry:980 br label %for.body981 982for.body: ; preds = %for.body, %entry983 %l1.02 = phi i16 [ 1, %entry ], [ %inc9, %for.body ]984 %mul = mul nsw i16 %l1.02, %stride985 %arrayidx6 = getelementptr inbounds [1 x i16], ptr @g, i16 0, i16 %mul986 %0 = load i16, ptr %arrayidx6, align 1987 %inc9 = add nuw nsw i16 %l1.02, 1988 %exitcond.not = icmp eq i16 %inc9, 16989 br i1 %exitcond.not, label %for.end, label %for.body990 991for.end: ; preds = %for.body992 ret void993}994 995; Make sure we do not crash while building the VPlan for the loop with the996; select below.997define i32 @PR48142(ptr %ptr.start, ptr %ptr.end) optsize {998; CHECK-LABEL: define i32 @PR48142(999; CHECK-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {1000; CHECK-NEXT: [[ENTRY:.*]]:1001; CHECK-NEXT: br label %[[FOR_BODY:.*]]1002; CHECK: [[FOR_BODY]]:1003; CHECK-NEXT: [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]1004; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]1005; CHECK-NEXT: [[CMP4:%.*]] = icmp slt i32 [[I_014]], 991006; CHECK-NEXT: [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]1007; CHECK-NEXT: store i32 0, ptr [[PTR_IV]], align 41008; CHECK-NEXT: [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 11009; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]1010; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]1011; CHECK: [[EXIT]]:1012; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]1013; CHECK-NEXT: ret i32 [[RES]]1014;1015; PGSO-LABEL: define i32 @PR48142(1016; PGSO-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {1017; PGSO-NEXT: [[ENTRY:.*]]:1018; PGSO-NEXT: br label %[[FOR_BODY:.*]]1019; PGSO: [[FOR_BODY]]:1020; PGSO-NEXT: [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]1021; PGSO-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]1022; PGSO-NEXT: [[CMP4:%.*]] = icmp slt i32 [[I_014]], 991023; PGSO-NEXT: [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]1024; PGSO-NEXT: store i32 0, ptr [[PTR_IV]], align 41025; PGSO-NEXT: [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 11026; PGSO-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]1027; PGSO-NEXT: br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]1028; PGSO: [[EXIT]]:1029; PGSO-NEXT: [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]1030; PGSO-NEXT: ret i32 [[RES]]1031;1032; NPGSO-LABEL: define i32 @PR48142(1033; NPGSO-SAME: ptr [[PTR_START:%.*]], ptr [[PTR_END:%.*]]) #[[ATTR0]] {1034; NPGSO-NEXT: [[ENTRY:.*]]:1035; NPGSO-NEXT: br label %[[FOR_BODY:.*]]1036; NPGSO: [[FOR_BODY]]:1037; NPGSO-NEXT: [[I_014:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]1038; NPGSO-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_START]], %[[ENTRY]] ], [ [[PTR_NEXT:%.*]], %[[FOR_BODY]] ]1039; NPGSO-NEXT: [[CMP4:%.*]] = icmp slt i32 [[I_014]], 991040; NPGSO-NEXT: [[COND]] = select i1 [[CMP4]], i32 99, i32 [[I_014]]1041; NPGSO-NEXT: store i32 0, ptr [[PTR_IV]], align 41042; NPGSO-NEXT: [[PTR_NEXT]] = getelementptr inbounds i32, ptr [[PTR_IV]], i64 11043; NPGSO-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[PTR_NEXT]], [[PTR_END]]1044; NPGSO-NEXT: br i1 [[CMP_NOT]], label %[[EXIT:.*]], label %[[FOR_BODY]]1045; NPGSO: [[EXIT]]:1046; NPGSO-NEXT: [[RES:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ]1047; NPGSO-NEXT: ret i32 [[RES]]1048;1049entry:1050 br label %for.body1051 1052for.body:1053 %i.014 = phi i32 [ 20, %entry ], [ %cond, %for.body ]1054 %ptr.iv = phi ptr [ %ptr.start, %entry ], [ %ptr.next, %for.body ]1055 %cmp4 = icmp slt i32 %i.014, 991056 %cond = select i1 %cmp4, i32 99, i32 %i.0141057 store i32 0, ptr %ptr.iv1058 %ptr.next = getelementptr inbounds i32, ptr %ptr.iv, i64 11059 %cmp.not = icmp eq ptr %ptr.next, %ptr.end1060 br i1 %cmp.not, label %exit, label %for.body1061 1062exit:1063 %res = phi i32 [ %cond, %for.body ]1064 ret i32 %res1065}1066 1067!llvm.module.flags = !{!0}1068!0 = !{i32 1, !"ProfileSummary", !1}1069!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}1070!2 = !{!"ProfileFormat", !"InstrProf"}1071!3 = !{!"TotalCount", i64 10000}1072!4 = !{!"MaxCount", i64 10}1073!5 = !{!"MaxInternalCount", i64 1}1074!6 = !{!"MaxFunctionCount", i64 1000}1075!7 = !{!"NumCounts", i64 3}1076!8 = !{!"NumFunctions", i64 3}1077!9 = !{!"DetailedSummary", !10}1078!10 = !{!11, !12, !13}1079!11 = !{i32 10000, i64 100, i32 1}1080!12 = !{i32 999000, i64 100, i32 1}1081!13 = !{i32 999999, i64 1, i32 2}1082!14 = !{!"function_entry_count", i64 0}1083!15 = distinct !{!15, !16}1084!16 = !{!"llvm.loop.vectorize.enable", i1 true}1085;.1086; CHECK: [[PROF14]] = !{!"function_entry_count", i64 0}1087; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}1088; CHECK: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}1089; CHECK: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}1090; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META16]], [[META17]]}1091; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}1092; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META16]], [[META17]]}1093; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]], [[META17]]}1094; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META16]], [[META17]]}1095; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]]}1096;.1097; PGSO: [[PROF14]] = !{!"function_entry_count", i64 0}1098; PGSO: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}1099; PGSO: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}1100; PGSO: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}1101; PGSO: [[LOOP18]] = distinct !{[[LOOP18]], [[META16]], [[META17]]}1102; PGSO: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}1103; PGSO: [[LOOP20]] = distinct !{[[LOOP20]], [[META16]], [[META17]]}1104; PGSO: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]], [[META17]]}1105; PGSO: [[LOOP22]] = distinct !{[[LOOP22]], [[META16]], [[META17]]}1106; PGSO: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]]}1107;.1108; NPGSO: [[PROF14]] = !{!"function_entry_count", i64 0}1109; NPGSO: [[LOOP15]] = distinct !{[[LOOP15]], [[META16:![0-9]+]], [[META17:![0-9]+]]}1110; NPGSO: [[META16]] = !{!"llvm.loop.isvectorized", i32 1}1111; NPGSO: [[META17]] = !{!"llvm.loop.unroll.runtime.disable"}1112; NPGSO: [[LOOP18]] = distinct !{[[LOOP18]], [[META17]], [[META16]]}1113; NPGSO: [[LOOP19]] = distinct !{[[LOOP19]], [[META16]], [[META17]]}1114; NPGSO: [[LOOP20]] = distinct !{[[LOOP20]], [[META16]], [[META17]]}1115; NPGSO: [[LOOP21]] = distinct !{[[LOOP21]], [[META16]]}1116; NPGSO: [[LOOP22]] = distinct !{[[LOOP22]], [[META16]], [[META17]]}1117; NPGSO: [[LOOP23]] = distinct !{[[LOOP23]], [[META16]], [[META17]]}1118; NPGSO: [[LOOP24]] = distinct !{[[LOOP24]], [[META17]], [[META16]]}1119; NPGSO: [[LOOP25]] = distinct !{[[LOOP25]], [[META16]], [[META17]]}1120; NPGSO: [[LOOP26]] = distinct !{[[LOOP26]], [[META16]], [[META17]]}1121; NPGSO: [[LOOP27]] = distinct !{[[LOOP27]], [[META16]]}1122;.1123