1112 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s3 4 5@p = external local_unnamed_addr global [257 x i32], align 166@q = external local_unnamed_addr global [257 x i32], align 167 8; Test case for PR43398.9 10define void @can_sink_after_store(i32 %x, ptr %ptr, i64 %tc) local_unnamed_addr #0 {11; CHECK-LABEL: @can_sink_after_store(12; CHECK-NEXT: entry:13; CHECK-NEXT: br label [[PREHEADER:%.*]]14; CHECK: preheader:15; CHECK-NEXT: [[IDX_PHI_TRANS:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 116; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[IDX_PHI_TRANS]], align 417; CHECK-NEXT: br label [[VECTOR_PH:%.*]]18; CHECK: vector.ph:19; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 020; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer21; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 322; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]23; CHECK: vector.body:24; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]25; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]26; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]27; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[OFFSET_IDX]]28; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, ptr [[TMP1]], align 429; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>30; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]31; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], [[WIDE_LOAD]]32; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[OFFSET_IDX]]33; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 434; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 435; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 199636; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]37; CHECK: middle.block:38; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 339; CHECK-NEXT: br label [[SCALAR_PH:%.*]]40; CHECK: scalar.ph:41; CHECK-NEXT: br label [[FOR:%.*]]42; CHECK: for:43; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[SCALAR_PH]] ], [ [[PRE_NEXT:%.*]], [[FOR]] ]44; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1997, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR]] ]45; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[SCALAR_RECUR]], [[X]]46; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[IV]]47; CHECK-NEXT: [[PRE_NEXT]] = load i32, ptr [[IDX_1]], align 448; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[ADD_1]], [[PRE_NEXT]]49; CHECK-NEXT: [[IDX_2:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[IV]]50; CHECK-NEXT: store i32 [[ADD_2]], ptr [[IDX_2]], align 451; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 152; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 200053; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR]], !llvm.loop [[LOOP3:![0-9]+]]54; CHECK: exit:55; CHECK-NEXT: ret void56;57 58entry:59 br label %preheader60 61preheader:62 %idx.phi.trans = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 163 %.pre = load i32, ptr %idx.phi.trans, align 464 br label %for65 66for:67 %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]68 %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]69 %add.1 = add i32 %pre.phi, %x70 %idx.1 = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 %iv71 %pre.next = load i32, ptr %idx.1, align 472 %add.2 = add i32 %add.1, %pre.next73 %idx.2 = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 %iv74 store i32 %add.2, ptr %idx.2, align 475 %iv.next = add nuw nsw i64 %iv, 176 %exitcond = icmp eq i64 %iv.next, 200077 br i1 %exitcond, label %exit, label %for78 79exit:80 ret void81}82 83; We can sink potential trapping instructions, as this will only delay the trap84; and not introduce traps on additional paths.85define void @sink_sdiv(i32 %x, ptr %ptr, i64 %tc) local_unnamed_addr #0 {86; CHECK-LABEL: @sink_sdiv(87; CHECK-NEXT: entry:88; CHECK-NEXT: br label [[PREHEADER:%.*]]89; CHECK: preheader:90; CHECK-NEXT: [[IDX_PHI_TRANS:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 191; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[IDX_PHI_TRANS]], align 492; CHECK-NEXT: br label [[VECTOR_PH:%.*]]93; CHECK: vector.ph:94; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 095; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer96; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 397; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]98; CHECK: vector.body:99; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]100; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]101; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]102; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[OFFSET_IDX]]103; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, ptr [[TMP1]], align 4104; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>105; CHECK-NEXT: [[TMP4:%.*]] = sdiv <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]106; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], [[WIDE_LOAD]]107; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[OFFSET_IDX]]108; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4109; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4110; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1996111; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]112; CHECK: middle.block:113; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3114; CHECK-NEXT: br label [[SCALAR_PH:%.*]]115; CHECK: scalar.ph:116; CHECK-NEXT: br label [[FOR:%.*]]117; CHECK: for:118; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[SCALAR_PH]] ], [ [[PRE_NEXT:%.*]], [[FOR]] ]119; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1997, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR]] ]120; CHECK-NEXT: [[DIV_1:%.*]] = sdiv i32 [[SCALAR_RECUR]], [[X]]121; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[IV]]122; CHECK-NEXT: [[PRE_NEXT]] = load i32, ptr [[IDX_1]], align 4123; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[DIV_1]], [[PRE_NEXT]]124; CHECK-NEXT: [[IDX_2:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[IV]]125; CHECK-NEXT: store i32 [[ADD_2]], ptr [[IDX_2]], align 4126; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1127; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 2000128; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR]], !llvm.loop [[LOOP5:![0-9]+]]129; CHECK: exit:130; CHECK-NEXT: ret void131;132 133entry:134 br label %preheader135 136preheader:137 %idx.phi.trans = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1138 %.pre = load i32, ptr %idx.phi.trans, align 4139 br label %for140 141for:142 %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]143 %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]144 %div.1 = sdiv i32 %pre.phi, %x145 %idx.1 = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 %iv146 %pre.next = load i32, ptr %idx.1, align 4147 %add.2 = add i32 %div.1, %pre.next148 %idx.2 = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 %iv149 store i32 %add.2, ptr %idx.2, align 4150 %iv.next = add nuw nsw i64 %iv, 1151 %exitcond = icmp eq i64 %iv.next, 2000152 br i1 %exitcond, label %exit, label %for153 154exit:155 ret void156}157 158; Sink users of %pre.phi recursively.159define void @can_sink_with_additional_user(i32 %x, ptr %ptr, i64 %tc) {160; CHECK-LABEL: @can_sink_with_additional_user(161; CHECK-NEXT: entry:162; CHECK-NEXT: br label [[PREHEADER:%.*]]163; CHECK: preheader:164; CHECK-NEXT: [[IDX_PHI_TRANS:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1165; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[IDX_PHI_TRANS]], align 4166; CHECK-NEXT: br label [[VECTOR_PH:%.*]]167; CHECK: vector.ph:168; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0169; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer170; CHECK-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[DOTPRE]], i32 3171; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]172; CHECK: vector.body:173; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]174; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]175; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]176; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[OFFSET_IDX]]177; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i32>, ptr [[TMP1]], align 4178; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>179; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP3]], [[BROADCAST_SPLAT]]180; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], [[BROADCAST_SPLAT]]181; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP4]], [[WIDE_LOAD]]182; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP5]], [[TMP6]]183; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[OFFSET_IDX]]184; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP8]], align 4185; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4186; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1996187; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]188; CHECK: middle.block:189; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3190; CHECK-NEXT: br label [[SCALAR_PH:%.*]]191; CHECK: scalar.ph:192; CHECK-NEXT: br label [[FOR:%.*]]193; CHECK: for:194; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[SCALAR_PH]] ], [ [[PRE_NEXT:%.*]], [[FOR]] ]195; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1997, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR]] ]196; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[SCALAR_RECUR]], [[X]]197; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[ADD_1]], [[X]]198; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[IV]]199; CHECK-NEXT: [[PRE_NEXT]] = load i32, ptr [[IDX_1]], align 4200; CHECK-NEXT: [[ADD_3:%.*]] = add i32 [[ADD_1]], [[PRE_NEXT]]201; CHECK-NEXT: [[ADD_4:%.*]] = add i32 [[ADD_2]], [[ADD_3]]202; CHECK-NEXT: [[IDX_2:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[IV]]203; CHECK-NEXT: store i32 [[ADD_4]], ptr [[IDX_2]], align 4204; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1205; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 2000206; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR]], !llvm.loop [[LOOP7:![0-9]+]]207; CHECK: exit:208; CHECK-NEXT: ret void209;210 211 212 213entry:214 br label %preheader215 216preheader:217 %idx.phi.trans = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1218 %.pre = load i32, ptr %idx.phi.trans, align 4219 br label %for220 221for:222 %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]223 %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]224 %add.1 = add i32 %pre.phi, %x225 %add.2 = add i32 %add.1, %x226 %idx.1 = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 %iv227 %pre.next = load i32, ptr %idx.1, align 4228 %add.3 = add i32 %add.1, %pre.next229 %add.4 = add i32 %add.2, %add.3230 %idx.2 = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 %iv231 store i32 %add.4, ptr %idx.2, align 4232 %iv.next = add nuw nsw i64 %iv, 1233 %exitcond = icmp eq i64 %iv.next, 2000234 br i1 %exitcond, label %exit, label %for235 236exit:237 ret void238}239 240; FIXME: We can sink a store, if we can guarantee that it does not alias any241; loads/stores in between.242define void @cannot_sink_store(i32 %x, ptr %ptr, i64 %tc) {243; CHECK-LABEL: @cannot_sink_store(244; CHECK-NEXT: entry:245; CHECK-NEXT: br label [[PREHEADER:%.*]]246; CHECK: preheader:247; CHECK-NEXT: [[IDX_PHI_TRANS:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1248; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[IDX_PHI_TRANS]], align 4249; CHECK-NEXT: br label [[FOR:%.*]]250; CHECK: for:251; CHECK-NEXT: [[PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], [[PREHEADER]] ], [ [[PRE_NEXT:%.*]], [[FOR]] ]252; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, [[PREHEADER]] ], [ [[IV_NEXT:%.*]], [[FOR]] ]253; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[PRE_PHI]], [[X:%.*]]254; CHECK-NEXT: store i32 [[ADD_1]], ptr [[PTR:%.*]], align 4255; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[IV]]256; CHECK-NEXT: [[PRE_NEXT]] = load i32, ptr [[IDX_1]], align 4257; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[ADD_1]], [[PRE_NEXT]]258; CHECK-NEXT: [[IDX_2:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[IV]]259; CHECK-NEXT: store i32 [[ADD_2]], ptr [[IDX_2]], align 4260; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1261; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 2000262; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR]]263; CHECK: exit:264; CHECK-NEXT: ret void265;266 267 268 269entry:270 br label %preheader271 272preheader:273 %idx.phi.trans = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1274 %.pre = load i32, ptr %idx.phi.trans, align 4275 br label %for276 277for:278 %pre.phi = phi i32 [ %.pre, %preheader ], [ %pre.next, %for ]279 %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]280 %add.1 = add i32 %pre.phi, %x281 store i32 %add.1, ptr %ptr282 %idx.1 = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 %iv283 %pre.next = load i32, ptr %idx.1, align 4284 %add.2 = add i32 %add.1, %pre.next285 %idx.2 = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 %iv286 store i32 %add.2, ptr %idx.2, align 4287 %iv.next = add nuw nsw i64 %iv, 1288 %exitcond = icmp eq i64 %iv.next, 2000289 br i1 %exitcond, label %exit, label %for290 291exit:292 ret void293}294 295; Some kinds of reductions are not detected by IVDescriptors. If we have a296; cycle, we cannot sink it.297define void @cannot_sink_reduction(i32 %x, ptr %ptr, i64 %tc) {298; CHECK-LABEL: @cannot_sink_reduction(299; CHECK-NEXT: entry:300; CHECK-NEXT: br label [[PREHEADER:%.*]]301; CHECK: preheader:302; CHECK-NEXT: [[IDX_PHI_TRANS:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1303; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr [[IDX_PHI_TRANS]], align 4304; CHECK-NEXT: br label [[FOR:%.*]]305; CHECK: for:306; CHECK-NEXT: [[PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], [[PREHEADER]] ], [ [[D:%.*]], [[FOR]] ]307; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, [[PREHEADER]] ], [ [[IV_NEXT:%.*]], [[FOR]] ]308; CHECK-NEXT: [[D]] = sdiv i32 [[PRE_PHI]], [[X:%.*]]309; CHECK-NEXT: [[IDX_1:%.*]] = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 [[IV]]310; CHECK-NEXT: [[PRE_NEXT:%.*]] = load i32, ptr [[IDX_1]], align 4311; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[X]], [[PRE_NEXT]]312; CHECK-NEXT: [[IDX_2:%.*]] = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 [[IV]]313; CHECK-NEXT: store i32 [[ADD_2]], ptr [[IDX_2]], align 4314; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1315; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 2000316; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT:%.*]], label [[FOR]]317; CHECK: exit:318; CHECK-NEXT: ret void319;320entry:321 br label %preheader322 323preheader:324 %idx.phi.trans = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 1325 %.pre = load i32, ptr %idx.phi.trans, align 4326 br label %for327 328for:329 %pre.phi = phi i32 [ %.pre, %preheader ], [ %d, %for ]330 %iv = phi i64 [ 1, %preheader ], [ %iv.next, %for ]331 %d = sdiv i32 %pre.phi, %x332 %idx.1 = getelementptr inbounds [257 x i32], ptr @p, i64 0, i64 %iv333 %pre.next = load i32, ptr %idx.1, align 4334 %add.2 = add i32 %x, %pre.next335 %idx.2 = getelementptr inbounds [257 x i32], ptr @q, i64 0, i64 %iv336 store i32 %add.2, ptr %idx.2, align 4337 %iv.next = add nuw nsw i64 %iv, 1338 %exitcond = icmp eq i64 %iv.next, 2000339 br i1 %exitcond, label %exit, label %for340 341exit:342 ret void343}344 345; Sink %tmp38 after %tmp60, then it enable the loop vectorization.346define void @instruction_with_2_FOR_operands(ptr noalias %A, ptr noalias %B, ptr noalias %C) {347; CHECK-LABEL: @instruction_with_2_FOR_operands(348; CHECK-NEXT: bb:349; CHECK-NEXT: br label [[VECTOR_PH:%.*]]350; CHECK: vector.ph:351; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]352; CHECK: vector.body:353; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]354; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT3:%.*]], [[VECTOR_BODY]] ]355; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 1.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], [[VECTOR_BODY]] ]356; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]]357; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[A:%.*]], align 4358; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i64 0359; CHECK-NEXT: [[BROADCAST_SPLAT]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer360; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR1]], <4 x float> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>361; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[B:%.*]], align 4362; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0363; CHECK-NEXT: [[BROADCAST_SPLAT3]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer364; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR]], <4 x float> [[BROADCAST_SPLAT3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>365; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP3]]366; CHECK-NEXT: store <4 x float> [[TMP6]], ptr [[TMP1]], align 4367; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4368; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000369; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]370; CHECK: middle.block:371; CHECK-NEXT: br label [[SCALAR_PH:%.*]]372; CHECK: scalar.ph:373; CHECK-NEXT: br label [[BB13:%.*]]374; CHECK: bb13:375; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi float [ [[TMP60:%.*]], [[BB13]] ], [ [[TMP4]], [[SCALAR_PH]] ]376; CHECK-NEXT: [[SCALAR_RECUR5:%.*]] = phi float [ [[TMP49:%.*]], [[BB13]] ], [ [[TMP2]], [[SCALAR_PH]] ]377; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[BB13]] ], [ 1000, [[SCALAR_PH]] ]378; CHECK-NEXT: [[TMP38:%.*]] = fmul fast float [[SCALAR_RECUR]], [[SCALAR_RECUR5]]379; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1380; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[IV]]381; CHECK-NEXT: [[TMP49]] = load float, ptr [[A]], align 4382; CHECK-NEXT: [[TMP60]] = load float, ptr [[B]], align 4383; CHECK-NEXT: store float [[TMP38]], ptr [[GEP]], align 4384; CHECK-NEXT: [[TMP12:%.*]] = icmp slt i64 [[IV]], 1000385; CHECK-NEXT: br i1 [[TMP12]], label [[BB13]], label [[BB74:%.*]], !llvm.loop [[LOOP9:![0-9]+]]386; CHECK: bb74:387; CHECK-NEXT: ret void388;389bb:390 br label %bb13391 392bb13: ; preds = %bb13, %bb393 %tmp37 = phi float [ %tmp60, %bb13 ], [ 0.0, %bb ]394 %tmp27 = phi float [ %tmp49, %bb13 ], [ 1.0, %bb ]395 %iv = phi i64 [ %iv.next, %bb13 ], [ 0, %bb ]396 %tmp38 = fmul fast float %tmp37, %tmp27397 %iv.next = add nuw nsw i64 %iv, 1398 %gep = getelementptr inbounds float, ptr %C, i64 %iv399 %tmp49 = load float, ptr %A, align 4400 %tmp60 = load float, ptr %B, align 4401 store float %tmp38, ptr %gep402 %tmp12 = icmp slt i64 %iv, 1000403 br i1 %tmp12, label %bb13, label %bb74404 405bb74: ; preds = %bb13406 ret void407}408 409define void @instruction_with_2_FOR_operands_and_multiple_other_uses(ptr noalias %dst.1, ptr noalias %dst.2, ptr noalias %dst.3, ptr noalias %for.ptr.1, ptr noalias %for.ptr.2) {410; CHECK-LABEL: @instruction_with_2_FOR_operands_and_multiple_other_uses(411; CHECK-NEXT: bb:412; CHECK-NEXT: br label [[VECTOR_PH:%.*]]413; CHECK: vector.ph:414; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]415; CHECK: vector.body:416; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]417; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT3:%.*]], [[VECTOR_BODY]] ]418; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], [[VECTOR_BODY]] ]419; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[FOR_PTR_2:%.*]], align 4420; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i64 0421; CHECK-NEXT: [[BROADCAST_SPLAT]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer422; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR1]], <4 x float> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>423; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[FOR_PTR_1:%.*]], align 4424; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0425; CHECK-NEXT: [[BROADCAST_SPLAT3]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer426; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR]], <4 x float> [[BROADCAST_SPLAT3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>427; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[TMP4]], splat (float 2.000000e+00)428; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[TMP4]], [[TMP2]]429; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[TMP4]], splat (float 1.000000e+00)430; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[DST_1:%.*]], i64 [[INDEX]]431; CHECK-NEXT: store <4 x float> [[TMP6]], ptr [[TMP8]], align 4432; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[DST_2:%.*]], i64 [[INDEX]]433; CHECK-NEXT: store <4 x float> [[TMP5]], ptr [[TMP10]], align 4434; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[DST_3:%.*]], i64 [[INDEX]]435; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[TMP12]], align 4436; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4437; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000438; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]439; CHECK: middle.block:440; CHECK-NEXT: br label [[SCALAR_PH:%.*]]441; CHECK: scalar.ph:442; CHECK-NEXT: br label [[LOOP:%.*]]443; CHECK: loop:444; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi float [ [[TMP3]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]445; CHECK-NEXT: [[SCALAR_RECUR5:%.*]] = phi float [ [[TMP1]], [[SCALAR_PH]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]446; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1000, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]447; CHECK-NEXT: [[FOR_1_USE_1:%.*]] = fmul fast float [[SCALAR_RECUR]], 2.000000e+00448; CHECK-NEXT: [[USED_BY_BOTH:%.*]] = fmul fast float [[SCALAR_RECUR]], [[SCALAR_RECUR5]]449; CHECK-NEXT: [[FOR_2_NEXT]] = load float, ptr [[FOR_PTR_2]], align 4450; CHECK-NEXT: [[FOR_1_USE_3:%.*]] = fadd fast float [[SCALAR_RECUR]], 1.000000e+00451; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1452; CHECK-NEXT: [[FOR_1_NEXT]] = load float, ptr [[FOR_PTR_1]], align 4453; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST_1]], i64 [[IV]]454; CHECK-NEXT: store float [[USED_BY_BOTH]], ptr [[GEP_DST_1]], align 4455; CHECK-NEXT: [[GEP_DST_2:%.*]] = getelementptr inbounds float, ptr [[DST_2]], i64 [[IV]]456; CHECK-NEXT: store float [[FOR_1_USE_1]], ptr [[GEP_DST_2]], align 4457; CHECK-NEXT: [[GEP_DST_3:%.*]] = getelementptr inbounds float, ptr [[DST_3]], i64 [[IV]]458; CHECK-NEXT: store float [[FOR_1_USE_3]], ptr [[GEP_DST_3]], align 4459; CHECK-NEXT: [[EC:%.*]] = icmp slt i64 [[IV]], 1000460; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]461; CHECK: exit:462; CHECK-NEXT: ret void463;464bb:465 br label %loop466 467loop:468 %for.1 = phi float [ 0.0, %bb ], [ %for.1.next, %loop]469 %for.2 = phi float [ 0.0, %bb ], [ %for.2.next, %loop]470 %iv = phi i64 [ 0, %bb ], [ %iv.next, %loop ]471 %for.1.use.1 = fmul fast float %for.1, 2.0472 %used.by.both = fmul fast float %for.1, %for.2473 %for.2.next = load float, ptr %for.ptr.2, align 4474 %for.1.use.3 = fadd fast float %for.1, 1.0475 %iv.next = add nuw nsw i64 %iv, 1476 %for.1.next = load float, ptr %for.ptr.1, align 4477 %gep.dst.1 = getelementptr inbounds float, ptr %dst.1, i64 %iv478 store float %used.by.both, ptr %gep.dst.1479 %gep.dst.2 = getelementptr inbounds float, ptr %dst.2, i64 %iv480 store float %for.1.use.1, ptr %gep.dst.2481 %gep.dst.3 = getelementptr inbounds float, ptr %dst.3, i64 %iv482 store float %for.1.use.3, ptr %gep.dst.3483 %ec = icmp slt i64 %iv, 1000484 br i1 %ec, label %loop, label %exit485 486exit:487 ret void488}489 490; Variation of @instruction_with_2_FOR_operands_and_multiple_other_uses, with491; multiple instructions in a chain from for.1 to %used.by.both.492define void @instruction_with_2_FOR_operands_and_multiple_other_uses_chain(ptr noalias %dst.1, ptr noalias %dst.2, ptr noalias %dst.3, ptr noalias %for.ptr.1, ptr noalias %for.ptr.2) {493; CHECK-LABEL: @instruction_with_2_FOR_operands_and_multiple_other_uses_chain(494; CHECK-NEXT: bb:495; CHECK-NEXT: br label [[VECTOR_PH:%.*]]496; CHECK: vector.ph:497; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]498; CHECK: vector.body:499; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]500; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT3:%.*]], [[VECTOR_BODY]] ]501; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x float> [ <float poison, float poison, float poison, float 0.000000e+00>, [[VECTOR_PH]] ], [ [[BROADCAST_SPLAT:%.*]], [[VECTOR_BODY]] ]502; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[FOR_PTR_2:%.*]], align 4503; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i64 0504; CHECK-NEXT: [[BROADCAST_SPLAT]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer505; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR1]], <4 x float> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>506; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[FOR_PTR_1:%.*]], align 4507; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0508; CHECK-NEXT: [[BROADCAST_SPLAT3]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT2]], <4 x float> poison, <4 x i32> zeroinitializer509; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[VECTOR_RECUR]], <4 x float> [[BROADCAST_SPLAT3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>510; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[TMP4]], splat (float 2.000000e+00)511; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x float> [[TMP5]], splat (float 2.000000e+00)512; CHECK-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[TMP6]], [[TMP2]]513; CHECK-NEXT: [[TMP8:%.*]] = fadd fast <4 x float> [[TMP4]], splat (float 1.000000e+00)514; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[DST_1:%.*]], i64 [[INDEX]]515; CHECK-NEXT: store <4 x float> [[TMP7]], ptr [[TMP9]], align 4516; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[DST_2:%.*]], i64 [[INDEX]]517; CHECK-NEXT: store <4 x float> [[TMP5]], ptr [[TMP11]], align 4518; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[DST_3:%.*]], i64 [[INDEX]]519; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[TMP13]], align 4520; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4521; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000522; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]523; CHECK: middle.block:524; CHECK-NEXT: br label [[SCALAR_PH:%.*]]525; CHECK: scalar.ph:526; CHECK-NEXT: br label [[LOOP:%.*]]527; CHECK: loop:528; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi float [ [[TMP3]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]529; CHECK-NEXT: [[SCALAR_RECUR5:%.*]] = phi float [ [[TMP1]], [[SCALAR_PH]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]530; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1000, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]531; CHECK-NEXT: [[FOR_1_USE_1:%.*]] = fmul fast float [[SCALAR_RECUR]], 2.000000e+00532; CHECK-NEXT: [[FOR_1_USE_C:%.*]] = fmul fast float [[FOR_1_USE_1]], 2.000000e+00533; CHECK-NEXT: [[USED_BY_BOTH:%.*]] = fmul fast float [[FOR_1_USE_C]], [[SCALAR_RECUR5]]534; CHECK-NEXT: [[FOR_2_NEXT]] = load float, ptr [[FOR_PTR_2]], align 4535; CHECK-NEXT: [[FOR_1_USE_3:%.*]] = fadd fast float [[SCALAR_RECUR]], 1.000000e+00536; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1537; CHECK-NEXT: [[FOR_1_NEXT]] = load float, ptr [[FOR_PTR_1]], align 4538; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST_1]], i64 [[IV]]539; CHECK-NEXT: store float [[USED_BY_BOTH]], ptr [[GEP_DST_1]], align 4540; CHECK-NEXT: [[GEP_DST_2:%.*]] = getelementptr inbounds float, ptr [[DST_2]], i64 [[IV]]541; CHECK-NEXT: store float [[FOR_1_USE_1]], ptr [[GEP_DST_2]], align 4542; CHECK-NEXT: [[GEP_DST_3:%.*]] = getelementptr inbounds float, ptr [[DST_3]], i64 [[IV]]543; CHECK-NEXT: store float [[FOR_1_USE_3]], ptr [[GEP_DST_3]], align 4544; CHECK-NEXT: [[EC:%.*]] = icmp slt i64 [[IV]], 1000545; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT:%.*]], !llvm.loop [[LOOP13:![0-9]+]]546; CHECK: exit:547; CHECK-NEXT: ret void548;549bb:550 br label %loop551 552loop:553 %for.1 = phi float [ 0.0, %bb ], [ %for.1.next, %loop]554 %for.2 = phi float [ 0.0, %bb ], [ %for.2.next, %loop]555 %iv = phi i64 [ 0, %bb ], [ %iv.next, %loop ]556 %for.1.use.1 = fmul fast float %for.1, 2.0557 %for.1.use.c = fmul fast float %for.1.use.1, 2.0558 %used.by.both = fmul fast float %for.1.use.c, %for.2559 %for.2.next = load float, ptr %for.ptr.2, align 4560 %for.1.use.3 = fadd fast float %for.1, 1.0561 %iv.next = add nuw nsw i64 %iv, 1562 %for.1.next = load float, ptr %for.ptr.1, align 4563 %gep.dst.1 = getelementptr inbounds float, ptr %dst.1, i64 %iv564 store float %used.by.both, ptr %gep.dst.1565 %gep.dst.2 = getelementptr inbounds float, ptr %dst.2, i64 %iv566 store float %for.1.use.1, ptr %gep.dst.2567 %gep.dst.3 = getelementptr inbounds float, ptr %dst.3, i64 %iv568 store float %for.1.use.3, ptr %gep.dst.3569 %ec = icmp slt i64 %iv, 1000570 br i1 %ec, label %loop, label %exit571 572exit:573 ret void574}575 576; The (first) reason `%first_time.1` cannot be sunk is because it appears outside577; the header and is not dominated by Previous. The fact that it feeds Previous578; is a second sinking-preventing reason.579define void @cannot_sink_phi(ptr %ptr) {580; CHECK-LABEL: @cannot_sink_phi(581; CHECK-NEXT: entry:582; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]583; CHECK: loop.header:584; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]585; CHECK-NEXT: [[FOR:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[FOR_NEXT:%.*]], [[LOOP_LATCH]] ]586; CHECK-NEXT: [[C_1:%.*]] = icmp ult i64 [[IV]], 500587; CHECK-NEXT: br i1 [[C_1]], label [[IF_TRUEBB:%.*]], label [[IF_FALSEBB:%.*]]588; CHECK: if.truebb:589; CHECK-NEXT: br label [[LOOP_LATCH]]590; CHECK: if.falsebb:591; CHECK-NEXT: br label [[LOOP_LATCH]]592; CHECK: loop.latch:593; CHECK-NEXT: [[FIRST_TIME_1:%.*]] = phi i32 [ 20, [[IF_TRUEBB]] ], [ [[FOR]], [[IF_FALSEBB]] ]594; CHECK-NEXT: [[C_2:%.*]] = icmp ult i64 [[IV]], 800595; CHECK-NEXT: [[FOR_NEXT]] = select i1 [[C_2]], i32 30, i32 [[FIRST_TIME_1]]596; CHECK-NEXT: [[PTR_IDX:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[IV]]597; CHECK-NEXT: store i32 [[FOR_NEXT]], ptr [[PTR_IDX]], align 4598; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1599; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1000600; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP_HEADER]]601; CHECK: exit:602; CHECK-NEXT: ret void603;604entry:605 br label %loop.header606 607loop.header:608 %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.latch ]609 %for = phi i32 [ 0, %entry ], [ %for.next, %loop.latch ]610 %c.1 = icmp ult i64 %iv, 500611 br i1 %c.1, label %if.truebb, label %if.falsebb612 613if.truebb:614 br label %loop.latch615 616if.falsebb:617 br label %loop.latch618 619loop.latch:620 %first_time.1 = phi i32 [ 20, %if.truebb ], [ %for, %if.falsebb ]621 %c.2 = icmp ult i64 %iv, 800622 %for.next = select i1 %c.2, i32 30, i32 %first_time.1623 %ptr.idx = getelementptr i32, ptr %ptr, i64 %iv624 store i32 %for.next, ptr %ptr.idx625 %iv.next = add nuw nsw i64 %iv, 1626 %exitcond.not = icmp eq i64 %iv.next, 1000627 br i1 %exitcond.not, label %exit, label %loop.header628 629exit:630 ret void631}632 633; A recurrence in a multiple exit loop.634define i16 @multiple_exit(ptr %p, i32 %n) {635; CHECK-LABEL: @multiple_exit(636; CHECK-NEXT: entry:637; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)638; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)639; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1640; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 4641; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]642; CHECK: vector.ph:643; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4644; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0645; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 4, i32 [[N_MOD_VF]]646; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]647; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]648; CHECK: vector.body:649; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]650; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]651; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64652; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]653; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP5]], align 2654; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>655; CHECK-NEXT: store <4 x i16> [[TMP7]], ptr [[TMP5]], align 4656; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4657; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]658; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]659; CHECK: middle.block:660; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3661; CHECK-NEXT: br label [[SCALAR_PH]]662; CHECK: scalar.ph:663; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]664; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]665; CHECK-NEXT: br label [[FOR_COND:%.*]]666; CHECK: for.cond:667; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]668; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_NEXT:%.*]], [[FOR_BODY]] ]669; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64670; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]671; CHECK-NEXT: [[REC_NEXT]] = load i16, ptr [[B]], align 2672; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]673; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]674; CHECK: for.body:675; CHECK-NEXT: store i16 [[SCALAR_RECUR]], ptr [[B]], align 4676; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1677; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096678; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP15:![0-9]+]]679; CHECK: if.end:680; CHECK-NEXT: [[REC_LCSSA:%.*]] = phi i16 [ [[SCALAR_RECUR]], [[FOR_BODY]] ], [ [[SCALAR_RECUR]], [[FOR_COND]] ]681; CHECK-NEXT: ret i16 [[REC_LCSSA]]682;683entry:684 br label %for.cond685 686for.cond:687 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]688 %rec = phi i16 [0, %entry], [ %rec.next, %for.body ]689 %iprom = sext i32 %i to i64690 %b = getelementptr inbounds i16, ptr %p, i64 %iprom691 %rec.next = load i16, ptr %b692 %cmp = icmp slt i32 %i, %n693 br i1 %cmp, label %for.body, label %if.end694 695for.body:696 store i16 %rec , ptr %b, align 4697 %inc = add nsw i32 %i, 1698 %cmp2 = icmp slt i32 %i, 2096699 br i1 %cmp2, label %for.cond, label %if.end700 701if.end:702 ret i16 %rec703}704 705 706; A multiple exit case where one of the exiting edges involves a value707; from the recurrence and one does not.708define i16 @multiple_exit2(ptr %p, i32 %n) {709; CHECK-LABEL: @multiple_exit2(710; CHECK-NEXT: entry:711; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 0)712; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[SMAX]], i32 2096)713; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1714; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], 4715; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]716; CHECK: vector.ph:717; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], 4718; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[N_MOD_VF]], 0719; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 4, i32 [[N_MOD_VF]]720; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP2]]721; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]722; CHECK: vector.body:723; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]724; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]725; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[INDEX]] to i64726; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[P:%.*]], i64 [[TMP4]]727; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x i16>, ptr [[TMP5]], align 2728; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>729; CHECK-NEXT: store <4 x i16> [[TMP7]], ptr [[TMP5]], align 4730; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4731; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]732; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]733; CHECK: middle.block:734; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[WIDE_LOAD]], i32 3735; CHECK-NEXT: br label [[SCALAR_PH]]736; CHECK: scalar.ph:737; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]738; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]739; CHECK-NEXT: br label [[FOR_COND:%.*]]740; CHECK: for.cond:741; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]742; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_NEXT:%.*]], [[FOR_BODY]] ]743; CHECK-NEXT: [[IPROM:%.*]] = sext i32 [[I]] to i64744; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]745; CHECK-NEXT: [[REC_NEXT]] = load i16, ptr [[B]], align 2746; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]747; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[IF_END:%.*]]748; CHECK: for.body:749; CHECK-NEXT: store i16 [[SCALAR_RECUR]], ptr [[B]], align 4750; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1751; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[I]], 2096752; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_COND]], label [[IF_END]], !llvm.loop [[LOOP17:![0-9]+]]753; CHECK: if.end:754; CHECK-NEXT: [[REC_LCSSA:%.*]] = phi i16 [ [[SCALAR_RECUR]], [[FOR_COND]] ], [ 10, [[FOR_BODY]] ]755; CHECK-NEXT: ret i16 [[REC_LCSSA]]756;757entry:758 br label %for.cond759 760for.cond:761 %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]762 %rec = phi i16 [0, %entry], [ %rec.next, %for.body ]763 %iprom = sext i32 %i to i64764 %b = getelementptr inbounds i16, ptr %p, i64 %iprom765 %rec.next = load i16, ptr %b766 %cmp = icmp slt i32 %i, %n767 br i1 %cmp, label %for.body, label %if.end768 769for.body:770 store i16 %rec , ptr %b, align 4771 %inc = add nsw i32 %i, 1772 %cmp2 = icmp slt i32 %i, 2096773 br i1 %cmp2, label %for.cond, label %if.end774 775if.end:776 %rec.lcssa = phi i16 [ %rec, %for.cond ], [ 10, %for.body ]777 ret i16 %rec.lcssa778}779 780; A test where the instructions to sink may not be visited in dominance order.781define void @sink_dominance(ptr %ptr, i32 %N) {782; CHECK-LABEL: @sink_dominance(783; CHECK-NEXT: entry:784; CHECK-NEXT: [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[N:%.*]], i32 1)785; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX1]], 4786; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]787; CHECK: vector.scevcheck:788; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)789; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[UMAX]], -1790; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0791; CHECK-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]792; CHECK: vector.ph:793; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[UMAX1]], 4794; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[UMAX1]], [[N_MOD_VF]]795; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]796; CHECK: vector.body:797; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]798; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]799; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i32 [[INDEX]]800; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4801; CHECK-NEXT: [[TMP5]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>802; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>803; CHECK-NEXT: [[TMP7:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i32>804; CHECK-NEXT: [[TMP8:%.*]] = icmp slt <4 x i32> [[TMP7]], splat (i32 213)805; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> [[TMP7]], <4 x i32> splat (i32 22)806; CHECK-NEXT: store <4 x i32> [[TMP9]], ptr [[TMP3]], align 4807; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4808; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]809; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]810; CHECK: middle.block:811; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3812; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]813; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]814; CHECK: scalar.ph:815; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]816; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]817; CHECK-NEXT: br label [[LOOP:%.*]]818; CHECK: loop:819; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]820; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]821; CHECK-NEXT: [[FOR_TRUNC:%.*]] = trunc i64 [[SCALAR_RECUR]] to i32822; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[FOR_TRUNC]], 213823; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[CMP]], i32 [[FOR_TRUNC]], i32 22824; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i32 [[IV]]825; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4826; CHECK-NEXT: [[FOR_NEXT]] = zext i32 [[LV]] to i64827; CHECK-NEXT: store i32 [[SELECT]], ptr [[GEP]], align 4828; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1829; CHECK-NEXT: [[CMP73:%.*]] = icmp ugt i32 [[N]], [[IV_NEXT]]830; CHECK-NEXT: br i1 [[CMP73]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]831; CHECK: exit:832; CHECK-NEXT: ret void833;834entry:835 br label %loop836 837loop:838 %for = phi i64 [ 0, %entry ], [ %for.next, %loop ]839 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]840 841 %for.trunc = trunc i64 %for to i32842 %cmp = icmp slt i32 %for.trunc, 213843 %select = select i1 %cmp, i32 %for.trunc, i32 22844 845 %gep = getelementptr inbounds i32, ptr %ptr, i32 %iv846 %lv = load i32, ptr %gep, align 4847 %for.next = zext i32 %lv to i64848 store i32 %select, ptr %gep849 850 %iv.next = add i32 %iv, 1851 %cmp73 = icmp ugt i32 %N, %iv.next852 br i1 %cmp73, label %loop, label %exit853 854exit:855 ret void856}857 858; Similar to @sink_dominance, but with 2 separate chains that merge at %select859; with a different number of instructions in between.860define void @sink_dominance_2(ptr %ptr, i32 %N) {861; CHECK-LABEL: @sink_dominance_2(862; CHECK-NEXT: entry:863; CHECK-NEXT: [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[N:%.*]], i32 1)864; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[UMAX1]], 4865; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]866; CHECK: vector.scevcheck:867; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[N]], i32 1)868; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[UMAX]], -1869; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0870; CHECK-NEXT: br i1 [[TMP1]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]871; CHECK: vector.ph:872; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[UMAX1]], 4873; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[UMAX1]], [[N_MOD_VF]]874; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]875; CHECK: vector.body:876; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]877; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]878; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i32 [[INDEX]]879; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4880; CHECK-NEXT: [[TMP5]] = zext <4 x i32> [[WIDE_LOAD]] to <4 x i64>881; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[TMP5]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>882; CHECK-NEXT: [[TMP7:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i32>883; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], splat (i32 2)884; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP8]], splat (i32 99)885; CHECK-NEXT: [[TMP10:%.*]] = icmp slt <4 x i32> [[TMP7]], splat (i32 213)886; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP7]], <4 x i32> [[TMP9]]887; CHECK-NEXT: store <4 x i32> [[TMP11]], ptr [[TMP3]], align 4888; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4889; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]890; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]891; CHECK: middle.block:892; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP5]], i32 3893; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[UMAX1]], [[N_VEC]]894; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]895; CHECK: scalar.ph:896; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]897; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]898; CHECK-NEXT: br label [[LOOP:%.*]]899; CHECK: loop:900; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]901; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]902; CHECK-NEXT: [[FOR_TRUNC:%.*]] = trunc i64 [[SCALAR_RECUR]] to i32903; CHECK-NEXT: [[STEP:%.*]] = add i32 [[FOR_TRUNC]], 2904; CHECK-NEXT: [[STEP_2:%.*]] = mul i32 [[STEP]], 99905; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[FOR_TRUNC]], 213906; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[CMP]], i32 [[FOR_TRUNC]], i32 [[STEP_2]]907; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i32 [[IV]]908; CHECK-NEXT: [[LV:%.*]] = load i32, ptr [[GEP]], align 4909; CHECK-NEXT: [[FOR_NEXT]] = zext i32 [[LV]] to i64910; CHECK-NEXT: store i32 [[SELECT]], ptr [[GEP]], align 4911; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1912; CHECK-NEXT: [[CMP73:%.*]] = icmp ugt i32 [[N]], [[IV_NEXT]]913; CHECK-NEXT: br i1 [[CMP73]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]914; CHECK: exit:915; CHECK-NEXT: ret void916;917entry:918 br label %loop919 920loop:921 %for = phi i64 [ 0, %entry ], [ %for.next, %loop ]922 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]923 924 %for.trunc = trunc i64 %for to i32925 %step = add i32 %for.trunc, 2926 %step.2 = mul i32 %step, 99927 928 %cmp = icmp slt i32 %for.trunc, 213929 %select = select i1 %cmp, i32 %for.trunc, i32 %step.2930 931 %gep = getelementptr inbounds i32, ptr %ptr, i32 %iv932 %lv = load i32, ptr %gep, align 4933 %for.next = zext i32 %lv to i64934 store i32 %select, ptr %gep935 936 %iv.next = add i32 %iv, 1937 %cmp73 = icmp ugt i32 %N, %iv.next938 br i1 %cmp73, label %loop, label %exit939 940exit:941 ret void942}943 944define void @cannot_sink_load_past_store(ptr %ptr, i32 %N) {945; CHECK-LABEL: @cannot_sink_load_past_store(946; CHECK-NEXT: entry:947; CHECK-NEXT: br label [[LOOP:%.*]]948; CHECK: loop:949; CHECK-NEXT: [[FOR:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]950; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]951; CHECK-NEXT: [[GEP_FOR:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[FOR]]952; CHECK-NEXT: [[LV_FOR:%.*]] = load i32, ptr [[GEP_FOR]], align 4953; CHECK-NEXT: [[FOR_TRUNC:%.*]] = trunc i64 [[FOR]] to i32954; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[LV_FOR]], [[FOR_TRUNC]]955; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[CMP]], i32 [[LV_FOR]], i32 22956; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i32 [[IV]]957; CHECK-NEXT: store i32 0, ptr [[GEP_IV]], align 4958; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1959; CHECK-NEXT: [[FOR_NEXT]] = zext i32 [[IV]] to i64960; CHECK-NEXT: [[CMP73:%.*]] = icmp ugt i32 [[N:%.*]], [[IV_NEXT]]961; CHECK-NEXT: br i1 [[CMP73]], label [[LOOP]], label [[EXIT:%.*]]962; CHECK: exit:963; CHECK-NEXT: ret void964;965entry:966 br label %loop967 968loop:969 %for = phi i64 [ 0, %entry ], [ %for.next, %loop ]970 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]971 972 %gep.for = getelementptr inbounds i32, ptr %ptr, i64 %for973 %lv.for = load i32, ptr %gep.for, align 4974 %for.trunc = trunc i64 %for to i32975 %cmp = icmp slt i32 %lv.for, %for.trunc976 %select = select i1 %cmp, i32 %lv.for, i32 22977 978 %gep.iv = getelementptr inbounds i32, ptr %ptr, i32 %iv979 store i32 0, ptr %gep.iv980 %iv.next = add i32 %iv, 1981 %for.next = zext i32 %iv to i64982 983 %cmp73 = icmp ugt i32 %N, %iv.next984 br i1 %cmp73, label %loop, label %exit985 986exit:987 ret void988}989 990define void @test_for_sink_instruction_after_same_incoming_1(ptr %ptr) {991; CHECK-LABEL: @test_for_sink_instruction_after_same_incoming_1(992; CHECK-NEXT: entry:993; CHECK-NEXT: br label [[VECTOR_PH:%.*]]994; CHECK: vector.ph:995; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]996; CHECK: vector.body:997; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]998; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 1.000000e+01>, [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]999; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 2.000000e+01>, [[VECTOR_PH]] ], [ [[WIDE_LOAD]], [[VECTOR_BODY]] ]1000; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]1001; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[PTR:%.*]], i64 [[OFFSET_IDX]]1002; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x double>, ptr [[TMP1]], align 81003; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>1004; CHECK-NEXT: [[TMP4:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[TMP3]]1005; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>1006; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x double> [[TMP4]], [[TMP5]]1007; CHECK-NEXT: store <4 x double> [[TMP6]], ptr [[TMP1]], align 81008; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41009; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9961010; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]1011; CHECK: middle.block:1012; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 31013; CHECK-NEXT: br label [[SCALAR_PH:%.*]]1014; CHECK: scalar.ph:1015; CHECK-NEXT: br label [[LOOP:%.*]]1016; CHECK: loop:1017; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT2]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]1018; CHECK-NEXT: [[SCALAR_RECUR4:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT2]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT]], [[LOOP]] ]1019; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 997, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]1020; CHECK-NEXT: [[ADD_1:%.*]] = fadd double 1.000000e+01, [[SCALAR_RECUR4]]1021; CHECK-NEXT: [[ADD_2:%.*]] = fadd double [[ADD_1]], [[SCALAR_RECUR]]1022; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11023; CHECK-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 [[IV]]1024; CHECK-NEXT: [[FOR_1_NEXT]] = load double, ptr [[GEP_PTR]], align 81025; CHECK-NEXT: store double [[ADD_2]], ptr [[GEP_PTR]], align 81026; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10001027; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]1028; CHECK: exit:1029; CHECK-NEXT: ret void1030;1031entry:1032 br label %loop1033 1034loop:1035 %for.1 = phi double [ 10.0, %entry ], [ %for.1.next, %loop ]1036 %for.2 = phi double [ 20.0, %entry ], [ %for.1.next, %loop ]1037 %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]1038 %add.1 = fadd double 10.0, %for.21039 %add.2 = fadd double %add.1, %for.11040 %iv.next = add nuw nsw i64 %iv, 11041 %gep.ptr = getelementptr inbounds double, ptr %ptr, i64 %iv1042 %for.1.next = load double, ptr %gep.ptr, align 81043 store double %add.2, ptr %gep.ptr1044 %exitcond.not = icmp eq i64 %iv.next, 10001045 br i1 %exitcond.not, label %exit, label %loop1046 1047exit:1048 ret void1049}1050 1051 1052define void @test_for_sink_instruction_after_same_incoming_2(ptr %ptr) {1053; CHECK-LABEL: @test_for_sink_instruction_after_same_incoming_2(1054; CHECK-NEXT: entry:1055; CHECK-NEXT: br label [[VECTOR_PH:%.*]]1056; CHECK: vector.ph:1057; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]1058; CHECK: vector.body:1059; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]1060; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 2.000000e+01>, [[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], [[VECTOR_BODY]] ]1061; CHECK-NEXT: [[VECTOR_RECUR1:%.*]] = phi <4 x double> [ <double poison, double poison, double poison, double 1.000000e+01>, [[VECTOR_PH]] ], [ [[WIDE_LOAD]], [[VECTOR_BODY]] ]1062; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]1063; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[PTR:%.*]], i64 [[OFFSET_IDX]]1064; CHECK-NEXT: [[WIDE_LOAD]] = load <4 x double>, ptr [[TMP1]], align 81065; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR1]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>1066; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[VECTOR_RECUR]], <4 x double> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>1067; CHECK-NEXT: [[TMP5:%.*]] = fadd <4 x double> splat (double 1.000000e+01), [[TMP4]]1068; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x double> [[TMP5]], [[TMP3]]1069; CHECK-NEXT: store <4 x double> [[TMP6]], ptr [[TMP1]], align 81070; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 41071; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9961072; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]1073; CHECK: middle.block:1074; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT2:%.*]] = extractelement <4 x double> [[WIDE_LOAD]], i32 31075; CHECK-NEXT: br label [[SCALAR_PH:%.*]]1076; CHECK: scalar.ph:1077; CHECK-NEXT: br label [[LOOP:%.*]]1078; CHECK: loop:1079; CHECK-NEXT: [[SCALAR_RECUR:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT2]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]1080; CHECK-NEXT: [[SCALAR_RECUR4:%.*]] = phi double [ [[VECTOR_RECUR_EXTRACT2]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT]], [[LOOP]] ]1081; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 997, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]1082; CHECK-NEXT: [[ADD_1:%.*]] = fadd double 1.000000e+01, [[SCALAR_RECUR]]1083; CHECK-NEXT: [[ADD_2:%.*]] = fadd double [[ADD_1]], [[SCALAR_RECUR4]]1084; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 11085; CHECK-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds double, ptr [[PTR]], i64 [[IV]]1086; CHECK-NEXT: [[FOR_1_NEXT]] = load double, ptr [[GEP_PTR]], align 81087; CHECK-NEXT: store double [[ADD_2]], ptr [[GEP_PTR]], align 81088; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10001089; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]1090; CHECK: exit:1091; CHECK-NEXT: ret void1092;1093entry:1094 br label %loop1095 1096loop:1097 %for.2 = phi double [ 20.0, %entry ], [ %for.1.next, %loop ]1098 %for.1 = phi double [ 10.0, %entry ], [ %for.1.next, %loop ]1099 %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]1100 %add.1 = fadd double 10.0, %for.21101 %add.2 = fadd double %add.1, %for.11102 %iv.next = add nuw nsw i64 %iv, 11103 %gep.ptr = getelementptr inbounds double, ptr %ptr, i64 %iv1104 %for.1.next = load double, ptr %gep.ptr, align 81105 store double %add.2, ptr %gep.ptr1106 %exitcond.not = icmp eq i64 %iv.next, 10001107 br i1 %exitcond.not, label %exit, label %loop1108 1109exit:1110 ret void1111}1112