1160 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 62; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s3 4define void @test(ptr %dst, ptr %src, ptr %cond) {5; CHECK-LABEL: define void @test(6; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {7; CHECK-NEXT: [[ENTRY:.*:]]8; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]9; CHECK: [[VECTOR_MEMCHECK]]:10; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 40011; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND]], i64 40012; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 40013; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]14; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND]], [[SCEVGEP]]15; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]16; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]17; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]18; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]19; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]20; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]21; CHECK: [[VECTOR_PH]]:22; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]23; CHECK: [[VECTOR_BODY]]:24; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]25; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 026; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 127; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]28; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]29; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]30; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META0:![0-9]+]]31; CHECK-NEXT: [[TMP15:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)32; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP8]], align 4, !alias.scope [[META3:![0-9]+]]33; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP9]], align 4, !alias.scope [[META3]]34; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 035; CHECK-NEXT: [[TMP36:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP7]], i32 136; CHECK-NEXT: [[TMP25:%.*]] = add <2 x i32> [[TMP36]], splat (i32 10)37; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP15]], <2 x i32> [[TMP36]], <2 x i32> [[TMP25]]38; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]39; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP37]], align 4, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]40; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 241; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i32 [[INDEX_NEXT]], 10042; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]43; CHECK: [[MIDDLE_BLOCK]]:44; CHECK-NEXT: br [[EXIT:label %.*]]45; CHECK: [[SCALAR_PH]]:46;47entry:48 br label %loop49 50loop:51 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]52 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv53 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv54 %l.c = load i32, ptr %gep.cond55 %c = icmp ule i32 %l.c, 1156 br i1 %c, label %then, label %else57 58then:59 %l.src = load i32, ptr %gep.src, align 460 br label %loop.latch61 62else:63 %l.src.2 = load i32, ptr %gep.src, align 464 %add = add i32 %l.src.2, 1065 br label %loop.latch66 67loop.latch:68 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]69 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv70 store i32 %merge, ptr %gep.dst, align 471 %iv.next = add nuw nsw i32 %iv, 172 %ec = icmp eq i32 %iv.next, 10073 br i1 %ec, label %exit, label %loop74 75exit:76 ret void77}78 79; Negative test: Different addresses - should NOT hoist80define void @different_addresses(ptr %dst, ptr %src1, ptr %src2, ptr %cond) {81; CHECK-LABEL: define void @different_addresses(82; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC1:%.*]], ptr [[SRC2:%.*]], ptr [[COND:%.*]]) {83; CHECK-NEXT: [[ENTRY:.*:]]84; CHECK-NEXT: [[SRC15:%.*]] = ptrtoint ptr [[SRC1]] to i6485; CHECK-NEXT: [[SRC23:%.*]] = ptrtoint ptr [[SRC2]] to i6486; CHECK-NEXT: [[COND2:%.*]] = ptrtoint ptr [[COND]] to i6487; CHECK-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i6488; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]89; CHECK: [[VECTOR_MEMCHECK]]:90; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[COND2]]91; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = icmp ult i64 [[TMP0]], 892; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[DST1]], [[SRC23]]93; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = icmp ult i64 [[TMP1]], 894; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]95; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[DST1]], [[SRC15]]96; CHECK-NEXT: [[FOUND_CONFLICT9:%.*]] = icmp ult i64 [[TMP2]], 897; CHECK-NEXT: [[CONFLICT_RDX10:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT9]]98; CHECK-NEXT: br i1 [[CONFLICT_RDX10]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]99; CHECK: [[VECTOR_PH]]:100; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]101; CHECK: [[VECTOR_BODY]]:102; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE13:.*]] ]103; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0104; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 1105; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]106; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP16]], align 4107; CHECK-NEXT: [[TMP11:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)108; CHECK-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[TMP11]], splat (i1 true)109; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0110; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]111; CHECK: [[PRED_LOAD_IF]]:112; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i32 [[TMP4]]113; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP20]], align 4114; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP15]], i32 0115; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]116; CHECK: [[PRED_LOAD_CONTINUE]]:117; CHECK-NEXT: [[TMP17:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP19]], %[[PRED_LOAD_IF]] ]118; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1119; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]120; CHECK: [[PRED_LOAD_IF8]]:121; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i32 [[TMP5]]122; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP25]], align 4123; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP29]], i32 1124; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]125; CHECK: [[PRED_LOAD_CONTINUE9]]:126; CHECK-NEXT: [[TMP22:%.*]] = phi <2 x i32> [ [[TMP17]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP21]], %[[PRED_LOAD_IF8]] ]127; CHECK-NEXT: [[TMP23:%.*]] = add <2 x i32> [[TMP22]], splat (i32 10)128; CHECK-NEXT: [[TMP28:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0129; CHECK-NEXT: br i1 [[TMP28]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]130; CHECK: [[PRED_LOAD_IF10]]:131; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i32 [[TMP4]]132; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP49]], align 4133; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x i32> poison, i32 [[TMP24]], i32 0134; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]135; CHECK: [[PRED_LOAD_CONTINUE11]]:136; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP30]], %[[PRED_LOAD_IF10]] ]137; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1138; CHECK-NEXT: br i1 [[TMP31]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13]]139; CHECK: [[PRED_LOAD_IF12]]:140; CHECK-NEXT: [[TMP52:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i32 [[TMP5]]141; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP52]], align 4142; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> [[TMP32]], i32 [[TMP26]], i32 1143; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE13]]144; CHECK: [[PRED_LOAD_CONTINUE13]]:145; CHECK-NEXT: [[TMP33:%.*]] = phi <2 x i32> [ [[TMP32]], %[[PRED_LOAD_CONTINUE11]] ], [ [[TMP27]], %[[PRED_LOAD_IF12]] ]146; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x i32> [[TMP33]], <2 x i32> [[TMP23]]147; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]148; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP34]], align 4149; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2150; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100151; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]152; CHECK: [[MIDDLE_BLOCK]]:153; CHECK-NEXT: br [[EXIT:label %.*]]154; CHECK: [[SCALAR_PH]]:155;156entry:157 br label %loop158 159loop:160 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]161 %gep.src.1 = getelementptr inbounds i32, ptr %src1, i32 %iv162 %gep.src.2 = getelementptr inbounds i32, ptr %src2, i32 %iv163 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv164 %l.c = load i32, ptr %gep.cond165 %c = icmp ule i32 %l.c, 11166 br i1 %c, label %then, label %else167 168then:169 %l.src = load i32, ptr %gep.src.1, align 4170 br label %loop.latch171 172else:173 %l.src.2 = load i32, ptr %gep.src.2, align 4174 %add = add i32 %l.src.2, 10175 br label %loop.latch176 177loop.latch:178 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]179 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv180 store i32 %merge, ptr %gep.dst, align 4181 %iv.next = add nuw nsw i32 %iv, 1182 %ec = icmp eq i32 %iv.next, 100183 br i1 %ec, label %exit, label %loop184 185exit:186 ret void187}188 189; Negative test: Non-complementary masks - should NOT hoist190define void @non_complementary_masks(ptr %dst, ptr %src, ptr %cond1, ptr %cond2) {191; CHECK-LABEL: define void @non_complementary_masks(192; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND1:%.*]], ptr [[COND2:%.*]]) {193; CHECK-NEXT: [[ENTRY:.*:]]194; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]195; CHECK: [[VECTOR_MEMCHECK]]:196; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 400197; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND1]], i64 400198; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[COND2]], i64 400199; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 400200; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]201; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND1]], [[SCEVGEP]]202; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]203; CHECK-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]204; CHECK-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[COND2]], [[SCEVGEP]]205; CHECK-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]206; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]207; CHECK-NEXT: [[BOUND07:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]]208; CHECK-NEXT: [[BOUND18:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]209; CHECK-NEXT: [[FOUND_CONFLICT9:%.*]] = and i1 [[BOUND07]], [[BOUND18]]210; CHECK-NEXT: [[CONFLICT_RDX10:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT9]]211; CHECK-NEXT: br i1 [[CONFLICT_RDX10]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]212; CHECK: [[VECTOR_PH]]:213; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]214; CHECK: [[VECTOR_BODY]]:215; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE17:.*]] ]216; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0217; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 1218; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]219; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]220; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP8]], i32 0221; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x ptr> [[TMP10]], ptr [[TMP9]], i32 1222; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[COND1]], i32 [[TMP4]]223; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[COND2]], i32 [[TMP4]]224; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META14:![0-9]+]]225; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <2 x i32>, ptr [[TMP25]], align 4, !alias.scope [[META17:![0-9]+]]226; CHECK-NEXT: [[TMP37:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)227; CHECK-NEXT: [[TMP38:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD11]], splat (i32 20)228; CHECK-NEXT: [[TMP18:%.*]] = xor <2 x i1> [[TMP37]], splat (i1 true)229; CHECK-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP18]], <2 x i1> [[TMP38]], <2 x i1> zeroinitializer230; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP19]], i32 0231; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]232; CHECK: [[PRED_LOAD_IF]]:233; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP8]], align 4, !alias.scope [[META19:![0-9]+]]234; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i32 0235; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]236; CHECK: [[PRED_LOAD_CONTINUE]]:237; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP22]], %[[PRED_LOAD_IF]] ]238; CHECK-NEXT: [[TMP39:%.*]] = extractelement <2 x i1> [[TMP19]], i32 1239; CHECK-NEXT: br i1 [[TMP39]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13:.*]]240; CHECK: [[PRED_LOAD_IF12]]:241; CHECK-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP9]], align 4, !alias.scope [[META19]]242; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP40]], i32 1243; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE13]]244; CHECK: [[PRED_LOAD_CONTINUE13]]:245; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x i32> [ [[TMP23]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP26]], %[[PRED_LOAD_IF12]] ]246; CHECK-NEXT: [[TMP28:%.*]] = add <2 x i32> [[TMP27]], splat (i32 10)247; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x i1> [[TMP37]], i32 0248; CHECK-NEXT: br i1 [[TMP33]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15:.*]]249; CHECK: [[PRED_LOAD_IF14]]:250; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP8]], align 4, !alias.scope [[META19]]251; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x i32> poison, i32 [[TMP29]], i32 0252; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE15]]253; CHECK: [[PRED_LOAD_CONTINUE15]]:254; CHECK-NEXT: [[TMP36:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP34]], %[[PRED_LOAD_IF14]] ]255; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP37]], i32 1256; CHECK-NEXT: br i1 [[TMP35]], label %[[PRED_LOAD_IF16:.*]], label %[[PRED_LOAD_CONTINUE17]]257; CHECK: [[PRED_LOAD_IF16]]:258; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP9]], align 4, !alias.scope [[META19]]259; CHECK-NEXT: [[TMP31:%.*]] = insertelement <2 x i32> [[TMP36]], i32 [[TMP30]], i32 1260; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE17]]261; CHECK: [[PRED_LOAD_CONTINUE17]]:262; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x i32> [ [[TMP36]], %[[PRED_LOAD_CONTINUE15]] ], [ [[TMP31]], %[[PRED_LOAD_IF16]] ]263; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP19]], <2 x i32> [[TMP28]], <2 x i32> zeroinitializer264; CHECK-NEXT: [[PREDPHI18:%.*]] = select <2 x i1> [[TMP37]], <2 x i32> [[TMP32]], <2 x i32> [[PREDPHI]]265; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]266; CHECK-NEXT: store <2 x i32> [[PREDPHI18]], ptr [[TMP41]], align 4, !alias.scope [[META21:![0-9]+]], !noalias [[META23:![0-9]+]]267; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2268; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100269; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]270; CHECK: [[MIDDLE_BLOCK]]:271; CHECK-NEXT: br [[EXIT:label %.*]]272; CHECK: [[SCALAR_PH]]:273;274entry:275 br label %loop276 277loop:278 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]279 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv280 %gep.cond1 = getelementptr inbounds i32, ptr %cond1, i32 %iv281 %gep.cond2 = getelementptr inbounds i32, ptr %cond2, i32 %iv282 %l.c1 = load i32, ptr %gep.cond1283 %l.c2 = load i32, ptr %gep.cond2284 %c1 = icmp ule i32 %l.c1, 11285 %c2 = icmp ule i32 %l.c2, 20286 br i1 %c1, label %then, label %else287 288then:289 %l.src = load i32, ptr %gep.src, align 4290 br label %loop.latch291 292else:293 br i1 %c2, label %else.then, label %loop.latch294 295else.then:296 %l.src.2 = load i32, ptr %gep.src, align 4297 %add = add i32 %l.src.2, 10298 br label %loop.latch299 300loop.latch:301 %merge = phi i32 [ %l.src, %then ], [ %add, %else.then ], [ 0, %else ]302 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv303 store i32 %merge, ptr %gep.dst, align 4304 %iv.next = add nuw nsw i32 %iv, 1305 %ec = icmp eq i32 %iv.next, 100306 br i1 %ec, label %exit, label %loop307 308exit:309 ret void310}311 312; Negative test: Different access sizes - should NOT hoist313; Both loads use the same pointer but have different types (i8 vs i32)314define void @different_access_sizes(ptr %dst, ptr %src, ptr %cond) {315; CHECK-LABEL: define void @different_access_sizes(316; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {317; CHECK-NEXT: [[ENTRY:.*:]]318; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]319; CHECK: [[VECTOR_MEMCHECK]]:320; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 400321; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND]], i64 400322; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 400323; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]324; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND]], [[SCEVGEP]]325; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]326; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]327; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]328; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]329; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]330; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]331; CHECK: [[VECTOR_PH]]:332; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]333; CHECK: [[VECTOR_BODY]]:334; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11:.*]] ]335; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0336; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 1337; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]338; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]339; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 0340; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x ptr> [[TMP8]], ptr [[TMP7]], i32 1341; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]342; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP10]], align 4, !alias.scope [[META26:![0-9]+]]343; CHECK-NEXT: [[TMP11:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)344; CHECK-NEXT: [[TMP12:%.*]] = xor <2 x i1> [[TMP11]], splat (i1 true)345; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP12]], i32 0346; CHECK-NEXT: br i1 [[TMP13]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]347; CHECK: [[PRED_LOAD_IF]]:348; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META29:![0-9]+]]349; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> poison, i32 [[TMP14]], i32 0350; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]351; CHECK: [[PRED_LOAD_CONTINUE]]:352; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP15]], %[[PRED_LOAD_IF]] ]353; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1354; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]355; CHECK: [[PRED_LOAD_IF6]]:356; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META29]]357; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[TMP18]], i32 1358; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]359; CHECK: [[PRED_LOAD_CONTINUE7]]:360; CHECK-NEXT: [[TMP20:%.*]] = phi <2 x i32> [ [[TMP16]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP19]], %[[PRED_LOAD_IF6]] ]361; CHECK-NEXT: [[TMP21:%.*]] = add <2 x i32> [[TMP20]], splat (i32 10)362; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0363; CHECK-NEXT: br i1 [[TMP22]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]364; CHECK: [[PRED_LOAD_IF8]]:365; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[TMP6]], align 4, !alias.scope [[META29]]366; CHECK-NEXT: [[TMP24:%.*]] = insertelement <2 x i8> poison, i8 [[TMP23]], i32 0367; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]368; CHECK: [[PRED_LOAD_CONTINUE9]]:369; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i8> [ poison, %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP24]], %[[PRED_LOAD_IF8]] ]370; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1371; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11]]372; CHECK: [[PRED_LOAD_IF10]]:373; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[TMP7]], align 4, !alias.scope [[META29]]374; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i8> [[TMP25]], i8 [[TMP27]], i32 1375; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]376; CHECK: [[PRED_LOAD_CONTINUE11]]:377; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP28]], %[[PRED_LOAD_IF10]] ]378; CHECK-NEXT: [[TMP30:%.*]] = zext <2 x i8> [[TMP29]] to <2 x i32>379; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP11]], <2 x i32> [[TMP30]], <2 x i32> [[TMP21]]380; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]381; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP31]], align 4, !alias.scope [[META31:![0-9]+]], !noalias [[META33:![0-9]+]]382; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2383; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100384; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]385; CHECK: [[MIDDLE_BLOCK]]:386; CHECK-NEXT: br [[EXIT:label %.*]]387; CHECK: [[SCALAR_PH]]:388;389entry:390 br label %loop391 392loop:393 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]394 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv395 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv396 %l.c = load i32, ptr %gep.cond397 %c = icmp ule i32 %l.c, 11398 br i1 %c, label %then, label %else399 400then:401 %l.src = load i8, ptr %gep.src, align 4402 %ext = zext i8 %l.src to i32403 br label %loop.latch404 405else:406 %l.src.2 = load i32, ptr %gep.src, align 4407 %add = add i32 %l.src.2, 10408 br label %loop.latch409 410loop.latch:411 %merge = phi i32 [ %ext, %then ], [ %add, %else ]412 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv413 store i32 %merge, ptr %gep.dst, align 4414 %iv.next = add nuw nsw i32 %iv, 1415 %ec = icmp eq i32 %iv.next, 100416 br i1 %ec, label %exit, label %loop417 418exit:419 ret void420}421 422; Make sure the minimum alignment is used when loads have different alignments.423define void @different_alignments_same_address(ptr %dst, ptr %src, ptr %cond) {424; CHECK-LABEL: define void @different_alignments_same_address(425; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {426; CHECK-NEXT: [[ENTRY:.*:]]427; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]428; CHECK: [[VECTOR_MEMCHECK]]:429; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 400430; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND]], i64 400431; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 400432; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]433; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND]], [[SCEVGEP]]434; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]435; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]436; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]437; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]438; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]439; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]440; CHECK: [[VECTOR_PH]]:441; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]442; CHECK: [[VECTOR_BODY]]:443; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]444; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0445; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 1446; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]447; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]448; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]449; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP20]], align 4, !alias.scope [[META36:![0-9]+]]450; CHECK-NEXT: [[TMP15:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)451; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP8]], align 2, !alias.scope [[META39:![0-9]+]]452; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP9]], align 2, !alias.scope [[META39]]453; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0454; CHECK-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP7]], i32 1455; CHECK-NEXT: [[TMP26:%.*]] = add <2 x i32> [[TMP25]], splat (i32 10)456; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP15]], <2 x i32> [[TMP25]], <2 x i32> [[TMP26]]457; CHECK-NEXT: [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]458; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP35]], align 4, !alias.scope [[META41:![0-9]+]], !noalias [[META43:![0-9]+]]459; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2460; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100461; CHECK-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]462; CHECK: [[MIDDLE_BLOCK]]:463; CHECK-NEXT: br [[EXIT:label %.*]]464; CHECK: [[SCALAR_PH]]:465;466entry:467 br label %loop468 469loop:470 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]471 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv472 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv473 %l.c = load i32, ptr %gep.cond474 %c = icmp ule i32 %l.c, 11475 br i1 %c, label %then, label %else476 477then:478 %l.src = load i32, ptr %gep.src, align 2479 br label %loop.latch480 481else:482 %l.src.2 = load i32, ptr %gep.src, align 4483 %add = add i32 %l.src.2, 10484 br label %loop.latch485 486loop.latch:487 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]488 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv489 store i32 %merge, ptr %gep.dst, align 4490 %iv.next = add nuw nsw i32 %iv, 1491 %ec = icmp eq i32 %iv.next, 100492 br i1 %ec, label %exit, label %loop493 494exit:495 ret void496}497 498; Negative test: Volatile loads - should NOT hoist499define void @volatile_load(ptr %dst, ptr %src, ptr %cond) {500; CHECK-LABEL: define void @volatile_load(501; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {502; CHECK-NEXT: [[ENTRY:.*]]:503; CHECK-NEXT: br label %[[LOOP:.*]]504; CHECK: [[LOOP]]:505; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]506; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]507; CHECK-NEXT: [[GEP_COND:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[IV]]508; CHECK-NEXT: [[L_C:%.*]] = load i32, ptr [[GEP_COND]], align 4509; CHECK-NEXT: [[C:%.*]] = icmp ule i32 [[L_C]], 11510; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]511; CHECK: [[THEN]]:512; CHECK-NEXT: [[L_SRC:%.*]] = load volatile i32, ptr [[GEP_SRC]], align 4513; CHECK-NEXT: br label %[[LOOP_LATCH]]514; CHECK: [[ELSE]]:515; CHECK-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC]], align 4516; CHECK-NEXT: [[ADD:%.*]] = add i32 [[L_SRC_2]], 10517; CHECK-NEXT: br label %[[LOOP_LATCH]]518; CHECK: [[LOOP_LATCH]]:519; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[L_SRC]], %[[THEN]] ], [ [[ADD]], %[[ELSE]] ]520; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV]]521; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_DST]], align 4522; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1523; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 100524; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]525; CHECK: [[EXIT]]:526; CHECK-NEXT: ret void527;528entry:529 br label %loop530 531loop:532 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]533 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv534 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv535 %l.c = load i32, ptr %gep.cond536 %c = icmp ule i32 %l.c, 11537 br i1 %c, label %then, label %else538 539then:540 %l.src = load volatile i32, ptr %gep.src, align 4541 br label %loop.latch542 543else:544 %l.src.2 = load i32, ptr %gep.src, align 4545 %add = add i32 %l.src.2, 10546 br label %loop.latch547 548loop.latch:549 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]550 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv551 store i32 %merge, ptr %gep.dst, align 4552 %iv.next = add nuw nsw i32 %iv, 1553 %ec = icmp eq i32 %iv.next, 100554 br i1 %ec, label %exit, label %loop555 556exit:557 ret void558}559 560; Test hoisting with duplicate GEPs: The same address is computed by different561; GEP instructions in different branches. The hoisting pass should use SCEV to562; recognize they compute the same address and hoist the load.563define void @duplicate_gep(ptr %dst, ptr %src, ptr %cond) {564; CHECK-LABEL: define void @duplicate_gep(565; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {566; CHECK-NEXT: [[ENTRY:.*:]]567; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]568; CHECK: [[VECTOR_MEMCHECK]]:569; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 400570; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND]], i64 400571; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 400572; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]573; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND]], [[SCEVGEP]]574; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]575; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]576; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]577; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]578; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]579; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]580; CHECK: [[VECTOR_PH]]:581; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]582; CHECK: [[VECTOR_BODY]]:583; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]584; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0585; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 1586; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]587; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 4, !alias.scope [[META46:![0-9]+]]588; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)589; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]590; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]591; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP10]], align 4, !alias.scope [[META49:![0-9]+]]592; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP15]], align 4, !alias.scope [[META49]]593; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0594; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP8]], i32 1595; CHECK-NEXT: [[TMP19:%.*]] = add <2 x i32> [[TMP29]], splat (i32 10)596; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i32> [[TMP29]], <2 x i32> [[TMP19]]597; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]598; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP30]], align 4, !alias.scope [[META51:![0-9]+]], !noalias [[META53:![0-9]+]]599; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2600; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100601; CHECK-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]602; CHECK: [[MIDDLE_BLOCK]]:603; CHECK-NEXT: br [[EXIT:label %.*]]604; CHECK: [[SCALAR_PH]]:605;606entry:607 br label %loop608 609loop:610 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]611 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv612 %l.c = load i32, ptr %gep.cond613 %c = icmp ule i32 %l.c, 11614 br i1 %c, label %then, label %else615 616then:617 %gep.src.then = getelementptr inbounds i32, ptr %src, i32 %iv618 %l.src = load i32, ptr %gep.src.then, align 4619 br label %loop.latch620 621else:622 %gep.src.else= getelementptr inbounds i32, ptr %src, i32 %iv623 %l.src.2 = load i32, ptr %gep.src.else, align 4624 %add = add i32 %l.src.2, 10625 br label %loop.latch626 627loop.latch:628 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]629 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv630 store i32 %merge, ptr %gep.dst, align 4631 %iv.next = add nuw nsw i32 %iv, 1632 %ec = icmp eq i32 %iv.next, 100633 br i1 %ec, label %exit, label %loop634 635exit:636 ret void637}638 639; Test with non-unit-stride loads: Loads have stride 16 (2 doubles * 8 bytes)640; instead of unit stride (8 bytes). The hoisting optimization should still work641; since both loads access the same address with the same stride.642define void @non_unit_stride_i64(ptr %dst, ptr %src, ptr %cond) {643; CHECK-LABEL: define void @non_unit_stride_i64(644; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[COND:%.*]]) {645; CHECK-NEXT: [[ENTRY:.*:]]646; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]647; CHECK: [[VECTOR_MEMCHECK]]:648; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 400649; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[COND]], i64 400650; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 796651; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]652; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[COND]], [[SCEVGEP]]653; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]654; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]655; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]656; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]657; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]658; CHECK-NEXT: br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]659; CHECK: [[VECTOR_PH]]:660; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]661; CHECK: [[VECTOR_BODY]]:662; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]663; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 0664; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 1665; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP6]]666; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4, !alias.scope [[META56:![0-9]+]]667; CHECK-NEXT: [[TMP9:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)668; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i32 [[TMP6]]669; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i32 [[TMP7]]670; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP12]], align 4, !alias.scope [[META59:![0-9]+]]671; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP17]], align 4, !alias.scope [[META59]]672; CHECK-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> poison, i32 [[TMP24]], i32 0673; CHECK-NEXT: [[TMP31:%.*]] = insertelement <2 x i32> [[TMP25]], i32 [[TMP10]], i32 1674; CHECK-NEXT: [[TMP21:%.*]] = add <2 x i32> [[TMP31]], splat (i32 10)675; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP9]], <2 x i32> [[TMP31]], <2 x i32> [[TMP21]]676; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP6]]677; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP32]], align 4, !alias.scope [[META61:![0-9]+]], !noalias [[META63:![0-9]+]]678; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2679; CHECK-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100680; CHECK-NEXT: br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP64:![0-9]+]]681; CHECK: [[MIDDLE_BLOCK]]:682; CHECK-NEXT: br [[EXIT:label %.*]]683; CHECK: [[SCALAR_PH]]:684;685entry:686 br label %loop687 688loop:689 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]690 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv691 %l.c = load i32, ptr %gep.cond692 %c = icmp ule i32 %l.c, 11693 br i1 %c, label %then, label %else694 695then:696 %gep.src.then = getelementptr inbounds i64, ptr %src, i32 %iv697 %l.src = load i32, ptr %gep.src.then, align 4698 br label %loop.latch699 700else:701 %gep.src.else= getelementptr inbounds i64, ptr %src, i32 %iv702 %l.src.2 = load i32, ptr %gep.src.else, align 4703 %add = add i32 %l.src.2, 10704 br label %loop.latch705 706loop.latch:707 %merge = phi i32 [ %l.src, %then ], [ %add, %else ]708 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv709 store i32 %merge, ptr %gep.dst, align 4710 %iv.next = add nuw nsw i32 %iv, 1711 %ec = icmp eq i32 %iv.next, 100712 br i1 %ec, label %exit, label %loop713 714exit:715 ret void716}717 718 719; Test that loads inside masked regions (without individual masks) are720; correctly detected and hoisted when they have complementary predicates.721define void @hoist_loads_in_masked_regions(ptr noalias %dst, ptr noalias %src, ptr %cond) {722; CHECK-LABEL: define void @hoist_loads_in_masked_regions(723; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr [[COND:%.*]]) {724; CHECK-NEXT: [[ENTRY:.*:]]725; CHECK-NEXT: br label %[[VECTOR_PH:.*]]726; CHECK: [[VECTOR_PH]]:727; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]728; CHECK: [[VECTOR_BODY]]:729; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]730; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[INDEX]]731; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[INDEX]]732; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4733; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)734; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4735; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD1]], <2 x i32> zeroinitializer736; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[INDEX]]737; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP3]], align 4738; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2739; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100740; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP66:![0-9]+]]741; CHECK: [[MIDDLE_BLOCK]]:742; CHECK-NEXT: br label %[[EXIT:.*]]743; CHECK: [[EXIT]]:744; CHECK-NEXT: ret void745;746entry:747 br label %loop748 749loop:750 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]751 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv752 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv753 %l.c = load i32, ptr %gep.cond754 %c = icmp ule i32 %l.c, 11755 br i1 %c, label %then, label %loop.latch756 757then:758 %l.src = load i32, ptr %gep.src, align 4759 br label %loop.latch760 761loop.latch:762 %merge = phi i32 [ %l.src, %then ], [ 0, %loop ]763 %l.src.2 = load i32, ptr %gep.src, align 4764 %add = add i32 %l.src.2, %merge765 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv766 store i32 %merge, ptr %gep.dst, align 4767 %iv.next = add nuw nsw i32 %iv, 1768 %ec = icmp eq i32 %iv.next, 100769 br i1 %ec, label %exit, label %loop770 771exit:772 ret void773}774; Test that when there are 3 or more regions with complementary predicates775; loading from the same address, all loads are hoisted and replaced, not just776; the first pair. This tests the K loop that continues searching after finding777; the initial complementary pair.778define void @hoist_multiple_complementary_loads(ptr noalias %dst, ptr noalias %src, ptr %cond) {779; CHECK-LABEL: define void @hoist_multiple_complementary_loads(780; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], ptr [[COND:%.*]]) {781; CHECK-NEXT: [[ENTRY:.*:]]782; CHECK-NEXT: br label %[[VECTOR_PH:.*]]783; CHECK: [[VECTOR_PH]]:784; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]785; CHECK: [[VECTOR_BODY]]:786; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE10:.*]] ]787; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0788; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1789; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP0]]790; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP43]], align 4791; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer792; CHECK-NEXT: [[TMP8:%.*]] = xor <2 x i1> [[TMP7]], splat (i1 true)793; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0794; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]795; CHECK: [[PRED_LOAD_IF]]:796; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP0]]797; CHECK-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP63]], align 4798; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[TMP64]], i32 0799; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]800; CHECK: [[PRED_LOAD_CONTINUE]]:801; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP12]], %[[PRED_LOAD_IF]] ]802; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1803; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]804; CHECK: [[PRED_LOAD_IF1]]:805; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP1]]806; CHECK-NEXT: [[TMP70:%.*]] = load i32, ptr [[TMP69]], align 4807; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> [[TMP13]], i32 [[TMP70]], i32 1808; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]809; CHECK: [[PRED_LOAD_CONTINUE2]]:810; CHECK-NEXT: [[TMP28:%.*]] = phi <2 x i32> [ [[TMP13]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP17]], %[[PRED_LOAD_IF1]] ]811; CHECK-NEXT: [[TMP15:%.*]] = add <2 x i32> [[TMP28]], splat (i32 1)812; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], splat (i32 32)813; CHECK-NEXT: [[TMP29:%.*]] = xor <2 x i1> [[TMP16]], splat (i1 true)814; CHECK-NEXT: [[TMP32:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP29]], <2 x i1> zeroinitializer815; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP32]], i32 0816; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]817; CHECK: [[PRED_LOAD_IF3]]:818; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP0]]819; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4820; CHECK-NEXT: [[TMP33:%.*]] = insertelement <2 x i32> poison, i32 [[TMP21]], i32 0821; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]822; CHECK: [[PRED_LOAD_CONTINUE4]]:823; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP33]], %[[PRED_LOAD_IF3]] ]824; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x i1> [[TMP32]], i32 1825; CHECK-NEXT: br i1 [[TMP24]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]826; CHECK: [[PRED_LOAD_IF5]]:827; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP1]]828; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4829; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP26]], i32 1830; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]831; CHECK: [[PRED_LOAD_CONTINUE6]]:832; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i32> [ [[TMP23]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP27]], %[[PRED_LOAD_IF5]] ]833; CHECK-NEXT: [[TMP22:%.*]] = mul <2 x i32> [[TMP18]], splat (i32 2)834; CHECK-NEXT: [[TMP30:%.*]] = select <2 x i1> [[TMP7]], <2 x i1> [[TMP16]], <2 x i1> zeroinitializer835; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[TMP30]], i32 0836; CHECK-NEXT: br i1 [[TMP31]], label %[[PRED_LOAD_IF7:.*]], label %[[PRED_LOAD_CONTINUE8:.*]]837; CHECK: [[PRED_LOAD_IF7]]:838; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP0]]839; CHECK-NEXT: [[TMP34:%.*]] = load i32, ptr [[TMP61]], align 4840; CHECK-NEXT: [[TMP38:%.*]] = insertelement <2 x i32> poison, i32 [[TMP34]], i32 0841; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE8]]842; CHECK: [[PRED_LOAD_CONTINUE8]]:843; CHECK-NEXT: [[TMP35:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE6]] ], [ [[TMP38]], %[[PRED_LOAD_IF7]] ]844; CHECK-NEXT: [[TMP36:%.*]] = extractelement <2 x i1> [[TMP30]], i32 1845; CHECK-NEXT: br i1 [[TMP36]], label %[[PRED_LOAD_IF9:.*]], label %[[PRED_LOAD_CONTINUE10]]846; CHECK: [[PRED_LOAD_IF9]]:847; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[TMP1]]848; CHECK-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP65]], align 4849; CHECK-NEXT: [[TMP44:%.*]] = insertelement <2 x i32> [[TMP35]], i32 [[TMP37]], i32 1850; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE10]]851; CHECK: [[PRED_LOAD_CONTINUE10]]:852; CHECK-NEXT: [[TMP45:%.*]] = phi <2 x i32> [ [[TMP35]], %[[PRED_LOAD_CONTINUE8]] ], [ [[TMP44]], %[[PRED_LOAD_IF9]] ]853; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP32]], <2 x i32> [[TMP22]], <2 x i32> [[TMP15]]854; CHECK-NEXT: [[TMP42:%.*]] = select <2 x i1> [[TMP30]], <2 x i32> [[TMP45]], <2 x i32> [[PREDPHI]]855; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP0]]856; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[TMP39]], i64 32857; CHECK-NEXT: store <2 x i32> [[TMP42]], ptr [[TMP40]], align 4858; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2859; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i32 [[INDEX_NEXT]], 100860; CHECK-NEXT: br i1 [[TMP46]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP67:![0-9]+]]861; CHECK: [[MIDDLE_BLOCK]]:862; CHECK-NEXT: br label %[[EXIT:.*]]863; CHECK: [[EXIT]]:864; CHECK-NEXT: ret void865;866entry:867 br label %loop868 869loop:870 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]871 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv872 %l.cond = load i32, ptr %gep.cond, align 4873 %c.1 = icmp ne i32 %l.cond, 0874 br i1 %c.1, label %check2, label %region3875 876check2:877 %c.2 = icmp ne i32 %l.cond, 32878 br i1 %c.2, label %region1, label %region2879 880region1:881 %gep.src.8.r1 = getelementptr inbounds i8, ptr %src, i32 %iv882 %val1 = load i32, ptr %gep.src.8.r1, align 4883 br label %loop.latch884 885region2:886 %gep.src.8.r2 = getelementptr inbounds i8, ptr %src, i32 %iv887 %val2 = load i32, ptr %gep.src.8.r2, align 4888 %mul = mul i32 %val2, 2889 br label %loop.latch890 891region3:892 %gep.src.8.r3 = getelementptr inbounds i8, ptr %src, i32 %iv893 %val3 = load i32, ptr %gep.src.8.r3, align 4894 %add = add i32 %val3, 1895 br label %loop.latch896 897loop.latch:898 %merge = phi i32 [ %val1, %region1 ], [ %mul, %region2 ], [ %add, %region3 ]899 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv900 %offset.dst = getelementptr inbounds i8, ptr %gep.dst, i64 32901 store i32 %merge, ptr %offset.dst, align 4902 %iv.next = add nuw nsw i32 %iv, 1903 %ec = icmp eq i32 %iv.next, 100904 br i1 %ec, label %exit, label %loop905 906exit:907 ret void908}909 910define void @hoist_predicated_load_with_chained_geps1(ptr %dst, ptr %src, i1 %cond) {911; CHECK-LABEL: define void @hoist_predicated_load_with_chained_geps1(912; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i1 [[COND:%.*]]) {913; CHECK-NEXT: [[ENTRY:.*:]]914; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]915; CHECK: [[VECTOR_MEMCHECK]]:916; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2917; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 8918; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 2210919; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]920; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]921; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]922; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]923; CHECK: [[VECTOR_PH]]:924; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]925; CHECK: [[VECTOR_BODY]]:926; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]927; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0928; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1929; CHECK-NEXT: [[TMP20:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]930; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i64 8931; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META68:![0-9]+]]932; CHECK-NEXT: store i16 [[TMP4]], ptr [[DST]], align 2, !alias.scope [[META71:![0-9]+]], !noalias [[META68]]933; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2934; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100935; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP73:![0-9]+]]936; CHECK: [[MIDDLE_BLOCK]]:937; CHECK-NEXT: br label %[[SCALAR_PH]]938; CHECK: [[SCALAR_PH]]:939;940entry:941 br label %loop.header942 943loop.header:944 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]945 br i1 %cond, label %then, label %else946 947then:948 %gep1 = getelementptr [11 x i16], ptr %src, i64 %iv949 %gep2 = getelementptr i8, ptr %gep1, i64 8950 %l.0 = load i16, ptr %gep2, align 2951 br label %loop.latch952 953else:954 %gep3 = getelementptr [11 x i16], ptr %src, i64 %iv955 %gep4 = getelementptr i8, ptr %gep3, i64 8956 %l.1 = load i16, ptr %gep4, align 2957 br label %loop.latch958 959loop.latch:960 %merge = phi i16 [ %l.0, %then ], [ %l.1, %else ]961 store i16 %merge, ptr %dst, align 2962 %iv.next = add i64 %iv, 1963 %ec = icmp eq i64 %iv, 100964 br i1 %ec, label %exit, label %loop.header965 966exit:967 ret void968}969 970define void @hoist_predicated_load_with_chained_geps2(ptr %dst, ptr %src, i1 %cond) {971; CHECK-LABEL: define void @hoist_predicated_load_with_chained_geps2(972; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i1 [[COND:%.*]]) {973; CHECK-NEXT: [[ENTRY:.*:]]974; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]975; CHECK: [[VECTOR_MEMCHECK]]:976; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2977; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 8978; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 2210979; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]980; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]981; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]982; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]983; CHECK: [[VECTOR_PH]]:984; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]985; CHECK: [[VECTOR_BODY]]:986; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]987; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 0988; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 1989; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [11 x i16], ptr [[SRC]], i64 [[TMP2]]990; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP4]], i64 8991; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP21]], align 2, !alias.scope [[META75:![0-9]+]]992; CHECK-NEXT: store i16 [[TMP5]], ptr [[DST]], align 2, !alias.scope [[META78:![0-9]+]], !noalias [[META75]]993; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2994; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100995; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP80:![0-9]+]]996; CHECK: [[MIDDLE_BLOCK]]:997; CHECK-NEXT: br label %[[SCALAR_PH]]998; CHECK: [[SCALAR_PH]]:999;1000entry:1001 br label %loop.header1002 1003loop.header:1004 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]1005 %gep1 = getelementptr [11 x i16], ptr %src, i64 %iv1006 br i1 %cond, label %then, label %else1007 1008then:1009 %gep2 = getelementptr i8, ptr %gep1, i64 81010 %l.0 = load i16, ptr %gep2, align 21011 br label %loop.latch1012 1013else:1014 %gep3 = getelementptr i8, ptr %gep1, i64 81015 %l.1 = load i16, ptr %gep3, align 21016 br label %loop.latch1017 1018loop.latch:1019 %merge = phi i16 [ %l.0, %then ], [ %l.1, %else ]1020 store i16 %merge, ptr %dst, align 21021 %iv.next = add i64 %iv, 11022 %ec = icmp eq i64 %iv, 1001023 br i1 %ec, label %exit, label %loop.header1024 1025exit:1026 ret void1027}1028 1029define void @hoist_all_three_loads_at_same_address(ptr %dst, ptr %src, ptr noalias %cond) {1030; CHECK-LABEL: define void @hoist_all_three_loads_at_same_address(1031; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr noalias [[COND:%.*]]) {1032; CHECK-NEXT: [[ENTRY:.*:]]1033; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]1034; CHECK: [[VECTOR_MEMCHECK]]:1035; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 4001036; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[SRC]], i64 4001037; CHECK-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP2]]1038; CHECK-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]1039; CHECK-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]1040; CHECK-NEXT: br i1 [[FOUND_CONFLICT5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]1041; CHECK: [[VECTOR_PH]]:1042; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]1043; CHECK: [[VECTOR_BODY]]:1044; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE11:.*]] ]1045; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 01046; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 11047; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP4]]1048; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP5]]1049; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 01050; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x ptr> [[TMP8]], ptr [[TMP7]], i32 11051; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[COND]], i32 [[TMP4]]1052; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP10]], align 41053; CHECK-NEXT: [[TMP11:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 11)1054; CHECK-NEXT: [[TMP12:%.*]] = icmp ule <2 x i32> [[WIDE_LOAD]], splat (i32 20)1055; CHECK-NEXT: [[TMP13:%.*]] = xor <2 x i1> [[TMP11]], splat (i1 true)1056; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i1> [[TMP12]], splat (i1 true)1057; CHECK-NEXT: [[TMP15:%.*]] = select <2 x i1> [[TMP13]], <2 x i1> [[TMP14]], <2 x i1> zeroinitializer1058; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP15]], i32 01059; CHECK-NEXT: br i1 [[TMP16]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]1060; CHECK: [[PRED_LOAD_IF]]:1061; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META82:![0-9]+]]1062; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 01063; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]1064; CHECK: [[PRED_LOAD_CONTINUE]]:1065; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]1066; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP15]], i32 11067; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]1068; CHECK: [[PRED_LOAD_IF2]]:1069; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META82]]1070; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP21]], i32 11071; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE3]]1072; CHECK: [[PRED_LOAD_CONTINUE3]]:1073; CHECK-NEXT: [[TMP23:%.*]] = phi <2 x i32> [ [[TMP19]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP22]], %[[PRED_LOAD_IF2]] ]1074; CHECK-NEXT: [[TMP24:%.*]] = add <2 x i32> [[TMP23]], splat (i32 10)1075; CHECK-NEXT: [[TMP25:%.*]] = select <2 x i1> [[TMP13]], <2 x i1> [[TMP12]], <2 x i1> zeroinitializer1076; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i1> [[TMP25]], i32 01077; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]1078; CHECK: [[PRED_LOAD_IF4]]:1079; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META82]]1080; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> poison, i32 [[TMP27]], i32 01081; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE5]]1082; CHECK: [[PRED_LOAD_CONTINUE5]]:1083; CHECK-NEXT: [[TMP29:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP28]], %[[PRED_LOAD_IF4]] ]1084; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i1> [[TMP25]], i32 11085; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]1086; CHECK: [[PRED_LOAD_IF6]]:1087; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META82]]1088; CHECK-NEXT: [[TMP32:%.*]] = insertelement <2 x i32> [[TMP29]], i32 [[TMP31]], i32 11089; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE7]]1090; CHECK: [[PRED_LOAD_CONTINUE7]]:1091; CHECK-NEXT: [[TMP33:%.*]] = phi <2 x i32> [ [[TMP29]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP32]], %[[PRED_LOAD_IF6]] ]1092; CHECK-NEXT: [[TMP34:%.*]] = mul <2 x i32> [[TMP33]], splat (i32 2)1093; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP11]], i32 01094; CHECK-NEXT: br i1 [[TMP35]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]1095; CHECK: [[PRED_LOAD_IF8]]:1096; CHECK-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP6]], align 4, !alias.scope [[META82]]1097; CHECK-NEXT: [[TMP37:%.*]] = insertelement <2 x i32> poison, i32 [[TMP36]], i32 01098; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE9]]1099; CHECK: [[PRED_LOAD_CONTINUE9]]:1100; CHECK-NEXT: [[TMP38:%.*]] = phi <2 x i32> [ poison, %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP37]], %[[PRED_LOAD_IF8]] ]1101; CHECK-NEXT: [[TMP39:%.*]] = extractelement <2 x i1> [[TMP11]], i32 11102; CHECK-NEXT: br i1 [[TMP39]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11]]1103; CHECK: [[PRED_LOAD_IF10]]:1104; CHECK-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP7]], align 4, !alias.scope [[META82]]1105; CHECK-NEXT: [[TMP41:%.*]] = insertelement <2 x i32> [[TMP38]], i32 [[TMP40]], i32 11106; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE11]]1107; CHECK: [[PRED_LOAD_CONTINUE11]]:1108; CHECK-NEXT: [[TMP42:%.*]] = phi <2 x i32> [ [[TMP38]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP41]], %[[PRED_LOAD_IF10]] ]1109; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP25]], <2 x i32> [[TMP34]], <2 x i32> [[TMP24]]1110; CHECK-NEXT: [[PREDPHI16:%.*]] = select <2 x i1> [[TMP11]], <2 x i32> [[TMP42]], <2 x i32> [[PREDPHI]]1111; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP4]]1112; CHECK-NEXT: store <2 x i32> [[PREDPHI16]], ptr [[TMP43]], align 4, !alias.scope [[META85:![0-9]+]], !noalias [[META82]]1113; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 21114; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1001115; CHECK-NEXT: br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP87:![0-9]+]]1116; CHECK: [[MIDDLE_BLOCK]]:1117; CHECK-NEXT: br [[EXIT:label %.*]]1118; CHECK: [[SCALAR_PH]]:1119;1120entry:1121 br label %loop1122 1123loop:1124 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]1125 %gep.src = getelementptr inbounds i32, ptr %src, i32 %iv1126 %gep.cond = getelementptr inbounds i32, ptr %cond, i32 %iv1127 %l.c = load i32, ptr %gep.cond1128 %c1 = icmp ule i32 %l.c, 111129 %c2 = icmp ule i32 %l.c, 201130 br i1 %c1, label %then, label %else1131 1132then:1133 %l.src.then = load i32, ptr %gep.src, align 41134 br label %loop.latch1135 1136else:1137 br i1 %c2, label %else.if, label %else.else1138 1139else.if:1140 %l.src.else.if = load i32, ptr %gep.src, align 41141 %mul = mul i32 %l.src.else.if, 21142 br label %loop.latch1143 1144else.else:1145 %l.src.else.else = load i32, ptr %gep.src, align 41146 %add = add i32 %l.src.else.else, 101147 br label %loop.latch1148 1149loop.latch:1150 %merge = phi i32 [ %l.src.then, %then ], [ %mul, %else.if ], [ %add, %else.else ]1151 %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv1152 store i32 %merge, ptr %gep.dst, align 41153 %iv.next = add nuw nsw i32 %iv, 11154 %ec = icmp eq i32 %iv.next, 1001155 br i1 %ec, label %exit, label %loop1156 1157exit:1158 ret void1159}1160