751 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"5 6declare void @init(ptr nocapture nofree)7 8; Test case where the predicated load in the loop has an access size of 2 but9; has an alignment of 4.10define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {11; CHECK-LABEL: @test_access_size_not_multiple_of_align(12; CHECK-NEXT: entry:13; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [163840 x i16], align 414; CHECK-NEXT: call void @init(ptr [[ALLOCA]])15; CHECK-NEXT: br label [[VECTOR_PH:%.*]]16; CHECK: vector.ph:17; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]18; CHECK: vector.body:19; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]20; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]21; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[INDEX]]22; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 123; CHECK-NEXT: [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer24; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 025; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]26; CHECK: pred.load.if:27; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 028; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP5]]29; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 430; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i16> poison, i16 [[TMP7]], i32 031; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]32; CHECK: pred.load.continue:33; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]34; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP3]], i32 135; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]36; CHECK: pred.load.if1:37; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 138; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP11]]39; CHECK-NEXT: [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 440; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> [[TMP9]], i16 [[TMP13]], i32 141; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]42; CHECK: pred.load.continue2:43; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i16> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]44; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP18]], <2 x i16> zeroinitializer45; CHECK-NEXT: [[TMP15]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]46; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 247; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 409648; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]49; CHECK: middle.block:50; CHECK-NEXT: [[TMP17:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP15]])51; CHECK-NEXT: br label [[LATCH:%.*]]52; CHECK: loop_exit:53; CHECK-NEXT: ret i16 [[TMP17]]54;55entry:56 %alloca = alloca [163840 x i16], align 457 call void @init(ptr %alloca)58 br label %loop59loop:60 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]61 %accum = phi i16 [ 0, %entry ], [ %accum.next, %latch ]62 %iv.next = add i64 %iv, 163 %test_addr = getelementptr inbounds i8, ptr %test_base, i64 %iv64 %l.t = load i8, ptr %test_addr65 %cmp = icmp sge i8 %l.t, 066 br i1 %cmp, label %pred, label %latch67pred:68 %addr = getelementptr inbounds i16, ptr %alloca, i64 %iv69 %val = load i16, ptr %addr, align 470 br label %latch71latch:72 %val.phi = phi i16 [0, %loop], [%val, %pred]73 %accum.next = add i16 %accum, %val.phi74 %exit = icmp eq i64 %iv, 409575 br i1 %exit, label %loop_exit, label %loop76 77loop_exit:78 ret i16 %accum.next79}80 81; Test case where the predicated load in the loop has an access size of 4 and82; an alignment of 4, but the start pointer is offset by 1.83define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %test_base) {84; CHECK-LABEL: @test_access_size_multiple_of_align_but_offset_by_1(85; CHECK-NEXT: entry:86; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [163840 x i32], align 487; CHECK-NEXT: call void @init(ptr [[ALLOCA]])88; CHECK-NEXT: [[START:%.*]] = getelementptr i8, ptr [[ALLOCA]], i64 289; CHECK-NEXT: br label [[VECTOR_PH:%.*]]90; CHECK: vector.ph:91; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]92; CHECK: vector.body:93; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]94; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[PRED_LOAD_CONTINUE2]] ]95; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[INDEX]]96; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 197; CHECK-NEXT: [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer98; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 099; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]100; CHECK: pred.load.if:101; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0102; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP5]]103; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4104; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0105; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]106; CHECK: pred.load.continue:107; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]108; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1109; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]110; CHECK: pred.load.if1:111; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1112; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[START]], i64 [[TMP11]]113; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4114; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1115; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]116; CHECK: pred.load.continue2:117; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]118; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP18]], <2 x i32> zeroinitializer119; CHECK-NEXT: [[TMP15]] = add <2 x i32> [[VEC_PHI]], [[PREDPHI]]120; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2121; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096122; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]123; CHECK: middle.block:124; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP15]])125; CHECK-NEXT: br label [[LATCH:%.*]]126; CHECK: loop_exit:127; CHECK-NEXT: ret i32 [[TMP17]]128;129entry:130 %alloca = alloca [163840 x i32], align 4131 call void @init(ptr %alloca)132 %start = getelementptr i8, ptr %alloca, i64 2133 br label %loop134loop:135 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]136 %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]137 %iv.next = add i64 %iv, 1138 %test_addr = getelementptr inbounds i8, ptr %test_base, i64 %iv139 %l.t = load i8, ptr %test_addr140 %cmp = icmp sge i8 %l.t, 0141 br i1 %cmp, label %pred, label %latch142pred:143 %addr = getelementptr inbounds i32, ptr %start, i64 %iv144 %val = load i32, ptr %addr, align 4145 br label %latch146latch:147 %val.phi = phi i32 [0, %loop], [%val, %pred]148 %accum.next = add i32 %accum, %val.phi149 %exit = icmp eq i64 %iv, 4095150 br i1 %exit, label %loop_exit, label %loop151 152loop_exit:153 ret i32 %accum.next154}155 156 157define i32 @loop_requires_scev_predicate(ptr %dest, i32 %end) {158; CHECK-LABEL: @loop_requires_scev_predicate(159; CHECK-NEXT: entry:160; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i32], align 4161; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i32], align 4162; CHECK-NEXT: call void @init(ptr [[P1]])163; CHECK-NEXT: call void @init(ptr [[P2]])164; CHECK-NEXT: [[END_CLAMPED:%.*]] = and i32 [[END:%.*]], 1023165; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[END]] to i10166; CHECK-NEXT: [[TMP1:%.*]] = zext i10 [[TMP0]] to i64167; CHECK-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 1)168; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 2169; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]170; CHECK: vector.scevcheck:171; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[END_CLAMPED]], i32 1)172; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[UMAX]], -1173; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8174; CHECK-NEXT: [[TMP4:%.*]] = add i8 1, [[TMP3]]175; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i8 [[TMP4]], 1176; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[TMP2]], 255177; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]178; CHECK-NEXT: br i1 [[TMP7]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]179; CHECK: vector.ph:180; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 2181; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]182; CHECK-NEXT: [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i8183; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]184; CHECK: vector.body:185; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]186; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]187; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP9]], align 4188; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer189; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]190; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <2 x i32>, ptr [[TMP12]], align 4191; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0192; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]193; CHECK: pred.store.if:194; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0195; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DEST:%.*]], i64 [[TMP8]]196; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 0197; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[WIDE_LOAD3]], i32 0198; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[TMP16]], [[TMP17]]199; CHECK-NEXT: store i32 [[TMP18]], ptr [[TMP15]], align 4200; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]201; CHECK: pred.store.continue:202; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x i1> [[TMP11]], i32 1203; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]204; CHECK: pred.store.if3:205; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], 1206; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 [[TMP20]]207; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1208; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i32> [[WIDE_LOAD3]], i32 1209; CHECK-NEXT: [[TMP24:%.*]] = add i32 [[TMP22]], [[TMP23]]210; CHECK-NEXT: store i32 [[TMP24]], ptr [[TMP21]], align 4211; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]212; CHECK: pred.store.continue4:213; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2214; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]215; CHECK-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]216; CHECK: middle.block:217; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]218; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]219; CHECK: scalar.ph:220; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]221; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]222; CHECK-NEXT: br label [[FOR_BODY:%.*]]223; CHECK: for.body:224; CHECK-NEXT: [[IND:%.*]] = phi i8 [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]225; CHECK-NEXT: [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]226; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[GEP_IND]]227; CHECK-NEXT: [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX]], align 4228; CHECK-NEXT: [[DOWORK:%.*]] = icmp ne i32 [[TMP26]], 0229; CHECK-NEXT: br i1 [[DOWORK]], label [[FOR_DOWORK:%.*]], label [[FOR_INC]]230; CHECK: for.dowork:231; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[GEP_IND]]232; CHECK-NEXT: [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4233; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP26]], [[TMP27]]234; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 [[GEP_IND]]235; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4236; CHECK-NEXT: br label [[FOR_INC]]237; CHECK: for.inc:238; CHECK-NEXT: [[IND_NEXT]] = add i8 [[IND]], 1239; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[IND_NEXT]] to i32240; CHECK-NEXT: [[GEP_IND_NEXT]] = add i64 [[GEP_IND]], 1241; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], [[END_CLAMPED]]242; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]243; CHECK: exit:244; CHECK-NEXT: ret i32 0245;246entry:247 %p1 = alloca [1024 x i32]248 %p2 = alloca [1024 x i32]249 call void @init(ptr %p1)250 call void @init(ptr %p2)251 %end.clamped = and i32 %end, 1023252 br label %for.body253 254for.body:255 %ind = phi i8 [ %ind.next, %for.inc ], [ 0, %entry ]256 %gep.ind = phi i64 [ %gep.ind.next, %for.inc ], [ 0, %entry ]257 %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %gep.ind258 %0 = load i32, ptr %arrayidx, align 4259 %dowork = icmp ne i32 %0, 0260 br i1 %dowork, label %for.dowork, label %for.inc261 262for.dowork:263 %arrayidx3 = getelementptr inbounds i32, ptr %p2, i64 %gep.ind264 %1 = load i32, ptr %arrayidx3, align 4265 %add = add i32 %0, %1266 %arrayidx5 = getelementptr inbounds i32, ptr %dest, i64 %gep.ind267 store i32 %add, ptr %arrayidx5, align 4268 br label %for.inc269 270for.inc:271 %ind.next = add i8 %ind, 1272 %conv = zext i8 %ind.next to i32273 %gep.ind.next = add i64 %gep.ind, 1274 %cmp = icmp ult i32 %conv, %end.clamped275 br i1 %cmp, label %for.body, label %exit276 277exit:278 ret i32 0279}280 281 282; Test reverse loops where we should be able to prove loads in predicated blocks283; are safe to load unconditionally.284define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {285; CHECK-LABEL: @test_rev_loops_deref_loads(286; CHECK-NEXT: entry:287; CHECK-NEXT: [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4288; CHECK-NEXT: [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4289; CHECK-NEXT: [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4290; CHECK-NEXT: call void @init(ptr [[LOCAL_SRC]])291; CHECK-NEXT: call void @init(ptr [[LOCAL_CMP]])292; CHECK-NEXT: br label [[VECTOR_PH:%.*]]293; CHECK: vector.ph:294; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]295; CHECK: vector.body:296; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]297; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]298; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFFSET_IDX]]299; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 0300; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 -1301; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4302; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>303; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <2 x i32> [[REVERSE]], splat (i32 3)304; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[OFFSET_IDX]]305; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 0306; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 -1307; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP8]], align 4308; CHECK-NEXT: [[REVERSE2:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD1]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>309; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0310; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]311; CHECK: pred.store.if:312; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0313; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]314; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[REVERSE2]], i32 0315; CHECK-NEXT: [[TMP12:%.*]] = shl nsw i32 [[TMP11]], 2316; CHECK-NEXT: store i32 [[TMP12]], ptr [[TMP10]], align 4317; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]318; CHECK: pred.store.continue:319; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1320; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]321; CHECK: pred.store.if3:322; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], -1323; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP14]]324; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i32> [[REVERSE2]], i32 1325; CHECK-NEXT: [[TMP17:%.*]] = shl nsw i32 [[TMP16]], 2326; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP15]], align 4327; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]328; CHECK: pred.store.continue4:329; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2330; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024331; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]332; CHECK: middle.block:333; CHECK-NEXT: br label [[FOR_INC:%.*]]334; CHECK: exit:335; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)336; CHECK-NEXT: ret void337;338entry:339 %local_dest = alloca [1024 x i32], align 4340 %local_src = alloca [1024 x i32], align 4341 %local_cmp = alloca [1024 x i32], align 4342 call void @init(ptr %local_src)343 call void @init(ptr %local_cmp)344 br label %for.body345 346for.body:347 %iv = phi i64 [ 1023, %entry ], [ %iv.next, %for.inc ]348 %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %iv349 %0 = load i32, ptr %arrayidx, align 4350 %cmp3.not = icmp eq i32 %0, 3351 br i1 %cmp3.not, label %for.inc, label %if.then352 353if.then:354 %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %iv355 %1 = load i32, ptr %arrayidx5, align 4356 %mul = shl nsw i32 %1, 2357 %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %iv358 store i32 %mul, ptr %arrayidx7, align 4359 br label %for.inc360 361for.inc:362 %iv.next = add nsw i64 %iv, -1363 %cmp2.not = icmp eq i64 %iv, 0364 br i1 %cmp2.not, label %exit, label %for.body365 366exit:367 call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)368 ret void369}370 371 372; Test reverse loops where we *cannot* prove loads in predicated blocks are safe373; to load unconditionally.374define void @test_rev_loops_non_deref_loads(ptr nocapture noundef writeonly %dest) {375; CHECK-LABEL: @test_rev_loops_non_deref_loads(376; CHECK-NEXT: entry:377; CHECK-NEXT: [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4378; CHECK-NEXT: [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4379; CHECK-NEXT: [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4380; CHECK-NEXT: call void @init(ptr [[LOCAL_SRC]])381; CHECK-NEXT: call void @init(ptr [[LOCAL_CMP]])382; CHECK-NEXT: br label [[VECTOR_PH:%.*]]383; CHECK: vector.ph:384; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]385; CHECK: vector.body:386; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]387; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1023, i64 1022>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]388; CHECK-NEXT: [[TMP0:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 -1)389; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0390; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[TMP1]]391; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0392; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 -1393; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4394; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>395; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <2 x i32> [[REVERSE]], splat (i32 3)396; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0397; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]398; CHECK: pred.store.if:399; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0400; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP8]]401; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4402; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0403; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP11]]404; CHECK-NEXT: [[TMP13:%.*]] = shl nsw i32 [[TMP10]], 2405; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4406; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]407; CHECK: pred.store.continue:408; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1409; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]410; CHECK: pred.store.if1:411; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1412; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP15]]413; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4414; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1415; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP18]]416; CHECK-NEXT: [[TMP20:%.*]] = shl nsw i32 [[TMP17]], 2417; CHECK-NEXT: store i32 [[TMP20]], ptr [[TMP19]], align 4418; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]419; CHECK: pred.store.continue2:420; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2421; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 -2)422; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024423; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]424; CHECK: middle.block:425; CHECK-NEXT: br label [[FOR_INC:%.*]]426; CHECK: exit:427; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)428; CHECK-NEXT: ret void429;430entry:431 %local_dest = alloca [1024 x i32], align 4432 %local_src = alloca [1024 x i32], align 4433 %local_cmp = alloca [1024 x i32], align 4434 call void @init(ptr %local_src)435 call void @init(ptr %local_cmp)436 br label %for.body437 438for.body:439 %iv = phi i64 [ 1023, %entry ], [ %iv.next, %for.inc ]440 %off = add i64 %iv, -1441 %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %off442 %0 = load i32, ptr %arrayidx, align 4443 %cmp3.not = icmp eq i32 %0, 3444 br i1 %cmp3.not, label %for.inc, label %if.then445 446if.then:447 %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %off448 %1 = load i32, ptr %arrayidx5, align 4449 %mul = shl nsw i32 %1, 2450 %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %off451 store i32 %mul, ptr %arrayidx7, align 4452 br label %for.inc453 454for.inc:455 %iv.next = add nsw i64 %iv, -1456 %cmp2.not = icmp eq i64 %iv, 0457 br i1 %cmp2.not, label %exit, label %for.body458 459exit:460 call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)461 ret void462}463 464 465; Test a loop with a positive step recurrence that has a strided access466define i16 @test_strided_access(i64 %len, ptr %test_base) {467; CHECK-LABEL: @test_strided_access(468; CHECK-NEXT: entry:469; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [163840 x i16], align 4470; CHECK-NEXT: call void @init(ptr [[ALLOCA]])471; CHECK-NEXT: br label [[VECTOR_PH:%.*]]472; CHECK: vector.ph:473; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]474; CHECK: vector.body:475; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]476; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]477; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]478; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE:%.*]], i64 [[INDEX]]479; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP1]], align 1480; CHECK-NEXT: [[TMP3:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer481; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 2)482; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0483; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[TMP4]], i32 1484; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP5]]485; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[ALLOCA]], i64 [[TMP7]]486; CHECK-NEXT: [[TMP9:%.*]] = load i16, ptr [[TMP6]], align 2487; CHECK-NEXT: [[TMP10:%.*]] = load i16, ptr [[TMP8]], align 2488; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i16> poison, i16 [[TMP9]], i32 0489; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> [[TMP11]], i16 [[TMP10]], i32 1490; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP12]], <2 x i16> zeroinitializer491; CHECK-NEXT: [[TMP13]] = add <2 x i16> [[VEC_PHI]], [[PREDPHI]]492; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2493; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)494; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096495; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]496; CHECK: middle.block:497; CHECK-NEXT: [[TMP15:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP13]])498; CHECK-NEXT: br label [[LATCH:%.*]]499; CHECK: loop_exit:500; CHECK-NEXT: ret i16 [[TMP15]]501;502entry:503 %alloca = alloca [163840 x i16], align 4504 call void @init(ptr %alloca)505 br label %loop506loop:507 %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]508 %accum = phi i16 [ 0, %entry ], [ %accum.next, %latch ]509 %iv.next = add i64 %iv, 1510 %test_addr = getelementptr inbounds i8, ptr %test_base, i64 %iv511 %l.t = load i8, ptr %test_addr512 %cmp = icmp sge i8 %l.t, 0513 br i1 %cmp, label %pred, label %latch514pred:515 %iv.stride = mul i64 %iv, 2516 %addr = getelementptr inbounds i16, ptr %alloca, i64 %iv.stride517 %val = load i16, ptr %addr, align 2518 br label %latch519latch:520 %val.phi = phi i16 [0, %loop], [%val, %pred]521 %accum.next = add i16 %accum, %val.phi522 %exit = icmp eq i64 %iv, 4095523 br i1 %exit, label %loop_exit, label %loop524 525loop_exit:526 ret i16 %accum.next527}528 529 530; Test a loop with a negative step recurrence that has a strided access531define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly %dest) {532; CHECK-LABEL: @test_rev_loops_strided_deref_loads(533; CHECK-NEXT: entry:534; CHECK-NEXT: [[LOCAL_DEST:%.*]] = alloca [1024 x i32], align 4535; CHECK-NEXT: [[LOCAL_SRC:%.*]] = alloca [1024 x i32], align 4536; CHECK-NEXT: [[LOCAL_CMP:%.*]] = alloca [1024 x i32], align 4537; CHECK-NEXT: call void @init(ptr [[LOCAL_SRC]])538; CHECK-NEXT: call void @init(ptr [[LOCAL_CMP]])539; CHECK-NEXT: br label [[VECTOR_PH:%.*]]540; CHECK: vector.ph:541; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]542; CHECK: vector.body:543; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE2:%.*]] ]544; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 511, i64 510>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE2]] ]545; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 511, [[INDEX]]546; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFFSET_IDX]]547; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 0548; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 -1549; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4550; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <2 x i32> [[WIDE_LOAD]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>551; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <2 x i32> [[REVERSE]], splat (i32 3)552; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[VEC_IND]], splat (i64 2)553; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0554; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1555; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP7]]556; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP9]]557; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP8]], align 4558; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4559; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x i32> poison, i32 [[TMP11]], i32 0560; CHECK-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP12]], i32 1561; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0562; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]563; CHECK: pred.store.if:564; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0565; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP0]]566; CHECK-NEXT: [[TMP15:%.*]] = shl nsw i32 [[TMP11]], 2567; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4568; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]569; CHECK: pred.store.continue:570; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1571; CHECK-NEXT: br i1 [[TMP16]], label [[PRED_STORE_IF1:%.*]], label [[PRED_STORE_CONTINUE2]]572; CHECK: pred.store.if1:573; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], -1574; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_DEST]], i64 0, i64 [[TMP17]]575; CHECK-NEXT: [[TMP19:%.*]] = shl nsw i32 [[TMP12]], 2576; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4577; CHECK-NEXT: br label [[PRED_STORE_CONTINUE2]]578; CHECK: pred.store.continue2:579; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2580; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i64> [[VEC_IND]], splat (i64 -2)581; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512582; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]583; CHECK: middle.block:584; CHECK-NEXT: br label [[FOR_INC:%.*]]585; CHECK: exit:586; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DEST:%.*]], ptr [[LOCAL_DEST]], i64 1024, i1 false)587; CHECK-NEXT: ret void588;589entry:590 %local_dest = alloca [1024 x i32], align 4591 %local_src = alloca [1024 x i32], align 4592 %local_cmp = alloca [1024 x i32], align 4593 call void @init(ptr %local_src)594 call void @init(ptr %local_cmp)595 br label %for.body596 597for.body:598 %iv = phi i64 [ 511, %entry ], [ %iv.next, %for.inc ]599 %arrayidx = getelementptr inbounds [1024 x i32], ptr %local_cmp, i64 0, i64 %iv600 %0 = load i32, ptr %arrayidx, align 4601 %cmp3.not = icmp eq i32 %0, 3602 br i1 %cmp3.not, label %for.inc, label %if.then603 604if.then:605 %iv.strided = mul i64 %iv, 2606 %arrayidx5 = getelementptr inbounds [1024 x i32], ptr %local_src, i64 0, i64 %iv.strided607 %1 = load i32, ptr %arrayidx5, align 4608 %mul = shl nsw i32 %1, 2609 %arrayidx7 = getelementptr inbounds [1024 x i32], ptr %local_dest, i64 0, i64 %iv610 store i32 %mul, ptr %arrayidx7, align 4611 br label %for.inc612 613for.inc:614 %iv.next = add nsw i64 %iv, -1615 %cmp2.not = icmp eq i64 %iv, 0616 br i1 %cmp2.not, label %exit, label %for.body617 618exit:619 call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %local_dest, i64 1024, i1 false)620 ret void621}622 623define void @adding_offset_overflows(i32 %n, ptr %A) {624; CHECK-LABEL: @adding_offset_overflows(625; CHECK-NEXT: entry:626; CHECK-NEXT: [[B:%.*]] = alloca [62 x i32], align 4627; CHECK-NEXT: [[C:%.*]] = alloca [144 x i32], align 4628; CHECK-NEXT: call void @init(ptr [[B]])629; CHECK-NEXT: call void @init(ptr [[C]])630; CHECK-NEXT: [[PRE:%.*]] = icmp slt i32 [[N:%.*]], 1631; CHECK-NEXT: br i1 [[PRE]], label [[EXIT:%.*]], label [[PH:%.*]]632; CHECK: ph:633; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64634; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1635; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2636; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]637; CHECK: vector.ph:638; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2639; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]640; CHECK-NEXT: [[TMP1:%.*]] = add i64 1, [[N_VEC]]641; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]642; CHECK: vector.body:643; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE3:%.*]] ]644; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]645; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[OFFSET_IDX]]646; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4647; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[WIDE_LOAD]], zeroinitializer648; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0649; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]650; CHECK: pred.load.if:651; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 0652; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[B]], i64 [[TMP15]]653; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4654; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0655; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]656; CHECK: pred.load.continue:657; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP18]], [[PRED_LOAD_IF]] ]658; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1659; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]660; CHECK: pred.load.if1:661; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 1662; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[B]], i64 [[TMP21]]663; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP22]], align 4664; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP13]], i32 1665; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]666; CHECK: pred.load.continue2:667; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], [[PRED_LOAD_IF1]] ]668; CHECK-NEXT: [[TMP5:%.*]] = sext <2 x i32> [[WIDE_LOAD1]] to <2 x i64>669; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0670; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]671; CHECK: pred.store.if:672; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0673; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[C]], i64 [[TMP7]]674; CHECK-NEXT: store i32 0, ptr [[TMP8]], align 4675; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]676; CHECK: pred.store.continue:677; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1678; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_STORE_IF2:%.*]], label [[PRED_STORE_CONTINUE3]]679; CHECK: pred.store.if3:680; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1681; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[C]], i64 [[TMP10]]682; CHECK-NEXT: store i32 0, ptr [[TMP11]], align 4683; CHECK-NEXT: br label [[PRED_STORE_CONTINUE3]]684; CHECK: pred.store.continue4:685; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2686; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]687; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]688; CHECK: middle.block:689; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]690; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]691; CHECK: scalar.ph:692; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], [[MIDDLE_BLOCK]] ], [ 1, [[PH]] ]693; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]694; CHECK: loop.header:695; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]696; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]697; CHECK-NEXT: [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 4698; CHECK-NEXT: [[C_1:%.*]] = icmp eq i32 [[L_A]], 0699; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_LATCH]], label [[IF_THEN:%.*]]700; CHECK: if.then:701; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]702; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_B]], align 4703; CHECK-NEXT: [[IDX_EXT:%.*]] = sext i32 [[L_IDX]] to i64704; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr i32, ptr [[C]], i64 [[IDX_EXT]]705; CHECK-NEXT: store i32 0, ptr [[GEP_C]], align 4706; CHECK-NEXT: br label [[LOOP_LATCH]]707; CHECK: loop.latch:708; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1709; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[WIDE_TRIP_COUNT]]710; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]711; CHECK: exit.loopexit:712; CHECK-NEXT: br label [[EXIT]]713; CHECK: exit:714; CHECK-NEXT: ret void715;716entry:717 %B = alloca [62 x i32], align 4718 %C = alloca [144 x i32], align 4719 call void @init(ptr %B)720 call void @init(ptr %C)721 %pre = icmp slt i32 %n, 1722 br i1 %pre, label %exit, label %ph723 724ph:725 %wide.trip.count = zext i32 %n to i64726 br label %loop.header727 728loop.header:729 %iv = phi i64 [ 1, %ph ], [ %iv.next, %loop.latch ]730 %gep.A = getelementptr i32, ptr %A, i64 %iv731 %l.A = load i32, ptr %gep.A, align 4732 %c.1 = icmp eq i32 %l.A, 0733 br i1 %c.1, label %loop.latch, label %if.then734 735if.then:736 %gep.B = getelementptr i32, ptr %B, i64 %iv737 %l.idx = load i32, ptr %gep.B, align 4738 %idx.ext = sext i32 %l.idx to i64739 %gep.C = getelementptr i32, ptr %C, i64 %idx.ext740 store i32 0, ptr %gep.C, align 4741 br label %loop.latch742 743loop.latch:744 %iv.next = add i64 %iv, 1745 %ec = icmp eq i64 %iv.next, %wide.trip.count746 br i1 %ec, label %exit, label %loop.header747 748exit:749 ret void750}751