503 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 42; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s3 4target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"5 6define void @test_versioned_with_sext_use(i32 %offset, ptr %dst) {7; CHECK-LABEL: define void @test_versioned_with_sext_use(8; CHECK-SAME: i32 [[OFFSET:%.*]], ptr [[DST:%.*]]) {9; CHECK-NEXT: entry:10; CHECK-NEXT: [[OFFSET_EXT:%.*]] = sext i32 [[OFFSET]] to i6411; CHECK-NEXT: br label [[OUTER_HEADER:%.*]]12; CHECK: outer.header.loopexit:13; CHECK-NEXT: [[IV_2_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_2_NEXT:%.*]], [[INNER_LOOP:%.*]] ]14; CHECK-NEXT: br label [[OUTER_HEADER]]15; CHECK: outer.header:16; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_2_NEXT_LCSSA]], [[OUTER_HEADER_LOOPEXIT:%.*]] ]17; CHECK-NEXT: [[C:%.*]] = call i1 @cond()18; CHECK-NEXT: br i1 [[C]], label [[INNER_LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]19; CHECK: inner.loop.preheader:20; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]21; CHECK: vector.scevcheck:22; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[OFFSET]], 123; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]24; CHECK: vector.ph:25; CHECK-NEXT: [[IND_END:%.*]] = add i64 [[IV_1]], 20026; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]27; CHECK: vector.body:28; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]29; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV_1]], [[INDEX]]30; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP3]]31; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP4]], align 832; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 433; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20034; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]35; CHECK: middle.block:36; CHECK-NEXT: br label [[SCALAR_PH]]37; CHECK: scalar.ph:38; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]39; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]40; CHECK-NEXT: br label [[INNER_LOOP]]41; CHECK: inner.loop:42; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]43; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]44; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_2]]45; CHECK-NEXT: store i32 0, ptr [[GEP]], align 846; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], [[OFFSET_EXT]]47; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 148; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 20049; CHECK-NEXT: br i1 [[EC]], label [[OUTER_HEADER_LOOPEXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]50; CHECK: exit:51; CHECK-NEXT: ret void52;53entry:54 %offset.ext = sext i32 %offset to i6455 br label %outer.header56 57outer.header:58 %iv.1 = phi i64 [ 0, %entry ], [ %iv.2.next, %inner.loop ]59 %c = call i1 @cond()60 br i1 %c, label %inner.loop, label %exit61 62inner.loop:63 %iv.2 = phi i64 [ %iv.1, %outer.header ], [ %iv.2.next, %inner.loop ]64 %iv.3 = phi i32 [ 0, %outer.header ], [ %iv.3.next, %inner.loop ]65 %gep = getelementptr i32, ptr %dst, i64 %iv.266 store i32 0, ptr %gep, align 867 %iv.2.next = add i64 %iv.2, %offset.ext68 %iv.3.next = add i32 %iv.3, 169 %ec = icmp eq i32 %iv.3, 20070 br i1 %ec, label %outer.header, label %inner.loop71 72exit:73 ret void74}75 76define void @test_versioned_with_zext_use(i32 %offset, ptr %dst) {77; CHECK-LABEL: define void @test_versioned_with_zext_use(78; CHECK-SAME: i32 [[OFFSET:%.*]], ptr [[DST:%.*]]) {79; CHECK-NEXT: entry:80; CHECK-NEXT: [[OFFSET_EXT:%.*]] = zext i32 [[OFFSET]] to i6481; CHECK-NEXT: br label [[OUTER_HEADER:%.*]]82; CHECK: outer.header.loopexit:83; CHECK-NEXT: [[IV_2_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_2_NEXT:%.*]], [[INNER_LOOP:%.*]] ]84; CHECK-NEXT: br label [[OUTER_HEADER]]85; CHECK: outer.header:86; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_2_NEXT_LCSSA]], [[OUTER_HEADER_LOOPEXIT:%.*]] ]87; CHECK-NEXT: [[C:%.*]] = call i1 @cond()88; CHECK-NEXT: br i1 [[C]], label [[INNER_LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]89; CHECK: inner.loop.preheader:90; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]91; CHECK: vector.scevcheck:92; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[OFFSET]], 193; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]94; CHECK: vector.ph:95; CHECK-NEXT: [[IND_END:%.*]] = add i64 [[IV_1]], 20096; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]97; CHECK: vector.body:98; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]99; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV_1]], [[INDEX]]100; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP3]]101; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP4]], align 8102; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4103; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200104; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]105; CHECK: middle.block:106; CHECK-NEXT: br label [[SCALAR_PH]]107; CHECK: scalar.ph:108; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]109; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]110; CHECK-NEXT: br label [[INNER_LOOP]]111; CHECK: inner.loop:112; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]113; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]114; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_2]]115; CHECK-NEXT: store i32 0, ptr [[GEP]], align 8116; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], [[OFFSET_EXT]]117; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1118; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 200119; CHECK-NEXT: br i1 [[EC]], label [[OUTER_HEADER_LOOPEXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP5:![0-9]+]]120; CHECK: exit:121; CHECK-NEXT: ret void122;123entry:124 %offset.ext = zext i32 %offset to i64125 br label %outer.header126 127outer.header:128 %iv.1 = phi i64 [ 0, %entry ], [ %iv.2.next, %inner.loop ]129 %c = call i1 @cond()130 br i1 %c, label %inner.loop, label %exit131 132inner.loop:133 %iv.2 = phi i64 [ %iv.1, %outer.header ], [ %iv.2.next, %inner.loop ]134 %iv.3 = phi i32 [ 0, %outer.header ], [ %iv.3.next, %inner.loop ]135 %gep = getelementptr i32, ptr %dst, i64 %iv.2136 store i32 0, ptr %gep, align 8137 %iv.2.next = add i64 %iv.2, %offset.ext138 %iv.3.next = add i32 %iv.3, 1139 %ec = icmp eq i32 %iv.3, 200140 br i1 %ec, label %outer.header, label %inner.loop141 142exit:143 ret void144}145 146define void @versioned_sext_use_in_gep(i32 %scale, ptr %dst, i64 %scale.2) {147; CHECK-LABEL: define void @versioned_sext_use_in_gep(148; CHECK-SAME: i32 [[SCALE:%.*]], ptr [[DST:%.*]], i64 [[SCALE_2:%.*]]) {149; CHECK-NEXT: entry:150; CHECK-NEXT: [[SCALE_EXT:%.*]] = sext i32 [[SCALE]] to i64151; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]152; CHECK: vector.scevcheck:153; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[SCALE]], 1154; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]155; CHECK: vector.ph:156; CHECK-NEXT: [[TMP83:%.*]] = getelementptr i8, ptr [[DST]], i64 [[SCALE_2]]157; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]158; CHECK: vector.body:159; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]160; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0161; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1162; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], 2163; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], 3164; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP10]]165; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]166; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP14]]167; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP16]]168; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP11]], align 8169; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP13]], align 8170; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP15]], align 8171; CHECK-NEXT: store ptr [[TMP83]], ptr [[TMP17]], align 8172; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4173; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256174; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]175; CHECK: middle.block:176; CHECK-NEXT: br label [[EXIT:%.*]]177; CHECK: scalar.ph:178; CHECK-NEXT: br label [[LOOP:%.*]]179; CHECK: loop:180; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]181; CHECK-NEXT: [[IV_MUL:%.*]] = mul i64 [[IV]], [[SCALE_EXT]]182; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_MUL]]183; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1184; CHECK-NEXT: [[SCALE_MUL:%.*]] = mul i64 [[SCALE_EXT]], [[SCALE_2]]185; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[SCALE_MUL]]186; CHECK-NEXT: store ptr [[GEP_2]], ptr [[GEP_1]], align 8187; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 256188; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]189; CHECK: exit:190; CHECK-NEXT: ret void191;192entry:193 %scale.ext = sext i32 %scale to i64194 br label %loop195 196loop:197 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]198 %iv.mul = mul i64 %iv, %scale.ext199 %gep.1 = getelementptr i8, ptr %dst, i64 %iv.mul200 %iv.next = add i64 %iv, 1201 %scale.mul = mul i64 %scale.ext, %scale.2202 %gep.2 = getelementptr i8, ptr %dst, i64 %scale.mul203 store ptr %gep.2, ptr %gep.1, align 8204 %ec = icmp eq i64 %iv.next, 256205 br i1 %ec, label %exit, label %loop206 207exit:208 ret void209}210 211declare i1 @cond()212 213define void @test_versioned_with_different_uses(i32 %offset, ptr noalias %dst.1, ptr %dst.2) {214; CHECK-LABEL: define void @test_versioned_with_different_uses(215; CHECK-SAME: i32 [[OFFSET:%.*]], ptr noalias [[DST_1:%.*]], ptr [[DST_2:%.*]]) {216; CHECK-NEXT: entry:217; CHECK-NEXT: [[OFFSET_EXT:%.*]] = zext i32 [[OFFSET]] to i64218; CHECK-NEXT: br label [[OUTER_HEADER:%.*]]219; CHECK: outer.header.loopexit:220; CHECK-NEXT: [[IV_2_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_2_NEXT:%.*]], [[INNER_LOOP:%.*]] ]221; CHECK-NEXT: br label [[OUTER_HEADER]]222; CHECK: outer.header:223; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_2_NEXT_LCSSA]], [[OUTER_HEADER_LOOPEXIT:%.*]] ]224; CHECK-NEXT: [[C:%.*]] = call i1 @cond()225; CHECK-NEXT: br i1 [[C]], label [[INNER_LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]226; CHECK: inner.loop.preheader:227; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]228; CHECK: vector.scevcheck:229; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[OFFSET]], 1230; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]231; CHECK: vector.ph:232; CHECK-NEXT: [[IND_END:%.*]] = add i64 [[IV_1]], 200233; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]234; CHECK: vector.body:235; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]236; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV_1]], [[INDEX]]237; CHECK-NEXT: [[OFFSET_IDX2:%.*]] = trunc i64 [[INDEX]] to i32238; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX2]], 0239; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX2]], 1240; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[OFFSET_IDX2]], 2241; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX2]], 3242; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP4]]243; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP5]]244; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP6]]245; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP7]]246; CHECK-NEXT: store i32 0, ptr [[TMP8]], align 8247; CHECK-NEXT: store i32 0, ptr [[TMP9]], align 8248; CHECK-NEXT: store i32 0, ptr [[TMP10]], align 8249; CHECK-NEXT: store i32 0, ptr [[TMP11]], align 8250; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST_2]], i64 [[TMP3]]251; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP12]], align 8252; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4253; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200254; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]255; CHECK: middle.block:256; CHECK-NEXT: br label [[SCALAR_PH]]257; CHECK: scalar.ph:258; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_1]], [[VECTOR_SCEVCHECK]] ]259; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]260; CHECK-NEXT: br label [[INNER_LOOP]]261; CHECK: inner.loop:262; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_2_NEXT]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]263; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], [[INNER_LOOP]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]264; CHECK-NEXT: [[IV_MUL:%.*]] = mul i32 [[IV_3]], [[OFFSET]]265; CHECK-NEXT: [[GEP_MUL:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[IV_MUL]]266; CHECK-NEXT: store i32 0, ptr [[GEP_MUL]], align 8267; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST_2]], i64 [[IV_2]]268; CHECK-NEXT: store i32 0, ptr [[GEP]], align 8269; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], [[OFFSET_EXT]]270; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1271; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 200272; CHECK-NEXT: br i1 [[EC]], label [[OUTER_HEADER_LOOPEXIT]], label [[INNER_LOOP]], !llvm.loop [[LOOP9:![0-9]+]]273; CHECK: exit:274; CHECK-NEXT: ret void275;276entry:277 %offset.ext = zext i32 %offset to i64278 br label %outer.header279 280outer.header:281 %iv.1 = phi i64 [ 0, %entry ], [ %iv.2.next, %inner.loop ]282 %c = call i1 @cond()283 br i1 %c, label %inner.loop, label %exit284 285inner.loop:286 %iv.2 = phi i64 [ %iv.1, %outer.header ], [ %iv.2.next, %inner.loop ]287 %iv.3 = phi i32 [ 0, %outer.header ], [ %iv.3.next, %inner.loop ]288 %iv.mul = mul i32 %iv.3, %offset289 %gep.mul = getelementptr i8, ptr %dst.1, i32 %iv.mul290 store i32 0, ptr %gep.mul, align 8291 %gep = getelementptr i32, ptr %dst.2, i64 %iv.2292 store i32 0, ptr %gep, align 8293 %iv.2.next = add i64 %iv.2, %offset.ext294 %iv.3.next = add i32 %iv.3, 1295 %ec = icmp eq i32 %iv.3, 200296 br i1 %ec, label %outer.header, label %inner.loop297 298exit:299 ret void300}301 302define void @test_versioned_with_non_ex_use(i32 %offset, ptr noalias %dst.1, ptr %dst.2) {303; CHECK-LABEL: define void @test_versioned_with_non_ex_use(304; CHECK-SAME: i32 [[OFFSET:%.*]], ptr noalias [[DST_1:%.*]], ptr [[DST_2:%.*]]) {305; CHECK-NEXT: entry:306; CHECK-NEXT: [[OFFSET_EXT:%.*]] = zext i32 [[OFFSET]] to i64307; CHECK-NEXT: [[ADD:%.*]] = add i32 [[OFFSET]], 3308; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]309; CHECK: vector.scevcheck:310; CHECK-NEXT: [[TMP0:%.*]] = sub i32 -3, [[OFFSET]]311; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[ADD]], 0312; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 [[TMP0]], i32 [[ADD]]313; CHECK-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP2]], i32 200)314; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0315; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1316; CHECK-NEXT: [[TMP3:%.*]] = sub i32 0, [[MUL_RESULT]]317; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i32 [[MUL_RESULT]], 0318; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[TMP3]], 0319; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP1]], i1 [[TMP5]], i1 [[TMP4]]320; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[MUL_OVERFLOW]]321; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[OFFSET]], 1322; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[IDENT_CHECK]]323; CHECK-NEXT: br i1 [[TMP8]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]324; CHECK: vector.ph:325; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[ADD]], i64 0326; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer327; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]328; CHECK: vector.body:329; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]330; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]331; CHECK-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]332; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP10]], i32 0333; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP10]], i32 1334; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP10]], i32 2335; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP10]], i32 3336; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP11]]337; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP13]]338; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP15]]339; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[TMP17]]340; CHECK-NEXT: store i32 0, ptr [[TMP12]], align 8341; CHECK-NEXT: store i32 0, ptr [[TMP14]], align 8342; CHECK-NEXT: store i32 0, ptr [[TMP16]], align 8343; CHECK-NEXT: store i32 0, ptr [[TMP18]], align 8344; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST_2]], i64 [[INDEX]]345; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr [[TMP20]], align 8346; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4347; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)348; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200349; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]350; CHECK: middle.block:351; CHECK-NEXT: br label [[SCALAR_PH]]352; CHECK: scalar.ph:353; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]354; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 200, [[MIDDLE_BLOCK]] ], [ 0, [[VECTOR_SCEVCHECK]] ]355; CHECK-NEXT: br label [[LOOP:%.*]]356; CHECK: loop:357; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]358; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV_3_NEXT:%.*]], [[LOOP]] ]359; CHECK-NEXT: [[IV_MUL:%.*]] = mul i32 [[IV_3]], [[ADD]]360; CHECK-NEXT: [[GEP_MUL:%.*]] = getelementptr i8, ptr [[DST_1]], i32 [[IV_MUL]]361; CHECK-NEXT: store i32 0, ptr [[GEP_MUL]], align 8362; CHECK-NEXT: [[IV_2_MUL:%.*]] = mul i64 [[IV_2]], [[OFFSET_EXT]]363; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST_2]], i64 [[IV_2_MUL]]364; CHECK-NEXT: store i32 0, ptr [[GEP]], align 8365; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], 1366; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1367; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_3]], 200368; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]369; CHECK: exit:370; CHECK-NEXT: ret void371;372entry:373 %offset.ext = zext i32 %offset to i64374 %add = add i32 %offset, 3375 br label %loop376 377loop:378 %iv.2 = phi i64 [ 0, %entry ], [ %iv.2.next, %loop ]379 %iv.3 = phi i32 [ 0, %entry ], [ %iv.3.next, %loop ]380 %iv.mul = mul i32 %iv.3, %add381 %gep.mul = getelementptr i8, ptr %dst.1, i32 %iv.mul382 store i32 0, ptr %gep.mul, align 8383 %iv.2.mul = mul i64 %iv.2, %offset.ext384 %gep = getelementptr i32, ptr %dst.2, i64 %iv.2.mul385 store i32 0, ptr %gep, align 8386 %iv.2.next = add i64 %iv.2, 1387 %iv.3.next = add i32 %iv.3, 1388 %ec = icmp eq i32 %iv.3, 200389 br i1 %ec, label %exit, label %loop390 391exit:392 ret void393}394 395; Test case to make sure that uses of versioned strides of type i1 are properly396; extended. From https://github.com/llvm/llvm-project/issues/91369.397; TODO: Better check (udiv i64 15, %g.64) after checking if %g == 1.398define void @zext_of_i1_stride(i1 %g, ptr %dst) mustprogress {399; CHECK-LABEL: define void @zext_of_i1_stride(400; CHECK-SAME: i1 [[G:%.*]], ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {401; CHECK-NEXT: entry:402; CHECK-NEXT: [[G_16:%.*]] = zext i1 [[G]] to i16403; CHECK-NEXT: [[G_64:%.*]] = zext i1 [[G]] to i64404; CHECK-NEXT: [[TMP0:%.*]] = udiv i64 15, [[G_64]]405; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1406; CHECK-NEXT: br label [[VECTOR_SCEVCHECK:%.*]]407; CHECK: vector.scevcheck:408; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i1 [[G]], true409; CHECK-NEXT: br i1 [[IDENT_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]410; CHECK: vector.ph:411; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]412; CHECK: vector.body:413; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]414; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[INDEX]]415; CHECK-NEXT: store <4 x i16> splat (i16 1), ptr [[TMP4]], align 2416; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4417; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16418; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]419; CHECK: middle.block:420; CHECK-NEXT: br label [[EXIT:%.*]]421; CHECK: scalar.ph:422; CHECK-NEXT: br label [[LOOP:%.*]]423; CHECK: loop:424; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]425; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]]426; CHECK-NEXT: store i16 [[G_16]], ptr [[GEP]], align 2427; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], [[G_64]]428; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 16429; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]430; CHECK: exit:431; CHECK-NEXT: ret void432;433entry:434 %g.16 = zext i1 %g to i16435 %g.64 = zext i1 %g to i64436 br label %loop437 438loop:439 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]440 %gep = getelementptr inbounds i16, ptr %dst, i64 %iv441 store i16 %g.16, ptr %gep, align 2442 %iv.next = add nuw nsw i64 %iv, %g.64443 %cmp = icmp ult i64 %iv.next, 16444 br i1 %cmp, label %loop, label %exit445 446exit:447 ret void448}449 450; Test case to make sure that uses of versioned strides of type i1 are properly451; extended.452define void @sext_of_i1_stride(i1 %g, ptr %dst) mustprogress {453; CHECK-LABEL: define void @sext_of_i1_stride(454; CHECK-SAME: i1 [[G:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {455; CHECK-NEXT: entry:456; CHECK-NEXT: [[G_16:%.*]] = sext i1 [[G]] to i16457; CHECK-NEXT: [[G_64:%.*]] = sext i1 [[G]] to i64458; CHECK-NEXT: br label [[LOOP:%.*]]459; CHECK: loop:460; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]461; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]]462; CHECK-NEXT: store i16 [[G_16]], ptr [[GEP]], align 2463; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], [[G_64]]464; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV_NEXT]], 16465; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]466; CHECK: exit:467; CHECK-NEXT: ret void468;469entry:470 %g.16 = sext i1 %g to i16471 %g.64 = sext i1 %g to i64472 br label %loop473 474loop:475 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]476 %gep = getelementptr inbounds i16, ptr %dst, i64 %iv477 store i16 %g.16, ptr %gep, align 2478 %iv.next = add nuw nsw i64 %iv, %g.64479 %cmp = icmp ult i64 %iv.next, 16480 br i1 %cmp, label %loop, label %exit481 482exit:483 ret void484}485 486 487;.488; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}489; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}490; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}491; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}492; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}493; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}494; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}495; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}496; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}497; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}498; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}499; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}500; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}501; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}502;.503