1234 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF8UF1 %s3; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=2 -S %s | FileCheck --check-prefixes=VF8UF2 %s4; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=16 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF16UF1 %s5 6target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"7 8; Check if the vector loop condition can be simplified to true for a given9; VF/IC combination.10define void @test_tc_less_than_16(ptr %A, i64 %N) {11; VF8UF1-LABEL: define void @test_tc_less_than_16(12; VF8UF1-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {13; VF8UF1-NEXT: [[ENTRY:.*]]:14; VF8UF1-NEXT: [[AND:%.*]] = and i64 [[N]], 1515; VF8UF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[AND]], 816; VF8UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]17; VF8UF1: [[VECTOR_PH]]:18; VF8UF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[AND]], 819; VF8UF1-NEXT: [[N_VEC:%.*]] = sub i64 [[AND]], [[N_MOD_VF]]20; VF8UF1-NEXT: [[TMP0:%.*]] = sub i64 [[AND]], [[N_VEC]]21; VF8UF1-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]22; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]23; VF8UF1: [[VECTOR_BODY]]:24; VF8UF1-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]25; VF8UF1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]26; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[NEXT_GEP]], align 127; VF8UF1-NEXT: [[TMP4:%.*]] = add nsw <8 x i8> [[WIDE_LOAD]], splat (i8 10)28; VF8UF1-NEXT: store <8 x i8> [[TMP4]], ptr [[NEXT_GEP]], align 129; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 830; VF8UF1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]31; VF8UF1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]32; VF8UF1: [[MIDDLE_BLOCK]]:33; VF8UF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[AND]], [[N_VEC]]34; VF8UF1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]35; VF8UF1: [[SCALAR_PH]]:36; VF8UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[AND]], %[[ENTRY]] ]37; VF8UF1-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]38; VF8UF1-NEXT: br label %[[LOOP:.*]]39; VF8UF1: [[LOOP]]:40; VF8UF1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]41; VF8UF1-NEXT: [[P_SRC:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]42; VF8UF1-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 143; VF8UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 144; VF8UF1-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 1045; VF8UF1-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 146; VF8UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -147; VF8UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 048; VF8UF1-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]49; VF8UF1: [[EXIT]]:50; VF8UF1-NEXT: ret void51;52; VF8UF2-LABEL: define void @test_tc_less_than_16(53; VF8UF2-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {54; VF8UF2-NEXT: [[ENTRY:.*]]:55; VF8UF2-NEXT: [[AND:%.*]] = and i64 [[N]], 1556; VF8UF2-NEXT: br label %[[LOOP:.*]]57; VF8UF2: [[LOOP]]:58; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ [[AND]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]59; VF8UF2-NEXT: [[P_SRC:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]60; VF8UF2-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 161; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 162; VF8UF2-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 1063; VF8UF2-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 164; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -165; VF8UF2-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 066; VF8UF2-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]67; VF8UF2: [[EXIT]]:68; VF8UF2-NEXT: ret void69;70; VF16UF1-LABEL: define void @test_tc_less_than_16(71; VF16UF1-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {72; VF16UF1-NEXT: [[ENTRY:.*]]:73; VF16UF1-NEXT: [[AND:%.*]] = and i64 [[N]], 1574; VF16UF1-NEXT: br label %[[LOOP:.*]]75; VF16UF1: [[LOOP]]:76; VF16UF1-NEXT: [[IV:%.*]] = phi i64 [ [[AND]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]77; VF16UF1-NEXT: [[P_SRC:%.*]] = phi ptr [ [[A]], %[[ENTRY]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]78; VF16UF1-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 179; VF16UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 180; VF16UF1-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 1081; VF16UF1-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 182; VF16UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -183; VF16UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 084; VF16UF1-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]85; VF16UF1: [[EXIT]]:86; VF16UF1-NEXT: ret void87;88entry:89 %and = and i64 %N, 1590 br label %loop91 92loop:93 %iv = phi i64 [ %and, %entry ], [ %iv.next, %loop ]94 %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]95 %p.src.next = getelementptr inbounds i8, ptr %p.src, i64 196 %l = load i8, ptr %p.src, align 197 %add = add nsw i8 %l, 1098 store i8 %add, ptr %p.src99 %iv.next = add nsw i64 %iv, -1100 %cmp = icmp eq i64 %iv.next, 0101 br i1 %cmp, label %exit, label %loop102 103exit:104 ret void105}106 107define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5, 10) %N) {108; VF8UF1-LABEL: define void @remove_loop_region_with_replicate_recipe(109; VF8UF1-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {110; VF8UF1-NEXT: [[ENTRY:.*:]]111; VF8UF1-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -2112; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]113; VF8UF1: [[VECTOR_PH]]:114; VF8UF1-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1115; VF8UF1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0116; VF8UF1-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer117; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]118; VF8UF1: [[VECTOR_BODY]]:119; VF8UF1-NEXT: [[TMP2:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT1]]120; VF8UF1-NEXT: [[TMP3:%.*]] = extractelement <8 x i1> [[TMP2]], i32 0121; VF8UF1-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]122; VF8UF1: [[PRED_STORE_IF]]:123; VF8UF1-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[DST]], i64 2124; VF8UF1-NEXT: store i16 0, ptr [[TMP4]], align 2125; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE]]126; VF8UF1: [[PRED_STORE_CONTINUE]]:127; VF8UF1-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP2]], i32 1128; VF8UF1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]129; VF8UF1: [[PRED_STORE_IF1]]:130; VF8UF1-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[DST]], i64 3131; VF8UF1-NEXT: store i16 0, ptr [[TMP6]], align 2132; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE2]]133; VF8UF1: [[PRED_STORE_CONTINUE2]]:134; VF8UF1-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP2]], i32 2135; VF8UF1-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]136; VF8UF1: [[PRED_STORE_IF3]]:137; VF8UF1-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST]], i64 4138; VF8UF1-NEXT: store i16 0, ptr [[TMP8]], align 2139; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE4]]140; VF8UF1: [[PRED_STORE_CONTINUE4]]:141; VF8UF1-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP2]], i32 3142; VF8UF1-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]143; VF8UF1: [[PRED_STORE_IF5]]:144; VF8UF1-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[DST]], i64 5145; VF8UF1-NEXT: store i16 0, ptr [[TMP10]], align 2146; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE6]]147; VF8UF1: [[PRED_STORE_CONTINUE6]]:148; VF8UF1-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[TMP2]], i32 4149; VF8UF1-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]150; VF8UF1: [[PRED_STORE_IF7]]:151; VF8UF1-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[DST]], i64 6152; VF8UF1-NEXT: store i16 0, ptr [[TMP12]], align 2153; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE8]]154; VF8UF1: [[PRED_STORE_CONTINUE8]]:155; VF8UF1-NEXT: [[TMP13:%.*]] = extractelement <8 x i1> [[TMP2]], i32 5156; VF8UF1-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]157; VF8UF1: [[PRED_STORE_IF9]]:158; VF8UF1-NEXT: [[TMP14:%.*]] = getelementptr i16, ptr [[DST]], i64 7159; VF8UF1-NEXT: store i16 0, ptr [[TMP14]], align 2160; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE10]]161; VF8UF1: [[PRED_STORE_CONTINUE10]]:162; VF8UF1-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP2]], i32 6163; VF8UF1-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]164; VF8UF1: [[PRED_STORE_IF11]]:165; VF8UF1-NEXT: [[TMP16:%.*]] = getelementptr i16, ptr [[DST]], i64 8166; VF8UF1-NEXT: store i16 0, ptr [[TMP16]], align 2167; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE12]]168; VF8UF1: [[PRED_STORE_CONTINUE12]]:169; VF8UF1-NEXT: [[TMP17:%.*]] = extractelement <8 x i1> [[TMP2]], i32 7170; VF8UF1-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]171; VF8UF1: [[PRED_STORE_IF13]]:172; VF8UF1-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[DST]], i64 9173; VF8UF1-NEXT: store i16 0, ptr [[TMP18]], align 2174; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE14]]175; VF8UF1: [[PRED_STORE_CONTINUE14]]:176; VF8UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]177; VF8UF1: [[MIDDLE_BLOCK]]:178; VF8UF1-NEXT: br label %[[EXIT:.*]]179; VF8UF1: [[EXIT]]:180; VF8UF1-NEXT: ret void181;182; VF8UF2-LABEL: define void @remove_loop_region_with_replicate_recipe(183; VF8UF2-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {184; VF8UF2-NEXT: [[ENTRY:.*:]]185; VF8UF2-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -2186; VF8UF2-NEXT: br label %[[VECTOR_PH:.*]]187; VF8UF2: [[VECTOR_PH]]:188; VF8UF2-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1189; VF8UF2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0190; VF8UF2-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer191; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]192; VF8UF2: [[VECTOR_BODY]]:193; VF8UF2-NEXT: [[TMP2:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT1]]194; VF8UF2-NEXT: [[TMP3:%.*]] = icmp ule <8 x i64> <i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[BROADCAST_SPLAT1]]195; VF8UF2-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP2]], i32 0196; VF8UF2-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]197; VF8UF2: [[PRED_STORE_IF]]:198; VF8UF2-NEXT: [[TMP5:%.*]] = getelementptr i16, ptr [[DST]], i64 2199; VF8UF2-NEXT: store i16 0, ptr [[TMP5]], align 2200; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE]]201; VF8UF2: [[PRED_STORE_CONTINUE]]:202; VF8UF2-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP2]], i32 1203; VF8UF2-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]204; VF8UF2: [[PRED_STORE_IF1]]:205; VF8UF2-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[DST]], i64 3206; VF8UF2-NEXT: store i16 0, ptr [[TMP7]], align 2207; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE2]]208; VF8UF2: [[PRED_STORE_CONTINUE2]]:209; VF8UF2-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP2]], i32 2210; VF8UF2-NEXT: br i1 [[TMP8]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]211; VF8UF2: [[PRED_STORE_IF3]]:212; VF8UF2-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[DST]], i64 4213; VF8UF2-NEXT: store i16 0, ptr [[TMP9]], align 2214; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE4]]215; VF8UF2: [[PRED_STORE_CONTINUE4]]:216; VF8UF2-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP2]], i32 3217; VF8UF2-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]218; VF8UF2: [[PRED_STORE_IF5]]:219; VF8UF2-NEXT: [[TMP11:%.*]] = getelementptr i16, ptr [[DST]], i64 5220; VF8UF2-NEXT: store i16 0, ptr [[TMP11]], align 2221; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE6]]222; VF8UF2: [[PRED_STORE_CONTINUE6]]:223; VF8UF2-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[TMP2]], i32 4224; VF8UF2-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]225; VF8UF2: [[PRED_STORE_IF7]]:226; VF8UF2-NEXT: [[TMP13:%.*]] = getelementptr i16, ptr [[DST]], i64 6227; VF8UF2-NEXT: store i16 0, ptr [[TMP13]], align 2228; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE8]]229; VF8UF2: [[PRED_STORE_CONTINUE8]]:230; VF8UF2-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP2]], i32 5231; VF8UF2-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]232; VF8UF2: [[PRED_STORE_IF9]]:233; VF8UF2-NEXT: [[TMP15:%.*]] = getelementptr i16, ptr [[DST]], i64 7234; VF8UF2-NEXT: store i16 0, ptr [[TMP15]], align 2235; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE10]]236; VF8UF2: [[PRED_STORE_CONTINUE10]]:237; VF8UF2-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP2]], i32 6238; VF8UF2-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]239; VF8UF2: [[PRED_STORE_IF11]]:240; VF8UF2-NEXT: [[TMP17:%.*]] = getelementptr i16, ptr [[DST]], i64 8241; VF8UF2-NEXT: store i16 0, ptr [[TMP17]], align 2242; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE12]]243; VF8UF2: [[PRED_STORE_CONTINUE12]]:244; VF8UF2-NEXT: [[TMP18:%.*]] = extractelement <8 x i1> [[TMP2]], i32 7245; VF8UF2-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]246; VF8UF2: [[PRED_STORE_IF13]]:247; VF8UF2-NEXT: [[TMP19:%.*]] = getelementptr i16, ptr [[DST]], i64 9248; VF8UF2-NEXT: store i16 0, ptr [[TMP19]], align 2249; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE14]]250; VF8UF2: [[PRED_STORE_CONTINUE14]]:251; VF8UF2-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP3]], i32 0252; VF8UF2-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]253; VF8UF2: [[PRED_STORE_IF15]]:254; VF8UF2-NEXT: [[TMP21:%.*]] = getelementptr i16, ptr [[DST]], i64 10255; VF8UF2-NEXT: store i16 0, ptr [[TMP21]], align 2256; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE16]]257; VF8UF2: [[PRED_STORE_CONTINUE16]]:258; VF8UF2-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1259; VF8UF2-NEXT: br i1 [[TMP22]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]260; VF8UF2: [[PRED_STORE_IF17]]:261; VF8UF2-NEXT: [[TMP23:%.*]] = getelementptr i16, ptr [[DST]], i64 11262; VF8UF2-NEXT: store i16 0, ptr [[TMP23]], align 2263; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE18]]264; VF8UF2: [[PRED_STORE_CONTINUE18]]:265; VF8UF2-NEXT: [[TMP24:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2266; VF8UF2-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]267; VF8UF2: [[PRED_STORE_IF19]]:268; VF8UF2-NEXT: [[TMP25:%.*]] = getelementptr i16, ptr [[DST]], i64 12269; VF8UF2-NEXT: store i16 0, ptr [[TMP25]], align 2270; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE20]]271; VF8UF2: [[PRED_STORE_CONTINUE20]]:272; VF8UF2-NEXT: [[TMP26:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3273; VF8UF2-NEXT: br i1 [[TMP26]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]274; VF8UF2: [[PRED_STORE_IF21]]:275; VF8UF2-NEXT: [[TMP27:%.*]] = getelementptr i16, ptr [[DST]], i64 13276; VF8UF2-NEXT: store i16 0, ptr [[TMP27]], align 2277; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE22]]278; VF8UF2: [[PRED_STORE_CONTINUE22]]:279; VF8UF2-NEXT: [[TMP28:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4280; VF8UF2-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]281; VF8UF2: [[PRED_STORE_IF23]]:282; VF8UF2-NEXT: [[TMP29:%.*]] = getelementptr i16, ptr [[DST]], i64 14283; VF8UF2-NEXT: store i16 0, ptr [[TMP29]], align 2284; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE24]]285; VF8UF2: [[PRED_STORE_CONTINUE24]]:286; VF8UF2-NEXT: [[TMP30:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5287; VF8UF2-NEXT: br i1 [[TMP30]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]288; VF8UF2: [[PRED_STORE_IF25]]:289; VF8UF2-NEXT: [[TMP31:%.*]] = getelementptr i16, ptr [[DST]], i64 15290; VF8UF2-NEXT: store i16 0, ptr [[TMP31]], align 2291; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE26]]292; VF8UF2: [[PRED_STORE_CONTINUE26]]:293; VF8UF2-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6294; VF8UF2-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]295; VF8UF2: [[PRED_STORE_IF27]]:296; VF8UF2-NEXT: [[TMP33:%.*]] = getelementptr i16, ptr [[DST]], i64 16297; VF8UF2-NEXT: store i16 0, ptr [[TMP33]], align 2298; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE28]]299; VF8UF2: [[PRED_STORE_CONTINUE28]]:300; VF8UF2-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7301; VF8UF2-NEXT: br i1 [[TMP34]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]302; VF8UF2: [[PRED_STORE_IF29]]:303; VF8UF2-NEXT: [[TMP35:%.*]] = getelementptr i16, ptr [[DST]], i64 17304; VF8UF2-NEXT: store i16 0, ptr [[TMP35]], align 2305; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE30]]306; VF8UF2: [[PRED_STORE_CONTINUE30]]:307; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]308; VF8UF2: [[MIDDLE_BLOCK]]:309; VF8UF2-NEXT: br label %[[EXIT:.*]]310; VF8UF2: [[EXIT]]:311; VF8UF2-NEXT: ret void312;313; VF16UF1-LABEL: define void @remove_loop_region_with_replicate_recipe(314; VF16UF1-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {315; VF16UF1-NEXT: [[ENTRY:.*:]]316; VF16UF1-NEXT: [[TMP0:%.*]] = add nsw i64 [[N]], -2317; VF16UF1-NEXT: br label %[[VECTOR_PH:.*]]318; VF16UF1: [[VECTOR_PH]]:319; VF16UF1-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1320; VF16UF1-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0321; VF16UF1-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT1]], <16 x i64> poison, <16 x i32> zeroinitializer322; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]323; VF16UF1: [[VECTOR_BODY]]:324; VF16UF1-NEXT: [[TMP2:%.*]] = icmp ule <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[BROADCAST_SPLAT1]]325; VF16UF1-NEXT: [[TMP3:%.*]] = extractelement <16 x i1> [[TMP2]], i32 0326; VF16UF1-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]327; VF16UF1: [[PRED_STORE_IF]]:328; VF16UF1-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[DST]], i64 2329; VF16UF1-NEXT: store i16 0, ptr [[TMP4]], align 2330; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE]]331; VF16UF1: [[PRED_STORE_CONTINUE]]:332; VF16UF1-NEXT: [[TMP5:%.*]] = extractelement <16 x i1> [[TMP2]], i32 1333; VF16UF1-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]334; VF16UF1: [[PRED_STORE_IF1]]:335; VF16UF1-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[DST]], i64 3336; VF16UF1-NEXT: store i16 0, ptr [[TMP6]], align 2337; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE2]]338; VF16UF1: [[PRED_STORE_CONTINUE2]]:339; VF16UF1-NEXT: [[TMP7:%.*]] = extractelement <16 x i1> [[TMP2]], i32 2340; VF16UF1-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]341; VF16UF1: [[PRED_STORE_IF3]]:342; VF16UF1-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST]], i64 4343; VF16UF1-NEXT: store i16 0, ptr [[TMP8]], align 2344; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE4]]345; VF16UF1: [[PRED_STORE_CONTINUE4]]:346; VF16UF1-NEXT: [[TMP9:%.*]] = extractelement <16 x i1> [[TMP2]], i32 3347; VF16UF1-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]348; VF16UF1: [[PRED_STORE_IF5]]:349; VF16UF1-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[DST]], i64 5350; VF16UF1-NEXT: store i16 0, ptr [[TMP10]], align 2351; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE6]]352; VF16UF1: [[PRED_STORE_CONTINUE6]]:353; VF16UF1-NEXT: [[TMP11:%.*]] = extractelement <16 x i1> [[TMP2]], i32 4354; VF16UF1-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]355; VF16UF1: [[PRED_STORE_IF7]]:356; VF16UF1-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[DST]], i64 6357; VF16UF1-NEXT: store i16 0, ptr [[TMP12]], align 2358; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE8]]359; VF16UF1: [[PRED_STORE_CONTINUE8]]:360; VF16UF1-NEXT: [[TMP13:%.*]] = extractelement <16 x i1> [[TMP2]], i32 5361; VF16UF1-NEXT: br i1 [[TMP13]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]362; VF16UF1: [[PRED_STORE_IF9]]:363; VF16UF1-NEXT: [[TMP14:%.*]] = getelementptr i16, ptr [[DST]], i64 7364; VF16UF1-NEXT: store i16 0, ptr [[TMP14]], align 2365; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE10]]366; VF16UF1: [[PRED_STORE_CONTINUE10]]:367; VF16UF1-NEXT: [[TMP15:%.*]] = extractelement <16 x i1> [[TMP2]], i32 6368; VF16UF1-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]369; VF16UF1: [[PRED_STORE_IF11]]:370; VF16UF1-NEXT: [[TMP16:%.*]] = getelementptr i16, ptr [[DST]], i64 8371; VF16UF1-NEXT: store i16 0, ptr [[TMP16]], align 2372; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE12]]373; VF16UF1: [[PRED_STORE_CONTINUE12]]:374; VF16UF1-NEXT: [[TMP17:%.*]] = extractelement <16 x i1> [[TMP2]], i32 7375; VF16UF1-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]376; VF16UF1: [[PRED_STORE_IF13]]:377; VF16UF1-NEXT: [[TMP18:%.*]] = getelementptr i16, ptr [[DST]], i64 9378; VF16UF1-NEXT: store i16 0, ptr [[TMP18]], align 2379; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE14]]380; VF16UF1: [[PRED_STORE_CONTINUE14]]:381; VF16UF1-NEXT: [[TMP19:%.*]] = extractelement <16 x i1> [[TMP2]], i32 8382; VF16UF1-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]383; VF16UF1: [[PRED_STORE_IF15]]:384; VF16UF1-NEXT: [[TMP20:%.*]] = getelementptr i16, ptr [[DST]], i64 10385; VF16UF1-NEXT: store i16 0, ptr [[TMP20]], align 2386; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE16]]387; VF16UF1: [[PRED_STORE_CONTINUE16]]:388; VF16UF1-NEXT: [[TMP21:%.*]] = extractelement <16 x i1> [[TMP2]], i32 9389; VF16UF1-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]390; VF16UF1: [[PRED_STORE_IF17]]:391; VF16UF1-NEXT: [[TMP22:%.*]] = getelementptr i16, ptr [[DST]], i64 11392; VF16UF1-NEXT: store i16 0, ptr [[TMP22]], align 2393; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE18]]394; VF16UF1: [[PRED_STORE_CONTINUE18]]:395; VF16UF1-NEXT: [[TMP23:%.*]] = extractelement <16 x i1> [[TMP2]], i32 10396; VF16UF1-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]397; VF16UF1: [[PRED_STORE_IF19]]:398; VF16UF1-NEXT: [[TMP24:%.*]] = getelementptr i16, ptr [[DST]], i64 12399; VF16UF1-NEXT: store i16 0, ptr [[TMP24]], align 2400; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE20]]401; VF16UF1: [[PRED_STORE_CONTINUE20]]:402; VF16UF1-NEXT: [[TMP25:%.*]] = extractelement <16 x i1> [[TMP2]], i32 11403; VF16UF1-NEXT: br i1 [[TMP25]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]404; VF16UF1: [[PRED_STORE_IF21]]:405; VF16UF1-NEXT: [[TMP26:%.*]] = getelementptr i16, ptr [[DST]], i64 13406; VF16UF1-NEXT: store i16 0, ptr [[TMP26]], align 2407; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE22]]408; VF16UF1: [[PRED_STORE_CONTINUE22]]:409; VF16UF1-NEXT: [[TMP27:%.*]] = extractelement <16 x i1> [[TMP2]], i32 12410; VF16UF1-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]411; VF16UF1: [[PRED_STORE_IF23]]:412; VF16UF1-NEXT: [[TMP28:%.*]] = getelementptr i16, ptr [[DST]], i64 14413; VF16UF1-NEXT: store i16 0, ptr [[TMP28]], align 2414; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE24]]415; VF16UF1: [[PRED_STORE_CONTINUE24]]:416; VF16UF1-NEXT: [[TMP29:%.*]] = extractelement <16 x i1> [[TMP2]], i32 13417; VF16UF1-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]418; VF16UF1: [[PRED_STORE_IF25]]:419; VF16UF1-NEXT: [[TMP30:%.*]] = getelementptr i16, ptr [[DST]], i64 15420; VF16UF1-NEXT: store i16 0, ptr [[TMP30]], align 2421; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE26]]422; VF16UF1: [[PRED_STORE_CONTINUE26]]:423; VF16UF1-NEXT: [[TMP31:%.*]] = extractelement <16 x i1> [[TMP2]], i32 14424; VF16UF1-NEXT: br i1 [[TMP31]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]425; VF16UF1: [[PRED_STORE_IF27]]:426; VF16UF1-NEXT: [[TMP32:%.*]] = getelementptr i16, ptr [[DST]], i64 16427; VF16UF1-NEXT: store i16 0, ptr [[TMP32]], align 2428; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE28]]429; VF16UF1: [[PRED_STORE_CONTINUE28]]:430; VF16UF1-NEXT: [[TMP33:%.*]] = extractelement <16 x i1> [[TMP2]], i32 15431; VF16UF1-NEXT: br i1 [[TMP33]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]432; VF16UF1: [[PRED_STORE_IF29]]:433; VF16UF1-NEXT: [[TMP34:%.*]] = getelementptr i16, ptr [[DST]], i64 17434; VF16UF1-NEXT: store i16 0, ptr [[TMP34]], align 2435; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE30]]436; VF16UF1: [[PRED_STORE_CONTINUE30]]:437; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]438; VF16UF1: [[MIDDLE_BLOCK]]:439; VF16UF1-NEXT: br label %[[EXIT:.*]]440; VF16UF1: [[EXIT]]:441; VF16UF1-NEXT: ret void442;443entry:444 br label %loop445 446loop:447 %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]448 %gep.dst = getelementptr i16, ptr %dst, i64 %iv449 store i16 0, ptr %gep.dst, align 2450 %iv.next = add i64 %iv, 1451 %ec = icmp eq i64 %iv.next, %N452 br i1 %ec, label %exit, label %loop453 454exit:455 ret void456}457 458declare i1 @cond()459 460define void @remove_loop_region_outer_loop(i64 range(i64 8, 17) %N, ptr noalias %src, ptr %dst) {461; VF8UF1-LABEL: define void @remove_loop_region_outer_loop(462; VF8UF1-SAME: i64 range(i64 8, 17) [[N:%.*]], ptr noalias [[SRC:%.*]], ptr [[DST:%.*]]) {463; VF8UF1-NEXT: [[ENTRY:.*]]:464; VF8UF1-NEXT: br label %[[OUTER_HEADER:.*]]465; VF8UF1: [[OUTER_HEADER]]:466; VF8UF1-NEXT: [[OUTER_IV:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]467; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]468; VF8UF1: [[VECTOR_PH]]:469; VF8UF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8470; VF8UF1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]471; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]472; VF8UF1: [[VECTOR_BODY]]:473; VF8UF1-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]474; VF8UF1-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[OUTER_IV]], i64 [[TMP0]]475; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1476; VF8UF1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]477; VF8UF1-NEXT: store <8 x i8> [[WIDE_LOAD]], ptr [[TMP3]], align 1478; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 8479; VF8UF1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]480; VF8UF1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]481; VF8UF1: [[MIDDLE_BLOCK]]:482; VF8UF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]483; VF8UF1-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH:.*]]484; VF8UF1: [[SCALAR_PH]]:485; VF8UF1-NEXT: br label %[[INNER:.*]]486; VF8UF1: [[INNER]]:487; VF8UF1-NEXT: [[INNER_IV:%.*]] = phi i64 [ [[N_VEC]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]488; VF8UF1-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[OUTER_IV]], i64 [[INNER_IV]]489; VF8UF1-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1490; VF8UF1-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INNER_IV]]491; VF8UF1-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1492; VF8UF1-NEXT: [[IV_NEXT]] = add i64 [[INNER_IV]], 1493; VF8UF1-NEXT: [[C_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]494; VF8UF1-NEXT: br i1 [[C_1]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP5:![0-9]+]]495; VF8UF1: [[OUTER_LATCH]]:496; VF8UF1-NEXT: [[OUTER_IV_NEXT]] = getelementptr i8, ptr [[OUTER_IV]], i64 1497; VF8UF1-NEXT: [[C_2:%.*]] = call i1 @cond()498; VF8UF1-NEXT: br i1 [[C_2]], label %[[OUTER_HEADER]], label %[[EXIT:.*]]499; VF8UF1: [[EXIT]]:500; VF8UF1-NEXT: ret void501;502; VF8UF2-LABEL: define void @remove_loop_region_outer_loop(503; VF8UF2-SAME: i64 range(i64 8, 17) [[N:%.*]], ptr noalias [[SRC:%.*]], ptr [[DST:%.*]]) {504; VF8UF2-NEXT: [[ENTRY:.*]]:505; VF8UF2-NEXT: br label %[[OUTER_HEADER:.*]]506; VF8UF2: [[OUTER_HEADER]]:507; VF8UF2-NEXT: [[TMP0:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]508; VF8UF2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16509; VF8UF2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]510; VF8UF2: [[VECTOR_PH]]:511; VF8UF2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16512; VF8UF2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]513; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]514; VF8UF2: [[VECTOR_BODY]]:515; VF8UF2-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8516; VF8UF2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1517; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1518; VF8UF2-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[DST]], i64 8519; VF8UF2-NEXT: store <8 x i8> [[WIDE_LOAD]], ptr [[DST]], align 1520; VF8UF2-NEXT: store <8 x i8> [[WIDE_LOAD1]], ptr [[TMP5]], align 1521; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]522; VF8UF2: [[MIDDLE_BLOCK]]:523; VF8UF2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]524; VF8UF2-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]525; VF8UF2: [[SCALAR_PH]]:526; VF8UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_HEADER]] ]527; VF8UF2-NEXT: br label %[[INNER:.*]]528; VF8UF2: [[INNER]]:529; VF8UF2-NEXT: [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]530; VF8UF2-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[INNER_IV]]531; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1532; VF8UF2-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INNER_IV]]533; VF8UF2-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1534; VF8UF2-NEXT: [[IV_NEXT]] = add i64 [[INNER_IV]], 1535; VF8UF2-NEXT: [[C_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]536; VF8UF2-NEXT: br i1 [[C_1]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP0:![0-9]+]]537; VF8UF2: [[OUTER_LATCH]]:538; VF8UF2-NEXT: [[OUTER_IV_NEXT]] = getelementptr i8, ptr [[TMP0]], i64 1539; VF8UF2-NEXT: [[C_2:%.*]] = call i1 @cond()540; VF8UF2-NEXT: br i1 [[C_2]], label %[[OUTER_HEADER]], label %[[EXIT:.*]]541; VF8UF2: [[EXIT]]:542; VF8UF2-NEXT: ret void543;544; VF16UF1-LABEL: define void @remove_loop_region_outer_loop(545; VF16UF1-SAME: i64 range(i64 8, 17) [[N:%.*]], ptr noalias [[SRC:%.*]], ptr [[DST:%.*]]) {546; VF16UF1-NEXT: [[ENTRY:.*]]:547; VF16UF1-NEXT: br label %[[OUTER_HEADER:.*]]548; VF16UF1: [[OUTER_HEADER]]:549; VF16UF1-NEXT: [[TMP1:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]550; VF16UF1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16551; VF16UF1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]552; VF16UF1: [[VECTOR_PH]]:553; VF16UF1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16554; VF16UF1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]555; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]556; VF16UF1: [[VECTOR_BODY]]:557; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1558; VF16UF1-NEXT: store <16 x i8> [[WIDE_LOAD]], ptr [[DST]], align 1559; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]560; VF16UF1: [[MIDDLE_BLOCK]]:561; VF16UF1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]562; VF16UF1-NEXT: br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]563; VF16UF1: [[SCALAR_PH]]:564; VF16UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[OUTER_HEADER]] ]565; VF16UF1-NEXT: br label %[[INNER:.*]]566; VF16UF1: [[INNER]]:567; VF16UF1-NEXT: [[INNER_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]568; VF16UF1-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[TMP1]], i64 [[INNER_IV]]569; VF16UF1-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1570; VF16UF1-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INNER_IV]]571; VF16UF1-NEXT: store i8 [[L]], ptr [[GEP_DST]], align 1572; VF16UF1-NEXT: [[IV_NEXT]] = add i64 [[INNER_IV]], 1573; VF16UF1-NEXT: [[C_1:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]574; VF16UF1-NEXT: br i1 [[C_1]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP0:![0-9]+]]575; VF16UF1: [[OUTER_LATCH]]:576; VF16UF1-NEXT: [[OUTER_IV_NEXT]] = getelementptr i8, ptr [[TMP1]], i64 1577; VF16UF1-NEXT: [[C_2:%.*]] = call i1 @cond()578; VF16UF1-NEXT: br i1 [[C_2]], label %[[OUTER_HEADER]], label %[[EXIT:.*]]579; VF16UF1: [[EXIT]]:580; VF16UF1-NEXT: ret void581;582entry:583 br label %outer.header584 585outer.header:586 %outer.iv = phi ptr [ %src, %entry ], [ %outer.iv.next, %outer.latch ]587 br label %inner588 589inner:590 %inner.iv = phi i64 [ 0, %outer.header ], [ %iv.next, %inner ]591 %gep.src = getelementptr i8, ptr %outer.iv, i64 %inner.iv592 %l = load i8, ptr %gep.src, align 1593 %gep.dst = getelementptr i8, ptr %dst, i64 %inner.iv594 store i8 %l, ptr %gep.dst, align 1595 %iv.next = add i64 %inner.iv, 1596 %c.1 = icmp eq i64 %iv.next, %N597 br i1 %c.1, label %outer.latch, label %inner598 599outer.latch:600 %outer.iv.next = getelementptr i8, ptr %outer.iv, i64 1601 %c.2 = call i1 @cond()602 br i1 %c.2, label %outer.header, label %exit603 604exit:605 ret void606}607 608declare void @llvm.assume(i1)609 610; Test case for https://github.com/llvm/llvm-project/issues/121897.611define void @scev_expand_step(i64 %x, ptr %dst) {612; VF8UF1-LABEL: define void @scev_expand_step(613; VF8UF1-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {614; VF8UF1-NEXT: [[ENTRY:.*:]]615; VF8UF1-NEXT: [[C:%.*]] = icmp eq i64 [[X]], 65536616; VF8UF1-NEXT: call void @llvm.assume(i1 [[C]])617; VF8UF1-NEXT: [[FR:%.*]] = freeze i64 [[X]]618; VF8UF1-NEXT: [[STEP:%.*]] = add i64 [[FR]], -65534619; VF8UF1-NEXT: [[TMP0:%.*]] = udiv i64 15, [[STEP]]620; VF8UF1-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1621; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]622; VF8UF1: [[VECTOR_PH]]:623; VF8UF1-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1624; VF8UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0625; VF8UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer626; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]627; VF8UF1: [[VECTOR_BODY]]:628; VF8UF1-NEXT: [[TMP3:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT]]629; VF8UF1-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i32 0630; VF8UF1-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]631; VF8UF1: [[PRED_STORE_IF]]:632; VF8UF1-NEXT: [[TMP5:%.*]] = mul i64 0, [[STEP]]633; VF8UF1-NEXT: [[TMP6:%.*]] = add i64 0, [[TMP5]]634; VF8UF1-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], [[STEP]]635; VF8UF1-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]636; VF8UF1-NEXT: store i8 0, ptr [[TMP8]], align 1637; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE]]638; VF8UF1: [[PRED_STORE_CONTINUE]]:639; VF8UF1-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1640; VF8UF1-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]641; VF8UF1: [[PRED_STORE_IF1]]:642; VF8UF1-NEXT: [[TMP10:%.*]] = mul i64 1, [[STEP]]643; VF8UF1-NEXT: [[TMP11:%.*]] = add i64 0, [[TMP10]]644; VF8UF1-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], [[STEP]]645; VF8UF1-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]646; VF8UF1-NEXT: store i8 0, ptr [[TMP13]], align 1647; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE2]]648; VF8UF1: [[PRED_STORE_CONTINUE2]]:649; VF8UF1-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2650; VF8UF1-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]651; VF8UF1: [[PRED_STORE_IF3]]:652; VF8UF1-NEXT: [[TMP15:%.*]] = mul i64 2, [[STEP]]653; VF8UF1-NEXT: [[TMP16:%.*]] = add i64 0, [[TMP15]]654; VF8UF1-NEXT: [[TMP17:%.*]] = add i64 [[TMP16]], [[STEP]]655; VF8UF1-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP17]]656; VF8UF1-NEXT: store i8 0, ptr [[TMP18]], align 1657; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE4]]658; VF8UF1: [[PRED_STORE_CONTINUE4]]:659; VF8UF1-NEXT: [[TMP19:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3660; VF8UF1-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]661; VF8UF1: [[PRED_STORE_IF5]]:662; VF8UF1-NEXT: [[TMP20:%.*]] = mul i64 3, [[STEP]]663; VF8UF1-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]664; VF8UF1-NEXT: [[TMP22:%.*]] = add i64 [[TMP21]], [[STEP]]665; VF8UF1-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP22]]666; VF8UF1-NEXT: store i8 0, ptr [[TMP23]], align 1667; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE6]]668; VF8UF1: [[PRED_STORE_CONTINUE6]]:669; VF8UF1-NEXT: [[TMP24:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4670; VF8UF1-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]671; VF8UF1: [[PRED_STORE_IF7]]:672; VF8UF1-NEXT: [[TMP25:%.*]] = mul i64 4, [[STEP]]673; VF8UF1-NEXT: [[TMP26:%.*]] = add i64 0, [[TMP25]]674; VF8UF1-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[STEP]]675; VF8UF1-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP27]]676; VF8UF1-NEXT: store i8 0, ptr [[TMP28]], align 1677; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE8]]678; VF8UF1: [[PRED_STORE_CONTINUE8]]:679; VF8UF1-NEXT: [[TMP29:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5680; VF8UF1-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]681; VF8UF1: [[PRED_STORE_IF9]]:682; VF8UF1-NEXT: [[TMP30:%.*]] = mul i64 5, [[STEP]]683; VF8UF1-NEXT: [[TMP31:%.*]] = add i64 0, [[TMP30]]684; VF8UF1-NEXT: [[TMP32:%.*]] = add i64 [[TMP31]], [[STEP]]685; VF8UF1-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP32]]686; VF8UF1-NEXT: store i8 0, ptr [[TMP33]], align 1687; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE10]]688; VF8UF1: [[PRED_STORE_CONTINUE10]]:689; VF8UF1-NEXT: [[TMP34:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6690; VF8UF1-NEXT: br i1 [[TMP34]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]691; VF8UF1: [[PRED_STORE_IF11]]:692; VF8UF1-NEXT: [[TMP35:%.*]] = mul i64 6, [[STEP]]693; VF8UF1-NEXT: [[TMP36:%.*]] = add i64 0, [[TMP35]]694; VF8UF1-NEXT: [[TMP37:%.*]] = add i64 [[TMP36]], [[STEP]]695; VF8UF1-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP37]]696; VF8UF1-NEXT: store i8 0, ptr [[TMP38]], align 1697; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE12]]698; VF8UF1: [[PRED_STORE_CONTINUE12]]:699; VF8UF1-NEXT: [[TMP39:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7700; VF8UF1-NEXT: br i1 [[TMP39]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]701; VF8UF1: [[PRED_STORE_IF13]]:702; VF8UF1-NEXT: [[TMP40:%.*]] = mul i64 7, [[STEP]]703; VF8UF1-NEXT: [[TMP41:%.*]] = add i64 0, [[TMP40]]704; VF8UF1-NEXT: [[IV_NEXT:%.*]] = add i64 [[TMP41]], [[STEP]]705; VF8UF1-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_NEXT]]706; VF8UF1-NEXT: store i8 0, ptr [[GEP_DST]], align 1707; VF8UF1-NEXT: br label %[[PRED_STORE_CONTINUE14]]708; VF8UF1: [[PRED_STORE_CONTINUE14]]:709; VF8UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]710; VF8UF1: [[MIDDLE_BLOCK]]:711; VF8UF1-NEXT: br label %[[EXIT:.*]]712; VF8UF1: [[EXIT]]:713; VF8UF1-NEXT: ret void714;715; VF8UF2-LABEL: define void @scev_expand_step(716; VF8UF2-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {717; VF8UF2-NEXT: [[ENTRY:.*:]]718; VF8UF2-NEXT: [[C:%.*]] = icmp eq i64 [[X]], 65536719; VF8UF2-NEXT: call void @llvm.assume(i1 [[C]])720; VF8UF2-NEXT: [[FR:%.*]] = freeze i64 [[X]]721; VF8UF2-NEXT: [[STEP:%.*]] = add i64 [[FR]], -65534722; VF8UF2-NEXT: [[TMP0:%.*]] = udiv i64 15, [[STEP]]723; VF8UF2-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1724; VF8UF2-NEXT: br label %[[VECTOR_PH:.*]]725; VF8UF2: [[VECTOR_PH]]:726; VF8UF2-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1727; VF8UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0728; VF8UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer729; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]730; VF8UF2: [[VECTOR_BODY]]:731; VF8UF2-NEXT: [[TMP3:%.*]] = icmp ule <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[BROADCAST_SPLAT]]732; VF8UF2-NEXT: [[TMP4:%.*]] = icmp ule <8 x i64> <i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[BROADCAST_SPLAT]]733; VF8UF2-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP3]], i32 0734; VF8UF2-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]735; VF8UF2: [[PRED_STORE_IF]]:736; VF8UF2-NEXT: [[TMP6:%.*]] = mul i64 0, [[STEP]]737; VF8UF2-NEXT: [[TMP7:%.*]] = add i64 0, [[TMP6]]738; VF8UF2-NEXT: [[TMP8:%.*]] = add i64 [[TMP7]], [[STEP]]739; VF8UF2-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]740; VF8UF2-NEXT: store i8 0, ptr [[TMP9]], align 1741; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE]]742; VF8UF2: [[PRED_STORE_CONTINUE]]:743; VF8UF2-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP3]], i32 1744; VF8UF2-NEXT: br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]745; VF8UF2: [[PRED_STORE_IF1]]:746; VF8UF2-NEXT: [[TMP11:%.*]] = mul i64 1, [[STEP]]747; VF8UF2-NEXT: [[TMP12:%.*]] = add i64 0, [[TMP11]]748; VF8UF2-NEXT: [[TMP13:%.*]] = add i64 [[TMP12]], [[STEP]]749; VF8UF2-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]]750; VF8UF2-NEXT: store i8 0, ptr [[TMP14]], align 1751; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE2]]752; VF8UF2: [[PRED_STORE_CONTINUE2]]:753; VF8UF2-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP3]], i32 2754; VF8UF2-NEXT: br i1 [[TMP15]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]755; VF8UF2: [[PRED_STORE_IF3]]:756; VF8UF2-NEXT: [[TMP16:%.*]] = mul i64 2, [[STEP]]757; VF8UF2-NEXT: [[TMP17:%.*]] = add i64 0, [[TMP16]]758; VF8UF2-NEXT: [[TMP18:%.*]] = add i64 [[TMP17]], [[STEP]]759; VF8UF2-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP18]]760; VF8UF2-NEXT: store i8 0, ptr [[TMP19]], align 1761; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE4]]762; VF8UF2: [[PRED_STORE_CONTINUE4]]:763; VF8UF2-NEXT: [[TMP20:%.*]] = extractelement <8 x i1> [[TMP3]], i32 3764; VF8UF2-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]765; VF8UF2: [[PRED_STORE_IF5]]:766; VF8UF2-NEXT: [[TMP21:%.*]] = mul i64 3, [[STEP]]767; VF8UF2-NEXT: [[TMP22:%.*]] = add i64 0, [[TMP21]]768; VF8UF2-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[STEP]]769; VF8UF2-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP23]]770; VF8UF2-NEXT: store i8 0, ptr [[TMP24]], align 1771; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE6]]772; VF8UF2: [[PRED_STORE_CONTINUE6]]:773; VF8UF2-NEXT: [[TMP25:%.*]] = extractelement <8 x i1> [[TMP3]], i32 4774; VF8UF2-NEXT: br i1 [[TMP25]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]775; VF8UF2: [[PRED_STORE_IF7]]:776; VF8UF2-NEXT: [[TMP26:%.*]] = mul i64 4, [[STEP]]777; VF8UF2-NEXT: [[TMP27:%.*]] = add i64 0, [[TMP26]]778; VF8UF2-NEXT: [[TMP28:%.*]] = add i64 [[TMP27]], [[STEP]]779; VF8UF2-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP28]]780; VF8UF2-NEXT: store i8 0, ptr [[TMP29]], align 1781; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE8]]782; VF8UF2: [[PRED_STORE_CONTINUE8]]:783; VF8UF2-NEXT: [[TMP30:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5784; VF8UF2-NEXT: br i1 [[TMP30]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]785; VF8UF2: [[PRED_STORE_IF9]]:786; VF8UF2-NEXT: [[TMP31:%.*]] = mul i64 5, [[STEP]]787; VF8UF2-NEXT: [[TMP32:%.*]] = add i64 0, [[TMP31]]788; VF8UF2-NEXT: [[TMP33:%.*]] = add i64 [[TMP32]], [[STEP]]789; VF8UF2-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP33]]790; VF8UF2-NEXT: store i8 0, ptr [[TMP34]], align 1791; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE10]]792; VF8UF2: [[PRED_STORE_CONTINUE10]]:793; VF8UF2-NEXT: [[TMP35:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6794; VF8UF2-NEXT: br i1 [[TMP35]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]795; VF8UF2: [[PRED_STORE_IF11]]:796; VF8UF2-NEXT: [[TMP36:%.*]] = mul i64 6, [[STEP]]797; VF8UF2-NEXT: [[TMP37:%.*]] = add i64 0, [[TMP36]]798; VF8UF2-NEXT: [[TMP38:%.*]] = add i64 [[TMP37]], [[STEP]]799; VF8UF2-NEXT: [[TMP39:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP38]]800; VF8UF2-NEXT: store i8 0, ptr [[TMP39]], align 1801; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE12]]802; VF8UF2: [[PRED_STORE_CONTINUE12]]:803; VF8UF2-NEXT: [[TMP40:%.*]] = extractelement <8 x i1> [[TMP3]], i32 7804; VF8UF2-NEXT: br i1 [[TMP40]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]805; VF8UF2: [[PRED_STORE_IF13]]:806; VF8UF2-NEXT: [[TMP41:%.*]] = mul i64 7, [[STEP]]807; VF8UF2-NEXT: [[TMP42:%.*]] = add i64 0, [[TMP41]]808; VF8UF2-NEXT: [[TMP43:%.*]] = add i64 [[TMP42]], [[STEP]]809; VF8UF2-NEXT: [[TMP44:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP43]]810; VF8UF2-NEXT: store i8 0, ptr [[TMP44]], align 1811; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE14]]812; VF8UF2: [[PRED_STORE_CONTINUE14]]:813; VF8UF2-NEXT: [[TMP45:%.*]] = extractelement <8 x i1> [[TMP4]], i32 0814; VF8UF2-NEXT: br i1 [[TMP45]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]815; VF8UF2: [[PRED_STORE_IF15]]:816; VF8UF2-NEXT: [[TMP46:%.*]] = mul i64 8, [[STEP]]817; VF8UF2-NEXT: [[TMP47:%.*]] = add i64 0, [[TMP46]]818; VF8UF2-NEXT: [[TMP48:%.*]] = add i64 [[TMP47]], [[STEP]]819; VF8UF2-NEXT: [[TMP49:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP48]]820; VF8UF2-NEXT: store i8 0, ptr [[TMP49]], align 1821; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE16]]822; VF8UF2: [[PRED_STORE_CONTINUE16]]:823; VF8UF2-NEXT: [[TMP50:%.*]] = extractelement <8 x i1> [[TMP4]], i32 1824; VF8UF2-NEXT: br i1 [[TMP50]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]825; VF8UF2: [[PRED_STORE_IF17]]:826; VF8UF2-NEXT: [[TMP51:%.*]] = mul i64 9, [[STEP]]827; VF8UF2-NEXT: [[TMP52:%.*]] = add i64 0, [[TMP51]]828; VF8UF2-NEXT: [[TMP53:%.*]] = add i64 [[TMP52]], [[STEP]]829; VF8UF2-NEXT: [[TMP54:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP53]]830; VF8UF2-NEXT: store i8 0, ptr [[TMP54]], align 1831; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE18]]832; VF8UF2: [[PRED_STORE_CONTINUE18]]:833; VF8UF2-NEXT: [[TMP55:%.*]] = extractelement <8 x i1> [[TMP4]], i32 2834; VF8UF2-NEXT: br i1 [[TMP55]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]835; VF8UF2: [[PRED_STORE_IF19]]:836; VF8UF2-NEXT: [[TMP56:%.*]] = mul i64 10, [[STEP]]837; VF8UF2-NEXT: [[TMP57:%.*]] = add i64 0, [[TMP56]]838; VF8UF2-NEXT: [[TMP58:%.*]] = add i64 [[TMP57]], [[STEP]]839; VF8UF2-NEXT: [[TMP59:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP58]]840; VF8UF2-NEXT: store i8 0, ptr [[TMP59]], align 1841; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE20]]842; VF8UF2: [[PRED_STORE_CONTINUE20]]:843; VF8UF2-NEXT: [[TMP60:%.*]] = extractelement <8 x i1> [[TMP4]], i32 3844; VF8UF2-NEXT: br i1 [[TMP60]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]845; VF8UF2: [[PRED_STORE_IF21]]:846; VF8UF2-NEXT: [[TMP61:%.*]] = mul i64 11, [[STEP]]847; VF8UF2-NEXT: [[TMP62:%.*]] = add i64 0, [[TMP61]]848; VF8UF2-NEXT: [[TMP63:%.*]] = add i64 [[TMP62]], [[STEP]]849; VF8UF2-NEXT: [[TMP64:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP63]]850; VF8UF2-NEXT: store i8 0, ptr [[TMP64]], align 1851; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE22]]852; VF8UF2: [[PRED_STORE_CONTINUE22]]:853; VF8UF2-NEXT: [[TMP65:%.*]] = extractelement <8 x i1> [[TMP4]], i32 4854; VF8UF2-NEXT: br i1 [[TMP65]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]855; VF8UF2: [[PRED_STORE_IF23]]:856; VF8UF2-NEXT: [[TMP66:%.*]] = mul i64 12, [[STEP]]857; VF8UF2-NEXT: [[TMP67:%.*]] = add i64 0, [[TMP66]]858; VF8UF2-NEXT: [[TMP68:%.*]] = add i64 [[TMP67]], [[STEP]]859; VF8UF2-NEXT: [[TMP69:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP68]]860; VF8UF2-NEXT: store i8 0, ptr [[TMP69]], align 1861; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE24]]862; VF8UF2: [[PRED_STORE_CONTINUE24]]:863; VF8UF2-NEXT: [[TMP70:%.*]] = extractelement <8 x i1> [[TMP4]], i32 5864; VF8UF2-NEXT: br i1 [[TMP70]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]865; VF8UF2: [[PRED_STORE_IF25]]:866; VF8UF2-NEXT: [[TMP71:%.*]] = mul i64 13, [[STEP]]867; VF8UF2-NEXT: [[TMP72:%.*]] = add i64 0, [[TMP71]]868; VF8UF2-NEXT: [[TMP73:%.*]] = add i64 [[TMP72]], [[STEP]]869; VF8UF2-NEXT: [[TMP74:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP73]]870; VF8UF2-NEXT: store i8 0, ptr [[TMP74]], align 1871; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE26]]872; VF8UF2: [[PRED_STORE_CONTINUE26]]:873; VF8UF2-NEXT: [[TMP75:%.*]] = extractelement <8 x i1> [[TMP4]], i32 6874; VF8UF2-NEXT: br i1 [[TMP75]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]875; VF8UF2: [[PRED_STORE_IF27]]:876; VF8UF2-NEXT: [[TMP76:%.*]] = mul i64 14, [[STEP]]877; VF8UF2-NEXT: [[TMP77:%.*]] = add i64 0, [[TMP76]]878; VF8UF2-NEXT: [[TMP78:%.*]] = add i64 [[TMP77]], [[STEP]]879; VF8UF2-NEXT: [[TMP79:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP78]]880; VF8UF2-NEXT: store i8 0, ptr [[TMP79]], align 1881; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE28]]882; VF8UF2: [[PRED_STORE_CONTINUE28]]:883; VF8UF2-NEXT: [[TMP80:%.*]] = extractelement <8 x i1> [[TMP4]], i32 7884; VF8UF2-NEXT: br i1 [[TMP80]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]885; VF8UF2: [[PRED_STORE_IF29]]:886; VF8UF2-NEXT: [[TMP81:%.*]] = mul i64 15, [[STEP]]887; VF8UF2-NEXT: [[TMP82:%.*]] = add i64 0, [[TMP81]]888; VF8UF2-NEXT: [[TMP83:%.*]] = add i64 [[TMP82]], [[STEP]]889; VF8UF2-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP83]]890; VF8UF2-NEXT: store i8 0, ptr [[GEP_DST]], align 1891; VF8UF2-NEXT: br label %[[PRED_STORE_CONTINUE30]]892; VF8UF2: [[PRED_STORE_CONTINUE30]]:893; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]894; VF8UF2: [[MIDDLE_BLOCK]]:895; VF8UF2-NEXT: br label %[[EXIT:.*]]896; VF8UF2: [[EXIT]]:897; VF8UF2-NEXT: ret void898;899; VF16UF1-LABEL: define void @scev_expand_step(900; VF16UF1-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {901; VF16UF1-NEXT: [[ENTRY:.*:]]902; VF16UF1-NEXT: [[C:%.*]] = icmp eq i64 [[X]], 65536903; VF16UF1-NEXT: call void @llvm.assume(i1 [[C]])904; VF16UF1-NEXT: [[FR:%.*]] = freeze i64 [[X]]905; VF16UF1-NEXT: [[STEP:%.*]] = add i64 [[FR]], -65534906; VF16UF1-NEXT: [[TMP0:%.*]] = udiv i64 15, [[STEP]]907; VF16UF1-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1908; VF16UF1-NEXT: br label %[[VECTOR_PH:.*]]909; VF16UF1: [[VECTOR_PH]]:910; VF16UF1-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1911; VF16UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0912; VF16UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer913; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]914; VF16UF1: [[VECTOR_BODY]]:915; VF16UF1-NEXT: [[TMP3:%.*]] = icmp ule <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, [[BROADCAST_SPLAT]]916; VF16UF1-NEXT: [[TMP4:%.*]] = extractelement <16 x i1> [[TMP3]], i32 0917; VF16UF1-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]918; VF16UF1: [[PRED_STORE_IF]]:919; VF16UF1-NEXT: [[TMP5:%.*]] = mul i64 0, [[STEP]]920; VF16UF1-NEXT: [[TMP6:%.*]] = add i64 0, [[TMP5]]921; VF16UF1-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], [[STEP]]922; VF16UF1-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]923; VF16UF1-NEXT: store i8 0, ptr [[TMP8]], align 1924; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE]]925; VF16UF1: [[PRED_STORE_CONTINUE]]:926; VF16UF1-NEXT: [[TMP9:%.*]] = extractelement <16 x i1> [[TMP3]], i32 1927; VF16UF1-NEXT: br i1 [[TMP9]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]928; VF16UF1: [[PRED_STORE_IF1]]:929; VF16UF1-NEXT: [[TMP10:%.*]] = mul i64 1, [[STEP]]930; VF16UF1-NEXT: [[TMP11:%.*]] = add i64 0, [[TMP10]]931; VF16UF1-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], [[STEP]]932; VF16UF1-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP12]]933; VF16UF1-NEXT: store i8 0, ptr [[TMP13]], align 1934; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE2]]935; VF16UF1: [[PRED_STORE_CONTINUE2]]:936; VF16UF1-NEXT: [[TMP14:%.*]] = extractelement <16 x i1> [[TMP3]], i32 2937; VF16UF1-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]938; VF16UF1: [[PRED_STORE_IF3]]:939; VF16UF1-NEXT: [[TMP15:%.*]] = mul i64 2, [[STEP]]940; VF16UF1-NEXT: [[TMP16:%.*]] = add i64 0, [[TMP15]]941; VF16UF1-NEXT: [[TMP17:%.*]] = add i64 [[TMP16]], [[STEP]]942; VF16UF1-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP17]]943; VF16UF1-NEXT: store i8 0, ptr [[TMP18]], align 1944; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE4]]945; VF16UF1: [[PRED_STORE_CONTINUE4]]:946; VF16UF1-NEXT: [[TMP19:%.*]] = extractelement <16 x i1> [[TMP3]], i32 3947; VF16UF1-NEXT: br i1 [[TMP19]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]948; VF16UF1: [[PRED_STORE_IF5]]:949; VF16UF1-NEXT: [[TMP20:%.*]] = mul i64 3, [[STEP]]950; VF16UF1-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]951; VF16UF1-NEXT: [[TMP22:%.*]] = add i64 [[TMP21]], [[STEP]]952; VF16UF1-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP22]]953; VF16UF1-NEXT: store i8 0, ptr [[TMP23]], align 1954; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE6]]955; VF16UF1: [[PRED_STORE_CONTINUE6]]:956; VF16UF1-NEXT: [[TMP24:%.*]] = extractelement <16 x i1> [[TMP3]], i32 4957; VF16UF1-NEXT: br i1 [[TMP24]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]958; VF16UF1: [[PRED_STORE_IF7]]:959; VF16UF1-NEXT: [[TMP25:%.*]] = mul i64 4, [[STEP]]960; VF16UF1-NEXT: [[TMP26:%.*]] = add i64 0, [[TMP25]]961; VF16UF1-NEXT: [[TMP27:%.*]] = add i64 [[TMP26]], [[STEP]]962; VF16UF1-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP27]]963; VF16UF1-NEXT: store i8 0, ptr [[TMP28]], align 1964; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE8]]965; VF16UF1: [[PRED_STORE_CONTINUE8]]:966; VF16UF1-NEXT: [[TMP29:%.*]] = extractelement <16 x i1> [[TMP3]], i32 5967; VF16UF1-NEXT: br i1 [[TMP29]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]968; VF16UF1: [[PRED_STORE_IF9]]:969; VF16UF1-NEXT: [[TMP30:%.*]] = mul i64 5, [[STEP]]970; VF16UF1-NEXT: [[TMP31:%.*]] = add i64 0, [[TMP30]]971; VF16UF1-NEXT: [[TMP32:%.*]] = add i64 [[TMP31]], [[STEP]]972; VF16UF1-NEXT: [[TMP33:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP32]]973; VF16UF1-NEXT: store i8 0, ptr [[TMP33]], align 1974; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE10]]975; VF16UF1: [[PRED_STORE_CONTINUE10]]:976; VF16UF1-NEXT: [[TMP34:%.*]] = extractelement <16 x i1> [[TMP3]], i32 6977; VF16UF1-NEXT: br i1 [[TMP34]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]978; VF16UF1: [[PRED_STORE_IF11]]:979; VF16UF1-NEXT: [[TMP35:%.*]] = mul i64 6, [[STEP]]980; VF16UF1-NEXT: [[TMP36:%.*]] = add i64 0, [[TMP35]]981; VF16UF1-NEXT: [[TMP37:%.*]] = add i64 [[TMP36]], [[STEP]]982; VF16UF1-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP37]]983; VF16UF1-NEXT: store i8 0, ptr [[TMP38]], align 1984; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE12]]985; VF16UF1: [[PRED_STORE_CONTINUE12]]:986; VF16UF1-NEXT: [[TMP39:%.*]] = extractelement <16 x i1> [[TMP3]], i32 7987; VF16UF1-NEXT: br i1 [[TMP39]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]988; VF16UF1: [[PRED_STORE_IF13]]:989; VF16UF1-NEXT: [[TMP40:%.*]] = mul i64 7, [[STEP]]990; VF16UF1-NEXT: [[TMP41:%.*]] = add i64 0, [[TMP40]]991; VF16UF1-NEXT: [[TMP42:%.*]] = add i64 [[TMP41]], [[STEP]]992; VF16UF1-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP42]]993; VF16UF1-NEXT: store i8 0, ptr [[TMP43]], align 1994; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE14]]995; VF16UF1: [[PRED_STORE_CONTINUE14]]:996; VF16UF1-NEXT: [[TMP44:%.*]] = extractelement <16 x i1> [[TMP3]], i32 8997; VF16UF1-NEXT: br i1 [[TMP44]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]998; VF16UF1: [[PRED_STORE_IF15]]:999; VF16UF1-NEXT: [[TMP45:%.*]] = mul i64 8, [[STEP]]1000; VF16UF1-NEXT: [[TMP46:%.*]] = add i64 0, [[TMP45]]1001; VF16UF1-NEXT: [[TMP47:%.*]] = add i64 [[TMP46]], [[STEP]]1002; VF16UF1-NEXT: [[TMP48:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP47]]1003; VF16UF1-NEXT: store i8 0, ptr [[TMP48]], align 11004; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE16]]1005; VF16UF1: [[PRED_STORE_CONTINUE16]]:1006; VF16UF1-NEXT: [[TMP49:%.*]] = extractelement <16 x i1> [[TMP3]], i32 91007; VF16UF1-NEXT: br i1 [[TMP49]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]1008; VF16UF1: [[PRED_STORE_IF17]]:1009; VF16UF1-NEXT: [[TMP50:%.*]] = mul i64 9, [[STEP]]1010; VF16UF1-NEXT: [[TMP51:%.*]] = add i64 0, [[TMP50]]1011; VF16UF1-NEXT: [[TMP52:%.*]] = add i64 [[TMP51]], [[STEP]]1012; VF16UF1-NEXT: [[TMP53:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP52]]1013; VF16UF1-NEXT: store i8 0, ptr [[TMP53]], align 11014; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE18]]1015; VF16UF1: [[PRED_STORE_CONTINUE18]]:1016; VF16UF1-NEXT: [[TMP54:%.*]] = extractelement <16 x i1> [[TMP3]], i32 101017; VF16UF1-NEXT: br i1 [[TMP54]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]1018; VF16UF1: [[PRED_STORE_IF19]]:1019; VF16UF1-NEXT: [[TMP55:%.*]] = mul i64 10, [[STEP]]1020; VF16UF1-NEXT: [[TMP56:%.*]] = add i64 0, [[TMP55]]1021; VF16UF1-NEXT: [[TMP57:%.*]] = add i64 [[TMP56]], [[STEP]]1022; VF16UF1-NEXT: [[TMP58:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP57]]1023; VF16UF1-NEXT: store i8 0, ptr [[TMP58]], align 11024; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE20]]1025; VF16UF1: [[PRED_STORE_CONTINUE20]]:1026; VF16UF1-NEXT: [[TMP59:%.*]] = extractelement <16 x i1> [[TMP3]], i32 111027; VF16UF1-NEXT: br i1 [[TMP59]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22:.*]]1028; VF16UF1: [[PRED_STORE_IF21]]:1029; VF16UF1-NEXT: [[TMP60:%.*]] = mul i64 11, [[STEP]]1030; VF16UF1-NEXT: [[TMP61:%.*]] = add i64 0, [[TMP60]]1031; VF16UF1-NEXT: [[TMP62:%.*]] = add i64 [[TMP61]], [[STEP]]1032; VF16UF1-NEXT: [[TMP63:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP62]]1033; VF16UF1-NEXT: store i8 0, ptr [[TMP63]], align 11034; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE22]]1035; VF16UF1: [[PRED_STORE_CONTINUE22]]:1036; VF16UF1-NEXT: [[TMP64:%.*]] = extractelement <16 x i1> [[TMP3]], i32 121037; VF16UF1-NEXT: br i1 [[TMP64]], label %[[PRED_STORE_IF23:.*]], label %[[PRED_STORE_CONTINUE24:.*]]1038; VF16UF1: [[PRED_STORE_IF23]]:1039; VF16UF1-NEXT: [[TMP65:%.*]] = mul i64 12, [[STEP]]1040; VF16UF1-NEXT: [[TMP66:%.*]] = add i64 0, [[TMP65]]1041; VF16UF1-NEXT: [[TMP67:%.*]] = add i64 [[TMP66]], [[STEP]]1042; VF16UF1-NEXT: [[TMP68:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP67]]1043; VF16UF1-NEXT: store i8 0, ptr [[TMP68]], align 11044; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE24]]1045; VF16UF1: [[PRED_STORE_CONTINUE24]]:1046; VF16UF1-NEXT: [[TMP69:%.*]] = extractelement <16 x i1> [[TMP3]], i32 131047; VF16UF1-NEXT: br i1 [[TMP69]], label %[[PRED_STORE_IF25:.*]], label %[[PRED_STORE_CONTINUE26:.*]]1048; VF16UF1: [[PRED_STORE_IF25]]:1049; VF16UF1-NEXT: [[TMP70:%.*]] = mul i64 13, [[STEP]]1050; VF16UF1-NEXT: [[TMP71:%.*]] = add i64 0, [[TMP70]]1051; VF16UF1-NEXT: [[TMP72:%.*]] = add i64 [[TMP71]], [[STEP]]1052; VF16UF1-NEXT: [[TMP73:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP72]]1053; VF16UF1-NEXT: store i8 0, ptr [[TMP73]], align 11054; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE26]]1055; VF16UF1: [[PRED_STORE_CONTINUE26]]:1056; VF16UF1-NEXT: [[TMP74:%.*]] = extractelement <16 x i1> [[TMP3]], i32 141057; VF16UF1-NEXT: br i1 [[TMP74]], label %[[PRED_STORE_IF27:.*]], label %[[PRED_STORE_CONTINUE28:.*]]1058; VF16UF1: [[PRED_STORE_IF27]]:1059; VF16UF1-NEXT: [[TMP75:%.*]] = mul i64 14, [[STEP]]1060; VF16UF1-NEXT: [[TMP76:%.*]] = add i64 0, [[TMP75]]1061; VF16UF1-NEXT: [[TMP77:%.*]] = add i64 [[TMP76]], [[STEP]]1062; VF16UF1-NEXT: [[TMP78:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP77]]1063; VF16UF1-NEXT: store i8 0, ptr [[TMP78]], align 11064; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE28]]1065; VF16UF1: [[PRED_STORE_CONTINUE28]]:1066; VF16UF1-NEXT: [[TMP79:%.*]] = extractelement <16 x i1> [[TMP3]], i32 151067; VF16UF1-NEXT: br i1 [[TMP79]], label %[[PRED_STORE_IF29:.*]], label %[[PRED_STORE_CONTINUE30:.*]]1068; VF16UF1: [[PRED_STORE_IF29]]:1069; VF16UF1-NEXT: [[TMP80:%.*]] = mul i64 15, [[STEP]]1070; VF16UF1-NEXT: [[TMP81:%.*]] = add i64 0, [[TMP80]]1071; VF16UF1-NEXT: [[TMP82:%.*]] = add i64 [[TMP81]], [[STEP]]1072; VF16UF1-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP82]]1073; VF16UF1-NEXT: store i8 0, ptr [[GEP_DST]], align 11074; VF16UF1-NEXT: br label %[[PRED_STORE_CONTINUE30]]1075; VF16UF1: [[PRED_STORE_CONTINUE30]]:1076; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]1077; VF16UF1: [[MIDDLE_BLOCK]]:1078; VF16UF1-NEXT: br label %[[EXIT:.*]]1079; VF16UF1: [[EXIT]]:1080; VF16UF1-NEXT: ret void1081;1082entry:1083 %c = icmp eq i64 %x, 655361084 call void @llvm.assume(i1 %c)1085 %fr = freeze i64 %x1086 %step = add i64 %fr, -655341087 br label %loop1088 1089loop:1090 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1091 %iv.next = add i64 %iv, %step1092 %gep.dst = getelementptr i8, ptr %dst, i64 %iv.next1093 store i8 0, ptr %gep.dst, align 11094 %ec = icmp slt i64 %iv.next, 161095 br i1 %ec, label %loop, label %exit1096 1097exit:1098 ret void1099}1100 1101define void @test_vector_tc_eq_16(ptr %A) {1102; VF8UF1-LABEL: define void @test_vector_tc_eq_16(1103; VF8UF1-SAME: ptr [[A:%.*]]) {1104; VF8UF1-NEXT: [[ENTRY:.*:]]1105; VF8UF1-NEXT: br label %[[VECTOR_PH:.*]]1106; VF8UF1: [[VECTOR_PH]]:1107; VF8UF1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 161108; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]]1109; VF8UF1: [[VECTOR_BODY]]:1110; VF8UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]1111; VF8UF1-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]1112; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[NEXT_GEP]], align 11113; VF8UF1-NEXT: [[TMP1:%.*]] = add nsw <8 x i8> [[WIDE_LOAD]], splat (i8 10)1114; VF8UF1-NEXT: store <8 x i8> [[TMP1]], ptr [[NEXT_GEP]], align 11115; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 81116; VF8UF1-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 161117; VF8UF1-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]1118; VF8UF1: [[MIDDLE_BLOCK]]:1119; VF8UF1-NEXT: br label %[[SCALAR_PH:.*]]1120; VF8UF1: [[SCALAR_PH]]:1121; VF8UF1-NEXT: br label %[[LOOP:.*]]1122; VF8UF1: [[LOOP]]:1123; VF8UF1-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]1124; VF8UF1-NEXT: [[P_SRC:%.*]] = phi ptr [ [[TMP0]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]1125; VF8UF1-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 11126; VF8UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 11127; VF8UF1-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 101128; VF8UF1-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 11129; VF8UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 11130; VF8UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 171131; VF8UF1-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]1132; VF8UF1: [[EXIT]]:1133; VF8UF1-NEXT: ret void1134;1135; VF8UF2-LABEL: define void @test_vector_tc_eq_16(1136; VF8UF2-SAME: ptr [[A:%.*]]) {1137; VF8UF2-NEXT: [[ENTRY:.*:]]1138; VF8UF2-NEXT: br label %[[VECTOR_PH:.*]]1139; VF8UF2: [[VECTOR_PH]]:1140; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 161141; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]]1142; VF8UF2: [[VECTOR_BODY]]:1143; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 81144; VF8UF2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[A]], align 11145; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 11146; VF8UF2-NEXT: [[TMP2:%.*]] = add nsw <8 x i8> [[WIDE_LOAD]], splat (i8 10)1147; VF8UF2-NEXT: [[TMP3:%.*]] = add nsw <8 x i8> [[WIDE_LOAD1]], splat (i8 10)1148; VF8UF2-NEXT: store <8 x i8> [[TMP2]], ptr [[A]], align 11149; VF8UF2-NEXT: store <8 x i8> [[TMP3]], ptr [[TMP1]], align 11150; VF8UF2-NEXT: br label %[[MIDDLE_BLOCK:.*]]1151; VF8UF2: [[MIDDLE_BLOCK]]:1152; VF8UF2-NEXT: br label %[[SCALAR_PH:.*]]1153; VF8UF2: [[SCALAR_PH]]:1154; VF8UF2-NEXT: br label %[[LOOP:.*]]1155; VF8UF2: [[LOOP]]:1156; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]1157; VF8UF2-NEXT: [[P_SRC:%.*]] = phi ptr [ [[TMP0]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]1158; VF8UF2-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 11159; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 11160; VF8UF2-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 101161; VF8UF2-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 11162; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 11163; VF8UF2-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 171164; VF8UF2-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]1165; VF8UF2: [[EXIT]]:1166; VF8UF2-NEXT: ret void1167;1168; VF16UF1-LABEL: define void @test_vector_tc_eq_16(1169; VF16UF1-SAME: ptr [[A:%.*]]) {1170; VF16UF1-NEXT: [[ENTRY:.*:]]1171; VF16UF1-NEXT: br label %[[VECTOR_PH:.*]]1172; VF16UF1: [[VECTOR_PH]]:1173; VF16UF1-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 161174; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]]1175; VF16UF1: [[VECTOR_BODY]]:1176; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[A]], align 11177; VF16UF1-NEXT: [[TMP1:%.*]] = add nsw <16 x i8> [[WIDE_LOAD]], splat (i8 10)1178; VF16UF1-NEXT: store <16 x i8> [[TMP1]], ptr [[A]], align 11179; VF16UF1-NEXT: br label %[[MIDDLE_BLOCK:.*]]1180; VF16UF1: [[MIDDLE_BLOCK]]:1181; VF16UF1-NEXT: br label %[[SCALAR_PH:.*]]1182; VF16UF1: [[SCALAR_PH]]:1183; VF16UF1-NEXT: br label %[[LOOP:.*]]1184; VF16UF1: [[LOOP]]:1185; VF16UF1-NEXT: [[IV:%.*]] = phi i64 [ 16, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]1186; VF16UF1-NEXT: [[P_SRC:%.*]] = phi ptr [ [[TMP0]], %[[SCALAR_PH]] ], [ [[P_SRC_NEXT:%.*]], %[[LOOP]] ]1187; VF16UF1-NEXT: [[P_SRC_NEXT]] = getelementptr inbounds i8, ptr [[P_SRC]], i64 11188; VF16UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 11189; VF16UF1-NEXT: [[ADD:%.*]] = add nsw i8 [[L]], 101190; VF16UF1-NEXT: store i8 [[ADD]], ptr [[P_SRC]], align 11191; VF16UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 11192; VF16UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 171193; VF16UF1-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]1194; VF16UF1: [[EXIT]]:1195; VF16UF1-NEXT: ret void1196;1197entry:1198 br label %loop1199 1200loop:1201 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1202 %p.src = phi ptr [ %A, %entry ], [ %p.src.next, %loop ]1203 %p.src.next = getelementptr inbounds i8, ptr %p.src, i64 11204 %l = load i8, ptr %p.src, align 11205 %add = add nsw i8 %l, 101206 store i8 %add, ptr %p.src1207 %iv.next = add nsw i64 %iv, 11208 %cmp = icmp eq i64 %iv.next, 171209 br i1 %cmp, label %exit, label %loop1210 1211exit:1212 ret void1213}1214;.1215; VF8UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}1216; VF8UF1: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}1217; VF8UF1: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}1218; VF8UF1: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}1219; VF8UF1: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}1220; VF8UF1: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}1221; VF8UF1: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}1222; VF8UF1: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}1223;.1224; VF8UF2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}1225; VF8UF2: [[META1]] = !{!"llvm.loop.unroll.runtime.disable"}1226; VF8UF2: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}1227; VF8UF2: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}1228;.1229; VF16UF1: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}1230; VF16UF1: [[META1]] = !{!"llvm.loop.unroll.runtime.disable"}1231; VF16UF1: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}1232; VF16UF1: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}1233;.1234