703 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 52; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s3 4define i64 @test_vectorize_select_umax_idx(ptr %src, i64 %n) {5; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx(6; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {7; CHECK-NEXT: [[ENTRY:.*]]:8; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 49; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]10; CHECK: [[VECTOR_PH]]:11; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 412; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]13; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]14; CHECK: [[VECTOR_BODY]]:15; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]16; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]17; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]18; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]19; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]20; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP1]], align 421; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]22; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])23; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]24; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], 425; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)26; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]27; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]28; CHECK: [[MIDDLE_BLOCK]]:29; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP3]])30; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i64 031; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer32; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]33; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)34; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])35; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP9]], -922337203685477580836; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP9]], i64 037; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]38; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]39; CHECK: [[SCALAR_PH]]:40; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]41; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]42; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]43; CHECK-NEXT: br label %[[LOOP:.*]]44; CHECK: [[LOOP]]:45; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]46; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]47; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]48; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]49; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 450; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]51; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])52; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]53; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 154; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]55; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]56; CHECK: [[EXIT]]:57; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]58; CHECK-NEXT: ret i64 [[RES]]59;60entry:61 br label %loop62 63loop:64 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]65 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]66 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]67 %gep = getelementptr i64, ptr %src, i64 %iv68 %l = load i64, ptr %gep69 %cmp = icmp ule i64 %max.val, %l70 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)71 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx72 %iv.next = add nuw nsw i64 %iv, 173 %exitcond.not = icmp eq i64 %iv.next, %n74 br i1 %exitcond.not, label %exit, label %loop75 76exit:77 %res = phi i64 [ %max.idx.next, %loop ]78 ret i64 %res79}80 81define i64 @test_vectorize_select_umax_idx_cond_flipped(ptr %src, i64 %n) {82; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_cond_flipped(83; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {84; CHECK-NEXT: [[ENTRY:.*]]:85; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 486; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]87; CHECK: [[VECTOR_PH]]:88; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 489; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]90; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]91; CHECK: [[VECTOR_BODY]]:92; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]93; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]94; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]95; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]96; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]97; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP]], align 498; CHECK-NEXT: [[TMP2:%.*]] = icmp uge <4 x i64> [[WIDE_LOAD]], [[VEC_PHI1]]99; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> [[VEC_PHI1]], <4 x i64> [[WIDE_LOAD]])100; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]101; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4102; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)103; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]104; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]105; CHECK: [[MIDDLE_BLOCK]]:106; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP3]])107; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i64 0108; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer109; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]110; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)111; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])112; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808113; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP9]], i64 0114; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]115; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]116; CHECK: [[SCALAR_PH]]:117; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]118; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]119; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]120; CHECK-NEXT: br label %[[LOOP:.*]]121; CHECK: [[LOOP]]:122; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]123; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]124; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]125; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]126; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP1]], align 4127; CHECK-NEXT: [[CMP:%.*]] = icmp uge i64 [[L]], [[MIN_VAL]]128; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])129; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV1]], i64 [[MAX_IDX]]130; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1131; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]132; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]133; CHECK: [[EXIT]]:134; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]135; CHECK-NEXT: ret i64 [[RES]]136;137entry:138 br label %loop139 140loop:141 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]142 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]143 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]144 %gep = getelementptr i64, ptr %src, i64 %iv145 %l = load i64, ptr %gep146 %cmp = icmp uge i64 %l, %max.val147 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)148 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx149 %iv.next = add nuw nsw i64 %iv, 1150 %exitcond.not = icmp eq i64 %iv.next, %n151 br i1 %exitcond.not, label %exit, label %loop152 153exit:154 %res = phi i64 [ %max.idx.next, %loop ]155 ret i64 %res156}157 158define i64 @test_vectorize_select_umax_idx_select_ops_flipped(ptr %src, i64 %n) {159; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_select_ops_flipped(160; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {161; CHECK-NEXT: [[ENTRY:.*]]:162; CHECK-NEXT: br label %[[LOOP:.*]]163; CHECK: [[LOOP]]:164; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]165; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]166; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]167; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]168; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4169; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]170; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])171; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[MIN_IDX]], i64 [[IV]]172; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1173; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]174; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]175; CHECK: [[EXIT]]:176; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]177; CHECK-NEXT: ret i64 [[RES]]178;179entry:180 br label %loop181 182loop:183 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]184 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]185 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]186 %gep = getelementptr i64, ptr %src, i64 %iv187 %l = load i64, ptr %gep188 %cmp = icmp ule i64 %l, %max.val189 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)190 %max.idx.next = select i1 %cmp, i64 %max.idx, i64 %iv191 %iv.next = add nuw nsw i64 %iv, 1192 %exitcond.not = icmp eq i64 %iv.next, %n193 br i1 %exitcond.not, label %exit, label %loop194 195exit:196 %res = phi i64 [ %max.idx.next, %loop ]197 ret i64 %res198}199 200define i64 @test_vectorize_select_umax_via_select_idx(ptr %src, i64 %n) {201; CHECK-LABEL: define i64 @test_vectorize_select_umax_via_select_idx(202; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {203; CHECK-NEXT: [[ENTRY:.*]]:204; CHECK-NEXT: br label %[[LOOP:.*]]205; CHECK: [[LOOP]]:206; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]207; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]208; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]209; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]210; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4211; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]212; CHECK-NEXT: [[MIN_VAL_NEXT]] = select i1 [[CMP]], i64 [[L]], i64 [[MIN_VAL]]213; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]214; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1215; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]216; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]217; CHECK: [[EXIT]]:218; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]219; CHECK-NEXT: ret i64 [[RES]]220;221entry:222 br label %loop223 224loop:225 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]226 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]227 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]228 %gep = getelementptr i64, ptr %src, i64 %iv229 %l = load i64, ptr %gep230 %cmp = icmp ule i64 %max.val, %l231 %max.val.next = select i1 %cmp, i64 %l, i64 %max.val232 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx233 %iv.next = add nuw nsw i64 %iv, 1234 %exitcond.not = icmp eq i64 %iv.next, %n235 br i1 %exitcond.not, label %exit, label %loop236 237exit:238 %res = phi i64 [ %max.idx.next, %loop ]239 ret i64 %res240}241 242define i64 @test_vectorize_select_umax_idx_all_exit_inst(ptr %src, ptr %umax, i64 %n) {243; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_all_exit_inst(244; CHECK-SAME: ptr [[SRC:%.*]], ptr [[UMAX:%.*]], i64 [[N:%.*]]) {245; CHECK-NEXT: [[ENTRY:.*]]:246; CHECK-NEXT: br label %[[LOOP:.*]]247; CHECK: [[LOOP]]:248; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]249; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]250; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]251; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]252; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4253; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]254; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])255; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]256; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1257; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]258; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]259; CHECK: [[EXIT]]:260; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]261; CHECK-NEXT: [[RES_UMAX:%.*]] = phi i64 [ [[MIN_VAL_NEXT]], %[[LOOP]] ]262; CHECK-NEXT: store i64 [[RES_UMAX]], ptr [[UMAX]], align 4263; CHECK-NEXT: ret i64 [[RES]]264;265entry:266 br label %loop267 268loop:269 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]270 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]271 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]272 %gep = getelementptr i64, ptr %src, i64 %iv273 %l = load i64, ptr %gep274 %cmp = icmp ule i64 %max.val, %l275 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)276 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx277 %iv.next = add nuw nsw i64 %iv, 1278 %exitcond.not = icmp eq i64 %iv.next, %n279 br i1 %exitcond.not, label %exit, label %loop280 281exit:282 %res = phi i64 [ %max.idx.next, %loop ]283 %res.umax = phi i64 [ %max.val.next, %loop ]284 store i64 %res.umax, ptr %umax285 ret i64 %res286}287 288define i64 @test_vectorize_select_umax_idx_min_ops_switched(ptr %src, i64 %n) {289; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_min_ops_switched(290; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {291; CHECK-NEXT: [[ENTRY:.*]]:292; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4293; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]294; CHECK: [[VECTOR_PH]]:295; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4296; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]297; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]298; CHECK: [[VECTOR_BODY]]:299; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]300; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]301; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -9223372036854775808), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]302; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]303; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV1]]304; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[GEP1]], align 4305; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_PHI1]], [[WIDE_LOAD]]306; CHECK-NEXT: [[TMP3]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> [[WIDE_LOAD]], <4 x i64> [[VEC_PHI1]])307; CHECK-NEXT: [[TMP4]] = select <4 x i1> [[TMP2]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]308; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], 4309; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)310; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]311; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]312; CHECK: [[MIDDLE_BLOCK]]:313; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> [[TMP3]])314; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i64 0315; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer316; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP3]], [[BROADCAST_SPLAT]]317; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i64> [[TMP4]], <4 x i64> splat (i64 -9223372036854775808)318; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> [[TMP8]])319; CHECK-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP9]], -9223372036854775808320; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP9]], i64 0321; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]322; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]323; CHECK: [[SCALAR_PH]]:324; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]325; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]326; CHECK-NEXT: [[BC_MERGE_RDX2:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]327; CHECK-NEXT: br label %[[LOOP:.*]]328; CHECK: [[LOOP]]:329; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]330; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]331; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]332; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]333; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4334; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]335; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[L]], i64 [[MIN_VAL]])336; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]337; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1338; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]339; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]340; CHECK: [[EXIT]]:341; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]342; CHECK-NEXT: ret i64 [[RES]]343;344entry:345 br label %loop346 347loop:348 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]349 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]350 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]351 %gep = getelementptr i64, ptr %src, i64 %iv352 %l = load i64, ptr %gep353 %cmp = icmp ule i64 %max.val, %l354 %max.val.next = tail call i64 @llvm.umax.i64(i64 %l, i64 %max.val)355 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx356 %iv.next = add nuw nsw i64 %iv, 1357 %exitcond.not = icmp eq i64 %iv.next, %n358 br i1 %exitcond.not, label %exit, label %loop359 360exit:361 %res = phi i64 [ %max.idx.next, %loop ]362 ret i64 %res363}364 365define i64 @test_not_vectorize_select_no_min_reduction(ptr %src, i64 %n) {366; CHECK-LABEL: define i64 @test_not_vectorize_select_no_min_reduction(367; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {368; CHECK-NEXT: [[ENTRY:.*]]:369; CHECK-NEXT: br label %[[LOOP:.*]]370; CHECK: [[LOOP]]:371; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]372; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]373; CHECK-NEXT: [[RED_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RED_VAL_NEXT:%.*]], %[[LOOP]] ]374; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]375; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4376; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[RED_VAL]], [[L]]377; CHECK-NEXT: [[RED_VAL_NEXT]] = add i64 [[RED_VAL]], [[L]]378; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]379; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1380; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]381; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]382; CHECK: [[EXIT]]:383; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]384; CHECK-NEXT: ret i64 [[RES]]385;386entry:387 br label %loop388 389loop:390 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]391 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]392 %red.val = phi i64 [ 0, %entry ], [ %red.val.next, %loop ]393 %gep = getelementptr i64, ptr %src, i64 %iv394 %l = load i64, ptr %gep395 %cmp = icmp ule i64 %red.val, %l396 %red.val.next = add i64 %red.val, %l397 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx398 %iv.next = add nuw nsw i64 %iv, 1399 %exitcond.not = icmp eq i64 %iv.next, %n400 br i1 %exitcond.not, label %exit, label %loop401 402exit:403 %res = phi i64 [ %max.idx.next, %loop ]404 ret i64 %res405}406 407define i64 @test_cmp_and_umax_use_different_values(ptr %src, i64 %x, i64 %n) {408; CHECK-LABEL: define i64 @test_cmp_and_umax_use_different_values(409; CHECK-SAME: ptr [[SRC:%.*]], i64 [[X:%.*]], i64 [[N:%.*]]) {410; CHECK-NEXT: [[ENTRY:.*]]:411; CHECK-NEXT: br label %[[LOOP:.*]]412; CHECK: [[LOOP]]:413; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]414; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]415; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]416; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]417; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4418; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[X]]419; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])420; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MAX_IDX]]421; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1422; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]423; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]424; CHECK: [[EXIT]]:425; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]426; CHECK-NEXT: ret i64 [[RES]]427;428entry:429 br label %loop430 431loop:432 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]433 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]434 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]435 %gep = getelementptr i64, ptr %src, i64 %iv436 %l = load i64, ptr %gep437 %cmp = icmp ule i64 %max.val, %x438 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)439 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx440 %iv.next = add nuw nsw i64 %iv, 1441 %exitcond.not = icmp eq i64 %iv.next, %n442 br i1 %exitcond.not, label %exit, label %loop443 444exit:445 %res = phi i64 [ %max.idx.next, %loop ]446 ret i64 %res447}448 449define i32 @test_vectorize_select_umax_idx_with_trunc(ptr %src, i64 %n) {450; CHECK-LABEL: define i32 @test_vectorize_select_umax_idx_with_trunc(451; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {452; CHECK-NEXT: [[ENTRY:.*]]:453; CHECK-NEXT: br label %[[LOOP:.*]]454; CHECK: [[LOOP]]:455; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]456; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]457; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]458; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]459; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4460; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]461; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])462; CHECK-NEXT: [[TRUNC:%.*]] = trunc i64 [[IV]] to i32463; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i32 [[TRUNC]], i32 [[MIN_IDX]]464; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1465; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]466; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]467; CHECK: [[EXIT]]:468; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]469; CHECK-NEXT: ret i32 [[RES]]470;471entry:472 br label %loop473 474loop:475 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]476 %max.idx = phi i32 [ 0, %entry ], [ %max.idx.next, %loop ]477 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]478 %gep = getelementptr i64, ptr %src, i64 %iv479 %l = load i64, ptr %gep480 %cmp = icmp ule i64 %max.val, %l481 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)482 %trunc = trunc i64 %iv to i32483 %max.idx.next = select i1 %cmp, i32 %trunc, i32 %max.idx484 %iv.next = add nuw nsw i64 %iv, 1485 %exitcond.not = icmp eq i64 %iv.next, %n486 br i1 %exitcond.not, label %exit, label %loop487 488exit:489 %res = phi i32 [ %max.idx.next, %loop ]490 ret i32 %res491}492 493define ptr @test_with_ptr_index(ptr %start, ptr %end) {494; CHECK-LABEL: define ptr @test_with_ptr_index(495; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {496; CHECK-NEXT: [[ENTRY:.*]]:497; CHECK-NEXT: br label %[[LOOP:.*]]498; CHECK: [[LOOP]]:499; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]500; CHECK-NEXT: [[MIN_IDX:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]501; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]502; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[IV]], align 4503; CHECK-NEXT: [[CMP7_US:%.*]] = icmp ule i64 [[L]], [[MIN_VAL]]504; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])505; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP7_US]], ptr [[IV]], ptr [[MIN_IDX]]506; CHECK-NEXT: [[IV_NEXT]] = getelementptr i32, ptr [[IV]], i64 1507; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]508; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]509; CHECK: [[EXIT]]:510; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[MIN_IDX_NEXT]], %[[LOOP]] ]511; CHECK-NEXT: ret ptr [[RES]]512;513entry:514 br label %loop515 516loop:517 %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ]518 %max.idx = phi ptr [ null, %entry ], [ %max.idx.next, %loop ]519 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]520 %l = load i64, ptr %iv521 %cmp7.us = icmp ule i64 %l, %max.val522 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)523 %max.idx.next = select i1 %cmp7.us, ptr %iv, ptr %max.idx524 %iv.next = getelementptr i32, ptr %iv, i64 1525 %exitcond.not = icmp eq ptr %iv.next, %end526 br i1 %exitcond.not, label %exit, label %loop527 528exit:529 %res = phi ptr [ %max.idx.next, %loop ]530 ret ptr %res531}532 533define i64 @test_no_vectorize_select_iv_decrement(ptr %src) {534; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_decrement(535; CHECK-SAME: ptr [[SRC:%.*]]) {536; CHECK-NEXT: [[ENTRY:.*]]:537; CHECK-NEXT: br label %[[LOOP:.*]]538; CHECK: [[LOOP]]:539; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]540; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]541; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]542; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]543; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4544; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]545; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])546; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]547; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], -1548; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0549; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]550; CHECK: [[EXIT]]:551; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]552; CHECK-NEXT: ret i64 [[RES]]553;554entry:555 br label %loop556 557loop:558 %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]559 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]560 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]561 %gep = getelementptr i64, ptr %src, i64 %iv562 %l = load i64, ptr %gep563 %cmp = icmp ule i64 %max.val, %l564 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)565 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx566 %iv.next = add nuw nsw i64 %iv, -1567 %exitcond.not = icmp eq i64 %iv.next, 0568 br i1 %exitcond.not, label %exit, label %loop569 570exit:571 %res = phi i64 [ %max.idx.next, %loop ]572 ret i64 %res573}574 575define i64 @test_no_vectorize_select_iv_sub(ptr %src) {576; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_sub(577; CHECK-SAME: ptr [[SRC:%.*]]) {578; CHECK-NEXT: [[ENTRY:.*]]:579; CHECK-NEXT: br label %[[LOOP:.*]]580; CHECK: [[LOOP]]:581; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1000, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]582; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]583; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]584; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]585; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4586; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]587; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])588; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]589; CHECK-NEXT: [[IV_NEXT]] = sub i64 [[IV]], 1590; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0591; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]592; CHECK: [[EXIT]]:593; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]594; CHECK-NEXT: ret i64 [[RES]]595;596entry:597 br label %loop598 599loop:600 %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ]601 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]602 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]603 %gep = getelementptr i64, ptr %src, i64 %iv604 %l = load i64, ptr %gep605 %cmp = icmp ule i64 %max.val, %l606 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)607 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx608 %iv.next = sub i64 %iv, 1609 %exitcond.not = icmp eq i64 %iv.next, 0610 br i1 %exitcond.not, label %exit, label %loop611 612exit:613 %res = phi i64 [ %max.idx.next, %loop ]614 ret i64 %res615}616 617define i64 @test_no_vectorize_select_iv_mul(ptr %src) {618; CHECK-LABEL: define i64 @test_no_vectorize_select_iv_mul(619; CHECK-SAME: ptr [[SRC:%.*]]) {620; CHECK-NEXT: [[ENTRY:.*]]:621; CHECK-NEXT: br label %[[LOOP:.*]]622; CHECK: [[LOOP]]:623; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]624; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]625; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]626; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]627; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4628; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MIN_VAL]], [[L]]629; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MIN_VAL]], i64 [[L]])630; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]631; CHECK-NEXT: [[IV_NEXT]] = mul i64 [[IV]], 2632; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 128633; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]634; CHECK: [[EXIT]]:635; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]636; CHECK-NEXT: ret i64 [[RES]]637;638entry:639 br label %loop640 641loop:642 %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]643 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]644 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]645 %gep = getelementptr i64, ptr %src, i64 %iv646 %l = load i64, ptr %gep647 %cmp = icmp ule i64 %max.val, %l648 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)649 %max.idx.next = select i1 %cmp, i64 %iv, i64 %max.idx650 %iv.next = mul i64 %iv, 2651 %exitcond.not = icmp eq i64 %iv.next, 128652 br i1 %exitcond.not, label %exit, label %loop653 654exit:655 %res = phi i64 [ %max.idx.next, %loop ]656 ret i64 %res657}658 659define i64 @test_vectorize_select_umax_idx_inc(ptr %src, i64 %n) {660; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_inc(661; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {662; CHECK-NEXT: [[ENTRY:.*]]:663; CHECK-NEXT: br label %[[LOOP:.*]]664; CHECK: [[LOOP]]:665; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]666; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]667; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]668; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]669; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4670; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MAX_VAL]], [[L]]671; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MAX_VAL]], i64 [[L]])672; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1673; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_NEXT]], i64 [[MAX_IDX]]674; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]675; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]676; CHECK: [[EXIT]]:677; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]678; CHECK-NEXT: ret i64 [[RES]]679;680entry:681 br label %loop682 683loop:684 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]685 %max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]686 %max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]687 %gep = getelementptr i64, ptr %src, i64 %iv688 %l = load i64, ptr %gep689 %cmp = icmp ule i64 %max.val, %l690 %max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)691 %iv.next = add nuw nsw i64 %iv, 1692 %max.idx.next = select i1 %cmp, i64 %iv.next, i64 %max.idx693 %exitcond.not = icmp eq i64 %iv.next, %n694 br i1 %exitcond.not, label %exit, label %loop695 696exit:697 %res = phi i64 [ %max.idx.next, %loop ]698 ret i64 %res699}700 701declare i64 @llvm.umax.i64(i64, i64)702declare i16 @llvm.umax.i16(i16, i16)703