237 lines · plain
1; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=CHECK2; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK3; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK4 5define i64 @smax_idx(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {6; CHECK-LABEL: @smax_idx(7; CHECK-NOT: vector.body:8;9entry:10 br label %for.body11 12for.body:13 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]14 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]15 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]16 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv17 %0 = load i64, ptr %arrayidx18 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)19 %cmp1 = icmp slt i64 %max.09, %020 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.01121 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 122 %exitcond.not = icmp eq i64 %indvars.iv.next, %n23 br i1 %exitcond.not, label %exit, label %for.body24 25exit:26 store i64 %1, ptr %res_max27 ret i64 %spec.select728}29 30;31; Check the different order of reduction phis.32;33define i64 @smax_idx_inverted_phi(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {34; CHECK-LABEL: @smax_idx_inverted_phi(35; CHECK-NOT: vector.body:36;37entry:38 br label %for.body39 40for.body:41 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]42 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]43 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]44 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv45 %0 = load i64, ptr %arrayidx46 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)47 %cmp1 = icmp slt i64 %max.09, %048 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.01149 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 150 %exitcond.not = icmp eq i64 %indvars.iv.next, %n51 br i1 %exitcond.not, label %exit, label %for.body52 53exit:54 store i64 %1, ptr %res_max55 ret i64 %spec.select756}57 58; Check if it is a min/max with index (MMI) pattern when the59; min/max value is not used outside the loop.60;61; Currently, the vectorizer checks if smax value is used outside62; the loop. However, even if only the index part has external users,63; and smax itself does not have external users, it can still form a64; MMI pattern.65;66define i64 @smax_idx_max_no_exit_user(ptr nocapture readonly %a, i64 %mm, i64 %ii, i64 %n) {67; CHECK-LABEL: @smax_idx_max_no_exit_user(68; CHECK-NOT: vector.body:69;70entry:71 br label %for.body72 73for.body:74 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]75 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]76 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]77 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv78 %0 = load i64, ptr %arrayidx79 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)80 %cmp1 = icmp slt i64 %max.09, %081 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.01182 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 183 %exitcond.not = icmp eq i64 %indvars.iv.next, %n84 br i1 %exitcond.not, label %exit, label %for.body85 86exit:87 ; %1 has no external users88 ret i64 %spec.select789}90 91; Check smax implemented by select(cmp()).92;93; Currently, MMI pattern does not support icmp with multiple users.94; TODO: It may be possible to reuse some of the methods in instcombine pass to95; check whether icmp can be duplicated.96;97define i64 @smax_idx_select_cmp(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {98; CHECK-LABEL: @smax_idx_select_cmp(99; CHECK-NOT: vector.body:100;101entry:102 br label %for.body103 104for.body:105 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]106 %max.09 = phi i64 [ %mm, %entry ], [ %spec.select, %for.body ]107 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]108 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv109 %0 = load i64, ptr %arrayidx110 %cmp1 = icmp slt i64 %max.09, %0111 %spec.select = select i1 %cmp1, i64 %0, i64 %max.09112 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011113 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1114 %exitcond.not = icmp eq i64 %indvars.iv.next, %n115 br i1 %exitcond.not, label %exit, label %for.body116 117exit:118 store i64 %spec.select, ptr %res_max119 ret i64 %spec.select7120}121 122;123; Check sge case.124;125define i64 @smax_idx_inverted_pred(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {126; CHECK-LABEL: @smax_idx_inverted_pred(127; CHECK-NOT: vector.body:128;129entry:130 br label %for.body131 132for.body:133 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]134 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]135 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]136 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv137 %0 = load i64, ptr %arrayidx138 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)139 %cmp1 = icmp sge i64 %0, %max.09140 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011141 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1142 %exitcond.not = icmp eq i64 %indvars.iv.next, %n143 br i1 %exitcond.not, label %exit, label %for.body144 145exit:146 store i64 %1, ptr %res_max147 ret i64 %spec.select7148}149 150;151; In such cases, the last index should be extracted.152;153define i64 @smax_idx_extract_last(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {154; CHECK-LABEL: @smax_idx_extract_last(155; CHECK-NOT: vector.body:156;157entry:158 br label %for.body159 160for.body:161 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]162 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]163 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]164 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv165 %0 = load i64, ptr %arrayidx166 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)167 %cmp1.not = icmp sgt i64 %max.09, %0168 %spec.select7 = select i1 %cmp1.not, i64 %idx.011, i64 %indvars.iv169 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1170 %exitcond.not = icmp eq i64 %indvars.iv.next, %n171 br i1 %exitcond.not, label %exit, label %for.body172 173exit:174 store i64 %1, ptr %res_max175 ret i64 %spec.select7176}177 178;179; The operands of smax intrinsic and icmp are not the same to be recognized as MMI.180;181define i64 @smax_idx_not_vec_1(ptr nocapture readonly %a, ptr nocapture readonly %b, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {182; CHECK-LABEL: @smax_idx_not_vec_1(183; CHECK-NOT: vector.body:184;185 entry:186 br label %for.body187 188for.body:189 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]190 %max.09 = phi i64 [ %mm, %entry ], [ %2, %for.body ]191 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]192 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv193 %0 = load i64, ptr %arrayidx194 %arrayidx.01 = getelementptr inbounds i64, ptr %b, i64 %indvars.iv195 %1 = load i64, ptr %arrayidx196 %2 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)197 %cmp1 = icmp slt i64 %max.09, %1198 %spec.select7 = select i1 %cmp1, i64 %indvars.iv, i64 %idx.011199 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1200 %exitcond.not = icmp eq i64 %indvars.iv.next, %n201 br i1 %exitcond.not, label %exit, label %for.body202 203exit:204 store i64 %2, ptr %res_max205 ret i64 %spec.select7206}207 208;209; It cannot be recognized as MMI when the operand of index select is not an induction variable.210;211define i64 @smax_idx_not_vec_2(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) {212; CHECK-LABEL: @smax_idx_not_vec_2(213; CHECK-NOT: vector.body:214;215entry:216 br label %for.body217 218for.body:219 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]220 %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ]221 %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ]222 %arrayidx = getelementptr inbounds i64, ptr %a, i64 %indvars.iv223 %0 = load i64, ptr %arrayidx224 %1 = tail call i64 @llvm.smax.i64(i64 %max.09, i64 %0)225 %cmp1 = icmp slt i64 %max.09, %0226 %spec.select7 = select i1 %cmp1, i64 123, i64 %idx.011227 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1228 %exitcond.not = icmp eq i64 %indvars.iv.next, %n229 br i1 %exitcond.not, label %exit, label %for.body230 231exit:232 store i64 %1, ptr %res_max233 ret i64 %spec.select7234}235 236declare i64 @llvm.smax.i64(i64, i64)237