258 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 52; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s3 4; Test that VPlan native path is able to widen select instruction in the5; innermost loop under different conditions when outer loop is marked to be6; vectorized. These conditions include following:7; * Inner and outer loop invariant select condition8; * Select condition depending on outer loop iteration variable.9; * Select condidition depending on inner loop iteration variable.10; * Select conditition depending on both outer and inner loop iteration11; variables.12 13define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, double %a, double %b) {14; CHECK-LABEL: define void @loop_invariant_select(15; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], i1 [[SELECT:%.*]], double [[A:%.*]], double [[B:%.*]]) {16; CHECK-NEXT: [[ENTRY:.*:]]17; CHECK-NEXT: br label %[[VECTOR_PH:.*]]18; CHECK: [[VECTOR_PH]]:19; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 020; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer21; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 022; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer23; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]24; CHECK: [[VECTOR_BODY]]:25; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]26; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]27; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]28; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]29; CHECK: [[FOR2_HEADER3]]:30; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP2:%.*]], %[[FOR2_HEADER3]] ]31; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SELECT]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]32; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP1]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))33; CHECK-NEXT: [[TMP2]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)34; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP2]], splat (i64 10000)35; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 036; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]37; CHECK: [[VECTOR_LATCH]]:38; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 439; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)40; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100041; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]42; CHECK: [[MIDDLE_BLOCK]]:43; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]44; CHECK: [[SCALAR_PH]]:45;46entry:47 br label %for1.header48 49for1.header:50 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]51 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar152 br label %for2.header53 54for2.header:55 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]56 ; Select condition is loop invariant for both inner and outer loop.57 %select.b = select i1 %select, double %a, double %b58 store double %select.b, ptr %ptr, align 859 %indvar21 = add nuw nsw i64 %indvar2, 160 %for2.cond = icmp eq i64 %indvar21, 1000061 br i1 %for2.cond, label %for1.latch, label %for2.header62 63for1.latch:64 %indvar11 = add nuw nsw i64 %indvar1, 165 %for1.cond = icmp eq i64 %indvar11, 100066 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !067 68exit:69 ret void70}71 72define void @outer_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {73; CHECK-LABEL: define void @outer_loop_dependant_select(74; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {75; CHECK-NEXT: [[ENTRY:.*:]]76; CHECK-NEXT: br label %[[VECTOR_PH:.*]]77; CHECK: [[VECTOR_PH]]:78; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 079; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer80; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 081; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer82; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]83; CHECK: [[VECTOR_BODY]]:84; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]85; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]86; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]87; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]88; CHECK: [[FOR2_HEADER3]]:89; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]90; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i1>91; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]92; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))93; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)94; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)95; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 096; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]97; CHECK: [[VECTOR_LATCH]]:98; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 499; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)100; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000101; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]102; CHECK: [[MIDDLE_BLOCK]]:103; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]104; CHECK: [[SCALAR_PH]]:105;106entry:107 br label %for1.header108 109for1.header:110 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]111 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1112 br label %for2.header113 114for2.header:115 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]116 %select = trunc i64 %indvar1 to i1117 ; Select condition only depends on outer loop iteration variable.118 %select.b = select i1 %select, double %a, double %b119 store double %select.b, ptr %ptr, align 8120 %indvar21 = add nuw nsw i64 %indvar2, 1121 %for2.cond = icmp eq i64 %indvar21, 10000122 br i1 %for2.cond, label %for1.latch, label %for2.header123 124for1.latch:125 %indvar11 = add nuw nsw i64 %indvar1, 1126 %for1.cond = icmp eq i64 %indvar11, 1000127 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0128 129exit:130 ret void131}132 133define void @inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {134; CHECK-LABEL: define void @inner_loop_dependant_select(135; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {136; CHECK-NEXT: [[ENTRY:.*:]]137; CHECK-NEXT: br label %[[VECTOR_PH:.*]]138; CHECK: [[VECTOR_PH]]:139; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0140; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer141; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0142; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer143; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]144; CHECK: [[VECTOR_BODY]]:145; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]146; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]147; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]148; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]149; CHECK: [[FOR2_HEADER3]]:150; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]151; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_PHI]] to <4 x i1>152; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]153; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))154; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)155; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)156; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0157; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]158; CHECK: [[VECTOR_LATCH]]:159; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4160; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)161; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000162; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]163; CHECK: [[MIDDLE_BLOCK]]:164; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]165; CHECK: [[SCALAR_PH]]:166;167entry:168 br label %for1.header169 170for1.header:171 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]172 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1173 br label %for2.header174 175for2.header:176 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]177 %select = trunc i64 %indvar2 to i1178 ; Select condition only depends on inner loop iteration variable.179 %select.b = select i1 %select, double %a, double %b180 store double %select.b, ptr %ptr, align 8181 %indvar21 = add nuw nsw i64 %indvar2, 1182 %for2.cond = icmp eq i64 %indvar21, 10000183 br i1 %for2.cond, label %for1.latch, label %for2.header184 185for1.latch:186 %indvar11 = add nuw nsw i64 %indvar1, 1187 %for1.cond = icmp eq i64 %indvar11, 1000188 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0189 190exit:191 ret void192}193 194define void @outer_and_inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {195; CHECK-LABEL: define void @outer_and_inner_loop_dependant_select(196; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {197; CHECK-NEXT: [[ENTRY:.*:]]198; CHECK-NEXT: br label %[[VECTOR_PH:.*]]199; CHECK: [[VECTOR_PH]]:200; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0201; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer202; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0203; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer204; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]205; CHECK: [[VECTOR_BODY]]:206; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]207; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]208; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]209; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]210; CHECK: [[FOR2_HEADER3]]:211; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR2_HEADER3]] ]212; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], [[VEC_PHI]]213; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i1>214; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]215; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP3]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))216; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)217; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 10000)218; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0219; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]220; CHECK: [[VECTOR_LATCH]]:221; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4222; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)223; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000224; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]225; CHECK: [[MIDDLE_BLOCK]]:226; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]227; CHECK: [[SCALAR_PH]]:228;229entry:230 br label %for1.header231 232for1.header:233 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]234 %ptr = getelementptr inbounds double, ptr %out, i64 %indvar1235 br label %for2.header236 237for2.header:238 %indvar2 = phi i64 [ 0, %for1.header ], [ %indvar21, %for2.header ]239 %sum = add nuw nsw i64 %indvar1, %indvar2240 %select = trunc i64 %sum to i1241 ; Select condition depends on both inner and outer loop iteration variables.242 %select.b = select i1 %select, double %a, double %b243 store double %select.b, ptr %ptr, align 8244 %indvar21 = add nuw nsw i64 %indvar2, 1245 %for2.cond = icmp eq i64 %indvar21, 10000246 br i1 %for2.cond, label %for1.latch, label %for2.header247 248for1.latch:249 %indvar11 = add nuw nsw i64 %indvar1, 1250 %for1.cond = icmp eq i64 %indvar11, 1000251 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0252 253exit:254 ret void255}256!0 = distinct !{!0, !1}257!1 = !{!"llvm.loop.vectorize.enable", i1 true}258