231 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 32; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-vector-interleave=2 -force-vector-width=1 -force-ordered-reductions -S %s | FileCheck %s3; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-vector-interleave=2 -force-vector-width=1 -force-ordered-reductions -force-tail-folding-style=data -S %s | FileCheck --check-prefix=CHECK-ALM %s4 5define float @pr70988() {6; CHECK-LABEL: define float @pr70988() {7; CHECK-NEXT: entry:8; CHECK-NEXT: br label [[VECTOR_PH:%.*]]9; CHECK: vector.ph:10; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]11; CHECK: vector.body:12; CHECK-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]13; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]14; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX1]], 015; CHECK-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX1]], 116; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 102017; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV2]], 102018; CHECK-NEXT: [[TMP2:%.*]] = select contract i1 [[TMP0]], float 1.000000e+00, float -0.000000e+0019; CHECK-NEXT: [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]20; CHECK-NEXT: [[TMP4:%.*]] = select contract i1 [[TMP1]], float 1.000000e+00, float -0.000000e+0021; CHECK-NEXT: [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]22; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 223; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 102224; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]25; CHECK: middle.block:26; CHECK-NEXT: br label [[LOOP:%.*]]27; CHECK: exit:28; CHECK-NEXT: ret float [[TMP5]]29;30; CHECK-ALM-LABEL: define float @pr70988() {31; CHECK-ALM-NEXT: entry:32; CHECK-ALM-NEXT: br label [[VECTOR_PH:%.*]]33; CHECK-ALM: vector.ph:34; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]]35; CHECK-ALM: vector.body:36; CHECK-ALM-NEXT: [[INDEX1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]37; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]38; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX1]], 139; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX1]], 102140; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK2:%.*]] = icmp ult i32 [[TMP1]], 102141; CHECK-ALM-NEXT: [[TMP2:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float 1.000000e+00, float -0.000000e+0042; CHECK-ALM-NEXT: [[TMP3:%.*]] = fadd contract float [[VEC_PHI]], [[TMP2]]43; CHECK-ALM-NEXT: [[TMP4:%.*]] = select contract i1 [[ACTIVE_LANE_MASK2]], float 1.000000e+00, float -0.000000e+0044; CHECK-ALM-NEXT: [[TMP5]] = fadd contract float [[TMP3]], [[TMP4]]45; CHECK-ALM-NEXT: [[INDEX_NEXT3]] = add nuw i32 [[INDEX1]], 246; CHECK-ALM-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT3]], 102247; CHECK-ALM-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]48; CHECK-ALM: middle.block:49; CHECK-ALM-NEXT: br label [[LOOP:%.*]]50; CHECK-ALM: exit:51; CHECK-ALM-NEXT: ret float [[TMP5]]52;53entry:54 br label %loop55 56loop:57 %index = phi i32 [ 0, %entry ], [ %index.next, %loop ]58 %rdx = phi float [ 0.000000e+00, %entry ], [ %rdx.next, %loop ]59 %rdx.next = fadd contract float %rdx, 1.000000e+0060 %index.next = add nuw nsw i32 %index, 161 %cond = icmp ult i32 %index.next, 102162 br i1 %cond, label %loop, label %exit63 64exit:65 %.lcssa = phi float [ %rdx.next, %loop ]66 ret float %.lcssa67}68 69define float @pr72720reduction_using_active_lane_mask(ptr %src) {70; CHECK-LABEL: define float @pr72720reduction_using_active_lane_mask(71; CHECK-SAME: ptr [[SRC:%.*]]) {72; CHECK-NEXT: entry:73; CHECK-NEXT: br label [[VECTOR_PH:%.*]]74; CHECK: vector.ph:75; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]76; CHECK: vector.body:77; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE3:%.*]] ]78; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[PRED_LOAD_CONTINUE3]] ]79; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 080; CHECK-NEXT: [[VEC_IV1:%.*]] = add i32 [[INDEX]], 181; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 1482; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV1]], 1483; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]84; CHECK: pred.load.if:85; CHECK-NEXT: [[TMP3:%.*]] = getelementptr float, ptr [[SRC]], i32 [[INDEX]]86; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 487; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]88; CHECK: pred.load.continue:89; CHECK-NEXT: [[TMP5:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_LOAD_IF]] ]90; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]91; CHECK: pred.load.if2:92; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 193; CHECK-NEXT: [[TMP7:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP6]]94; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 495; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]96; CHECK: pred.load.continue3:97; CHECK-NEXT: [[TMP9:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], [[PRED_LOAD_IF2]] ]98; CHECK-NEXT: [[TMP10:%.*]] = select contract i1 [[TMP0]], float [[TMP5]], float -0.000000e+0099; CHECK-NEXT: [[TMP11:%.*]] = fadd contract float [[VEC_PHI]], [[TMP10]]100; CHECK-NEXT: [[TMP12:%.*]] = select contract i1 [[TMP1]], float [[TMP9]], float -0.000000e+00101; CHECK-NEXT: [[TMP13]] = fadd contract float [[TMP11]], [[TMP12]]102; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2103; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16104; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]105; CHECK: middle.block:106; CHECK-NEXT: br label [[LOOP:%.*]]107; CHECK: exit:108; CHECK-NEXT: ret float [[TMP13]]109;110; CHECK-ALM-LABEL: define float @pr72720reduction_using_active_lane_mask(111; CHECK-ALM-SAME: ptr [[SRC:%.*]]) {112; CHECK-ALM-NEXT: entry:113; CHECK-ALM-NEXT: br label [[VECTOR_PH:%.*]]114; CHECK-ALM: vector.ph:115; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]]116; CHECK-ALM: vector.body:117; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE3:%.*]] ]118; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[PRED_LOAD_CONTINUE3]] ]119; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1120; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX]], 15121; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 15122; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]123; CHECK-ALM: pred.load.if:124; CHECK-ALM-NEXT: [[TMP2:%.*]] = getelementptr float, ptr [[SRC]], i32 [[INDEX]]125; CHECK-ALM-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP2]], align 4126; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE]]127; CHECK-ALM: pred.load.continue:128; CHECK-ALM-NEXT: [[TMP4:%.*]] = phi float [ poison, [[VECTOR_BODY]] ], [ [[TMP3]], [[PRED_LOAD_IF]] ]129; CHECK-ALM-NEXT: br i1 [[ACTIVE_LANE_MASK1]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3]]130; CHECK-ALM: pred.load.if2:131; CHECK-ALM-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP1]]132; CHECK-ALM-NEXT: [[TMP6:%.*]] = load float, ptr [[TMP5]], align 4133; CHECK-ALM-NEXT: br label [[PRED_LOAD_CONTINUE3]]134; CHECK-ALM: pred.load.continue3:135; CHECK-ALM-NEXT: [[TMP7:%.*]] = phi float [ poison, [[PRED_LOAD_CONTINUE]] ], [ [[TMP6]], [[PRED_LOAD_IF2]] ]136; CHECK-ALM-NEXT: [[TMP8:%.*]] = select contract i1 [[ACTIVE_LANE_MASK]], float [[TMP4]], float -0.000000e+00137; CHECK-ALM-NEXT: [[TMP9:%.*]] = fadd contract float [[VEC_PHI]], [[TMP8]]138; CHECK-ALM-NEXT: [[TMP10:%.*]] = select contract i1 [[ACTIVE_LANE_MASK1]], float [[TMP7]], float -0.000000e+00139; CHECK-ALM-NEXT: [[TMP11]] = fadd contract float [[TMP9]], [[TMP10]]140; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2141; CHECK-ALM-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16142; CHECK-ALM-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]143; CHECK-ALM: middle.block:144; CHECK-ALM-NEXT: br label [[LOOP:%.*]]145; CHECK-ALM: exit:146; CHECK-ALM-NEXT: ret float [[TMP11]]147;148entry:149 br label %loop150 151loop:152 %iv = phi i32 [ 0, %entry ], [ %narrow, %loop ]153 %rdx = phi float [ 0.0, %entry ], [ %rdx.next, %loop ]154 %narrow = add nuw nsw i32 %iv, 1155 %gep = getelementptr float, ptr %src, i32 %iv156 %l = load float, ptr %gep, align 4157 %rdx.next = fadd contract float %rdx, %l158 %ec = icmp ult i32 %narrow, 15159 br i1 %ec, label %loop, label %exit160 161exit:162 %.lcssa = phi float [ %rdx.next, %loop ]163 ret float %.lcssa164}165 166define float @fadd_reduction_with_live_in(float %inc) {167; CHECK-LABEL: define float @fadd_reduction_with_live_in(168; CHECK-SAME: float [[INC:%.*]]) {169; CHECK-NEXT: entry:170; CHECK-NEXT: br label [[VECTOR_PH:%.*]]171; CHECK: vector.ph:172; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]173; CHECK: vector.body:174; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]175; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]176; CHECK-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0177; CHECK-NEXT: [[VEC_IV1:%.*]] = add i32 [[INDEX]], 1178; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[VEC_IV]], 1000179; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i32 [[VEC_IV1]], 1000180; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP0]], float [[INC]], float -0.000000e+00181; CHECK-NEXT: [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]182; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP1]], float [[INC]], float -0.000000e+00183; CHECK-NEXT: [[TMP5]] = fadd float [[TMP3]], [[TMP4]]184; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2185; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002186; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]187; CHECK: middle.block:188; CHECK-NEXT: br label [[LOOP:%.*]]189; CHECK: exit:190; CHECK-NEXT: ret float [[TMP5]]191;192; CHECK-ALM-LABEL: define float @fadd_reduction_with_live_in(193; CHECK-ALM-SAME: float [[INC:%.*]]) {194; CHECK-ALM-NEXT: entry:195; CHECK-ALM-NEXT: br label [[VECTOR_PH:%.*]]196; CHECK-ALM: vector.ph:197; CHECK-ALM-NEXT: br label [[VECTOR_BODY:%.*]]198; CHECK-ALM: vector.body:199; CHECK-ALM-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]200; CHECK-ALM-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]201; CHECK-ALM-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1202; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK:%.*]] = icmp ult i32 [[INDEX]], 1001203; CHECK-ALM-NEXT: [[ACTIVE_LANE_MASK1:%.*]] = icmp ult i32 [[TMP1]], 1001204; CHECK-ALM-NEXT: [[TMP2:%.*]] = select i1 [[ACTIVE_LANE_MASK]], float [[INC]], float -0.000000e+00205; CHECK-ALM-NEXT: [[TMP3:%.*]] = fadd float [[VEC_PHI]], [[TMP2]]206; CHECK-ALM-NEXT: [[TMP4:%.*]] = select i1 [[ACTIVE_LANE_MASK1]], float [[INC]], float -0.000000e+00207; CHECK-ALM-NEXT: [[TMP5]] = fadd float [[TMP3]], [[TMP4]]208; CHECK-ALM-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2209; CHECK-ALM-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1002210; CHECK-ALM-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]211; CHECK-ALM: middle.block:212; CHECK-ALM-NEXT: br label [[LOOP:%.*]]213; CHECK-ALM: exit:214; CHECK-ALM-NEXT: ret float [[TMP5]]215;216entry:217 br label %loop218 219loop:220 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]221 %sum = phi float [ 0.000000e+00, %entry ], [ %sum.next, %loop ]222 %sum.next = fadd float %sum, %inc223 %iv.next = add i32 %iv, 1224 %ec = icmp eq i32 %iv, 1000225 br i1 %ec, label %exit, label %loop226 227exit:228 %lcssa = phi float [ %sum.next, %loop ]229 ret float %lcssa230}231