313 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=loop-vectorize -force-vector-width=2 -S %s | FileCheck %s3 4@src = external global [32 x i16], align 15@dst = external global [32 x i16], align 16 7; The load in the loop does not need predication, because the accessed memory8; is de-referenceable for all loop iterations.9define void @single_incoming_phi_no_blend_mask(i64 %a, i64 %b) {10; CHECK-LABEL: @single_incoming_phi_no_blend_mask(11; CHECK-NEXT: entry:12; CHECK-NEXT: br label [[VECTOR_PH:%.*]]13; CHECK: vector.ph:14; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 015; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer16; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]17; CHECK: vector.body:18; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]19; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]20; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i1621; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP1]]22; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP3]], align 123; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]24; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x i16> splat (i16 1), <2 x i16> [[WIDE_LOAD]]25; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]26; CHECK-NEXT: store <2 x i16> [[PREDPHI]], ptr [[TMP6]], align 227; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 228; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)29; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 3230; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: middle.block:32; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]33; CHECK: exit:34; CHECK-NEXT: ret void35;36entry:37 br label %loop.header38 39loop.header:40 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]41 %iv.trunc = trunc i64 %iv to i1642 br label %loop.cond43 44loop.cond:45 %blend = phi i16 [ %iv.trunc, %loop.header ]46 %src.ptr = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 %blend47 %lv = load i16, ptr %src.ptr, align 148 %cmp.b = icmp sgt i64 %iv, %a49 br i1 %cmp.b, label %loop.next, label %loop.latch50 51loop.next:52 br label %loop.latch53 54loop.latch:55 %res = phi i16 [ %lv, %loop.cond ], [ 1, %loop.next ]56 %dst.ptr = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 %iv57 store i16 %res, ptr %dst.ptr58 %iv.next = add nuw nsw i64 %iv, 159 %cmp439 = icmp ult i64 %iv, 3160 br i1 %cmp439, label %loop.header, label %exit61 62exit:63 ret void64}65 66; The load in the loop does not need predication, because the accessed memory67; is de-referenceable for all loop iterations.68define void @single_incoming_phi_with_blend_mask(i64 %a, i64 %b) {69; CHECK-LABEL: @single_incoming_phi_with_blend_mask(70; CHECK-NEXT: entry:71; CHECK-NEXT: br label [[VECTOR_PH:%.*]]72; CHECK: vector.ph:73; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 074; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer75; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]76; CHECK: vector.body:77; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]78; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]79; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i1680; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]81; CHECK-NEXT: [[TMP4:%.*]] = getelementptr [32 x i16], ptr @src, i16 0, i16 [[TMP1]]82; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP4]], align 183; CHECK-NEXT: [[TMP6:%.*]] = icmp sle <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]84; CHECK-NEXT: [[TMP7:%.*]] = select <2 x i1> [[TMP3]], <2 x i1> [[TMP6]], <2 x i1> zeroinitializer85; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i16> [[WIDE_LOAD]], <2 x i16> splat (i16 1)86; CHECK-NEXT: [[PREDPHI1:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[PREDPHI]], <2 x i16> zeroinitializer87; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]88; CHECK-NEXT: store <2 x i16> [[PREDPHI1]], ptr [[TMP9]], align 289; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 290; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)91; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 3292; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]93; CHECK: middle.block:94; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]95; CHECK: exit:96; CHECK-NEXT: ret void97;98entry:99 br label %loop.header100 101loop.header:102 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]103 %iv.trunc = trunc i64 %iv to i16104 %cmp.a = icmp ugt i64 %iv, %a105 br i1 %cmp.a, label %loop.cond, label %loop.latch106 107loop.cond:108 %blend = phi i16 [ %iv.trunc, %loop.header ]109 %src.ptr = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 %blend110 %lv = load i16, ptr %src.ptr, align 1111 %cmp.b = icmp sgt i64 %iv, %a112 br i1 %cmp.b, label %loop.next, label %loop.latch113 114loop.next:115 br label %loop.latch116 117loop.latch:118 %res = phi i16 [ 0, %loop.header ], [ %lv, %loop.cond ], [ 1, %loop.next ]119 %dst.ptr = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 %iv120 store i16 %res, ptr %dst.ptr121 %iv.next = add nuw nsw i64 %iv, 1122 %cmp439 = icmp ult i64 %iv, 31123 br i1 %cmp439, label %loop.header, label %exit124 125exit:126 ret void127}128 129define void @multiple_incoming_phi_with_blend_mask(i64 %a, ptr noalias %dst) {130; CHECK-LABEL: @multiple_incoming_phi_with_blend_mask(131; CHECK-NEXT: entry:132; CHECK-NEXT: br label [[VECTOR_PH:%.*]]133; CHECK: vector.ph:134; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 0135; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer136; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]137; CHECK: vector.body:138; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]139; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]140; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]141; CHECK-NEXT: [[VEC_IND3:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], [[VECTOR_BODY]] ]142; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]143; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> [[VEC_IND3]], <2 x i16> [[VEC_IND1]]144; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 0145; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1146; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP2]]147; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP4]]148; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[TMP3]], align 1149; CHECK-NEXT: [[TMP7:%.*]] = load i16, ptr [[TMP5]], align 1150; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i32 0151; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP7]], i32 1152; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[DST:%.*]], i64 [[INDEX]]153; CHECK-NEXT: store <2 x i16> [[TMP9]], ptr [[TMP10]], align 2154; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2155; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)156; CHECK-NEXT: [[VEC_IND_NEXT2]] = add <2 x i16> [[VEC_IND1]], splat (i16 2)157; CHECK-NEXT: [[VEC_IND_NEXT4]] = add <2 x i16> [[VEC_IND3]], splat (i16 2)158; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32159; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]160; CHECK: middle.block:161; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]162; CHECK: exit:163; CHECK-NEXT: ret void164;165entry:166 br label %loop.header167 168loop.header:169 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]170 %iv.trunc = trunc i64 %iv to i16171 %iv.trunc.2 = trunc i64 %iv to i16172 %cmp.a = icmp ugt i64 %iv, %a173 br i1 %cmp.a, label %loop.next, label %loop.latch174 175loop.next:176 br label %loop.latch177 178loop.latch:179 %blend = phi i16 [ %iv.trunc, %loop.header ], [ %iv.trunc.2, %loop.next ]180 %src.ptr = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 %blend181 %lv = load i16, ptr %src.ptr, align 1182 %dst.ptr = getelementptr inbounds i16, ptr %dst, i64 %iv183 store i16 %lv, ptr %dst.ptr184 %iv.next = add nuw nsw i64 %iv, 1185 %cmp439 = icmp ult i64 %iv, 31186 br i1 %cmp439, label %loop.header, label %exit187 188exit:189 ret void190}191 192; The load in the loop needs predication, because the accessed memory is not193; de-referencable for all iterations of the loop.194define void @single_incoming_needs_predication(i64 %a, i64 %b) {195; CHECK-LABEL: @single_incoming_needs_predication(196; CHECK-NEXT: entry:197; CHECK-NEXT: br label [[VECTOR_PH:%.*]]198; CHECK: vector.ph:199; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[A:%.*]], i64 0200; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer201; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]202; CHECK: vector.body:203; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]204; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE2]] ]205; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i16206; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]207; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0208; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]209; CHECK: pred.load.if:210; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[TMP1]], 0211; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP4]]212; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[TMP5]], align 1213; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i32 0214; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]215; CHECK: pred.load.continue:216; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x i16> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ]217; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1218; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]219; CHECK: pred.load.if1:220; CHECK-NEXT: [[TMP10:%.*]] = add i16 [[TMP1]], 1221; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 [[TMP10]]222; CHECK-NEXT: [[TMP12:%.*]] = load i16, ptr [[TMP11]], align 1223; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i16> [[TMP8]], i16 [[TMP12]], i32 1224; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]]225; CHECK: pred.load.continue2:226; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]227; CHECK-NEXT: [[TMP15:%.*]] = icmp sle <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]228; CHECK-NEXT: [[TMP16:%.*]] = select <2 x i1> [[TMP2]], <2 x i1> [[TMP15]], <2 x i1> zeroinitializer229; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP16]], <2 x i16> [[TMP14]], <2 x i16> splat (i16 1)230; CHECK-NEXT: [[PREDPHI3:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> [[PREDPHI]], <2 x i16> zeroinitializer231; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]232; CHECK-NEXT: store <2 x i16> [[PREDPHI3]], ptr [[TMP18]], align 2233; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2234; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)235; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64236; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]237; CHECK: middle.block:238; CHECK-NEXT: br label [[LOOP_LATCH:%.*]]239; CHECK: exit:240; CHECK-NEXT: ret void241;242entry:243 br label %loop.header244 245loop.header:246 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]247 %iv.trunc = trunc i64 %iv to i16248 %cmp.a = icmp ugt i64 %iv, %a249 br i1 %cmp.a, label %loop.cond, label %loop.latch250 251loop.cond:252 %blend = phi i16 [ %iv.trunc, %loop.header ]253 %src.ptr = getelementptr inbounds [32 x i16], ptr @src, i16 0, i16 %blend254 %lv = load i16, ptr %src.ptr, align 1255 %cmp.b = icmp sgt i64 %iv, %a256 br i1 %cmp.b, label %loop.next, label %loop.latch257 258loop.next:259 br label %loop.latch260 261loop.latch:262 %res = phi i16 [ 0, %loop.header ], [ %lv, %loop.cond ], [ 1, %loop.next ]263 %dst.ptr = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 %iv264 store i16 %res, ptr %dst.ptr265 %iv.next = add nuw nsw i64 %iv, 1266 %cmp439 = icmp ult i64 %iv, 63267 br i1 %cmp439, label %loop.header, label %exit268 269exit:270 ret void271}272 273; Test case for PR44800.274define void @duplicated_incoming_blocks_blend(i32 %x, ptr %ptr) {275; CHECK-LABEL: @duplicated_incoming_blocks_blend(276; CHECK-NEXT: entry:277; CHECK-NEXT: br label [[VECTOR_PH:%.*]]278; CHECK: vector.ph:279; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]280; CHECK: vector.body:281; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]282; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]283; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i32 [[INDEX]]284; CHECK-NEXT: store <2 x i32> [[VEC_IND]], ptr [[TMP1]], align 4285; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2286; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw <2 x i32> [[VEC_IND]], splat (i32 2)287; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000288; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]289; CHECK: middle.block:290; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]291; CHECK: exit:292; CHECK-NEXT: ret void293;294entry:295 br label %loop.header296 297loop.header:298 %iv = phi i32 [ 0 , %entry ], [ %add.i, %loop.latch ]299 %c.0 = icmp ugt i32 %iv, %x300 br i1 %c.0, label %loop.latch, label %loop.latch301 302loop.latch:303 %p = phi i32 [ %iv, %loop.header ], [ %iv, %loop.header ]304 %gep.ptr = getelementptr i32, ptr %ptr, i32 %p305 store i32 %p, ptr %gep.ptr306 %add.i = add nsw i32 %p, 1307 %cmp = icmp slt i32 %add.i, 1000308 br i1 %cmp, label %loop.header, label %exit309 310exit:311 ret void312}313