223 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 52; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s3 4define void @wide_phi_2_predecessors(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {5; CHECK-LABEL: define void @wide_phi_2_predecessors(6; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {7; CHECK-NEXT: [[ENTRY:.*:]]8; CHECK-NEXT: br label %[[VECTOR_PH:.*]]9; CHECK: [[VECTOR_PH]]:10; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 011; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer12; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]13; CHECK: [[VECTOR_BODY]]:14; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]15; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]16; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]17; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))18; CHECK-NEXT: br label %[[INNER_HEADER1:.*]]19; CHECK: [[INNER_HEADER1]]:20; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH3:.*]] ]21; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH3]] ]22; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]23; CHECK-NEXT: br i1 [[COND]], label %[[THEN2:.*]], label %[[INNER_LATCH3]]24; CHECK: [[THEN2]]:25; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)26; CHECK-NEXT: br label %[[INNER_LATCH3]]27; CHECK: [[INNER_LATCH3]]:28; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[THEN2]] ], [ zeroinitializer, %[[INNER_HEADER1]] ]29; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]30; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]31; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)32; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)33; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 034; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]35; CHECK: [[VECTOR_LATCH]]:36; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER_LATCH3]] ]37; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP10]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))38; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 439; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)40; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100041; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]42; CHECK: [[MIDDLE_BLOCK]]:43; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]44; CHECK: [[SCALAR_PH]]:45; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]46; CHECK: [[OUTER_HEADER]]:47; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]48; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]49; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 450; CHECK-NEXT: br label %[[INNER_HEADER:.*]]51; CHECK: [[INNER_HEADER]]:52; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]53; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]54; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]55; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]56; CHECK: [[THEN]]:57; CHECK-NEXT: [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 858; CHECK-NEXT: br label %[[INNER_LATCH]]59; CHECK: [[INNER_LATCH]]:60; CHECK-NEXT: [[P:%.*]] = phi i64 [ [[L_B]], %[[THEN]] ], [ 0, %[[INNER_HEADER]] ]61; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]62; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]63; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 164; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 100065; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]66; CHECK: [[OUTER_LATCH]]:67; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]68; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 869; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 170; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100071; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]72; CHECK: [[EXIT]]:73; CHECK-NEXT: ret void74;75entry:76 br label %outer.header77 78outer.header: ; preds = %outer.latch, %outer.header.lr.ph79 %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]80 %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv81 store i32 %c, ptr %gep.A, align 482 br label %inner.header83 84inner.header:85 %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]86 %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]87 %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv88 br i1 %cond, label %then, label %inner.latch89 90then:91 %l.b = load i64, ptr %gep.B, align 892 br label %inner.latch93 94inner.latch:95 %p = phi i64 [ %l.b, %then ], [ 0, %inner.header ]96 %add.1 = add nsw i64 %p, %outer.iv97 %red.next = add nsw i64 %add.1, %red98 %inner.iv.next = add nuw nsw i64 %inner.iv, 199 %inner.ec = icmp eq i64 %inner.iv.next, 1000100 br i1 %inner.ec, label %outer.latch, label %inner.header101 102outer.latch:103 store i64 %red.next, ptr %gep.A, align 8104 %outer.iv.next = add nuw nsw i64 %outer.iv, 1105 %outer.ec = icmp eq i64 %outer.iv.next, 1000106 br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1107 108exit:109 ret void110}111 112define void @wide_phi_2_predecessors_phi_ops_swapped(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {113; CHECK-LABEL: define void @wide_phi_2_predecessors_phi_ops_swapped(114; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {115; CHECK-NEXT: [[ENTRY:.*:]]116; CHECK-NEXT: br label %[[VECTOR_PH:.*]]117; CHECK: [[VECTOR_PH]]:118; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0119; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer120; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]121; CHECK: [[VECTOR_BODY]]:122; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]123; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]124; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]125; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))126; CHECK-NEXT: br label %[[INNER_HEADER1:.*]]127; CHECK: [[INNER_HEADER1]]:128; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH3:.*]] ]129; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH3]] ]130; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]131; CHECK-NEXT: br i1 [[COND]], label %[[THEN2:.*]], label %[[INNER_LATCH3]]132; CHECK: [[THEN2]]:133; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)134; CHECK-NEXT: br label %[[INNER_LATCH3]]135; CHECK: [[INNER_LATCH3]]:136; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[THEN2]] ], [ zeroinitializer, %[[INNER_HEADER1]] ]137; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]138; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]139; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)140; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)141; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0142; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]143; CHECK: [[VECTOR_LATCH]]:144; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i64> [ [[TMP3]], %[[INNER_LATCH3]] ]145; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP10]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))146; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4147; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)148; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000149; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]150; CHECK: [[MIDDLE_BLOCK]]:151; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]152; CHECK: [[SCALAR_PH]]:153; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]154; CHECK: [[OUTER_HEADER]]:155; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]156; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]157; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 4158; CHECK-NEXT: br label %[[INNER_HEADER:.*]]159; CHECK: [[INNER_HEADER]]:160; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]161; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]162; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]163; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]164; CHECK: [[THEN]]:165; CHECK-NEXT: [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 8166; CHECK-NEXT: br label %[[INNER_LATCH]]167; CHECK: [[INNER_LATCH]]:168; CHECK-NEXT: [[P:%.*]] = phi i64 [ 0, %[[INNER_HEADER]] ], [ [[L_B]], %[[THEN]] ]169; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]170; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]171; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1172; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000173; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]174; CHECK: [[OUTER_LATCH]]:175; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]176; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 8177; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1178; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000179; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]180; CHECK: [[EXIT]]:181; CHECK-NEXT: ret void182;183entry:184 br label %outer.header185 186outer.header: ; preds = %outer.latch, %outer.header.lr.ph187 %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]188 %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv189 store i32 %c, ptr %gep.A, align 4190 br label %inner.header191 192inner.header:193 %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]194 %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]195 %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv196 br i1 %cond, label %then, label %inner.latch197 198then:199 %l.b = load i64, ptr %gep.B, align 8200 br label %inner.latch201 202inner.latch:203 %p = phi i64 [ 0, %inner.header ], [ %l.b, %then ]204 %add.1 = add nsw i64 %p, %outer.iv205 %red.next = add nsw i64 %add.1, %red206 %inner.iv.next = add nuw nsw i64 %inner.iv, 1207 %inner.ec = icmp eq i64 %inner.iv.next, 1000208 br i1 %inner.ec, label %outer.latch, label %inner.header209 210outer.latch:211 store i64 %red.next, ptr %gep.A, align 8212 %outer.iv.next = add nuw nsw i64 %outer.iv, 1213 %outer.ec = icmp eq i64 %outer.iv.next, 1000214 br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1215 216exit:217 ret void218}219 220!1 = distinct !{!1, !2, !3}221!2 = !{!"llvm.loop.vectorize.width", i32 4}222!3 = !{!"llvm.loop.vectorize.enable", i1 true}223