157 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s3 4; Test that VPlan native path is able to widen call intructions like5; llvm.sqrt.* intrincis calls.6 7declare double @llvm.sqrt.f64(double %0)8define void @widen_call_instruction(ptr noalias nocapture readonly %a.in, ptr noalias nocapture readonly %b.in, ptr noalias nocapture %c.out) {9; CHECK-LABEL: define void @widen_call_instruction(10; CHECK-SAME: ptr noalias readonly captures(none) [[A_IN:%.*]], ptr noalias readonly captures(none) [[B_IN:%.*]], ptr noalias captures(none) [[C_OUT:%.*]]) {11; CHECK-NEXT: [[ENTRY:.*:]]12; CHECK-NEXT: br label %[[VECTOR_PH:.*]]13; CHECK: [[VECTOR_PH]]:14; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]15; CHECK: [[VECTOR_BODY]]:16; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]17; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]18; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[A_IN]], <4 x i64> [[VEC_IND]]19; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true), <4 x double> poison)20; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[B_IN]], <4 x i64> [[VEC_IND]]21; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x double> poison)22; CHECK-NEXT: [[TMP2:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[WIDE_MASKED_GATHER1]])23; CHECK-NEXT: br label %[[FOR2_HEADER2:.*]]24; CHECK: [[FOR2_HEADER2]]:25; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR2_HEADER2]] ]26; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x double> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER2]] ]27; CHECK-NEXT: [[TMP3]] = fadd <4 x double> [[TMP2]], [[VEC_PHI3]]28; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i32> [[VEC_PHI]], splat (i32 1)29; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[TMP4]], splat (i32 10000)30; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 031; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER2]]32; CHECK: [[VECTOR_LATCH]]:33; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x double> [ [[TMP3]], %[[FOR2_HEADER2]] ]34; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[C_OUT]], <4 x i64> [[VEC_IND]]35; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP9]], <4 x ptr> align 8 [[TMP7]], <4 x i1> splat (i1 true))36; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 437; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)38; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100039; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]40; CHECK: [[MIDDLE_BLOCK]]:41; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]42; CHECK: [[SCALAR_PH]]:43; CHECK-NEXT: br label %[[FOR1_HEADER:.*]]44; CHECK: [[FOR1_HEADER]]:45; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[INDVAR11:%.*]], %[[FOR1_LATCH:.*]] ]46; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds double, ptr [[A_IN]], i64 [[INDVAR1]]47; CHECK-NEXT: [[A:%.*]] = load double, ptr [[A_PTR]], align 848; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds double, ptr [[B_IN]], i64 [[INDVAR1]]49; CHECK-NEXT: [[B:%.*]] = load double, ptr [[B_PTR]], align 850; CHECK-NEXT: [[B_SQRT:%.*]] = call double @llvm.sqrt.f64(double [[B]])51; CHECK-NEXT: br label %[[FOR2_HEADER:.*]]52; CHECK: [[FOR2_HEADER]]:53; CHECK-NEXT: [[INDVAR2:%.*]] = phi i32 [ 0, %[[FOR1_HEADER]] ], [ [[INDVAR21:%.*]], %[[FOR2_HEADER]] ]54; CHECK-NEXT: [[A_REDUCTION:%.*]] = phi double [ [[A]], %[[FOR1_HEADER]] ], [ [[A_REDUCTION1:%.*]], %[[FOR2_HEADER]] ]55; CHECK-NEXT: [[A_REDUCTION1]] = fadd double [[B_SQRT]], [[A_REDUCTION]]56; CHECK-NEXT: [[INDVAR21]] = add nuw nsw i32 [[INDVAR2]], 157; CHECK-NEXT: [[FOR2_COND:%.*]] = icmp eq i32 [[INDVAR21]], 1000058; CHECK-NEXT: br i1 [[FOR2_COND]], label %[[FOR1_LATCH]], label %[[FOR2_HEADER]]59; CHECK: [[FOR1_LATCH]]:60; CHECK-NEXT: [[A_REDUCTION1_LCSSA:%.*]] = phi double [ [[A_REDUCTION1]], %[[FOR2_HEADER]] ]61; CHECK-NEXT: [[C_PTR:%.*]] = getelementptr inbounds double, ptr [[C_OUT]], i64 [[INDVAR1]]62; CHECK-NEXT: store double [[A_REDUCTION1_LCSSA]], ptr [[C_PTR]], align 863; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 164; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 100065; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT]], label %[[FOR1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]66; CHECK: [[EXIT]]:67; CHECK-NEXT: ret void68;69entry:70 br label %for1.header71 72for1.header:73 %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]74 %a.ptr = getelementptr inbounds double, ptr %a.in, i64 %indvar175 %a = load double, ptr %a.ptr, align 876 %b.ptr = getelementptr inbounds double, ptr %b.in, i64 %indvar177 %b = load double, ptr %b.ptr, align 878 %b.sqrt = call double @llvm.sqrt.f64(double %b)79 br label %for2.header80 81for2.header:82 %indvar2 = phi i32 [ 0, %for1.header ], [ %indvar21, %for2.header ]83 %a.reduction = phi double [ %a, %for1.header ], [ %a.reduction1, %for2.header ]84 %a.reduction1 = fadd double %b.sqrt, %a.reduction85 %indvar21 = add nuw nsw i32 %indvar2, 186 %for2.cond = icmp eq i32 %indvar21, 1000087 br i1 %for2.cond, label %for1.latch, label %for2.header88 89for1.latch:90 %c.ptr = getelementptr inbounds double, ptr %c.out, i64 %indvar191 store double %a.reduction1, ptr %c.ptr, align 892 %indvar11 = add nuw nsw i64 %indvar1, 193 %for1.cond = icmp eq i64 %indvar11, 100094 br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !095 96exit:97 ret void98}99 100; Check we do not try to widen non-intrinsic calls,101; https://github.com/llvm/llvm-project/issues/131071.102define void @call_to_non_intrinsic() {103; CHECK-LABEL: define void @call_to_non_intrinsic() {104; CHECK-NEXT: [[ENTRY:.*]]:105; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]106; CHECK: [[OUTER_HEADER]]:107; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]108; CHECK-NEXT: br label %[[INNER_HEADER:.*]]109; CHECK: [[INNER_HEADER]]:110; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_HEADER]] ]111; CHECK-NEXT: call void @use()112; CHECK-NEXT: [[INNER_IV_NEXT]] = add i64 [[INNER_IV]], 1113; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 100114; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]115; CHECK: [[OUTER_LATCH]]:116; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1117; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100118; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]119; CHECK: [[EXIT]]:120; CHECK-NEXT: ret void121;122entry:123 br label %outer.header124 125outer.header:126 %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]127 br label %inner.header128 129inner.header:130 %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.header ]131 call void @use()132 %inner.iv.next = add i64 %inner.iv, 1133 %inner.ec = icmp eq i64 %inner.iv.next, 100134 br i1 %inner.ec, label %outer.latch, label %inner.header135 136outer.latch:137 %outer.iv.next = add i64 %outer.iv, 1138 %outer.ec = icmp eq i64 %outer.iv.next, 100139 br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !0140 141exit:142 ret void143}144 145declare void @use()146 147!0 = distinct !{!0, !1}148!1 = !{!"llvm.loop.vectorize.enable", i1 true}149;.150; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}151; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}152; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}153; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}154; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}155; CHECK: [[META5]] = !{!"llvm.loop.vectorize.enable", i1 true}156;.157