215 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -p loop-vectorize -S %s | FileCheck %s3 4target triple = "arm64-apple-macosx11.0.0"5 6define void @fshl_operand_first_order_recurrence(ptr %dst, ptr noalias %src) {7; CHECK-LABEL: define void @fshl_operand_first_order_recurrence(8; CHECK-SAME: ptr [[DST:%.*]], ptr noalias [[SRC:%.*]]) {9; CHECK-NEXT: [[ENTRY:.*:]]10; CHECK-NEXT: br label %[[VECTOR_PH:.*]]11; CHECK: [[VECTOR_PH]]:12; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]13; CHECK: [[VECTOR_BODY]]:14; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]15; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i64> [ <i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD1:%.*]], %[[VECTOR_BODY]] ]16; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]17; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 218; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 819; CHECK-NEXT: [[WIDE_LOAD1]] = load <2 x i64>, ptr [[TMP5]], align 820; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[VECTOR_RECUR]], <2 x i64> [[WIDE_LOAD]], <2 x i32> <i32 1, i32 2>21; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[WIDE_LOAD]], <2 x i64> [[WIDE_LOAD1]], <2 x i32> <i32 1, i32 2>22; CHECK-NEXT: [[TMP8:%.*]] = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> splat (i64 1), <2 x i64> [[TMP6]], <2 x i64> splat (i64 1))23; CHECK-NEXT: [[TMP9:%.*]] = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> splat (i64 1), <2 x i64> [[TMP7]], <2 x i64> splat (i64 1))24; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]25; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 226; CHECK-NEXT: store <2 x i64> [[TMP8]], ptr [[TMP10]], align 827; CHECK-NEXT: store <2 x i64> [[TMP9]], ptr [[TMP13]], align 828; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 429; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10030; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: [[MIDDLE_BLOCK]]:32; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[WIDE_LOAD1]], i32 133; CHECK-NEXT: br label %[[SCALAR_PH:.*]]34; CHECK: [[SCALAR_PH]]:35; CHECK-NEXT: br label %[[LOOP:.*]]36; CHECK: [[LOOP]]:37; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]38; CHECK-NEXT: [[RECUR:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]39; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]40; CHECK-NEXT: [[L]] = load i64, ptr [[GEP_SRC]], align 841; CHECK-NEXT: [[OR:%.*]] = tail call i64 @llvm.fshl.i64(i64 1, i64 [[RECUR]], i64 1)42; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]43; CHECK-NEXT: store i64 [[OR]], ptr [[GEP_DST]], align 844; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 145; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 10046; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]47; CHECK: [[EXIT]]:48; CHECK-NEXT: ret void49;50entry:51 br label %loop52 53loop:54 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]55 %recur = phi i64 [ 0, %entry ], [ %l, %loop ]56 %gep.src = getelementptr inbounds i64, ptr %src, i64 %iv57 %l = load i64, ptr %gep.src, align 858 %or = tail call i64 @llvm.fshl.i64(i64 1, i64 %recur, i64 1)59 %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv60 store i64 %or, ptr %gep.dst, align 861 %iv.next = add i64 %iv, 162 %exitcond.not = icmp eq i64 %iv, 10063 br i1 %exitcond.not, label %exit, label %loop64 65exit:66 ret void67}68 69; Test case for https://github.com/llvm/llvm-project/issues/107016.70define void @powi_call(ptr %P) {71; CHECK-LABEL: define void @powi_call(72; CHECK-SAME: ptr [[P:%.*]]) {73; CHECK-NEXT: [[ENTRY:.*:]]74; CHECK-NEXT: br label %[[VECTOR_PH:.*]]75; CHECK: [[VECTOR_PH]]:76; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]77; CHECK: [[VECTOR_BODY]]:78; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[P]], align 879; CHECK-NEXT: [[TMP3:%.*]] = call <2 x double> @llvm.powi.v2f64.i32(<2 x double> [[WIDE_LOAD]], i32 3)80; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[P]], align 881; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]82; CHECK: [[MIDDLE_BLOCK]]:83; CHECK-NEXT: br label %[[EXIT:.*]]84; CHECK: [[EXIT]]:85; CHECK-NEXT: ret void86;87entry:88 br label %loop89 90loop:91 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]92 %gep = getelementptr inbounds double, ptr %P, i64 %iv93 %l = load double, ptr %gep94 %powi = tail call double @llvm.powi.f64.i32(double %l, i32 3)95 store double %powi, ptr %gep, align 896 %iv.next = add i64 %iv, 197 %ec = icmp eq i64 %iv, 198 br i1 %ec, label %exit, label %loop99 100exit:101 ret void102}103 104define void @call_scalarized(ptr noalias %src, ptr noalias %dst) {105; CHECK-LABEL: define void @call_scalarized(106; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {107; CHECK-NEXT: [[ENTRY:.*]]:108; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]109; CHECK: [[LOOP_HEADER]]:110; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 100, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]111; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -1112; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV_NEXT]]113; CHECK-NEXT: [[L:%.*]] = load double, ptr [[GEP_SRC]], align 8114; CHECK-NEXT: [[CMP295:%.*]] = fcmp une double [[L]], 4.000000e+00115; CHECK-NEXT: [[CMP299:%.*]] = fcmp ugt double [[L]], 0.000000e+00116; CHECK-NEXT: [[OR_COND:%.*]] = or i1 [[CMP295]], [[CMP299]]117; CHECK-NEXT: br i1 [[OR_COND]], label %[[LOOP_LATCH]], label %[[THEN:.*]]118; CHECK: [[THEN]]:119; CHECK-NEXT: [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[L]])120; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr double, ptr [[DST]], i64 [[IV_NEXT]]121; CHECK-NEXT: store double [[SQRT]], ptr [[GEP_DST]], align 8122; CHECK-NEXT: br label %[[LOOP_LATCH]]123; CHECK: [[LOOP_LATCH]]:124; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 0125; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]126; CHECK: [[EXIT]]:127; CHECK-NEXT: ret void128;129entry:130 br label %loop.header131 132loop.header:133 %iv = phi i64 [ 100, %entry ], [ %iv.next, %loop.latch ]134 %iv.next = add i64 %iv, -1135 %gep.src = getelementptr double, ptr %src, i64 %iv.next136 %l = load double, ptr %gep.src, align 8137 %cmp295 = fcmp une double %l, 4.000000e+00138 %cmp299 = fcmp ugt double %l, 0.000000e+00139 %or.cond = or i1 %cmp295, %cmp299140 br i1 %or.cond, label %loop.latch, label %then141 142then:143 %sqrt = call double @llvm.sqrt.f64(double %l)144 %gep.dst = getelementptr double, ptr %dst, i64 %iv.next145 store double %sqrt, ptr %gep.dst, align 8146 br label %loop.latch147 148loop.latch:149 %tobool.not = icmp eq i64 %iv.next, 0150 br i1 %tobool.not, label %exit, label %loop.header151 152exit:153 ret void154}155 156define void @call_forced_scalar(ptr %src.1, ptr %src.2, ptr noalias %dst.1, ptr noalias %dst.2) {157; CHECK-LABEL: define void @call_forced_scalar(158; CHECK-SAME: ptr [[SRC_1:%.*]], ptr [[SRC_2:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[DST_2:%.*]]) {159; CHECK-NEXT: [[ENTRY:.*]]:160; CHECK-NEXT: br label %[[LOOP:.*]]161; CHECK: [[LOOP]]:162; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]163; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC_1]], align 4164; CHECK-NEXT: [[SMAX:%.*]] = tail call i32 @llvm.smax.i32(i32 [[TMP0]], i32 0)165; CHECK-NEXT: [[UMIN:%.*]] = tail call i32 @llvm.umin.i32(i32 [[SMAX]], i32 1)166; CHECK-NEXT: [[UMIN_EXT:%.*]] = zext i32 [[UMIN]] to i64167; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr i8, ptr [[SRC_2]], i64 [[UMIN_EXT]]168; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[GEP_SRC_2]], align 1169; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[TMP1]] to i32170; CHECK-NEXT: [[MUL:%.*]] = mul i32 3, [[L_EXT]]171; CHECK-NEXT: store i32 [[MUL]], ptr [[DST_1]], align 4172; CHECK-NEXT: [[GEP_DST_2:%.*]] = getelementptr i32, ptr [[DST_2]], i64 [[IV]]173; CHECK-NEXT: store i32 0, ptr [[GEP_DST_2]], align 4174; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1175; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 0176; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]177; CHECK: [[EXIT]]:178; CHECK-NEXT: ret void179;180entry:181 br label %loop182 183loop:184 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]185 %0 = load i32, ptr %src.1, align 4186 %smax = tail call i32 @llvm.smax.i32(i32 %0, i32 0)187 %umin = tail call i32 @llvm.umin.i32(i32 %smax, i32 1)188 %umin.ext = zext i32 %umin to i64189 %gep.src.2 = getelementptr i8, ptr %src.2, i64 %umin.ext190 %1 = load i8, ptr %gep.src.2, align 1191 %l.ext = zext i8 %1 to i32192 %mul = mul i32 3, %l.ext193 store i32 %mul, ptr %dst.1, align 4194 %gep.dst.2 = getelementptr i32, ptr %dst.2, i64 %iv195 store i32 0, ptr %gep.dst.2, align 4196 %iv.next = add i64 %iv, 1197 %ec = icmp eq i64 %iv.next, 0198 br i1 %ec, label %exit, label %loop199 200exit:201 ret void202}203 204declare i32 @llvm.smax.i32(i32, i32)205declare i32 @llvm.umin.i32(i32, i32)206declare double @llvm.sqrt.f64(double)207declare double @llvm.powi.f64.i32(double, i32)208declare i64 @llvm.fshl.i64(i64, i64, i64)209;.210; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}211; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}212; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}213; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}214;.215