336 lines · plain
1# RUN: llc --verify-machineinstrs -mtriple=aarch64 -o - %s -run-pass pipeliner -aarch64-enable-pipeliner -debug-only=pipeliner -pipeliner-max-stages=50 -pipeliner-max-mii=50 -pipeliner-enable-copytophi=0 -pipeliner-ii-search-range=30 2>&1 | FileCheck %s2# REQUIRES: asserts3 4# Test that each instruction must be scheduled between the early cycle and the late cycle. Previously there were cases where an instruction is scheduled outside of the valid range. See issue #93936 for details.5 6# CHECK: {{^ *}}Try to schedule with 477# CHECK: {{^ *}}Inst (11) %48:fpr128 = LDRQui %35:gpr64sp, 0 :: (load (s128) from %ir.lsr.iv63, align 4, !tbaa !0)8# CHECK-EMPTY:9# CHECK-NEXT: {{^ *}}es: ffffffe9 ls: ffffffe910# CHECK-NEXT: {{^ *}}Trying to insert node between -23 and -23 II: 4711# CHECK-NEXT: {{^ *}}failed to insert at cycle -23 %48:fpr128 = LDRQui %35:gpr64sp, 0 :: (load (s128) from %ir.lsr.iv63, align 4, !tbaa !0)12 13--- |14 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"15 16 define dso_local void @f(ptr nocapture noundef writeonly %a, ptr nocapture noundef readonly %b, ptr nocapture noundef readonly %c, ptr nocapture noundef readonly %d, ptr nocapture noundef readonly %e, float noundef %f, i32 noundef %N) local_unnamed_addr {17 entry:18 %cmp16 = icmp sgt i32 %N, 019 br i1 %cmp16, label %for.body.preheader, label %for.cond.cleanup20 21 for.body.preheader: ; preds = %entry22 %wide.trip.count = zext nneg i32 %N to i6423 %min.iters.check = icmp ult i32 %N, 824 br i1 %min.iters.check, label %for.body.preheader37, label %vector.memcheck25 26 vector.memcheck: ; preds = %for.body.preheader27 %0 = ptrtoint ptr %a to i6428 %1 = ptrtoint ptr %b to i6429 %2 = ptrtoint ptr %c to i6430 %3 = ptrtoint ptr %d to i6431 %4 = ptrtoint ptr %e to i6432 %5 = sub i64 %0, %133 %diff.check = icmp ult i64 %5, 3234 %6 = sub i64 %0, %235 %diff.check22 = icmp ult i64 %6, 3236 %conflict.rdx = or i1 %diff.check, %diff.check2237 %7 = sub i64 %0, %338 %diff.check24 = icmp ult i64 %7, 3239 %conflict.rdx25 = or i1 %conflict.rdx, %diff.check2440 %8 = sub i64 %0, %441 %diff.check27 = icmp ult i64 %8, 3242 %conflict.rdx28 = or i1 %conflict.rdx25, %diff.check2743 br i1 %conflict.rdx28, label %for.body.preheader37, label %vector.ph44 45 vector.ph: ; preds = %vector.memcheck46 %n.vec = and i64 %wide.trip.count, 214748364047 %broadcast.splatinsert = insertelement <4 x float> poison, float %f, i64 048 %broadcast.splat = shufflevector <4 x float> %broadcast.splatinsert, <4 x float> poison, <4 x i32> zeroinitializer49 %scevgep54 = getelementptr i8, ptr %b, i64 1650 %scevgep58 = getelementptr i8, ptr %a, i64 1651 %scevgep62 = getelementptr i8, ptr %c, i64 1652 %scevgep66 = getelementptr i8, ptr %e, i64 1653 %scevgep70 = getelementptr i8, ptr %d, i64 1654 br label %vector.body55 56 vector.body: ; preds = %vector.body, %vector.ph57 %lsr.iv71 = phi ptr [ %scevgep72, %vector.body ], [ %scevgep70, %vector.ph ]58 %lsr.iv67 = phi ptr [ %scevgep68, %vector.body ], [ %scevgep66, %vector.ph ]59 %lsr.iv63 = phi ptr [ %scevgep64, %vector.body ], [ %scevgep62, %vector.ph ]60 %lsr.iv59 = phi ptr [ %scevgep60, %vector.body ], [ %scevgep58, %vector.ph ]61 %lsr.iv55 = phi ptr [ %scevgep56, %vector.body ], [ %scevgep54, %vector.ph ]62 %lsr.iv52 = phi i64 [ %lsr.iv.next53, %vector.body ], [ %n.vec, %vector.ph ]63 %scevgep57 = getelementptr i8, ptr %lsr.iv55, i64 -1664 %wide.load = load <4 x float>, ptr %scevgep57, align 4, !tbaa !665 %wide.load29 = load <4 x float>, ptr %lsr.iv55, align 4, !tbaa !666 %9 = fmul <4 x float> %wide.load, %broadcast.splat67 %10 = fmul <4 x float> %wide.load29, %broadcast.splat68 %scevgep65 = getelementptr i8, ptr %lsr.iv63, i64 -1669 %wide.load30 = load <4 x float>, ptr %scevgep65, align 4, !tbaa !670 %wide.load31 = load <4 x float>, ptr %lsr.iv63, align 4, !tbaa !671 %scevgep73 = getelementptr i8, ptr %lsr.iv71, i64 -1672 %wide.load32 = load <4 x float>, ptr %scevgep73, align 4, !tbaa !673 %wide.load33 = load <4 x float>, ptr %lsr.iv71, align 4, !tbaa !674 %11 = fsub <4 x float> %wide.load30, %wide.load3275 %12 = fsub <4 x float> %wide.load31, %wide.load3376 %13 = fmul <4 x float> %9, %1177 %14 = fmul <4 x float> %10, %1278 %scevgep69 = getelementptr i8, ptr %lsr.iv67, i64 -1679 %wide.load34 = load <4 x float>, ptr %scevgep69, align 4, !tbaa !680 %wide.load35 = load <4 x float>, ptr %lsr.iv67, align 4, !tbaa !681 %15 = fdiv <4 x float> %13, %wide.load3482 %16 = fdiv <4 x float> %14, %wide.load3583 %scevgep61 = getelementptr i8, ptr %lsr.iv59, i64 -1684 store <4 x float> %15, ptr %scevgep61, align 4, !tbaa !685 store <4 x float> %16, ptr %lsr.iv59, align 4, !tbaa !686 %lsr.iv.next53 = add nsw i64 %lsr.iv52, -887 %scevgep56 = getelementptr i8, ptr %lsr.iv55, i64 3288 %scevgep60 = getelementptr i8, ptr %lsr.iv59, i64 3289 %scevgep64 = getelementptr i8, ptr %lsr.iv63, i64 3290 %scevgep68 = getelementptr i8, ptr %lsr.iv67, i64 3291 %scevgep72 = getelementptr i8, ptr %lsr.iv71, i64 3292 %17 = icmp eq i64 %lsr.iv.next53, 093 br i1 %17, label %middle.block, label %vector.body, !llvm.loop !1094 95 middle.block: ; preds = %vector.body96 %cmp.n = icmp eq i64 %n.vec, %wide.trip.count97 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader3798 99 for.body.preheader37: ; preds = %vector.memcheck, %for.body.preheader, %middle.block100 %indvars.iv.ph = phi i64 [ %n.vec, %middle.block ], [ 0, %for.body.preheader ], [ 0, %vector.memcheck ]101 %18 = shl nuw nsw i64 %indvars.iv.ph, 2102 %scevgep = getelementptr i8, ptr %a, i64 %18103 %scevgep39 = getelementptr i8, ptr %e, i64 %18104 %scevgep42 = getelementptr i8, ptr %d, i64 %18105 %scevgep45 = getelementptr i8, ptr %c, i64 %18106 %scevgep48 = getelementptr i8, ptr %b, i64 %18107 %19 = sub i64 %wide.trip.count, %indvars.iv.ph108 br label %for.body109 110 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry111 ret void112 113 for.body: ; preds = %for.body.preheader37, %for.body114 %lsr.iv51 = phi i64 [ %19, %for.body.preheader37 ], [ %lsr.iv.next, %for.body ]115 %lsr.iv49 = phi ptr [ %scevgep48, %for.body.preheader37 ], [ %scevgep50, %for.body ]116 %lsr.iv46 = phi ptr [ %scevgep45, %for.body.preheader37 ], [ %scevgep47, %for.body ]117 %lsr.iv43 = phi ptr [ %scevgep42, %for.body.preheader37 ], [ %scevgep44, %for.body ]118 %lsr.iv40 = phi ptr [ %scevgep39, %for.body.preheader37 ], [ %scevgep41, %for.body ]119 %lsr.iv = phi ptr [ %scevgep, %for.body.preheader37 ], [ %scevgep38, %for.body ]120 %20 = load float, ptr %lsr.iv49, align 4, !tbaa !6121 %mul = fmul float %20, %f122 %21 = load float, ptr %lsr.iv46, align 4, !tbaa !6123 %22 = load float, ptr %lsr.iv43, align 4, !tbaa !6124 %sub = fsub float %21, %22125 %mul5 = fmul float %mul, %sub126 %23 = load float, ptr %lsr.iv40, align 4, !tbaa !6127 %div = fdiv float %mul5, %23128 store float %div, ptr %lsr.iv, align 4, !tbaa !6129 %scevgep38 = getelementptr i8, ptr %lsr.iv, i64 4130 %scevgep41 = getelementptr i8, ptr %lsr.iv40, i64 4131 %scevgep44 = getelementptr i8, ptr %lsr.iv43, i64 4132 %scevgep47 = getelementptr i8, ptr %lsr.iv46, i64 4133 %scevgep50 = getelementptr i8, ptr %lsr.iv49, i64 4134 %lsr.iv.next = add i64 %lsr.iv51, -1135 %exitcond.not = icmp eq i64 %lsr.iv.next, 0136 br i1 %exitcond.not, label %for.cond.cleanup, label %for.body137 }138 139 !6 = !{!7, !7, i64 0}140 !7 = !{!"float", !8, i64 0}141 !8 = !{!"omnipotent char", !9, i64 0}142 !9 = !{!"Simple C/C++ TBAA"}143 !10 = distinct !{!10, !11, !12, !13}144 !11 = !{!"llvm.loop.mustprogress"}145 !12 = !{!"llvm.loop.isvectorized", i32 1}146 !13 = !{!"llvm.loop.unroll.runtime.disable"}147 !14 = distinct !{!14, !11, !12}148 149...150---151name: f152tracksRegLiveness: true153liveins:154 - { reg: '$x0', virtual-reg: '%39' }155 - { reg: '$x1', virtual-reg: '%40' }156 - { reg: '$x2', virtual-reg: '%41' }157 - { reg: '$x3', virtual-reg: '%42' }158 - { reg: '$x4', virtual-reg: '%43' }159 - { reg: '$s0', virtual-reg: '%44' }160 - { reg: '$w5', virtual-reg: '%45' }161body: |162 bb.0.entry:163 successors: %bb.1, %bb.7164 liveins: $x0, $x1, $x2, $x3, $x4, $s0, $w5165 166 %45:gpr32common = COPY $w5167 %44:fpr32 = COPY $s0168 %43:gpr64common = COPY $x4169 %42:gpr64common = COPY $x3170 %41:gpr64common = COPY $x2171 %40:gpr64common = COPY $x1172 %39:gpr64common = COPY $x0173 dead $wzr = SUBSWri %45, 1, 0, implicit-def $nzcv174 Bcc 11, %bb.7, implicit $nzcv175 B %bb.1176 177 bb.1.for.body.preheader:178 successors: %bb.12, %bb.2179 180 %48:gpr32 = ORRWrs $wzr, %45, 0181 %0:gpr64 = SUBREG_TO_REG 0, killed %48, %subreg.sub_32182 dead $wzr = SUBSWri %45, 8, 0, implicit-def $nzcv183 Bcc 2, %bb.2, implicit $nzcv184 185 bb.12:186 %49:gpr64all = COPY $xzr187 %47:gpr64all = COPY %49188 B %bb.6189 190 bb.2.vector.memcheck:191 successors: %bb.6, %bb.11192 193 %55:gpr64common = SUBXrr %39, %40194 %59:gpr64all = COPY $xzr195 %51:gpr64all = COPY %59196 dead $xzr = SUBSXri killed %55, 32, 0, implicit-def $nzcv197 Bcc 3, %bb.6, implicit $nzcv198 B %bb.11199 200 bb.11.vector.memcheck:201 successors: %bb.6, %bb.10202 203 %56:gpr64common = SUBXrr %39, %41204 dead $xzr = SUBSXri %56, 32, 0, implicit-def $nzcv205 Bcc 3, %bb.6, implicit $nzcv206 B %bb.10207 208 bb.10.vector.memcheck:209 successors: %bb.6, %bb.9210 211 %57:gpr64common = SUBXrr %39, %42212 dead $xzr = SUBSXri %57, 32, 0, implicit-def $nzcv213 Bcc 3, %bb.6, implicit $nzcv214 B %bb.9215 216 bb.9.vector.memcheck:217 successors: %bb.6, %bb.3218 219 %58:gpr64common = SUBXrr %39, %43220 dead $xzr = SUBSXri %58, 32, 0, implicit-def $nzcv221 Bcc 3, %bb.6, implicit $nzcv222 B %bb.3223 224 bb.3.vector.ph:225 %64:gpr64common = ANDXri %0, 8027226 %1:gpr64 = COPY %64227 %66:fpr128 = IMPLICIT_DEF228 %65:fpr128 = INSERT_SUBREG %66, %44, %subreg.ssub229 %67:gpr64sp = ADDXri %40, 16, 0230 %3:gpr64all = COPY %67231 %68:gpr64sp = ADDXri %39, 16, 0232 %4:gpr64all = COPY %68233 %69:gpr64sp = ADDXri %41, 16, 0234 %5:gpr64all = COPY %69235 %70:gpr64sp = ADDXri %43, 16, 0236 %6:gpr64all = COPY %70237 %71:gpr64sp = ADDXri %42, 16, 0238 %7:gpr64all = COPY %71239 240 bb.4.vector.body:241 successors: %bb.5, %bb.4242 243 %8:gpr64sp = PHI %7, %bb.3, %19, %bb.4244 %9:gpr64sp = PHI %6, %bb.3, %18, %bb.4245 %10:gpr64sp = PHI %5, %bb.3, %17, %bb.4246 %11:gpr64sp = PHI %4, %bb.3, %16, %bb.4247 %12:gpr64sp = PHI %3, %bb.3, %15, %bb.4248 %13:gpr64sp = PHI %1, %bb.3, %14, %bb.4249 %72:fpr128 = LDURQi %12, -16 :: (load (s128) from %ir.scevgep57, align 4, !tbaa !6)250 %73:fpr128 = LDRQui %12, 0 :: (load (s128) from %ir.lsr.iv55, align 4, !tbaa !6)251 %74:fpr128 = nofpexcept FMULv4i32_indexed killed %72, %65, 0, implicit $fpcr252 %75:fpr128 = nofpexcept FMULv4i32_indexed killed %73, %65, 0, implicit $fpcr253 %76:fpr128 = LDURQi %10, -16 :: (load (s128) from %ir.scevgep65, align 4, !tbaa !6)254 %77:fpr128 = LDRQui %10, 0 :: (load (s128) from %ir.lsr.iv63, align 4, !tbaa !6)255 %78:fpr128 = LDURQi %8, -16 :: (load (s128) from %ir.scevgep73, align 4, !tbaa !6)256 %79:fpr128 = LDRQui %8, 0 :: (load (s128) from %ir.lsr.iv71, align 4, !tbaa !6)257 %80:fpr128 = nofpexcept FSUBv4f32 killed %76, killed %78, implicit $fpcr258 %81:fpr128 = nofpexcept FSUBv4f32 killed %77, killed %79, implicit $fpcr259 %82:fpr128 = nofpexcept FMULv4f32 killed %74, killed %80, implicit $fpcr260 %83:fpr128 = nofpexcept FMULv4f32 killed %75, killed %81, implicit $fpcr261 %84:fpr128 = LDURQi %9, -16 :: (load (s128) from %ir.scevgep69, align 4, !tbaa !6)262 %85:fpr128 = LDRQui %9, 0 :: (load (s128) from %ir.lsr.iv67, align 4, !tbaa !6)263 %86:fpr128 = nofpexcept FDIVv4f32 killed %82, killed %84, implicit $fpcr264 %87:fpr128 = nofpexcept FDIVv4f32 killed %83, killed %85, implicit $fpcr265 STURQi killed %86, %11, -16 :: (store (s128) into %ir.scevgep61, align 4, !tbaa !6)266 STRQui killed %87, %11, 0 :: (store (s128) into %ir.lsr.iv59, align 4, !tbaa !6)267 %88:gpr64 = nsw SUBSXri %13, 8, 0, implicit-def $nzcv268 %14:gpr64all = COPY %88269 %89:gpr64sp = ADDXri %12, 32, 0270 %15:gpr64all = COPY %89271 %90:gpr64sp = ADDXri %11, 32, 0272 %16:gpr64all = COPY %90273 %91:gpr64sp = ADDXri %10, 32, 0274 %17:gpr64all = COPY %91275 %92:gpr64sp = ADDXri %9, 32, 0276 %18:gpr64all = COPY %92277 %93:gpr64sp = ADDXri %8, 32, 0278 %19:gpr64all = COPY %93279 Bcc 1, %bb.4, implicit $nzcv280 B %bb.5281 282 bb.5.middle.block:283 dead $xzr = SUBSXrr %64, %0, implicit-def $nzcv284 Bcc 0, %bb.7, implicit $nzcv285 B %bb.6286 287 bb.6.for.body.preheader37:288 %20:gpr64 = PHI %47, %bb.12, %51, %bb.2, %51, %bb.11, %51, %bb.10, %51, %bb.9, %1, %bb.5289 %95:gpr64 = nuw nsw UBFMXri %20, 62, 61290 %96:gpr64 = ADDXrr %39, %95291 %21:gpr64all = COPY %96292 %97:gpr64 = ADDXrr %43, %95293 %22:gpr64all = COPY %97294 %98:gpr64 = ADDXrr %42, %95295 %23:gpr64all = COPY %98296 %99:gpr64 = ADDXrr %41, %95297 %24:gpr64all = COPY %99298 %100:gpr64 = ADDXrr %40, %95299 %25:gpr64all = COPY %100300 %101:gpr64 = SUBXrr %0, %20301 %26:gpr64all = COPY %101302 B %bb.8303 304 bb.7.for.cond.cleanup:305 RET_ReallyLR306 307 bb.8.for.body:308 successors: %bb.7, %bb.8309 310 %27:gpr64sp = PHI %26, %bb.6, %38, %bb.8311 %28:gpr64sp = PHI %25, %bb.6, %37, %bb.8312 %29:gpr64sp = PHI %24, %bb.6, %36, %bb.8313 %30:gpr64sp = PHI %23, %bb.6, %35, %bb.8314 %31:gpr64sp = PHI %22, %bb.6, %34, %bb.8315 %32:gpr64sp = PHI %21, %bb.6, %33, %bb.8316 early-clobber %102:gpr64sp, %103:fpr32 = LDRSpost %28, 4 :: (load (s32) from %ir.lsr.iv49, !tbaa !6)317 %104:fpr32 = nofpexcept FMULSrr killed %103, %44, implicit $fpcr318 early-clobber %105:gpr64sp, %106:fpr32 = LDRSpost %29, 4 :: (load (s32) from %ir.lsr.iv46, !tbaa !6)319 early-clobber %107:gpr64sp, %108:fpr32 = LDRSpost %30, 4 :: (load (s32) from %ir.lsr.iv43, !tbaa !6)320 %109:fpr32 = nofpexcept FSUBSrr killed %106, killed %108, implicit $fpcr321 %110:fpr32 = nofpexcept FMULSrr killed %104, killed %109, implicit $fpcr322 early-clobber %111:gpr64sp, %112:fpr32 = LDRSpost %31, 4 :: (load (s32) from %ir.lsr.iv40, !tbaa !6)323 %113:fpr32 = nofpexcept FDIVSrr killed %110, killed %112, implicit $fpcr324 early-clobber %114:gpr64sp = STRSpost killed %113, %32, 4 :: (store (s32) into %ir.lsr.iv, !tbaa !6)325 %33:gpr64all = COPY %114326 %34:gpr64all = COPY %111327 %35:gpr64all = COPY %107328 %36:gpr64all = COPY %105329 %37:gpr64all = COPY %102330 %115:gpr64 = SUBSXri %27, 1, 0, implicit-def $nzcv331 %38:gpr64all = COPY %115332 Bcc 0, %bb.7, implicit $nzcv333 B %bb.8334 335...336