185 lines · plain
1# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs -enable-subreg-liveness -start-before=greedy %s -o - | FileCheck %s2 3# No available group of four strided x4 registers, fall back on default allocation order4---5name: form_4x_tuple_many_live6tracksRegLiveness: true7stack:8 - { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,9 stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,10 debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }11body: |12 bb.0.entry:13 liveins: $x0, $x1, $z0, $z1714 15 ; CHECK-LABEL: form_4x_tuple_many_live16 ; CHECK: stp d11, d10, [sp, #-48]!17 ; CHECK-NEXT: stp d9, d8, [sp, #16]18 ; CHECK-NEXT: str x29, [sp, #32]19 ; CHECK-NEXT: addvl sp, sp, #-220 ; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 48 + 16 * VG21 ; CHECK-NEXT: .cfi_offset w29, -1622 ; CHECK-NEXT: .cfi_offset b8, -2423 ; CHECK-NEXT: .cfi_offset b9, -3224 ; CHECK-NEXT: .cfi_offset b10, -4025 ; CHECK-NEXT: .cfi_offset b11, -4826 ; CHECK-NEXT: lsl x9, x1, #127 ; CHECK-NEXT: ptrue pn8.b28 ; CHECK-NEXT: mov w8, wzr29 ; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]30 ; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x1]31 ; CHECK-NEXT: ptrue p0.b32 ; CHECK-NEXT: add x10, x9, x133 ; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x9]34 ; CHECK-NEXT: ld1b { z4.b - z7.b }, pn8/z, [x0, x10]35 ; CHECK-NEXT: mov z8.d, z16.d36 ; CHECK-NEXT: mov z9.d, z18.d37 ; CHECK-NEXT: mov z21.d, z22.d38 ; CHECK-NEXT: mov z10.d, z19.d39 ; CHECK-NEXT: mov z22.d, z23.d40 ; CHECK-NEXT: mov z25.d, z26.d41 ; CHECK-NEXT: mov z11.d, z4.d42 ; CHECK-NEXT: mov z23.d, z5.d43 ; CHECK-NEXT: mov z26.d, z27.d44 ; CHECK-NEXT: mov z27.d, z6.d45 ; CHECK-NEXT: mov z29.d, z30.d46 ; CHECK-NEXT: mov z30.d, z31.d47 ; CHECK-NEXT: mov z31.d, z7.d48 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]49 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]50 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]51 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]52 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]53 ; CHECK-NEXT: st1b { z17.b }, p0, [x0]54 ; CHECK-NEXT: addvl sp, sp, #255 ; CHECK-NEXT: ldp d9, d8, [sp, #16]56 ; CHECK-NEXT: ldr x29, [sp, #32]57 ; CHECK-NEXT: ldp d11, d10, [sp], #4858 ; CHECK-NEXT: ret59 60 %0:gpr64common = COPY $x061 %1:gpr64 = COPY $x162 %2:zpr = COPY $z063 %3:zpr = COPY $z1764 %5:matrixindexgpr32_8_11 = COPY $wzr65 %6:gpr64 = UBFMXri %1, 63, 6266 %pred:pnr_p8to15 = PTRUE_C_B implicit $vg67 %7:ppr_3b = PTRUE_B 31, implicit $vg68 %8:gpr64 = ADDXrr %6, %169 %9:zpr4stridedorcontiguous = LD1B_4Z_IMM_PSEUDO %pred, %0, 070 %10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %171 %11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %672 %12:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %873 %13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub0, %10.zsub0, %11.zsub0, %12.zsub074 %14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub1, %10.zsub1, %11.zsub1, %12.zsub175 %15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub2, %10.zsub2, %11.zsub2, %12.zsub276 %16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub3, %10.zsub3, %11.zsub3, %12.zsub377 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 078 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 079 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 080 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 081 ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)82 ST1B_IMM %3, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)83 RET_ReallyLR84...85 86# First multi-vector load to be allocated is not the first operand of the FORM_TRANSPOSED pseudo87---88name: form_4x_tuple_allocation_order89tracksRegLiveness: true90stack:91 - { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,92 stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,93 debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }94body: |95 bb.0.entry:96 liveins: $x0, $x1, $z097 98 ; CHECK: str x29, [sp, #-16]!99 ; CHECK-NEXT: addvl sp, sp, #-2100 ; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG101 ; CHECK-NEXT: .cfi_offset w29, -16102 ; CHECK-NEXT: lsl x9, x1, #1103 ; CHECK-NEXT: ptrue pn8.b104 ; CHECK-NEXT: mov w8, wzr105 ; CHECK-NEXT: ptrue p0.b106 ; CHECK-NEXT: add x10, x9, x1107 ; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]108 ; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]109 ; CHECK-NEXT: ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]110 ; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]111 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]112 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]113 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]114 ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]115 ; CHECK-NEXT: st1b { z0.b }, p0, [x0]116 ; CHECK-NEXT: addvl sp, sp, #2117 ; CHECK-NEXT: ldr x29, [sp], #16118 ; CHECK-NEXT: ret119 120 %0:gpr64common = COPY $x0121 %1:gpr64 = COPY $x1122 %2:zpr = COPY $z0123 %5:matrixindexgpr32_8_11 = COPY $wzr124 %6:gpr64 = UBFMXri %1, 63, 62125 %pred:pnr_p8to15 = PTRUE_C_B implicit $vg126 %7:ppr_3b = PTRUE_B 31, implicit $vg127 %8:gpr64 = ADDXrr %6, %1128 %9:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %8129 %10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %6130 %11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %1131 %12:zpr4stridedorcontiguous = LD1B_4Z_IMM_PSEUDO %pred, %0, 0132 %13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub0, %11.zsub0, %10.zsub0, %9.zsub0133 %14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub1, %11.zsub1, %10.zsub1, %9.zsub1134 %15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub2, %11.zsub2, %10.zsub2, %9.zsub2135 %16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub3, %11.zsub3, %10.zsub3, %9.zsub3136 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 0137 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 0138 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 0139 $za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 0140 ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)141 RET_ReallyLR142...143 144# Strided order is [ $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 ]145# Ensure we don't allocate $z23_z31 & $z0_z8 although they are consecutive146---147 name: udot_form_2x_tuple_live_reg_order148 tracksRegLiveness: true149 body: |150 bb.0.entry:151 liveins: $x0, $x1, $z16, $z17, $z18, $z19, $z20, $z21, $z22152 153 ; CHECK: stp d9, d8, [sp, #-16]!154 ; CHECK-NEXT: .cfi_def_cfa_offset 16155 ; CHECK-NEXT: .cfi_offset b8, -8156 ; CHECK-NEXT: .cfi_offset b9, -16157 ; CHECK-NEXT: ptrue pn8.b158 ; CHECK-NEXT: mov w8, wzr159 ; CHECK-NEXT: ld1b { z0.b, z8.b }, pn8/z, [x0]160 ; CHECK-NEXT: ld1b { z1.b, z9.b }, pn8/z, [x0, x1]161 ; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b162 ; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b163 ; CHECK-NEXT: ldp d9, d8, [sp], #16164 ; CHECK-NEXT: ret165 166 %0:gpr64 = COPY $x1167 %1:gpr64common = COPY $x0168 %2:zpr = COPY $z16169 %3:zpr = COPY $z17170 %4:zpr = COPY $z18171 %5:zpr = COPY $z19172 %6:zpr = COPY $z20173 %7:zpr = COPY $z21174 %8:zpr = COPY $z22175 %9:matrixindexgpr32_8_11 = COPY $wzr176 %10:pnr_p8to15 = PTRUE_C_B implicit $vg177 %11:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO %10, %1, 0178 %12:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO %10, %1, %0179 %13:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub0, %12.zsub0180 %14:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub1, %12.zsub1181 $za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %13, undef %15:zpr_4b182 $za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %14, undef %16:zpr_4b183 RET_ReallyLR184...185