376 lines · plain
1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2// See https://llvm.org/LICENSE.txt for license information.3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4 5// This patch implements the support routines for the SME ABI,6// described here:7// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines8 9#include "../assembly.h"10 11.set FEAT_SVE_BIT, 3012.set FEAT_SME_BIT, 4213.set FEAT_SME2_BIT, 5714.set FEAT_SME2_MASK, 1 << 5715.set SVCR_PSTATE_SM_BIT, 016 17#if !defined(__APPLE__)18#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features)19#define CPU_FEATS_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_cpu_features)20#else21// MachO requires @page/@pageoff directives because the global is defined22// in a different file. Otherwise this file may fail to build.23#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features)@page24#define CPU_FEATS_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_cpu_features)@pageoff25#endif26 27.arch armv9-a+sme228 29// Utility function which calls a system's abort() routine. Because the function30// is streaming-compatible it should disable streaming-SVE mode before calling31// abort(). Note that there is no need to preserve any state before the call,32// because the function does not return.33DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)34 .cfi_startproc35 .variant_pcs FUNC_SYMBOL(SYMBOL_NAME(do_abort))36 BTI_C37 stp x29, x30, [sp, #-32]!38 cntd x039 // Store VG to a stack location that we describe with .cfi_offset40 str x0, [sp, #16]41 .cfi_def_cfa_offset 3242 .cfi_offset w30, -2443 .cfi_offset w29, -3244 .cfi_offset 46, -1645 bl FUNC_SYMBOL(SYMBOL_NAME(__arm_sme_state))46 tbz x0, #0, 2f471:48 smstop sm492:50 // We can't make this into a tail-call because the unwinder would51 // need to restore the value of VG.52 bl FUNC_SYMBOL(SYMBOL_NAME(abort))53 .cfi_endproc54END_COMPILERRT_FUNCTION(do_abort)55 56// __arm_sme_state fills the result registers based on a local57// that is set as part of the compiler-rt startup code.58// __aarch64_has_sme_and_tpidr2_el059DEFINE_COMPILERRT_FUNCTION(__arm_sme_state)60 .variant_pcs __arm_sme_state61 BTI_C62 mov x0, xzr63 mov x1, xzr64 65 adrp x16, CPU_FEATS_SYMBOL66 ldr x16, [x16, CPU_FEATS_SYMBOL_OFFSET]67 tbz x16, #FEAT_SME_BIT, 1f680:69 orr x0, x0, #0xC00000000000000070 mrs x16, SVCR71 bfxil x0, x16, #0, #272 mrs x1, TPIDR2_EL0731:74 ret75END_COMPILERRT_FUNCTION(__arm_sme_state)76 77DEFINE_COMPILERRT_FUNCTION(__arm_tpidr2_restore)78 .variant_pcs __arm_tpidr2_restore79 BTI_C80 // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific81 // manner.82 mrs x14, TPIDR2_EL083 cbnz x14, 2f84 85 // If any of the reserved bytes in the first 16 bytes of BLK are nonzero,86 // the subroutine [..] aborts in some platform-defined manner.87 ldrh w14, [x0, #10]88 cbnz w14, 2f89 ldr w14, [x0, #12]90 cbnz w14, 2f91 92 // If BLK.za_save_buffer is NULL, the subroutine does nothing.93 ldr x16, [x0]94 cbz x16, 1f95 96 // If BLK.num_za_save_slices is zero, the subroutine does nothing.97 ldrh w14, [x0, #8]98 cbz x14, 1f99 100 mov x15, xzr1010:102 ldr za[w15,0], [x16]103 addsvl x16, x16, #1104 add x15, x15, #1105 cmp x14, x15106 b.ne 0b1071:108 ret1092:110 b FUNC_SYMBOL(SYMBOL_NAME(do_abort))111END_COMPILERRT_FUNCTION(__arm_tpidr2_restore)112 113DEFINE_COMPILERRT_FUNCTION(__arm_tpidr2_save)114 .variant_pcs __arm_tpidr2_save115 BTI_C116 // If the current thread does not have access to TPIDR2_EL0, the subroutine117 // does nothing.118 adrp x14, CPU_FEATS_SYMBOL119 ldr x14, [x14, CPU_FEATS_SYMBOL_OFFSET]120 tbz x14, #FEAT_SME_BIT, 1f121 122 // If TPIDR2_EL0 is null, the subroutine does nothing.123 mrs x16, TPIDR2_EL0124 cbz x16, 1f125 126 // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are127 // nonzero, the subroutine [..] aborts in some platform-defined manner.128 ldrh w14, [x16, #10]129 cbnz w14, 2f130 ldr w14, [x16, #12]131 cbnz w14, 2f132 133 // If num_za_save_slices is zero, the subroutine does nothing.134 ldrh w14, [x16, #8]135 cbz x14, 1f136 137 // If za_save_buffer is NULL, the subroutine does nothing.138 ldr x16, [x16]139 cbz x16, 1f140 141 mov x15, xzr1420:143 str za[w15,0], [x16]144 addsvl x16, x16, #1145 add x15, x15, #1146 cmp x14, x15147 b.ne 0b1481:149 ret1502:151 b FUNC_SYMBOL(SYMBOL_NAME(do_abort))152END_COMPILERRT_FUNCTION(__arm_tpidr2_save)153 154DEFINE_COMPILERRT_FUNCTION(__arm_za_disable)155 .cfi_startproc156 .variant_pcs __arm_za_disable157 BTI_C158 // If the current thread does not have access to SME, the subroutine does159 // nothing.160 adrp x14, CPU_FEATS_SYMBOL161 ldr x14, [x14, CPU_FEATS_SYMBOL_OFFSET]162 tbz x14, #FEAT_SME_BIT, 0f163 164 // Otherwise, the subroutine behaves as if it did the following:165 // * Call __arm_tpidr2_save.166 stp x29, x30, [sp, #-16]!167 .cfi_def_cfa_offset 16168 mov x29, sp169 .cfi_def_cfa w29, 16170 .cfi_offset w30, -8171 .cfi_offset w29, -16172 bl FUNC_SYMBOL(SYMBOL_NAME(__arm_tpidr2_save))173 174 // * Set TPIDR2_EL0 to null.175 msr TPIDR2_EL0, xzr176 177 // * Set PSTATE.ZA to 0.178 smstop za179 180 .cfi_def_cfa wsp, 16181 ldp x29, x30, [sp], #16182 .cfi_def_cfa_offset 0183 .cfi_restore w30184 .cfi_restore w291850:186 ret187 .cfi_endproc188END_COMPILERRT_FUNCTION(__arm_za_disable)189 190DEFINE_COMPILERRT_FUNCTION(__arm_get_current_vg)191 .variant_pcs __arm_get_current_vg192 BTI_C193 194 adrp x17, CPU_FEATS_SYMBOL195 ldr x17, [x17, CPU_FEATS_SYMBOL_OFFSET]196 tbnz w17, #FEAT_SVE_BIT, 1f197 tbz x17, #FEAT_SME_BIT, 2f1980:199 mrs x17, SVCR200 tbz x17, #SVCR_PSTATE_SM_BIT, 2f2011:202 cntd x0203 ret2042:205 mov x0, xzr206 ret207END_COMPILERRT_FUNCTION(__arm_get_current_vg)208 209// The diagram below describes the layout used in the following routines:210// * __arm_sme_state_size211// * __arm_sme_save212// * __arm_sme_restore213//214// +---------------------------------+215// | ... |216// | ZA buffer |217// | ... |218// +---------------------------------+ <- @96219// | ZT0 contents |220// +---------------------------------+ <- @32221// | byte 15-10: zero (reserved) |222// | byte 9-8: num_za_save_slices | TPIDR2 block223// | byte 7-0: za_save_buffer |224// +---------------------------------+ <- @16225// | bit 127-1: zero (reserved) | Internal state for __arm_sme_save/restore226// | bit 0: VALID |227// +---------------------------------+ <- @0228 229DEFINE_COMPILERRT_FUNCTION(__arm_sme_state_size)230 .variant_pcs __arm_sme_state_size231 BTI_C232 233 // Test if SME is available and ZA state is 'active'.234 adrp x17, CPU_FEATS_SYMBOL235 ldr x17, [x17, CPU_FEATS_SYMBOL_OFFSET]236 tbz x17, #FEAT_SME_BIT, 0f237 mrs x16, SVCR238 tbz x16, #1, 0f239 mrs x16, TPIDR2_EL0240 cbnz x16, 0f241 242 // Size = HAS_FEAT_SME2 ? 96 : 32243 tst x17, #FEAT_SME2_MASK244 mov w17, #32245 mov w16, #96246 csel x16, x17, x16, eq247 248 // Size = Size + (SVLB * SVLB)249 rdsvl x17, #1250 madd x0, x17, x17, x16251 ret252 2530:254 // Default case, 16 bytes is minimum (to encode VALID bit, multiple of 16 bytes)255 mov w0, #16256 ret257END_COMPILERRT_FUNCTION(__arm_sme_state_size)258 259DEFINE_COMPILERRT_FUNCTION(__arm_sme_save)260 .variant_pcs __arm_sme_save261 BTI_C262 263 // If PTR is not 16-byte aligned, abort.264 tst x0, #0xF265 b.ne 3f266 267 // Clear internal state bits268 stp xzr, xzr, [x0]269 270 // If SME is not available, PSTATE.ZA = 0 or TPIDR2_EL0 != 0, return.271 adrp x17, CPU_FEATS_SYMBOL272 ldr x17, [x17, CPU_FEATS_SYMBOL_OFFSET]273 tbz x17, #FEAT_SME_BIT, 2f274 mrs x16, SVCR275 tbz x16, #1, 2f276 mrs x16, TPIDR2_EL0277 cbnz x16, 2f278 279 # ZA or ZT0 need saving, we can now set internal VALID bit to 1280 mov w16, #1281 str x16, [x0]282 283 add x16, x0, #32284 tbz x17, #FEAT_SME2_BIT, 1f285 286 // Store ZT0287 str zt0, [x16]288 add x16, x16, #64289 2901:291 // Set up lazy-save (x16 = pointer to buffer)292 rdsvl x17, #1293 str x16, [x0, #16]!294 strh w17, [x0, #8]295 strh wzr, [x0, #10]296 str wzr, [x0, #12]297 msr TPIDR2_EL0, x0298 2992:300 // Do nothing301 ret302 3033:304 b FUNC_SYMBOL(SYMBOL_NAME(do_abort))305END_COMPILERRT_FUNCTION(__arm_sme_save)306 307DEFINE_COMPILERRT_FUNCTION(__arm_sme_restore)308 .cfi_startproc309 .variant_pcs __arm_sme_restore310 BTI_C311 312 stp x29, x30, [sp, #-16]!313 .cfi_def_cfa_offset 16314 mov x29, sp315 .cfi_def_cfa w29, 16316 .cfi_offset w30, -8317 .cfi_offset w29, -16318 319 // If PTR is not 16-byte aligned, abort.320 tst x0, #0xF321 b.ne 3f322 323 // If the VALID bit is 0, return early.324 ldr x16, [x0]325 cbz x16, 2f326 327 // If SME is not available, abort.328 adrp x17, CPU_FEATS_SYMBOL329 ldr x17, [x17, CPU_FEATS_SYMBOL_OFFSET]330 tbz x17, #FEAT_SME_BIT, 3f331 332 // If TPIDR2_EL0 != nullptr, no lazy-save was committed, try to reload zt0.333 mrs x16, TPIDR2_EL0334 cbnz x16, 1f335 336 // If TPIDR2_EL0 == nullptr and PSTATE.ZA = 1 (<=> ZA state is 'active'),337 // abort.338 mrs x16, SVCR339 tbnz x16, #1, 3f340 341 // Restore za.342 smstart za343 add x0, x0, #16344 bl __arm_tpidr2_restore345 sub x0, x0, #16346 3471:348 smstart za349 msr TPIDR2_EL0, xzr350 351 // Check if zt0 needs restoring.352 tbz x17, #FEAT_SME2_BIT, 2f353 354 // Restore zt0.355 add x16, x0, #32356 ldr zt0, [x16]357 3582:359 // Do nothing360 .cfi_def_cfa wsp, 16361 ldp x29, x30, [sp], #16362 .cfi_def_cfa_offset 0363 .cfi_restore w30364 .cfi_restore w29365 ret366 3673:368 b FUNC_SYMBOL(SYMBOL_NAME(do_abort))369 .cfi_endproc370END_COMPILERRT_FUNCTION(__arm_sme_restore)371 372NO_EXEC_STACK_DIRECTIVE373 374// GNU property note for BTI, PAC, and GCS375GNU_PROPERTY_BTI_PAC_GCS376