271 lines · plain
1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2// See https://llvm.org/LICENSE.txt for license information.3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4 5#include "assembly.h"6 7// Out-of-line LSE atomics helpers. Ported from libgcc library.8// N = {1, 2, 4, 8}9// M = {1, 2, 4, 8, 16}10// ORDER = {'relax', 'acq', 'rel', 'acq_rel', 'sync'}11// Routines implemented:12//13// iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)14// iN __aarch64_swpN_ORDER(iN val, iN *ptr)15// iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)16// iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)17// iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)18// iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)19//20// Routines may modify temporary registers tmp0, tmp1, tmp2,21// return value x0 and the flags only.22 23#if defined(__aarch64__) || defined(__arm64ec__)24 25#ifdef HAS_ASM_LSE26.arch armv8-a+lse27#else28.arch armv8-a29#endif30 31#if !defined(__APPLE__)32HIDDEN(__aarch64_have_lse_atomics)33#else34HIDDEN(___aarch64_have_lse_atomics)35#endif36 37// Generate mnemonics for38// L_cas: SIZE: 1,2,4,8,16 MODEL: 1,2,3,4,539// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8 MODEL: 1,2,3,4,540 41#if SIZE == 142#define S b43#define UXT uxtb44#define B 0x0000000045#elif SIZE == 246#define S h47#define UXT uxth48#define B 0x4000000049#elif SIZE == 4 || SIZE == 8 || SIZE == 1650#define S51#define UXT mov52#if SIZE == 453#define B 0x8000000054#elif SIZE == 855#define B 0xc000000056#endif57#else58#error59#endif // SIZE60 61#if MODEL == 162#define SUFF _relax63#define A64#define L65#define M 0x00000066#define N 0x00000067#define BARRIER68#elif MODEL == 269#define SUFF _acq70#define A a71#define L72#define M 0x40000073#define N 0x80000074#define BARRIER75#elif MODEL == 376#define SUFF _rel77#define A78#define L l79#define M 0x00800080#define N 0x40000081#define BARRIER82#elif MODEL == 483#define SUFF _acq_rel84#define A a85#define L l86#define M 0x40800087#define N 0xc0000088#define BARRIER89#elif MODEL == 590#define SUFF _sync91#ifdef L_swp92// swp has _acq semantics.93#define A a94#define L95#define M 0x40000096#define N 0x80000097#else98// All other _sync functions have _seq semantics.99#define A a100#define L l101#define M 0x408000102#define N 0xc00000103#endif104#define BARRIER dmb ish105#else106#error107#endif // MODEL108 109// Define register size.110#define x(N) GLUE2(x, N)111#define w(N) GLUE2(w, N)112#if SIZE < 8113#define s(N) w(N)114#else115#define s(N) x(N)116#endif117 118#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)119#if MODEL == 5120// Drop A for _sync functions.121#define LDXR GLUE3(ld, xr, S)122#else123#define LDXR GLUE4(ld, A, xr, S)124#endif125#define STXR GLUE4(st, L, xr, S)126 127// Define temporary registers.128#define tmp0 16129#define tmp1 17130#define tmp2 15131 132// Macro for branch to label if no LSE available133.macro JUMP_IF_NOT_LSE label134#if !defined(__APPLE__)135 adrp x(tmp0), __aarch64_have_lse_atomics136 ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]137#else138 adrp x(tmp0), ___aarch64_have_lse_atomics@page139 ldrb w(tmp0), [x(tmp0), ___aarch64_have_lse_atomics@pageoff]140#endif141 cbz w(tmp0), \label142.endm143 144#ifdef L_cas145DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas))146 JUMP_IF_NOT_LSE 8f147#if SIZE < 16148#ifdef HAS_ASM_LSE149#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2]150#else151#define CAS .inst 0x08a07c41 + B + M152#endif153 CAS // s(0), s(1), [x2]154 ret1558:156 UXT s(tmp0), s(0)1570:158 LDXR s(0), [x2]159 cmp s(0), s(tmp0)160 bne 1f161 STXR w(tmp1), s(1), [x2]162 cbnz w(tmp1), 0b1631:164 BARRIER165 ret166#else167#if MODEL == 5168// Drop A for _sync functions.169#define LDXP GLUE2(ld, xp)170#else171#define LDXP GLUE3(ld, A, xp)172#endif173#define STXP GLUE3(st, L, xp)174#ifdef HAS_ASM_LSE175#define CASP GLUE3(casp, A, L) x0, x1, x2, x3, [x4]176#else177#define CASP .inst 0x48207c82 + M178#endif179 180 CASP // x0, x1, x2, x3, [x4]181 ret1828:183 mov x(tmp0), x0184 mov x(tmp1), x11850:186 LDXP x0, x1, [x4]187 cmp x0, x(tmp0)188 ccmp x1, x(tmp1), #0, eq189 bne 1f190 STXP w(tmp2), x2, x3, [x4]191 cbnz w(tmp2), 0b1921:193 BARRIER194 ret195#endif196END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))197#endif // L_cas198 199#ifdef L_swp200#ifdef HAS_ASM_LSE201#define SWP GLUE4(swp, A, L, S) s(0), s(0), [x1]202#else203#define SWP .inst 0x38208020 + B + N204#endif205DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))206 JUMP_IF_NOT_LSE 8f207 SWP // s(0), s(0), [x1]208 ret2098:210 mov s(tmp0), s(0)2110:212 LDXR s(0), [x1]213 STXR w(tmp1), s(tmp0), [x1]214 cbnz w(tmp1), 0b215 BARRIER216 ret217END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))218#endif // L_swp219 220#if defined(L_ldadd) || defined(L_ldclr) || \221 defined(L_ldeor) || defined(L_ldset)222 223#ifdef L_ldadd224#define LDNM ldadd225#define OP add226#define OPN 0x0000227#elif defined(L_ldclr)228#define LDNM ldclr229#define OP bic230#define OPN 0x1000231#elif defined(L_ldeor)232#define LDNM ldeor233#define OP eor234#define OPN 0x2000235#elif defined(L_ldset)236#define LDNM ldset237#define OP orr238#define OPN 0x3000239#else240#error241#endif242 243#ifdef HAS_ASM_LSE244#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1]245#else246#define LDOP .inst 0x38200020 + OPN + B + N247#endif248 249DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))250 JUMP_IF_NOT_LSE 8f251 LDOP // s(0), s(0), [x1]252 ret2538:254 mov s(tmp0), s(0)2550:256 LDXR s(0), [x1]257 OP s(tmp1), s(0), s(tmp0)258 STXR w(tmp2), s(tmp1), [x1]259 cbnz w(tmp2), 0b260 BARRIER261 ret262END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))263#endif // L_ldadd L_ldclr L_ldeor L_ldset264 265NO_EXEC_STACK_DIRECTIVE266 267// GNU property note for BTI, PAC, and GCS268GNU_PROPERTY_BTI_PAC_GCS269 270#endif // defined(__aarch64__) || defined(__arm64ec__)271