634 lines · plain
1//=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// AArch64 Atomic operand code-gen constructs.10//11//===----------------------------------------------------------------------===//12 13//===----------------------------------14// Atomic fences15//===----------------------------------16let AddedComplexity = 15 in17def : Pat<(atomic_fence (timm), 0), (MEMBARRIER)>;18def : Pat<(atomic_fence (i64 4), (timm)), (DMB (i32 0x9))>;19def : Pat<(atomic_fence (timm), (timm)), (DMB (i32 0xb))>;20 21//===----------------------------------22// Atomic loads23//===----------------------------------24 25// When they're actually atomic, only one addressing mode (GPR64sp) is26// supported, but when they're relaxed and anything can be used, all the27// standard modes would be valid and may give efficiency gains.28 29// An atomic load operation that does not need either acquire or release30// semantics.31class relaxed_load<PatFrags base>32 : PatFrag<(ops node:$ptr), (base node:$ptr)> {33 let IsAtomic = 1;34 let IsAtomicOrderingAcquireOrStronger = 0;35}36 37// A atomic load operation that actually needs acquire semantics.38class acquiring_load<PatFrags base>39 : PatFrag<(ops node:$ptr), (base node:$ptr)> {40 let IsAtomic = 1;41 let IsAtomicOrderingAcquire = 1;42}43 44// An atomic load operation that needs sequential consistency.45class seq_cst_load<PatFrags base>46 : PatFrag<(ops node:$ptr), (base node:$ptr)> {47 let IsAtomic = 1;48 let IsAtomicOrderingSequentiallyConsistent = 1;49}50 51let Predicates = [HasRCPC] in {52 // v8.3 Release Consistent Processor Consistent support, optional in v8.2.53 // 8-bit loads54 def : Pat<(acquiring_load<atomic_load_azext_8> GPR64sp:$ptr), (LDAPRB GPR64sp:$ptr)>;55 // 16-bit loads56 def : Pat<(acquiring_load<atomic_load_azext_16> GPR64sp:$ptr), (LDAPRH GPR64sp:$ptr)>;57 // 32-bit loads58 def : Pat<(acquiring_load<atomic_load_nonext_32> GPR64sp:$ptr), (LDAPRW GPR64sp:$ptr)>;59 // 64-bit loads60 def : Pat<(acquiring_load<atomic_load_nonext_64> GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>;61}62 63// 8-bit loads64def : Pat<(seq_cst_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;65def : Pat<(acquiring_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;66def : Pat<(relaxed_load<atomic_load_azext_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,67 ro_Wextend8:$offset)),68 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;69def : Pat<(relaxed_load<atomic_load_azext_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,70 ro_Xextend8:$offset)),71 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;72def : Pat<(relaxed_load<atomic_load_azext_8> (am_indexed8 GPR64sp:$Rn,73 uimm12s1:$offset)),74 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;75def : Pat<(relaxed_load<atomic_load_azext_8>76 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),77 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;78 79// 16-bit loads80def : Pat<(seq_cst_load<atomic_load_azext_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;81def : Pat<(acquiring_load<atomic_load_azext_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;82def : Pat<(relaxed_load<atomic_load_azext_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,83 ro_Wextend16:$extend)),84 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;85def : Pat<(relaxed_load<atomic_load_azext_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,86 ro_Xextend16:$extend)),87 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;88def : Pat<(relaxed_load<atomic_load_azext_16> (am_indexed16 GPR64sp:$Rn,89 uimm12s2:$offset)),90 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;91def : Pat<(relaxed_load<atomic_load_azext_16>92 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),93 (LDURHHi GPR64sp:$Rn, simm9:$offset)>;94 95// 32-bit loads96def : Pat<(seq_cst_load<atomic_load_nonext_32> GPR64sp:$ptr),97 (LDARW GPR64sp:$ptr)>;98def : Pat<(acquiring_load<atomic_load_nonext_32> GPR64sp:$ptr),99 (LDARW GPR64sp:$ptr)>;100def : Pat<(relaxed_load<atomic_load_nonext_32>101 (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)),102 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;103def : Pat<(relaxed_load<atomic_load_nonext_32>104 (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)),105 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;106def : Pat<(relaxed_load<atomic_load_nonext_32>107 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset)),108 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;109def : Pat<(relaxed_load<atomic_load_nonext_32>110 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),111 (LDURWi GPR64sp:$Rn, simm9:$offset)>;112 113// 64-bit loads114def : Pat<(seq_cst_load<atomic_load_nonext_64> GPR64sp:$ptr),115 (LDARX GPR64sp:$ptr)>;116def : Pat<(acquiring_load<atomic_load_nonext_64> GPR64sp:$ptr),117 (LDARX GPR64sp:$ptr)>;118def : Pat<(relaxed_load<atomic_load_nonext_64>119 (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)),120 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;121def : Pat<(relaxed_load<atomic_load_nonext_64>122 (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)),123 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;124def : Pat<(relaxed_load<atomic_load_nonext_64>125 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset)),126 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;127def : Pat<(relaxed_load<atomic_load_nonext_64>128 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),129 (LDURXi GPR64sp:$Rn, simm9:$offset)>;130 131// FP 32-bit loads132def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_nonext_32>133 (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend))))),134 (LDRSroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;135def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_nonext_32>136 (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend))))),137 (LDRSroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;138def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_nonext_32>139 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset))))),140 (LDRSui GPR64sp:$Rn, uimm12s4:$offset)>;141def : Pat<(f32 (bitconvert (i32 (relaxed_load<atomic_load_nonext_32>142 (am_unscaled32 GPR64sp:$Rn, simm9:$offset))))),143 (LDURSi GPR64sp:$Rn, simm9:$offset)>;144 145// FP 64-bit loads146def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_nonext_64>147 (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend))))),148 (LDRDroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;149def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_nonext_64>150 (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend))))),151 (LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;152def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_nonext_64>153 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))))),154 (LDRDui GPR64sp:$Rn, uimm12s8:$offset)>;155def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_nonext_64>156 (am_unscaled64 GPR64sp:$Rn, simm9:$offset))))),157 (LDURDi GPR64sp:$Rn, simm9:$offset)>;158 159//===----------------------------------160// Atomic stores161//===----------------------------------162 163// When they're actually atomic, only one addressing mode (GPR64sp) is164// supported, but when they're relaxed and anything can be used, all the165// standard modes would be valid and may give efficiency gains.166 167// A store operation that actually needs release semantics.168class releasing_store<PatFrag base>169 : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {170 let IsAtomic = 1;171 let IsAtomicOrderingReleaseOrStronger = 1;172}173 174// An atomic store operation that doesn't actually need to be atomic on AArch64.175class relaxed_store<PatFrag base>176 : PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {177 let IsAtomic = 1;178 let IsAtomicOrderingReleaseOrStronger = 0;179}180 181// 8-bit stores182def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),183 (STLRB GPR32:$val, GPR64sp:$ptr)>;184def : Pat<(relaxed_store<atomic_store_8>185 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),186 GPR32:$val),187 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;188def : Pat<(relaxed_store<atomic_store_8>189 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),190 GPR32:$val),191 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;192def : Pat<(relaxed_store<atomic_store_8>193 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),194 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;195def : Pat<(relaxed_store<atomic_store_8>196 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),197 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;198 199// 16-bit stores200def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),201 (STLRH GPR32:$val, GPR64sp:$ptr)>;202def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,203 ro_Wextend16:$extend),204 GPR32:$val),205 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;206def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,207 ro_Xextend16:$extend),208 GPR32:$val),209 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;210def : Pat<(relaxed_store<atomic_store_16>211 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),212 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;213def : Pat<(relaxed_store<atomic_store_16>214 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),215 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;216 217// 32-bit stores218def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),219 (STLRW GPR32:$val, GPR64sp:$ptr)>;220def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,221 ro_Wextend32:$extend),222 GPR32:$val),223 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;224def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,225 ro_Xextend32:$extend),226 GPR32:$val),227 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;228def : Pat<(relaxed_store<atomic_store_32>229 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),230 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;231def : Pat<(relaxed_store<atomic_store_32>232 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),233 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;234 235// 64-bit stores236def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),237 (STLRX GPR64:$val, GPR64sp:$ptr)>;238def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,239 ro_Wextend64:$extend),240 GPR64:$val),241 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;242def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,243 ro_Xextend64:$extend),244 GPR64:$val),245 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;246def : Pat<(relaxed_store<atomic_store_64>247 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),248 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;249def : Pat<(relaxed_store<atomic_store_64>250 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),251 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;252 253// FP 32-bit stores254def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,255 ro_Wextend32:$extend),256 (i32 (bitconvert (f32 FPR32Op:$val)))),257 (STRSroW FPR32Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;258def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,259 ro_Xextend32:$extend),260 (i32 (bitconvert (f32 FPR32Op:$val)))),261 (STRSroX FPR32Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;262def : Pat<(relaxed_store<atomic_store_32>263 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),264 (STRSui FPR32Op:$val, GPR64sp:$Rn, uimm12s4:$offset)>;265def : Pat<(relaxed_store<atomic_store_32>266 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), (i32 (bitconvert (f32 FPR32Op:$val)))),267 (STURSi FPR32Op:$val, GPR64sp:$Rn, simm9:$offset)>;268 269// FP 64-bit stores270def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,271 ro_Wextend64:$extend),272 (i64 (bitconvert (f64 FPR64Op:$val)))),273 (STRDroW FPR64Op:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;274def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,275 ro_Xextend64:$extend),276 (i64 (bitconvert (f64 FPR64Op:$val)))),277 (STRDroX FPR64Op:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;278def : Pat<(relaxed_store<atomic_store_64>279 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),280 (STRDui FPR64Op:$val, GPR64sp:$Rn, uimm12s8:$offset)>;281def : Pat<(relaxed_store<atomic_store_64>282 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),283 (STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;284 285//===----------------------------------286// Low-level exclusive operations287//===----------------------------------288 289// Load-exclusives.290 291def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{292 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;293}]> {294 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];295}296 297def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{298 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;299}]> {300 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];301}302 303def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{304 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;305}]> {306 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];307}308 309def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{310 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;311}]> {312 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];313}314 315def : Pat<(ldxr_1 GPR64sp:$addr),316 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;317def : Pat<(ldxr_2 GPR64sp:$addr),318 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;319def : Pat<(ldxr_4 GPR64sp:$addr),320 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;321def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;322 323def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),324 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;325def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),326 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;327def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),328 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;329 330// Load-exclusives.331 332def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{333 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;334}]> {335 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];336}337 338def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{339 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;340}]> {341 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];342}343 344def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{345 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;346}]> {347 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];348}349 350def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{351 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;352}]> {353 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];354}355 356def : Pat<(ldaxr_1 GPR64sp:$addr),357 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;358def : Pat<(ldaxr_2 GPR64sp:$addr),359 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;360def : Pat<(ldaxr_4 GPR64sp:$addr),361 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;362def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;363 364def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),365 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;366def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),367 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;368def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),369 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;370 371// Store-exclusives.372 373def stxr_1 : PatFrag<(ops node:$val, node:$ptr),374 (int_aarch64_stxr node:$val, node:$ptr), [{375 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;376}]> {377 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];378}379 380def stxr_2 : PatFrag<(ops node:$val, node:$ptr),381 (int_aarch64_stxr node:$val, node:$ptr), [{382 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;383}]> {384 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];385}386 387def stxr_4 : PatFrag<(ops node:$val, node:$ptr),388 (int_aarch64_stxr node:$val, node:$ptr), [{389 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;390}]> {391 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];392}393 394def stxr_8 : PatFrag<(ops node:$val, node:$ptr),395 (int_aarch64_stxr node:$val, node:$ptr), [{396 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;397}]> {398 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];399}400 401 402def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),403 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;404def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),405 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;406def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),407 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;408def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),409 (STXRX GPR64:$val, GPR64sp:$addr)>;410 411def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),412 (STXRB GPR32:$val, GPR64sp:$addr)>;413def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),414 (STXRH GPR32:$val, GPR64sp:$addr)>;415def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),416 (STXRW GPR32:$val, GPR64sp:$addr)>;417 418def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),419 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;420def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),421 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;422def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),423 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;424 425// Store-release-exclusives.426 427def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),428 (int_aarch64_stlxr node:$val, node:$ptr), [{429 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;430}]> {431 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];432}433 434def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),435 (int_aarch64_stlxr node:$val, node:$ptr), [{436 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;437}]> {438 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];439}440 441def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),442 (int_aarch64_stlxr node:$val, node:$ptr), [{443 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;444}]> {445 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];446}447 448def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),449 (int_aarch64_stlxr node:$val, node:$ptr), [{450 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;451}]> {452 let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];453}454 455 456def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),457 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;458def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),459 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;460def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),461 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;462def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),463 (STLXRX GPR64:$val, GPR64sp:$addr)>;464 465def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),466 (STLXRB GPR32:$val, GPR64sp:$addr)>;467def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),468 (STLXRH GPR32:$val, GPR64sp:$addr)>;469def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),470 (STLXRW GPR32:$val, GPR64sp:$addr)>;471 472def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),473 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;474def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),475 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;476def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),477 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;478 479 480// And clear exclusive.481 482def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;483 484//===----------------------------------485// Atomic cmpxchg for -O0486//===----------------------------------487 488// The fast register allocator used during -O0 inserts spills to cover any VRegs489// live across basic block boundaries. When this happens between an LDXR and an490// STXR it can clear the exclusive monitor, causing all cmpxchg attempts to491// fail.492 493// Unfortunately, this means we have to have an alternative (expanded494// post-regalloc) path for -O0 compilations. Fortunately this path can be495// significantly more naive than the standard expansion: we conservatively496// assume seq_cst, strong cmpxchg and omit clrex on failure.497 498let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",499 mayLoad = 1, mayStore = 1 in {500def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),501 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,502 Sched<[WriteAtomic]>;503 504def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),505 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,506 Sched<[WriteAtomic]>;507 508def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),509 (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,510 Sched<[WriteAtomic]>;511 512def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),513 (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,514 Sched<[WriteAtomic]>;515}516 517let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",518 mayLoad = 1, mayStore = 1 in {519class cmp_swap_128 : Pseudo<(outs GPR64common:$RdLo, GPR64common:$RdHi,520 GPR32common:$scratch),521 (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,522 GPR64:$newLo, GPR64:$newHi), []>,523 Sched<[WriteAtomic]>;524def CMP_SWAP_128 : cmp_swap_128;525def CMP_SWAP_128_RELEASE : cmp_swap_128;526def CMP_SWAP_128_ACQUIRE : cmp_swap_128;527def CMP_SWAP_128_MONOTONIC : cmp_swap_128;528}529 530// v8.1 Atomic instructions:531let Predicates = [HasLSE] in {532 defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;533 defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;534 defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;535 defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">;536 defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;537 defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">;538 defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">;539 defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">;540 defm : LDOPregister_patterns<"SWP", "atomic_swap">;541 defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;542 543 // These two patterns are only needed for global isel, selection dag isel544 // converts atomic load-sub into a sub and atomic load-add, and likewise for545 // and -> clr.546 defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">;547 defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;548}549 550defm atomic_load_fadd : binary_atomic_op_fp<atomic_load_fadd>;551defm atomic_load_fmin : binary_atomic_op_fp<atomic_load_fmin>;552defm atomic_load_fmax : binary_atomic_op_fp<atomic_load_fmax>;553 554defm atomic_load_fminimum : binary_atomic_op_fp<atomic_load_fminimum>;555defm atomic_load_fmaximum : binary_atomic_op_fp<atomic_load_fmaximum>;556 557let Predicates = [HasLSFE] in {558 defm : LDFPOPregister_patterns<"LDFADD", "atomic_load_fadd">;559 defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;560 defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;561 defm : LDFPOPregister_patterns<"LDFMAX", "atomic_load_fmaximum">;562 defm : LDFPOPregister_patterns<"LDFMIN", "atomic_load_fminimum">;563 564 defm : LDBFPOPregister_patterns<"LDBFADD", "atomic_load_fadd">;565 defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;566 defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;567 defm : LDBFPOPregister_patterns<"LDBFMAX", "atomic_load_fmaximum">;568 defm : LDBFPOPregister_patterns<"LDBFMIN", "atomic_load_fminimum">;569}570 571// v8.9a/v9.4a FEAT_LRCPC patterns572let Predicates = [HasRCPC3, HasNEON] in {573 // LDAP1 loads574 def : Pat<(vector_insert (v2i64 VecListOne128:$Rd),575 (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)), (i64 VectorIndexD:$idx)),576 (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;577 def : Pat<(vector_insert (v2f64 VecListOne128:$Rd),578 (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)),579 (LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;580 def : Pat<(v1i64 (scalar_to_vector581 (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))),582 (EXTRACT_SUBREG (LDAP1 (v2i64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;583 def : Pat<(v1f64 (scalar_to_vector584 (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))))),585 (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;586 587 // STL1 stores588 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,589 (i64 (vector_extract (v2i64 VecListOne128:$Vt), VectorIndexD:$idx))),590 (STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;591 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,592 (i64 (bitconvert (f64 (vector_extract (v2f64 VecListOne128:$Vt), VectorIndexD:$idx))))),593 (STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;594 // The v1i64 version of the vldap1_lane_* intrinsic is represented as a595 // vector_insert -> vector_extract -> atomic store sequence, which is captured596 // by the patterns above. We only need to cover the v1f64 case manually.597 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,598 (i64 (bitconvert (v1f64 VecListOne64:$Vt)))),599 (STL1 (SUBREG_TO_REG (i64 0), VecListOne64:$Vt, dsub), (i64 0), GPR64sp:$Rn)>;600}601 602// v8.4a FEAT_LRCPC2 patterns603let Predicates = [HasRCPC_IMMO, UseLDAPUR] in {604 // Load-Acquire RCpc Register unscaled loads605 def : Pat<(acquiring_load<atomic_load_azext_8>606 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),607 (LDAPURBi GPR64sp:$Rn, simm9:$offset)>;608 def : Pat<(acquiring_load<atomic_load_azext_16>609 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),610 (LDAPURHi GPR64sp:$Rn, simm9:$offset)>;611 def : Pat<(acquiring_load<atomic_load_nonext_32>612 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),613 (LDAPURi GPR64sp:$Rn, simm9:$offset)>;614 def : Pat<(acquiring_load<atomic_load_nonext_64>615 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),616 (LDAPURXi GPR64sp:$Rn, simm9:$offset)>;617}618 619let Predicates = [HasRCPC_IMMO] in {620 // Store-Release Register unscaled stores621 def : Pat<(releasing_store<atomic_store_8>622 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),623 (STLURBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;624 def : Pat<(releasing_store<atomic_store_16>625 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),626 (STLURHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;627 def : Pat<(releasing_store<atomic_store_32>628 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),629 (STLURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;630 def : Pat<(releasing_store<atomic_store_64>631 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),632 (STLURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;633}634