1003 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "(?!^\s*lda.*\bsp\b)^\s*.*\bsp\b" --filter "^\s*(ld|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"2; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.8a -mattr=+rcpc-immo -global-isel=true -global-isel-abort=2 -O0 | FileCheck %s --check-prefixes=CHECK,GISEL3; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR4; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo,avoid-ldapur -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR5; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=neoverse-v2 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR6; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=neoverse-v3 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR7; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x3 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR8; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x4 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR9; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mcpu=cortex-x925 -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR10; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v9a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-AVOIDLDAPUR11; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.8a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-NOAVOIDLDAPUR12; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v9.3a -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-NOAVOIDLDAPUR13 14define i8 @load_atomic_i8_aligned_unordered(ptr %ptr) {15; CHECK-LABEL: load_atomic_i8_aligned_unordered:16; CHECK: ldrb w0, [x0, #4]17 %gep = getelementptr inbounds i8, ptr %ptr, i32 418 %r = load atomic i8, ptr %gep unordered, align 119 ret i8 %r20}21 22define i8 @load_atomic_i8_aligned_unordered_const(ptr readonly %ptr) {23; CHECK-LABEL: load_atomic_i8_aligned_unordered_const:24; CHECK: ldrb w0, [x0, #4]25 %gep = getelementptr inbounds i8, ptr %ptr, i32 426 %r = load atomic i8, ptr %gep unordered, align 127 ret i8 %r28}29 30define i8 @load_atomic_i8_aligned_monotonic(ptr %ptr) {31; CHECK-LABEL: load_atomic_i8_aligned_monotonic:32; CHECK: ldrb w0, [x0, #4]33 %gep = getelementptr inbounds i8, ptr %ptr, i32 434 %r = load atomic i8, ptr %gep monotonic, align 135 ret i8 %r36}37 38define i8 @load_atomic_i8_aligned_monotonic_const(ptr readonly %ptr) {39; CHECK-LABEL: load_atomic_i8_aligned_monotonic_const:40; CHECK: ldrb w0, [x0, #4]41 %gep = getelementptr inbounds i8, ptr %ptr, i32 442 %r = load atomic i8, ptr %gep monotonic, align 143 ret i8 %r44}45 46define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {47; GISEL-LABEL: load_atomic_i8_aligned_acquire:48; GISEL: add x8, x0, #449; GISEL: ldaprb w0, [x8]50;51; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire:52; SDAG-NOAVOIDLDAPUR: ldapurb w0, [x0, #4]53;54; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire:55; SDAG-AVOIDLDAPUR: add x8, x0, #456; SDAG-AVOIDLDAPUR: ldaprb w0, [x8]57 %gep = getelementptr inbounds i8, ptr %ptr, i32 458 %r = load atomic i8, ptr %gep acquire, align 159 ret i8 %r60}61 62define i8 @load_atomic_i8_aligned_acquire_const(ptr readonly %ptr) {63; GISEL-LABEL: load_atomic_i8_aligned_acquire_const:64; GISEL: add x8, x0, #465; GISEL: ldaprb w0, [x8]66;67; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire_const:68; SDAG-NOAVOIDLDAPUR: ldapurb w0, [x0, #4]69;70; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_aligned_acquire_const:71; SDAG-AVOIDLDAPUR: add x8, x0, #472; SDAG-AVOIDLDAPUR: ldaprb w0, [x8]73 %gep = getelementptr inbounds i8, ptr %ptr, i32 474 %r = load atomic i8, ptr %gep acquire, align 175 ret i8 %r76}77 78define i8 @load_atomic_i8_aligned_seq_cst(ptr %ptr) {79; CHECK-LABEL: load_atomic_i8_aligned_seq_cst:80; CHECK: add x8, x0, #481; CHECK: ldarb w0, [x8]82 %gep = getelementptr inbounds i8, ptr %ptr, i32 483 %r = load atomic i8, ptr %gep seq_cst, align 184 ret i8 %r85}86 87define i8 @load_atomic_i8_aligned_seq_cst_const(ptr readonly %ptr) {88; CHECK-LABEL: load_atomic_i8_aligned_seq_cst_const:89; CHECK: add x8, x0, #490; CHECK: ldarb w0, [x8]91 %gep = getelementptr inbounds i8, ptr %ptr, i32 492 %r = load atomic i8, ptr %gep seq_cst, align 193 ret i8 %r94}95 96define i16 @load_atomic_i16_aligned_unordered(ptr %ptr) {97; CHECK-LABEL: load_atomic_i16_aligned_unordered:98; CHECK: ldrh w0, [x0, #8]99 %gep = getelementptr inbounds i16, ptr %ptr, i32 4100 %r = load atomic i16, ptr %gep unordered, align 2101 ret i16 %r102}103 104define i16 @load_atomic_i16_aligned_unordered_const(ptr readonly %ptr) {105; CHECK-LABEL: load_atomic_i16_aligned_unordered_const:106; CHECK: ldrh w0, [x0, #8]107 %gep = getelementptr inbounds i16, ptr %ptr, i32 4108 %r = load atomic i16, ptr %gep unordered, align 2109 ret i16 %r110}111 112define i16 @load_atomic_i16_aligned_monotonic(ptr %ptr) {113; CHECK-LABEL: load_atomic_i16_aligned_monotonic:114; CHECK: ldrh w0, [x0, #8]115 %gep = getelementptr inbounds i16, ptr %ptr, i32 4116 %r = load atomic i16, ptr %gep monotonic, align 2117 ret i16 %r118}119 120define i16 @load_atomic_i16_aligned_monotonic_const(ptr readonly %ptr) {121; CHECK-LABEL: load_atomic_i16_aligned_monotonic_const:122; CHECK: ldrh w0, [x0, #8]123 %gep = getelementptr inbounds i16, ptr %ptr, i32 4124 %r = load atomic i16, ptr %gep monotonic, align 2125 ret i16 %r126}127 128define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {129; GISEL-LABEL: load_atomic_i16_aligned_acquire:130; GISEL: add x8, x0, #8131; GISEL: ldaprh w0, [x8]132;133; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire:134; SDAG-NOAVOIDLDAPUR: ldapurh w0, [x0, #8]135;136; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire:137; SDAG-AVOIDLDAPUR: add x8, x0, #8138; SDAG-AVOIDLDAPUR: ldaprh w0, [x8]139 %gep = getelementptr inbounds i16, ptr %ptr, i32 4140 %r = load atomic i16, ptr %gep acquire, align 2141 ret i16 %r142}143 144define i16 @load_atomic_i16_aligned_acquire_const(ptr readonly %ptr) {145; GISEL-LABEL: load_atomic_i16_aligned_acquire_const:146; GISEL: add x8, x0, #8147; GISEL: ldaprh w0, [x8]148;149; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire_const:150; SDAG-NOAVOIDLDAPUR: ldapurh w0, [x0, #8]151;152; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_aligned_acquire_const:153; SDAG-AVOIDLDAPUR: add x8, x0, #8154; SDAG-AVOIDLDAPUR: ldaprh w0, [x8]155 %gep = getelementptr inbounds i16, ptr %ptr, i32 4156 %r = load atomic i16, ptr %gep acquire, align 2157 ret i16 %r158}159 160define i16 @load_atomic_i16_aligned_seq_cst(ptr %ptr) {161; CHECK-LABEL: load_atomic_i16_aligned_seq_cst:162; CHECK: add x8, x0, #8163; CHECK: ldarh w0, [x8]164 %gep = getelementptr inbounds i16, ptr %ptr, i32 4165 %r = load atomic i16, ptr %gep seq_cst, align 2166 ret i16 %r167}168 169define i16 @load_atomic_i16_aligned_seq_cst_const(ptr readonly %ptr) {170; CHECK-LABEL: load_atomic_i16_aligned_seq_cst_const:171; CHECK: add x8, x0, #8172; CHECK: ldarh w0, [x8]173 %gep = getelementptr inbounds i16, ptr %ptr, i32 4174 %r = load atomic i16, ptr %gep seq_cst, align 2175 ret i16 %r176}177 178define i32 @load_atomic_i32_aligned_unordered(ptr %ptr) {179; CHECK-LABEL: load_atomic_i32_aligned_unordered:180; CHECK: ldr w0, [x0, #16]181 %gep = getelementptr inbounds i32, ptr %ptr, i32 4182 %r = load atomic i32, ptr %gep unordered, align 4183 ret i32 %r184}185 186define i32 @load_atomic_i32_aligned_unordered_const(ptr readonly %ptr) {187; CHECK-LABEL: load_atomic_i32_aligned_unordered_const:188; CHECK: ldr w0, [x0, #16]189 %gep = getelementptr inbounds i32, ptr %ptr, i32 4190 %r = load atomic i32, ptr %gep unordered, align 4191 ret i32 %r192}193 194define i32 @load_atomic_i32_aligned_monotonic(ptr %ptr) {195; CHECK-LABEL: load_atomic_i32_aligned_monotonic:196; CHECK: ldr w0, [x0, #16]197 %gep = getelementptr inbounds i32, ptr %ptr, i32 4198 %r = load atomic i32, ptr %gep monotonic, align 4199 ret i32 %r200}201 202define i32 @load_atomic_i32_aligned_monotonic_const(ptr readonly %ptr) {203; CHECK-LABEL: load_atomic_i32_aligned_monotonic_const:204; CHECK: ldr w0, [x0, #16]205 %gep = getelementptr inbounds i32, ptr %ptr, i32 4206 %r = load atomic i32, ptr %gep monotonic, align 4207 ret i32 %r208}209 210define i32 @load_atomic_i32_aligned_acquire(ptr %ptr) {211; GISEL-LABEL: load_atomic_i32_aligned_acquire:212; GISEL: ldapur w0, [x0, #16]213;214; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire:215; SDAG-NOAVOIDLDAPUR: ldapur w0, [x0, #16]216;217; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire:218; SDAG-AVOIDLDAPUR: add x8, x0, #16219; SDAG-AVOIDLDAPUR: ldapr w0, [x8]220 %gep = getelementptr inbounds i32, ptr %ptr, i32 4221 %r = load atomic i32, ptr %gep acquire, align 4222 ret i32 %r223}224 225define i32 @load_atomic_i32_aligned_acquire_const(ptr readonly %ptr) {226; GISEL-LABEL: load_atomic_i32_aligned_acquire_const:227; GISEL: ldapur w0, [x0, #16]228;229; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire_const:230; SDAG-NOAVOIDLDAPUR: ldapur w0, [x0, #16]231;232; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_aligned_acquire_const:233; SDAG-AVOIDLDAPUR: add x8, x0, #16234; SDAG-AVOIDLDAPUR: ldapr w0, [x8]235 %gep = getelementptr inbounds i32, ptr %ptr, i32 4236 %r = load atomic i32, ptr %gep acquire, align 4237 ret i32 %r238}239 240define i32 @load_atomic_i32_aligned_seq_cst(ptr %ptr) {241; CHECK-LABEL: load_atomic_i32_aligned_seq_cst:242; CHECK: add x8, x0, #16243; CHECK: ldar w0, [x8]244 %gep = getelementptr inbounds i32, ptr %ptr, i32 4245 %r = load atomic i32, ptr %gep seq_cst, align 4246 ret i32 %r247}248 249define i32 @load_atomic_i32_aligned_seq_cst_const(ptr readonly %ptr) {250; CHECK-LABEL: load_atomic_i32_aligned_seq_cst_const:251; CHECK: add x8, x0, #16252; CHECK: ldar w0, [x8]253 %gep = getelementptr inbounds i32, ptr %ptr, i32 4254 %r = load atomic i32, ptr %gep seq_cst, align 4255 ret i32 %r256}257 258define i64 @load_atomic_i64_aligned_unordered(ptr %ptr) {259; CHECK-LABEL: load_atomic_i64_aligned_unordered:260; CHECK: ldr x0, [x0, #32]261 %gep = getelementptr inbounds i64, ptr %ptr, i32 4262 %r = load atomic i64, ptr %gep unordered, align 8263 ret i64 %r264}265 266define i64 @load_atomic_i64_aligned_unordered_const(ptr readonly %ptr) {267; CHECK-LABEL: load_atomic_i64_aligned_unordered_const:268; CHECK: ldr x0, [x0, #32]269 %gep = getelementptr inbounds i64, ptr %ptr, i32 4270 %r = load atomic i64, ptr %gep unordered, align 8271 ret i64 %r272}273 274define i64 @load_atomic_i64_aligned_monotonic(ptr %ptr) {275; CHECK-LABEL: load_atomic_i64_aligned_monotonic:276; CHECK: ldr x0, [x0, #32]277 %gep = getelementptr inbounds i64, ptr %ptr, i32 4278 %r = load atomic i64, ptr %gep monotonic, align 8279 ret i64 %r280}281 282define i64 @load_atomic_i64_aligned_monotonic_const(ptr readonly %ptr) {283; CHECK-LABEL: load_atomic_i64_aligned_monotonic_const:284; CHECK: ldr x0, [x0, #32]285 %gep = getelementptr inbounds i64, ptr %ptr, i32 4286 %r = load atomic i64, ptr %gep monotonic, align 8287 ret i64 %r288}289 290define i64 @load_atomic_i64_aligned_acquire(ptr %ptr) {291; GISEL-LABEL: load_atomic_i64_aligned_acquire:292; GISEL: ldapur x0, [x0, #32]293;294; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire:295; SDAG-NOAVOIDLDAPUR: ldapur x0, [x0, #32]296;297; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire:298; SDAG-AVOIDLDAPUR: add x8, x0, #32299; SDAG-AVOIDLDAPUR: ldapr x0, [x8]300 %gep = getelementptr inbounds i64, ptr %ptr, i32 4301 %r = load atomic i64, ptr %gep acquire, align 8302 ret i64 %r303}304 305define i64 @load_atomic_i64_aligned_acquire_const(ptr readonly %ptr) {306; GISEL-LABEL: load_atomic_i64_aligned_acquire_const:307; GISEL: ldapur x0, [x0, #32]308;309; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire_const:310; SDAG-NOAVOIDLDAPUR: ldapur x0, [x0, #32]311;312; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_aligned_acquire_const:313; SDAG-AVOIDLDAPUR: add x8, x0, #32314; SDAG-AVOIDLDAPUR: ldapr x0, [x8]315 %gep = getelementptr inbounds i64, ptr %ptr, i32 4316 %r = load atomic i64, ptr %gep acquire, align 8317 ret i64 %r318}319 320define i64 @load_atomic_i64_aligned_seq_cst(ptr %ptr) {321; CHECK-LABEL: load_atomic_i64_aligned_seq_cst:322; CHECK: add x8, x0, #32323; CHECK: ldar x0, [x8]324 %gep = getelementptr inbounds i64, ptr %ptr, i32 4325 %r = load atomic i64, ptr %gep seq_cst, align 8326 ret i64 %r327}328 329define i64 @load_atomic_i64_aligned_seq_cst_const(ptr readonly %ptr) {330; CHECK-LABEL: load_atomic_i64_aligned_seq_cst_const:331; CHECK: add x8, x0, #32332; CHECK: ldar x0, [x8]333 %gep = getelementptr inbounds i64, ptr %ptr, i32 4334 %r = load atomic i64, ptr %gep seq_cst, align 8335 ret i64 %r336}337 338define i128 @load_atomic_i128_aligned_unordered(ptr %ptr) {339; CHECK-LABEL: load_atomic_i128_aligned_unordered:340; CHECK: ldp x0, x1, [x0, #64]341 %gep = getelementptr inbounds i128, ptr %ptr, i32 4342 %r = load atomic i128, ptr %gep unordered, align 16343 ret i128 %r344}345 346define i128 @load_atomic_i128_aligned_unordered_const(ptr readonly %ptr) {347; CHECK-LABEL: load_atomic_i128_aligned_unordered_const:348; CHECK: ldp x0, x1, [x0, #64]349 %gep = getelementptr inbounds i128, ptr %ptr, i32 4350 %r = load atomic i128, ptr %gep unordered, align 16351 ret i128 %r352}353 354define i128 @load_atomic_i128_aligned_monotonic(ptr %ptr) {355; CHECK-LABEL: load_atomic_i128_aligned_monotonic:356; CHECK: ldp x0, x1, [x0, #64]357 %gep = getelementptr inbounds i128, ptr %ptr, i32 4358 %r = load atomic i128, ptr %gep monotonic, align 16359 ret i128 %r360}361 362define i128 @load_atomic_i128_aligned_monotonic_const(ptr readonly %ptr) {363; CHECK-LABEL: load_atomic_i128_aligned_monotonic_const:364; CHECK: ldp x0, x1, [x0, #64]365 %gep = getelementptr inbounds i128, ptr %ptr, i32 4366 %r = load atomic i128, ptr %gep monotonic, align 16367 ret i128 %r368}369 370define i128 @load_atomic_i128_aligned_acquire(ptr %ptr) {371; CHECK-LABEL: load_atomic_i128_aligned_acquire:372; CHECK: ldp x0, x1, [x0, #64]373; CHECK: dmb ishld374 %gep = getelementptr inbounds i128, ptr %ptr, i32 4375 %r = load atomic i128, ptr %gep acquire, align 16376 ret i128 %r377}378 379define i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) {380; CHECK-LABEL: load_atomic_i128_aligned_acquire_const:381; CHECK: ldp x0, x1, [x0, #64]382; CHECK: dmb ishld383 %gep = getelementptr inbounds i128, ptr %ptr, i32 4384 %r = load atomic i128, ptr %gep acquire, align 16385 ret i128 %r386}387 388define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {389; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:390; CHECK: ldp x0, x1, [x0, #64]391; CHECK: dmb ish392 %gep = getelementptr inbounds i128, ptr %ptr, i32 4393 %r = load atomic i128, ptr %gep seq_cst, align 16394 ret i128 %r395}396 397define i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {398; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:399; CHECK: ldp x0, x1, [x0, #64]400; CHECK: dmb ish401 %gep = getelementptr inbounds i128, ptr %ptr, i32 4402 %r = load atomic i128, ptr %gep seq_cst, align 16403 ret i128 %r404}405 406define i8 @load_atomic_i8_unaligned_unordered(ptr %ptr) {407; CHECK-LABEL: load_atomic_i8_unaligned_unordered:408; CHECK: ldrb w0, [x0, #4]409 %gep = getelementptr inbounds i8, ptr %ptr, i32 4410 %r = load atomic i8, ptr %gep unordered, align 1411 ret i8 %r412}413 414define i8 @load_atomic_i8_unaligned_unordered_const(ptr readonly %ptr) {415; CHECK-LABEL: load_atomic_i8_unaligned_unordered_const:416; CHECK: ldrb w0, [x0, #4]417 %gep = getelementptr inbounds i8, ptr %ptr, i32 4418 %r = load atomic i8, ptr %gep unordered, align 1419 ret i8 %r420}421 422define i8 @load_atomic_i8_unaligned_monotonic(ptr %ptr) {423; CHECK-LABEL: load_atomic_i8_unaligned_monotonic:424; CHECK: ldrb w0, [x0, #4]425 %gep = getelementptr inbounds i8, ptr %ptr, i32 4426 %r = load atomic i8, ptr %gep monotonic, align 1427 ret i8 %r428}429 430define i8 @load_atomic_i8_unaligned_monotonic_const(ptr readonly %ptr) {431; CHECK-LABEL: load_atomic_i8_unaligned_monotonic_const:432; CHECK: ldrb w0, [x0, #4]433 %gep = getelementptr inbounds i8, ptr %ptr, i32 4434 %r = load atomic i8, ptr %gep monotonic, align 1435 ret i8 %r436}437 438define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {439; GISEL-LABEL: load_atomic_i8_unaligned_acquire:440; GISEL: add x8, x0, #4441; GISEL: ldaprb w0, [x8]442;443; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire:444; SDAG-NOAVOIDLDAPUR: ldapurb w0, [x0, #4]445;446; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire:447; SDAG-AVOIDLDAPUR: add x8, x0, #4448; SDAG-AVOIDLDAPUR: ldaprb w0, [x8]449 %gep = getelementptr inbounds i8, ptr %ptr, i32 4450 %r = load atomic i8, ptr %gep acquire, align 1451 ret i8 %r452}453 454define i8 @load_atomic_i8_unaligned_acquire_const(ptr readonly %ptr) {455; GISEL-LABEL: load_atomic_i8_unaligned_acquire_const:456; GISEL: add x8, x0, #4457; GISEL: ldaprb w0, [x8]458;459; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire_const:460; SDAG-NOAVOIDLDAPUR: ldapurb w0, [x0, #4]461;462; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_unaligned_acquire_const:463; SDAG-AVOIDLDAPUR: add x8, x0, #4464; SDAG-AVOIDLDAPUR: ldaprb w0, [x8]465 %gep = getelementptr inbounds i8, ptr %ptr, i32 4466 %r = load atomic i8, ptr %gep acquire, align 1467 ret i8 %r468}469 470define i8 @load_atomic_i8_unaligned_seq_cst(ptr %ptr) {471; CHECK-LABEL: load_atomic_i8_unaligned_seq_cst:472; CHECK: add x8, x0, #4473; CHECK: ldarb w0, [x8]474 %gep = getelementptr inbounds i8, ptr %ptr, i32 4475 %r = load atomic i8, ptr %gep seq_cst, align 1476 ret i8 %r477}478 479define i8 @load_atomic_i8_unaligned_seq_cst_const(ptr readonly %ptr) {480; CHECK-LABEL: load_atomic_i8_unaligned_seq_cst_const:481; CHECK: add x8, x0, #4482; CHECK: ldarb w0, [x8]483 %gep = getelementptr inbounds i8, ptr %ptr, i32 4484 %r = load atomic i8, ptr %gep seq_cst, align 1485 ret i8 %r486}487 488define i16 @load_atomic_i16_unaligned_unordered(ptr %ptr) {489; GISEL-LABEL: load_atomic_i16_unaligned_unordered:490; GISEL: add x1, x8, #4491; GISEL: bl __atomic_load492;493; SDAG-LABEL: load_atomic_i16_unaligned_unordered:494; SDAG: add x1, x0, #4495; SDAG: bl __atomic_load496 %gep = getelementptr inbounds i8, ptr %ptr, i32 4497 %r = load atomic i16, ptr %gep unordered, align 1498 ret i16 %r499}500 501define i16 @load_atomic_i16_unaligned_unordered_const(ptr readonly %ptr) {502; GISEL-LABEL: load_atomic_i16_unaligned_unordered_const:503; GISEL: add x1, x8, #4504; GISEL: bl __atomic_load505;506; SDAG-LABEL: load_atomic_i16_unaligned_unordered_const:507; SDAG: add x1, x0, #4508; SDAG: bl __atomic_load509 %gep = getelementptr inbounds i8, ptr %ptr, i32 4510 %r = load atomic i16, ptr %gep unordered, align 1511 ret i16 %r512}513 514define i16 @load_atomic_i16_unaligned_monotonic(ptr %ptr) {515; GISEL-LABEL: load_atomic_i16_unaligned_monotonic:516; GISEL: add x1, x8, #8517; GISEL: bl __atomic_load518;519; SDAG-LABEL: load_atomic_i16_unaligned_monotonic:520; SDAG: add x1, x0, #8521; SDAG: bl __atomic_load522 %gep = getelementptr inbounds i16, ptr %ptr, i32 4523 %r = load atomic i16, ptr %gep monotonic, align 1524 ret i16 %r525}526 527define i16 @load_atomic_i16_unaligned_monotonic_const(ptr readonly %ptr) {528; GISEL-LABEL: load_atomic_i16_unaligned_monotonic_const:529; GISEL: add x1, x8, #8530; GISEL: bl __atomic_load531;532; SDAG-LABEL: load_atomic_i16_unaligned_monotonic_const:533; SDAG: add x1, x0, #8534; SDAG: bl __atomic_load535 %gep = getelementptr inbounds i16, ptr %ptr, i32 4536 %r = load atomic i16, ptr %gep monotonic, align 1537 ret i16 %r538}539 540define i16 @load_atomic_i16_unaligned_acquire(ptr %ptr) {541; GISEL-LABEL: load_atomic_i16_unaligned_acquire:542; GISEL: add x1, x8, #8543; GISEL: bl __atomic_load544;545; SDAG-LABEL: load_atomic_i16_unaligned_acquire:546; SDAG: add x1, x0, #8547; SDAG: bl __atomic_load548 %gep = getelementptr inbounds i16, ptr %ptr, i32 4549 %r = load atomic i16, ptr %gep acquire, align 1550 ret i16 %r551}552 553define i16 @load_atomic_i16_unaligned_acquire_const(ptr readonly %ptr) {554; GISEL-LABEL: load_atomic_i16_unaligned_acquire_const:555; GISEL: add x1, x8, #8556; GISEL: bl __atomic_load557;558; SDAG-LABEL: load_atomic_i16_unaligned_acquire_const:559; SDAG: add x1, x0, #8560; SDAG: bl __atomic_load561 %gep = getelementptr inbounds i16, ptr %ptr, i32 4562 %r = load atomic i16, ptr %gep acquire, align 1563 ret i16 %r564}565 566define i16 @load_atomic_i16_unaligned_seq_cst(ptr %ptr) {567; GISEL-LABEL: load_atomic_i16_unaligned_seq_cst:568; GISEL: add x1, x8, #8569; GISEL: bl __atomic_load570;571; SDAG-LABEL: load_atomic_i16_unaligned_seq_cst:572; SDAG: add x1, x0, #8573; SDAG: bl __atomic_load574 %gep = getelementptr inbounds i16, ptr %ptr, i32 4575 %r = load atomic i16, ptr %gep seq_cst, align 1576 ret i16 %r577}578 579define i16 @load_atomic_i16_unaligned_seq_cst_const(ptr readonly %ptr) {580; GISEL-LABEL: load_atomic_i16_unaligned_seq_cst_const:581; GISEL: add x1, x8, #8582; GISEL: bl __atomic_load583;584; SDAG-LABEL: load_atomic_i16_unaligned_seq_cst_const:585; SDAG: add x1, x0, #8586; SDAG: bl __atomic_load587 %gep = getelementptr inbounds i16, ptr %ptr, i32 4588 %r = load atomic i16, ptr %gep seq_cst, align 1589 ret i16 %r590}591 592define i32 @load_atomic_i32_unaligned_unordered(ptr %ptr) {593; GISEL-LABEL: load_atomic_i32_unaligned_unordered:594; GISEL: add x1, x8, #16595; GISEL: bl __atomic_load596;597; SDAG-LABEL: load_atomic_i32_unaligned_unordered:598; SDAG: add x1, x0, #16599; SDAG: bl __atomic_load600 %gep = getelementptr inbounds i32, ptr %ptr, i32 4601 %r = load atomic i32, ptr %gep unordered, align 1602 ret i32 %r603}604 605define i32 @load_atomic_i32_unaligned_unordered_const(ptr readonly %ptr) {606; GISEL-LABEL: load_atomic_i32_unaligned_unordered_const:607; GISEL: add x1, x8, #16608; GISEL: bl __atomic_load609;610; SDAG-LABEL: load_atomic_i32_unaligned_unordered_const:611; SDAG: add x1, x0, #16612; SDAG: bl __atomic_load613 %gep = getelementptr inbounds i32, ptr %ptr, i32 4614 %r = load atomic i32, ptr %gep unordered, align 1615 ret i32 %r616}617 618define i32 @load_atomic_i32_unaligned_monotonic(ptr %ptr) {619; GISEL-LABEL: load_atomic_i32_unaligned_monotonic:620; GISEL: add x1, x8, #16621; GISEL: bl __atomic_load622;623; SDAG-LABEL: load_atomic_i32_unaligned_monotonic:624; SDAG: add x1, x0, #16625; SDAG: bl __atomic_load626 %gep = getelementptr inbounds i32, ptr %ptr, i32 4627 %r = load atomic i32, ptr %gep monotonic, align 1628 ret i32 %r629}630 631define i32 @load_atomic_i32_unaligned_monotonic_const(ptr readonly %ptr) {632; GISEL-LABEL: load_atomic_i32_unaligned_monotonic_const:633; GISEL: add x1, x8, #16634; GISEL: bl __atomic_load635;636; SDAG-LABEL: load_atomic_i32_unaligned_monotonic_const:637; SDAG: add x1, x0, #16638; SDAG: bl __atomic_load639 %gep = getelementptr inbounds i32, ptr %ptr, i32 4640 %r = load atomic i32, ptr %gep monotonic, align 1641 ret i32 %r642}643 644define i32 @load_atomic_i32_unaligned_acquire(ptr %ptr) {645; GISEL-LABEL: load_atomic_i32_unaligned_acquire:646; GISEL: add x1, x8, #16647; GISEL: bl __atomic_load648;649; SDAG-LABEL: load_atomic_i32_unaligned_acquire:650; SDAG: add x1, x0, #16651; SDAG: bl __atomic_load652 %gep = getelementptr inbounds i32, ptr %ptr, i32 4653 %r = load atomic i32, ptr %gep acquire, align 1654 ret i32 %r655}656 657define i32 @load_atomic_i32_unaligned_acquire_const(ptr readonly %ptr) {658; GISEL-LABEL: load_atomic_i32_unaligned_acquire_const:659; GISEL: add x1, x8, #16660; GISEL: bl __atomic_load661;662; SDAG-LABEL: load_atomic_i32_unaligned_acquire_const:663; SDAG: add x1, x0, #16664; SDAG: bl __atomic_load665 %gep = getelementptr inbounds i32, ptr %ptr, i32 4666 %r = load atomic i32, ptr %gep acquire, align 1667 ret i32 %r668}669 670define i32 @load_atomic_i32_unaligned_seq_cst(ptr %ptr) {671; GISEL-LABEL: load_atomic_i32_unaligned_seq_cst:672; GISEL: add x1, x8, #16673; GISEL: bl __atomic_load674;675; SDAG-LABEL: load_atomic_i32_unaligned_seq_cst:676; SDAG: add x1, x0, #16677; SDAG: bl __atomic_load678 %gep = getelementptr inbounds i32, ptr %ptr, i32 4679 %r = load atomic i32, ptr %gep seq_cst, align 1680 ret i32 %r681}682 683define i32 @load_atomic_i32_unaligned_seq_cst_const(ptr readonly %ptr) {684; GISEL-LABEL: load_atomic_i32_unaligned_seq_cst_const:685; GISEL: add x1, x8, #16686; GISEL: bl __atomic_load687;688; SDAG-LABEL: load_atomic_i32_unaligned_seq_cst_const:689; SDAG: add x1, x0, #16690; SDAG: bl __atomic_load691 %gep = getelementptr inbounds i32, ptr %ptr, i32 4692 %r = load atomic i32, ptr %gep seq_cst, align 1693 ret i32 %r694}695 696define i64 @load_atomic_i64_unaligned_unordered(ptr %ptr) {697; GISEL-LABEL: load_atomic_i64_unaligned_unordered:698; GISEL: add x1, x8, #32699; GISEL: bl __atomic_load700;701; SDAG-LABEL: load_atomic_i64_unaligned_unordered:702; SDAG: add x1, x0, #32703; SDAG: bl __atomic_load704 %gep = getelementptr inbounds i64, ptr %ptr, i32 4705 %r = load atomic i64, ptr %gep unordered, align 1706 ret i64 %r707}708 709define i64 @load_atomic_i64_unaligned_unordered_const(ptr readonly %ptr) {710; GISEL-LABEL: load_atomic_i64_unaligned_unordered_const:711; GISEL: add x1, x8, #32712; GISEL: bl __atomic_load713;714; SDAG-LABEL: load_atomic_i64_unaligned_unordered_const:715; SDAG: add x1, x0, #32716; SDAG: bl __atomic_load717 %gep = getelementptr inbounds i64, ptr %ptr, i32 4718 %r = load atomic i64, ptr %gep unordered, align 1719 ret i64 %r720}721 722define i64 @load_atomic_i64_unaligned_monotonic(ptr %ptr) {723; GISEL-LABEL: load_atomic_i64_unaligned_monotonic:724; GISEL: add x1, x8, #32725; GISEL: bl __atomic_load726;727; SDAG-LABEL: load_atomic_i64_unaligned_monotonic:728; SDAG: add x1, x0, #32729; SDAG: bl __atomic_load730 %gep = getelementptr inbounds i64, ptr %ptr, i32 4731 %r = load atomic i64, ptr %gep monotonic, align 1732 ret i64 %r733}734 735define i64 @load_atomic_i64_unaligned_monotonic_const(ptr readonly %ptr) {736; GISEL-LABEL: load_atomic_i64_unaligned_monotonic_const:737; GISEL: add x1, x8, #32738; GISEL: bl __atomic_load739;740; SDAG-LABEL: load_atomic_i64_unaligned_monotonic_const:741; SDAG: add x1, x0, #32742; SDAG: bl __atomic_load743 %gep = getelementptr inbounds i64, ptr %ptr, i32 4744 %r = load atomic i64, ptr %gep monotonic, align 1745 ret i64 %r746}747 748define i64 @load_atomic_i64_unaligned_acquire(ptr %ptr) {749; GISEL-LABEL: load_atomic_i64_unaligned_acquire:750; GISEL: add x1, x8, #32751; GISEL: bl __atomic_load752;753; SDAG-LABEL: load_atomic_i64_unaligned_acquire:754; SDAG: add x1, x0, #32755; SDAG: bl __atomic_load756 %gep = getelementptr inbounds i64, ptr %ptr, i32 4757 %r = load atomic i64, ptr %gep acquire, align 1758 ret i64 %r759}760 761define i64 @load_atomic_i64_unaligned_acquire_const(ptr readonly %ptr) {762; GISEL-LABEL: load_atomic_i64_unaligned_acquire_const:763; GISEL: add x1, x8, #32764; GISEL: bl __atomic_load765;766; SDAG-LABEL: load_atomic_i64_unaligned_acquire_const:767; SDAG: add x1, x0, #32768; SDAG: bl __atomic_load769 %gep = getelementptr inbounds i64, ptr %ptr, i32 4770 %r = load atomic i64, ptr %gep acquire, align 1771 ret i64 %r772}773 774define i64 @load_atomic_i64_unaligned_seq_cst(ptr %ptr) {775; GISEL-LABEL: load_atomic_i64_unaligned_seq_cst:776; GISEL: add x1, x8, #32777; GISEL: bl __atomic_load778;779; SDAG-LABEL: load_atomic_i64_unaligned_seq_cst:780; SDAG: add x1, x0, #32781; SDAG: bl __atomic_load782 %gep = getelementptr inbounds i64, ptr %ptr, i32 4783 %r = load atomic i64, ptr %gep seq_cst, align 1784 ret i64 %r785}786 787define i64 @load_atomic_i64_unaligned_seq_cst_const(ptr readonly %ptr) {788; GISEL-LABEL: load_atomic_i64_unaligned_seq_cst_const:789; GISEL: add x1, x8, #32790; GISEL: bl __atomic_load791;792; SDAG-LABEL: load_atomic_i64_unaligned_seq_cst_const:793; SDAG: add x1, x0, #32794; SDAG: bl __atomic_load795 %gep = getelementptr inbounds i64, ptr %ptr, i32 4796 %r = load atomic i64, ptr %gep seq_cst, align 1797 ret i64 %r798}799 800define i128 @load_atomic_i128_unaligned_unordered(ptr %ptr) {801; GISEL-LABEL: load_atomic_i128_unaligned_unordered:802; GISEL: add x1, x8, #64803; GISEL: bl __atomic_load804;805; SDAG-LABEL: load_atomic_i128_unaligned_unordered:806; SDAG: add x1, x0, #64807; SDAG: bl __atomic_load808 %gep = getelementptr inbounds i128, ptr %ptr, i32 4809 %r = load atomic i128, ptr %gep unordered, align 1810 ret i128 %r811}812 813define i128 @load_atomic_i128_unaligned_unordered_const(ptr readonly %ptr) {814; GISEL-LABEL: load_atomic_i128_unaligned_unordered_const:815; GISEL: add x1, x8, #64816; GISEL: bl __atomic_load817;818; SDAG-LABEL: load_atomic_i128_unaligned_unordered_const:819; SDAG: add x1, x0, #64820; SDAG: bl __atomic_load821 %gep = getelementptr inbounds i128, ptr %ptr, i32 4822 %r = load atomic i128, ptr %gep unordered, align 1823 ret i128 %r824}825 826define i128 @load_atomic_i128_unaligned_monotonic(ptr %ptr) {827; GISEL-LABEL: load_atomic_i128_unaligned_monotonic:828; GISEL: add x1, x8, #64829; GISEL: bl __atomic_load830;831; SDAG-LABEL: load_atomic_i128_unaligned_monotonic:832; SDAG: add x1, x0, #64833; SDAG: bl __atomic_load834 %gep = getelementptr inbounds i128, ptr %ptr, i32 4835 %r = load atomic i128, ptr %gep monotonic, align 1836 ret i128 %r837}838 839define i128 @load_atomic_i128_unaligned_monotonic_const(ptr readonly %ptr) {840; GISEL-LABEL: load_atomic_i128_unaligned_monotonic_const:841; GISEL: add x1, x8, #64842; GISEL: bl __atomic_load843;844; SDAG-LABEL: load_atomic_i128_unaligned_monotonic_const:845; SDAG: add x1, x0, #64846; SDAG: bl __atomic_load847 %gep = getelementptr inbounds i128, ptr %ptr, i32 4848 %r = load atomic i128, ptr %gep monotonic, align 1849 ret i128 %r850}851 852define i128 @load_atomic_i128_unaligned_acquire(ptr %ptr) {853; GISEL-LABEL: load_atomic_i128_unaligned_acquire:854; GISEL: add x1, x8, #64855; GISEL: bl __atomic_load856;857; SDAG-LABEL: load_atomic_i128_unaligned_acquire:858; SDAG: add x1, x0, #64859; SDAG: bl __atomic_load860 %gep = getelementptr inbounds i128, ptr %ptr, i32 4861 %r = load atomic i128, ptr %gep acquire, align 1862 ret i128 %r863}864 865define i128 @load_atomic_i128_unaligned_acquire_const(ptr readonly %ptr) {866; GISEL-LABEL: load_atomic_i128_unaligned_acquire_const:867; GISEL: add x1, x8, #64868; GISEL: bl __atomic_load869;870; SDAG-LABEL: load_atomic_i128_unaligned_acquire_const:871; SDAG: add x1, x0, #64872; SDAG: bl __atomic_load873 %gep = getelementptr inbounds i128, ptr %ptr, i32 4874 %r = load atomic i128, ptr %gep acquire, align 1875 ret i128 %r876}877 878define i128 @load_atomic_i128_unaligned_seq_cst(ptr %ptr) {879; GISEL-LABEL: load_atomic_i128_unaligned_seq_cst:880; GISEL: add x1, x8, #64881; GISEL: bl __atomic_load882;883; SDAG-LABEL: load_atomic_i128_unaligned_seq_cst:884; SDAG: add x1, x0, #64885; SDAG: bl __atomic_load886 %gep = getelementptr inbounds i128, ptr %ptr, i32 4887 %r = load atomic i128, ptr %gep seq_cst, align 1888 ret i128 %r889}890 891define i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %ptr) {892; GISEL-LABEL: load_atomic_i128_unaligned_seq_cst_const:893; GISEL: add x1, x8, #64894; GISEL: bl __atomic_load895;896; SDAG-LABEL: load_atomic_i128_unaligned_seq_cst_const:897; SDAG: add x1, x0, #64898; SDAG: bl __atomic_load899 %gep = getelementptr inbounds i128, ptr %ptr, i32 4900 %r = load atomic i128, ptr %gep seq_cst, align 1901 ret i128 %r902}903 904define i8 @load_atomic_i8_from_gep() {905; GISEL-LABEL: load_atomic_i8_from_gep:906; GISEL: bl init907; GISEL: add x8, x8, #1908; GISEL: ldaprb w0, [x8]909;910; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i8_from_gep:911; SDAG-NOAVOIDLDAPUR: bl init912; SDAG-NOAVOIDLDAPUR: ldapurb w0, [sp, #13]913;914; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i8_from_gep:915; SDAG-AVOIDLDAPUR: bl init916; SDAG-AVOIDLDAPUR: orr x8, x19, #0x1917; SDAG-AVOIDLDAPUR: ldaprb w0, [x8]918 %a = alloca [3 x i8]919 call void @init(ptr %a)920 %arrayidx = getelementptr [3 x i8], ptr %a, i64 0, i64 1921 %l = load atomic i8, ptr %arrayidx acquire, align 8922 ret i8 %l923}924 925define i16 @load_atomic_i16_from_gep() {926; GISEL-LABEL: load_atomic_i16_from_gep:927; GISEL: bl init928; GISEL: add x8, x8, #2929; GISEL: ldaprh w0, [x8]930;931; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i16_from_gep:932; SDAG-NOAVOIDLDAPUR: bl init933; SDAG-NOAVOIDLDAPUR: ldapurh w0, [sp, #10]934;935; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i16_from_gep:936; SDAG-AVOIDLDAPUR: bl init937; SDAG-AVOIDLDAPUR: orr x8, x19, #0x2938; SDAG-AVOIDLDAPUR: ldaprh w0, [x8]939 %a = alloca [3 x i16]940 call void @init(ptr %a)941 %arrayidx = getelementptr [3 x i16], ptr %a, i64 0, i64 1942 %l = load atomic i16, ptr %arrayidx acquire, align 8943 ret i16 %l944}945 946define i32 @load_atomic_i32_from_gep() {947; GISEL-LABEL: load_atomic_i32_from_gep:948; GISEL: bl init949; GISEL: ldapur w0, [x8, #4]950;951; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:952; SDAG-NOAVOIDLDAPUR: bl init953; SDAG-NOAVOIDLDAPUR: ldapur w0, [sp, #8]954;955; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i32_from_gep:956; SDAG-AVOIDLDAPUR: bl init957; SDAG-AVOIDLDAPUR: add x8, x19, #4958; SDAG-AVOIDLDAPUR: ldapr w0, [x8]959 %a = alloca [3 x i32]960 call void @init(ptr %a)961 %arrayidx = getelementptr [3 x i32], ptr %a, i64 0, i64 1962 %l = load atomic i32, ptr %arrayidx acquire, align 8963 ret i32 %l964}965 966define i64 @load_atomic_i64_from_gep() {967; GISEL-LABEL: load_atomic_i64_from_gep:968; GISEL: bl init969; GISEL: ldapur x0, [x8, #8]970;971; SDAG-NOAVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:972; SDAG-NOAVOIDLDAPUR: bl init973; SDAG-NOAVOIDLDAPUR: ldapur x0, [sp, #16]974;975; SDAG-AVOIDLDAPUR-LABEL: load_atomic_i64_from_gep:976; SDAG-AVOIDLDAPUR: bl init977; SDAG-AVOIDLDAPUR: add x8, x19, #8978; SDAG-AVOIDLDAPUR: ldapr x0, [x8]979 %a = alloca [3 x i64]980 call void @init(ptr %a)981 %arrayidx = getelementptr [3 x i64], ptr %a, i64 0, i64 1982 %l = load atomic i64, ptr %arrayidx acquire, align 8983 ret i64 %l984}985 986define i128 @load_atomic_i128_from_gep() {987; GISEL-LABEL: load_atomic_i128_from_gep:988; GISEL: bl init989; GISEL: ldp x0, x1, [x8, #16]990; GISEL: dmb ishld991;992; SDAG-LABEL: load_atomic_i128_from_gep:993; SDAG: bl init994; SDAG: dmb ishld995 %a = alloca [3 x i128]996 call void @init(ptr %a)997 %arrayidx = getelementptr [3 x i128], ptr %a, i64 0, i64 1998 %l = load atomic i128, ptr %arrayidx acquire, align 16999 ret i128 %l1000}1001 1002declare void @init(ptr)1003