390 lines · plain
1; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck -enable-var-scope %s2; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck -enable-var-scope %s -check-prefix=OUTLINE-ATOMICS3 4define i32 @val_compare_and_swap(ptr %p, i32 %cmp, i32 %new) #0 {5; OUTLINE-ATOMICS: bl __aarch64_cas4_acq6; CHECK-LABEL: val_compare_and_swap:7; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x08; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:9; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]]10; CHECK-NEXT: cmp [[RESULT]], w111; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]]12; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]]13; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]]14; CHECK-NEXT: ret15; CHECK-NEXT: [[FAILBB]]:16; CHECK-NEXT: clrex17; CHECK-NEXT: ret18 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acquire acquire19 %val = extractvalue { i32, i1 } %pair, 020 ret i32 %val21}22 23define i32 @val_compare_and_swap_from_load(ptr %p, i32 %cmp, ptr %pnew) #0 {24; OUTLINE-ATOMICS: bl __aarch64_cas4_acq25; CHECK-LABEL: val_compare_and_swap_from_load:26; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2]27; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:28; CHECK-NEXT: ldaxr w[[RESULT:[0-9]+]], [x0]29; CHECK-NEXT: cmp w[[RESULT]], w130; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]]31; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0]32; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]]33; CHECK-NEXT: mov x0, x[[RESULT]]34; CHECK-NEXT: ret35; CHECK-NEXT: [[FAILBB]]:36; CHECK-NEXT: clrex37; CHECK-NEXT: mov x0, x[[RESULT]]38; CHECK-NEXT: ret39 %new = load i32, ptr %pnew40 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acquire acquire41 %val = extractvalue { i32, i1 } %pair, 042 ret i32 %val43}44 45define i32 @val_compare_and_swap_rel(ptr %p, i32 %cmp, i32 %new) #0 {46; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel47; CHECK-LABEL: val_compare_and_swap_rel:48; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x049; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:50; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]]51; CHECK-NEXT: cmp [[RESULT]], w152; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]]53; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]]54; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]]55; CHECK-NEXT: ret56; CHECK-NEXT: [[FAILBB]]:57; CHECK-NEXT: clrex58; CHECK-NEXT: ret59 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acq_rel monotonic60 %val = extractvalue { i32, i1 } %pair, 061 ret i32 %val62}63 64define i64 @val_compare_and_swap_64(ptr %p, i64 %cmp, i64 %new) #0 {65; OUTLINE-ATOMICS: bl __aarch64_cas8_relax66; CHECK-LABEL: val_compare_and_swap_64:67; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x068; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]:69; CHECK-NEXT: ldxr [[RESULT:x[0-9]+]], [x[[ADDR]]]70; CHECK-NEXT: cmp [[RESULT]], x171; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]]72; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]]73; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]]74; CHECK-NEXT: ret75; CHECK-NEXT: [[FAILBB]]:76; CHECK-NEXT: clrex77; CHECK-NEXT: ret78 %pair = cmpxchg ptr %p, i64 %cmp, i64 %new monotonic monotonic79 %val = extractvalue { i64, i1 } %pair, 080 ret i64 %val81}82 83define i32 @fetch_and_nand(ptr %p) #0 {84; CHECK-LABEL: fetch_and_nand:85; CHECK: [[TRYBB:.?LBB[0-9_]+]]:86; CHECK: ldxr w[[DEST_REG:[0-9]+]], [x0]87; CHECK: mvn [[TMP_REG:w[0-9]+]], w[[DEST_REG]]88; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], [[TMP_REG]], #0xfffffff889; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]]90; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0]91; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]]92; CHECK: mov x0, x[[DEST_REG]]93 %val = atomicrmw nand ptr %p, i32 7 release94 ret i32 %val95}96 97define i64 @fetch_and_nand_64(ptr %p) #0 {98; CHECK-LABEL: fetch_and_nand_64:99; CHECK: mov x[[ADDR:[0-9]+]], x0100; CHECK: [[TRYBB:.?LBB[0-9_]+]]:101; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]]102; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]]103; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8104; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]]105; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]]106 107 %val = atomicrmw nand ptr %p, i64 7 acq_rel108 ret i64 %val109}110 111define i32 @fetch_and_or(ptr %p) #0 {112; OUTLINE-ATOMICS: bl __aarch64_ldset4_acq_rel113; CHECK-LABEL: fetch_and_or:114; CHECK: mov [[OLDVAL_REG:w[0-9]+]], #5115; CHECK: [[TRYBB:.?LBB[0-9_]+]]:116; CHECK: ldaxr w[[DEST_REG:[0-9]+]], [x0]117; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], w[[DEST_REG]], [[OLDVAL_REG]]118; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]]119; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0]120; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]]121; CHECK: mov x0, x[[DEST_REG]]122 %val = atomicrmw or ptr %p, i32 5 seq_cst123 ret i32 %val124}125 126define i64 @fetch_and_or_64(ptr %p) #0 {127; OUTLINE-ATOMICS: bl __aarch64_ldset8_relax128; CHECK: fetch_and_or_64:129; CHECK: mov x[[ADDR:[0-9]+]], x0130; CHECK: [[TRYBB:.?LBB[0-9_]+]]:131; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]]132; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7133; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]]134; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]]135 %val = atomicrmw or ptr %p, i64 7 monotonic136 ret i64 %val137}138 139define void @acquire_fence() #0 {140 fence acquire141 ret void142 ; CHECK-LABEL: acquire_fence:143 ; CHECK: dmb ishld144}145 146define void @release_fence() #0 {147 fence release148 ret void149 ; CHECK-LABEL: release_fence:150 ; CHECK: dmb ish{{$}}151}152 153define void @seq_cst_fence() #0 {154 fence seq_cst155 ret void156 ; CHECK-LABEL: seq_cst_fence:157 ; CHECK: dmb ish{{$}}158}159 160define i32 @atomic_load(ptr %p) #0 {161 %r = load atomic i32, ptr %p seq_cst, align 4162 ret i32 %r163 ; CHECK-LABEL: atomic_load:164 ; CHECK: ldar165}166 167define i8 @atomic_load_relaxed_8(ptr %p, i32 %off32) #0 {168; CHECK-LABEL: atomic_load_relaxed_8:169 %ptr_unsigned = getelementptr i8, ptr %p, i32 4095170 %val_unsigned = load atomic i8, ptr %ptr_unsigned monotonic, align 1171; CHECK: ldrb {{w[0-9]+}}, [x0, #4095]172 173 %ptr_regoff = getelementptr i8, ptr %p, i32 %off32174 %val_regoff = load atomic i8, ptr %ptr_regoff unordered, align 1175 %tot1 = add i8 %val_unsigned, %val_regoff176; CHECK: ldrb {{w[0-9]+}}, [x0, w1, sxtw]177 178 %ptr_unscaled = getelementptr i8, ptr %p, i32 -256179 %val_unscaled = load atomic i8, ptr %ptr_unscaled monotonic, align 1180 %tot2 = add i8 %tot1, %val_unscaled181; CHECK: ldurb {{w[0-9]+}}, [x0, #-256]182 183 %ptr_random = getelementptr i8, ptr %p, i32 1191936 ; 0x123000 (i.e. ADD imm)184 %val_random = load atomic i8, ptr %ptr_random unordered, align 1185 %tot3 = add i8 %tot2, %val_random186; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12187; CHECK: ldrb {{w[0-9]+}}, [x[[ADDR]]]188 189 ret i8 %tot3190}191 192define i16 @atomic_load_relaxed_16(ptr %p, i32 %off32) #0 {193; CHECK-LABEL: atomic_load_relaxed_16:194 %ptr_unsigned = getelementptr i16, ptr %p, i32 4095195 %val_unsigned = load atomic i16, ptr %ptr_unsigned monotonic, align 2196; CHECK: ldrh {{w[0-9]+}}, [x0, #8190]197 198 %ptr_regoff = getelementptr i16, ptr %p, i32 %off32199 %val_regoff = load atomic i16, ptr %ptr_regoff unordered, align 2200 %tot1 = add i16 %val_unsigned, %val_regoff201; CHECK: ldrh {{w[0-9]+}}, [x0, w1, sxtw #1]202 203 %ptr_unscaled = getelementptr i16, ptr %p, i32 -128204 %val_unscaled = load atomic i16, ptr %ptr_unscaled monotonic, align 2205 %tot2 = add i16 %tot1, %val_unscaled206; CHECK: ldurh {{w[0-9]+}}, [x0, #-256]207 208 %ptr_random = getelementptr i16, ptr %p, i32 595968 ; 0x123000/2 (i.e. ADD imm)209 %val_random = load atomic i16, ptr %ptr_random unordered, align 2210 %tot3 = add i16 %tot2, %val_random211; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12212; CHECK: ldrh {{w[0-9]+}}, [x[[ADDR]]]213 214 ret i16 %tot3215}216 217define i32 @atomic_load_relaxed_32(ptr %p, i32 %off32) #0 {218; CHECK-LABEL: atomic_load_relaxed_32:219 %ptr_unsigned = getelementptr i32, ptr %p, i32 4095220 %val_unsigned = load atomic i32, ptr %ptr_unsigned monotonic, align 4221; CHECK: ldr {{w[0-9]+}}, [x0, #16380]222 223 %ptr_regoff = getelementptr i32, ptr %p, i32 %off32224 %val_regoff = load atomic i32, ptr %ptr_regoff unordered, align 4225 %tot1 = add i32 %val_unsigned, %val_regoff226; CHECK: ldr {{w[0-9]+}}, [x0, w1, sxtw #2]227 228 %ptr_unscaled = getelementptr i32, ptr %p, i32 -64229 %val_unscaled = load atomic i32, ptr %ptr_unscaled monotonic, align 4230 %tot2 = add i32 %tot1, %val_unscaled231; CHECK: ldur {{w[0-9]+}}, [x0, #-256]232 233 %ptr_random = getelementptr i32, ptr %p, i32 297984 ; 0x123000/4 (i.e. ADD imm)234 %val_random = load atomic i32, ptr %ptr_random unordered, align 4235 %tot3 = add i32 %tot2, %val_random236; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12237; CHECK: ldr {{w[0-9]+}}, [x[[ADDR]]]238 239 ret i32 %tot3240}241 242define i64 @atomic_load_relaxed_64(ptr %p, i32 %off32) #0 {243; CHECK-LABEL: atomic_load_relaxed_64:244 %ptr_unsigned = getelementptr i64, ptr %p, i32 4095245 %val_unsigned = load atomic i64, ptr %ptr_unsigned monotonic, align 8246; CHECK: ldr {{x[0-9]+}}, [x0, #32760]247 248 %ptr_regoff = getelementptr i64, ptr %p, i32 %off32249 %val_regoff = load atomic i64, ptr %ptr_regoff unordered, align 8250 %tot1 = add i64 %val_unsigned, %val_regoff251; CHECK: ldr {{x[0-9]+}}, [x0, w1, sxtw #3]252 253 %ptr_unscaled = getelementptr i64, ptr %p, i32 -32254 %val_unscaled = load atomic i64, ptr %ptr_unscaled monotonic, align 8255 %tot2 = add i64 %tot1, %val_unscaled256; CHECK: ldur {{x[0-9]+}}, [x0, #-256]257 258 %ptr_random = getelementptr i64, ptr %p, i32 148992 ; 0x123000/8 (i.e. ADD imm)259 %val_random = load atomic i64, ptr %ptr_random unordered, align 8260 %tot3 = add i64 %tot2, %val_random261; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12262; CHECK: ldr {{x[0-9]+}}, [x[[ADDR]]]263 264 ret i64 %tot3265}266 267 268define void @atomc_store(ptr %p) #0 {269 store atomic i32 4, ptr %p seq_cst, align 4270 ret void271 ; CHECK-LABEL: atomc_store:272 ; CHECK: stlr273}274 275define void @atomic_store_relaxed_8(ptr %p, i32 %off32, i8 %val) #0 {276; CHECK-LABEL: atomic_store_relaxed_8:277 %ptr_unsigned = getelementptr i8, ptr %p, i32 4095278 store atomic i8 %val, ptr %ptr_unsigned monotonic, align 1279; CHECK: strb {{w[0-9]+}}, [x0, #4095]280 281 %ptr_regoff = getelementptr i8, ptr %p, i32 %off32282 store atomic i8 %val, ptr %ptr_regoff unordered, align 1283; CHECK: strb {{w[0-9]+}}, [x0, w1, sxtw]284 285 %ptr_unscaled = getelementptr i8, ptr %p, i32 -256286 store atomic i8 %val, ptr %ptr_unscaled monotonic, align 1287; CHECK: sturb {{w[0-9]+}}, [x0, #-256]288 289 %ptr_random = getelementptr i8, ptr %p, i32 1191936 ; 0x123000 (i.e. ADD imm)290 store atomic i8 %val, ptr %ptr_random unordered, align 1291; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12292; CHECK: strb {{w[0-9]+}}, [x[[ADDR]]]293 294 ret void295}296 297define void @atomic_store_relaxed_16(ptr %p, i32 %off32, i16 %val) #0 {298; CHECK-LABEL: atomic_store_relaxed_16:299 %ptr_unsigned = getelementptr i16, ptr %p, i32 4095300 store atomic i16 %val, ptr %ptr_unsigned monotonic, align 2301; CHECK: strh {{w[0-9]+}}, [x0, #8190]302 303 %ptr_regoff = getelementptr i16, ptr %p, i32 %off32304 store atomic i16 %val, ptr %ptr_regoff unordered, align 2305; CHECK: strh {{w[0-9]+}}, [x0, w1, sxtw #1]306 307 %ptr_unscaled = getelementptr i16, ptr %p, i32 -128308 store atomic i16 %val, ptr %ptr_unscaled monotonic, align 2309; CHECK: sturh {{w[0-9]+}}, [x0, #-256]310 311 %ptr_random = getelementptr i16, ptr %p, i32 595968 ; 0x123000/2 (i.e. ADD imm)312 store atomic i16 %val, ptr %ptr_random unordered, align 2313; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12314; CHECK: strh {{w[0-9]+}}, [x[[ADDR]]]315 316 ret void317}318 319define void @atomic_store_relaxed_32(ptr %p, i32 %off32, i32 %val) #0 {320; CHECK-LABEL: atomic_store_relaxed_32:321 %ptr_unsigned = getelementptr i32, ptr %p, i32 4095322 store atomic i32 %val, ptr %ptr_unsigned monotonic, align 4323; CHECK: str {{w[0-9]+}}, [x0, #16380]324 325 %ptr_regoff = getelementptr i32, ptr %p, i32 %off32326 store atomic i32 %val, ptr %ptr_regoff unordered, align 4327; CHECK: str {{w[0-9]+}}, [x0, w1, sxtw #2]328 329 %ptr_unscaled = getelementptr i32, ptr %p, i32 -64330 store atomic i32 %val, ptr %ptr_unscaled monotonic, align 4331; CHECK: stur {{w[0-9]+}}, [x0, #-256]332 333 %ptr_random = getelementptr i32, ptr %p, i32 297984 ; 0x123000/4 (i.e. ADD imm)334 store atomic i32 %val, ptr %ptr_random unordered, align 4335; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12336; CHECK: str {{w[0-9]+}}, [x[[ADDR]]]337 338 ret void339}340 341define void @atomic_store_relaxed_64(ptr %p, i32 %off32, i64 %val) #0 {342; OUTLINE-ATOMICS: bl __aarch64_ldadd4_acq_rel343; CHECK-LABEL: atomic_store_relaxed_64:344 %ptr_unsigned = getelementptr i64, ptr %p, i32 4095345 store atomic i64 %val, ptr %ptr_unsigned monotonic, align 8346; CHECK: str {{x[0-9]+}}, [x0, #32760]347 348 %ptr_regoff = getelementptr i64, ptr %p, i32 %off32349 store atomic i64 %val, ptr %ptr_regoff unordered, align 8350; CHECK: str {{x[0-9]+}}, [x0, w1, sxtw #3]351 352 %ptr_unscaled = getelementptr i64, ptr %p, i32 -32353 store atomic i64 %val, ptr %ptr_unscaled monotonic, align 8354; CHECK: stur {{x[0-9]+}}, [x0, #-256]355 356 %ptr_random = getelementptr i64, ptr %p, i32 148992 ; 0x123000/8 (i.e. ADD imm)357 store atomic i64 %val, ptr %ptr_random unordered, align 8358; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12359; CHECK: str {{x[0-9]+}}, [x[[ADDR]]]360 361 ret void362}363 364; rdar://11531169365; rdar://11531308366 367%"class.X::Atomic" = type { %struct.x_atomic_t }368%struct.x_atomic_t = type { i32 }369 370@counter = external hidden global %"class.X::Atomic", align 4371 372define i32 @next_id() nounwind optsize ssp align 2 {373entry:374 %0 = atomicrmw add ptr @counter, i32 1 seq_cst375 %add.i = add i32 %0, 1376 %tobool = icmp eq i32 %add.i, 0377 br i1 %tobool, label %if.else, label %return378 379if.else: ; preds = %entry380 %1 = atomicrmw add ptr @counter, i32 1 seq_cst381 %add.i2 = add i32 %1, 1382 br label %return383 384return: ; preds = %if.else, %entry385 %retval.0 = phi i32 [ %add.i2, %if.else ], [ %add.i, %entry ]386 ret i32 %retval.0387}388 389attributes #0 = { nounwind }390