887 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=atomic-expand %s | FileCheck %s3 4define i16 @test_atomicrmw_xchg_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {5; CHECK-LABEL: @test_atomicrmw_xchg_i16_global_system(6; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)7; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i648; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 39; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 310; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i3211; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]12; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -113; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i3214; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]15; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 416; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]17; CHECK: atomicrmw.start:18; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]19; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]20; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]21; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] seq_cst seq_cst, align 422; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 123; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 024; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]25; CHECK: atomicrmw.end:26; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]27; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i1628; CHECK-NEXT: ret i16 [[EXTRACTED]]29;30 %res = atomicrmw xchg ptr addrspace(1) %ptr, i16 %value seq_cst31 ret i16 %res32}33 34define i16 @test_atomicrmw_xchg_i16_global_system_align4(ptr addrspace(1) %ptr, i16 %value) {35; CHECK-LABEL: @test_atomicrmw_xchg_i16_global_system_align4(36; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i3237; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 438; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]39; CHECK: atomicrmw.start:40; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]41; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[LOADED]], -6553642; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]43; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP4]] seq_cst seq_cst, align 444; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 145; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 046; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]47; CHECK: atomicrmw.end:48; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i1649; CHECK-NEXT: ret i16 [[EXTRACTED]]50;51 %res = atomicrmw xchg ptr addrspace(1) %ptr, i16 %value seq_cst, align 452 ret i16 %res53}54 55define i16 @test_atomicrmw_add_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {56; CHECK-LABEL: @test_atomicrmw_add_i16_global_system(57; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)58; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i6459; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 360; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 361; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i3262; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]63; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -164; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i3265; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]66; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 467; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]68; CHECK: atomicrmw.start:69; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]70; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]71; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]72; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]73; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]74; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 475; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 176; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 077; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]78; CHECK: atomicrmw.end:79; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]80; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i1681; CHECK-NEXT: ret i16 [[EXTRACTED]]82;83 %res = atomicrmw add ptr addrspace(1) %ptr, i16 %value seq_cst84 ret i16 %res85}86 87define i16 @test_atomicrmw_add_i16_global_system_align4(ptr addrspace(1) %ptr, i16 %value) {88; CHECK-LABEL: @test_atomicrmw_add_i16_global_system_align4(89; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i3290; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 491; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]92; CHECK: atomicrmw.start:93; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]94; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]95; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[NEW]], 6553596; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -6553697; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]98; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] seq_cst seq_cst, align 499; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1100; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0101; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]102; CHECK: atomicrmw.end:103; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16104; CHECK-NEXT: ret i16 [[EXTRACTED]]105;106 %res = atomicrmw add ptr addrspace(1) %ptr, i16 %value seq_cst, align 4107 ret i16 %res108}109 110define i16 @test_atomicrmw_sub_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {111; CHECK-LABEL: @test_atomicrmw_sub_i16_global_system(112; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)113; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64114; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3115; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3116; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32117; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]118; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1119; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32120; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]121; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4122; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]123; CHECK: atomicrmw.start:124; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]125; CHECK-NEXT: [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]126; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]127; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]128; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]129; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4130; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1131; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0132; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]133; CHECK: atomicrmw.end:134; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]135; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16136; CHECK-NEXT: ret i16 [[EXTRACTED]]137;138 %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value seq_cst139 ret i16 %res140}141 142define i16 @test_atomicrmw_and_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {143; CHECK-LABEL: @test_atomicrmw_and_i16_global_system(144; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)145; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64146; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3147; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3148; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32149; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]150; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1151; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32152; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]153; CHECK-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]154; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4155; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]156; CHECK: atomicrmw.start:157; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]158; CHECK-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]159; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4160; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1161; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0162; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]163; CHECK: atomicrmw.end:164; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]165; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16166; CHECK-NEXT: ret i16 [[EXTRACTED]]167;168 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value seq_cst169 ret i16 %res170}171 172define i16 @test_atomicrmw_nand_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {173; CHECK-LABEL: @test_atomicrmw_nand_i16_global_system(174; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)175; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64176; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3177; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3178; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32179; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]180; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1181; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32182; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]183; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4184; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]185; CHECK: atomicrmw.start:186; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]187; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[VALOPERAND_SHIFTED]]188; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP5]], -1189; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[NEW]], [[MASK]]190; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[LOADED]], [[INV_MASK]]191; CHECK-NEXT: [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP6]]192; CHECK-NEXT: [[TMP9:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP8]] seq_cst seq_cst, align 4193; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP9]], 1194; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP9]], 0195; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]196; CHECK: atomicrmw.end:197; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]198; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16199; CHECK-NEXT: ret i16 [[EXTRACTED]]200;201 %res = atomicrmw nand ptr addrspace(1) %ptr, i16 %value seq_cst202 ret i16 %res203}204 205define i16 @test_atomicrmw_or_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {206; CHECK-LABEL: @test_atomicrmw_or_i16_global_system(207; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)208; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64209; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3210; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3211; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32212; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]213; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1214; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32215; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]216; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4217; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]218; CHECK: atomicrmw.start:219; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]220; CHECK-NEXT: [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]221; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4222; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1223; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0224; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]225; CHECK: atomicrmw.end:226; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]227; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16228; CHECK-NEXT: ret i16 [[EXTRACTED]]229;230 %res = atomicrmw or ptr addrspace(1) %ptr, i16 %value seq_cst231 ret i16 %res232}233 234define i16 @test_atomicrmw_xor_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {235; CHECK-LABEL: @test_atomicrmw_xor_i16_global_system(236; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)237; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64238; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3239; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3240; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32241; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]242; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1243; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32244; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]245; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4246; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]247; CHECK: atomicrmw.start:248; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]249; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]250; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4251; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1252; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0253; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]254; CHECK: atomicrmw.end:255; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]256; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16257; CHECK-NEXT: ret i16 [[EXTRACTED]]258;259 %res = atomicrmw xor ptr addrspace(1) %ptr, i16 %value seq_cst260 ret i16 %res261}262 263define i16 @test_atomicrmw_max_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {264; CHECK-LABEL: @test_atomicrmw_max_i16_global_system(265; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)266; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64267; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3268; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3269; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32270; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]271; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1272; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4273; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]274; CHECK: atomicrmw.start:275; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]276; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]277; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16278; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt i16 [[EXTRACTED]], [[VALUE:%.*]]279; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]280; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32281; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]282; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]283; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]284; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4285; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1286; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0287; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]288; CHECK: atomicrmw.end:289; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]290; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16291; CHECK-NEXT: ret i16 [[EXTRACTED3]]292;293 %res = atomicrmw max ptr addrspace(1) %ptr, i16 %value seq_cst294 ret i16 %res295}296 297define i16 @test_atomicrmw_min_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {298; CHECK-LABEL: @test_atomicrmw_min_i16_global_system(299; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)300; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64301; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3302; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3303; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32304; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]305; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1306; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4307; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]308; CHECK: atomicrmw.start:309; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]310; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]311; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16312; CHECK-NEXT: [[TMP4:%.*]] = icmp sle i16 [[EXTRACTED]], [[VALUE:%.*]]313; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]314; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32315; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]316; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]317; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]318; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4319; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1320; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0321; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]322; CHECK: atomicrmw.end:323; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]324; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16325; CHECK-NEXT: ret i16 [[EXTRACTED3]]326;327 %res = atomicrmw min ptr addrspace(1) %ptr, i16 %value seq_cst328 ret i16 %res329}330 331define i16 @test_atomicrmw_umax_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {332; CHECK-LABEL: @test_atomicrmw_umax_i16_global_system(333; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)334; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64335; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3336; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3337; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32338; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]339; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1340; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4341; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]342; CHECK: atomicrmw.start:343; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]344; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]345; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16346; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]347; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]348; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32349; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]350; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]351; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]352; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4353; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1354; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0355; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]356; CHECK: atomicrmw.end:357; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]358; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16359; CHECK-NEXT: ret i16 [[EXTRACTED3]]360;361 %res = atomicrmw umax ptr addrspace(1) %ptr, i16 %value seq_cst362 ret i16 %res363}364 365define i16 @test_atomicrmw_umin_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {366; CHECK-LABEL: @test_atomicrmw_umin_i16_global_system(367; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)368; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64369; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3370; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3371; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32372; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]373; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1374; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4375; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]376; CHECK: atomicrmw.start:377; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]378; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]379; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16380; CHECK-NEXT: [[TMP4:%.*]] = icmp ule i16 [[EXTRACTED]], [[VALUE:%.*]]381; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]382; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32383; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]384; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]385; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]386; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4387; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1388; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0389; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]390; CHECK: atomicrmw.end:391; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]392; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16393; CHECK-NEXT: ret i16 [[EXTRACTED3]]394;395 %res = atomicrmw umin ptr addrspace(1) %ptr, i16 %value seq_cst396 ret i16 %res397}398 399define i16 @test_cmpxchg_i16_global_system(ptr addrspace(1) %out, i16 %in, i16 %old) {400; CHECK-LABEL: @test_cmpxchg_i16_global_system(401; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr addrspace(1) [[OUT:%.*]], i64 4402; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[GEP]], i64 -4)403; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[GEP]] to i64404; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3405; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3406; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32407; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]408; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1409; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[IN:%.*]] to i32410; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]411; CHECK-NEXT: [[TMP5:%.*]] = zext i16 [[OLD:%.*]] to i32412; CHECK-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], [[SHIFTAMT]]413; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4414; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], [[INV_MASK]]415; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]416; CHECK: partword.cmpxchg.loop:417; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[TMP0:%.*]] ], [ [[TMP15:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]418; CHECK-NEXT: [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP4]]419; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP6]]420; CHECK-NEXT: [[TMP12:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[TMP11]], i32 [[TMP10]] seq_cst seq_cst, align 4421; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { i32, i1 } [[TMP12]], 0422; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP12]], 1423; CHECK-NEXT: br i1 [[TMP14]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]424; CHECK: partword.cmpxchg.failure:425; CHECK-NEXT: [[TMP15]] = and i32 [[TMP13]], [[INV_MASK]]426; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP9]], [[TMP15]]427; CHECK-NEXT: br i1 [[TMP16]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]428; CHECK: partword.cmpxchg.end:429; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP13]], [[SHIFTAMT]]430; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16431; CHECK-NEXT: [[TMP17:%.*]] = insertvalue { i16, i1 } poison, i16 [[EXTRACTED]], 0432; CHECK-NEXT: [[TMP18:%.*]] = insertvalue { i16, i1 } [[TMP17]], i1 [[TMP14]], 1433; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP18]], 0434; CHECK-NEXT: ret i16 [[EXTRACT]]435;436 %gep = getelementptr i16, ptr addrspace(1) %out, i64 4437 %res = cmpxchg ptr addrspace(1) %gep, i16 %old, i16 %in seq_cst seq_cst438 %extract = extractvalue {i16, i1} %res, 0439 ret i16 %extract440}441 442define i16 @test_cmpxchg_i16_global_system_align4(ptr addrspace(1) %out, i16 %in, i16 %old) {443; CHECK-LABEL: @test_cmpxchg_i16_global_system_align4(444; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr addrspace(1) [[OUT:%.*]], i64 4445; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[IN:%.*]] to i32446; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[OLD:%.*]] to i32447; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4448; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], -65536449; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]450; CHECK: partword.cmpxchg.loop:451; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[TMP11:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]452; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[TMP1]]453; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP5]], [[TMP2]]454; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[GEP]], i32 [[TMP7]], i32 [[TMP6]] seq_cst seq_cst, align 4455; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0456; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1457; CHECK-NEXT: br i1 [[TMP10]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]458; CHECK: partword.cmpxchg.failure:459; CHECK-NEXT: [[TMP11]] = and i32 [[TMP9]], -65536460; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i32 [[TMP5]], [[TMP11]]461; CHECK-NEXT: br i1 [[TMP12]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]462; CHECK: partword.cmpxchg.end:463; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP9]] to i16464; CHECK-NEXT: [[TMP13:%.*]] = insertvalue { i16, i1 } poison, i16 [[EXTRACTED]], 0465; CHECK-NEXT: [[TMP14:%.*]] = insertvalue { i16, i1 } [[TMP13]], i1 [[TMP10]], 1466; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP14]], 0467; CHECK-NEXT: ret i16 [[EXTRACT]]468;469 %gep = getelementptr i16, ptr addrspace(1) %out, i64 4470 %res = cmpxchg ptr addrspace(1) %gep, i16 %old, i16 %in seq_cst seq_cst, align 4471 %extract = extractvalue {i16, i1} %res, 0472 ret i16 %extract473}474 475define i16 @test_atomicrmw_inc_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {476; CHECK-LABEL: @test_atomicrmw_inc_i16_global_system(477; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)478; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64479; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3480; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3481; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32482; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]483; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1484; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4485; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]486; CHECK: atomicrmw.start:487; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]488; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]489; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16490; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[EXTRACTED]], 1491; CHECK-NEXT: [[TMP5:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]492; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 0, i16 [[TMP4]]493; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32494; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]495; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]496; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]497; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4498; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1499; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0500; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]501; CHECK: atomicrmw.end:502; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]503; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16504; CHECK-NEXT: ret i16 [[EXTRACTED3]]505;506 %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i16 %value seq_cst507 ret i16 %res508}509 510define i16 @test_atomicrmw_inc_i16_global_system_align4(ptr addrspace(1) %ptr, i16 %value) {511; CHECK-LABEL: @test_atomicrmw_inc_i16_global_system_align4(512; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4513; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]514; CHECK: atomicrmw.start:515; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]516; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16517; CHECK-NEXT: [[TMP2:%.*]] = add i16 [[EXTRACTED]], 1518; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]519; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP2]]520; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32521; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536522; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]523; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4524; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1525; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0526; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]527; CHECK: atomicrmw.end:528; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16529; CHECK-NEXT: ret i16 [[EXTRACTED1]]530;531 %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i16 %value seq_cst, align 4532 ret i16 %res533}534 535define i16 @test_atomicrmw_inc_i16_flat_system(ptr %ptr, i16 %value) {536; CHECK-LABEL: @test_atomicrmw_inc_i16_flat_system(537; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)538; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64539; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3540; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3541; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32542; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]543; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1544; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4545; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]546; CHECK: atomicrmw.start:547; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]548; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]549; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16550; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[EXTRACTED]], 1551; CHECK-NEXT: [[TMP5:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]552; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 0, i16 [[TMP4]]553; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32554; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]555; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]556; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]557; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4558; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1559; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0560; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]561; CHECK: atomicrmw.end:562; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]563; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16564; CHECK-NEXT: ret i16 [[EXTRACTED3]]565;566 %res = atomicrmw uinc_wrap ptr %ptr, i16 %value seq_cst567 ret i16 %res568}569 570define i16 @test_atomicrmw_inc_i16_flat_system_align4(ptr %ptr, i16 %value) {571; CHECK-LABEL: @test_atomicrmw_inc_i16_flat_system_align4(572; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4573; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]574; CHECK: atomicrmw.start:575; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]576; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16577; CHECK-NEXT: [[TMP2:%.*]] = add i16 [[EXTRACTED]], 1578; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]579; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP2]]580; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32581; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536582; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]583; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4584; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1585; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0586; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]587; CHECK: atomicrmw.end:588; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16589; CHECK-NEXT: ret i16 [[EXTRACTED1]]590;591 %res = atomicrmw uinc_wrap ptr %ptr, i16 %value seq_cst, align 4592 ret i16 %res593}594 595define i16 @test_atomicrmw_dec_i16_global_system(ptr addrspace(1) %ptr, i16 %value) {596; CHECK-LABEL: @test_atomicrmw_dec_i16_global_system(597; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)598; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64599; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3600; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3601; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32602; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]603; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1604; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4605; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]606; CHECK: atomicrmw.start:607; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]608; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]609; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16610; CHECK-NEXT: [[TMP4:%.*]] = sub i16 [[EXTRACTED]], 1611; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i16 [[EXTRACTED]], 0612; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]613; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]614; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP7]], i16 [[VALUE]], i16 [[TMP4]]615; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32616; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]617; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]618; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]619; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4620; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1621; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0622; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]623; CHECK: atomicrmw.end:624; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]625; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16626; CHECK-NEXT: ret i16 [[EXTRACTED3]]627;628 %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i16 %value seq_cst629 ret i16 %res630}631 632define i16 @test_atomicrmw_dec_i16_global_system_align4(ptr addrspace(1) %ptr, i16 %value) {633; CHECK-LABEL: @test_atomicrmw_dec_i16_global_system_align4(634; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4635; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]636; CHECK: atomicrmw.start:637; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]638; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16639; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[EXTRACTED]], 1640; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[EXTRACTED]], 0641; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]642; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]643; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 [[VALUE]], i16 [[TMP2]]644; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32645; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536646; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]647; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4648; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1649; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0650; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]651; CHECK: atomicrmw.end:652; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16653; CHECK-NEXT: ret i16 [[EXTRACTED1]]654;655 %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i16 %value seq_cst, align 4656 ret i16 %res657}658 659define i16 @test_atomicrmw_dec_i16_flat_system(ptr %ptr, i16 %value) {660; CHECK-LABEL: @test_atomicrmw_dec_i16_flat_system(661; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)662; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64663; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3664; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3665; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32666; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]667; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1668; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4669; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]670; CHECK: atomicrmw.start:671; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]672; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]673; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16674; CHECK-NEXT: [[TMP4:%.*]] = sub i16 [[EXTRACTED]], 1675; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i16 [[EXTRACTED]], 0676; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]677; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]678; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP7]], i16 [[VALUE]], i16 [[TMP4]]679; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32680; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]681; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]682; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]683; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4684; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1685; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0686; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]687; CHECK: atomicrmw.end:688; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]689; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16690; CHECK-NEXT: ret i16 [[EXTRACTED3]]691;692 %res = atomicrmw udec_wrap ptr %ptr, i16 %value seq_cst693 ret i16 %res694}695 696define i16 @test_atomicrmw_dec_i16_flat_system_align4(ptr %ptr, i16 %value) {697; CHECK-LABEL: @test_atomicrmw_dec_i16_flat_system_align4(698; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4699; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]700; CHECK: atomicrmw.start:701; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]702; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16703; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[EXTRACTED]], 1704; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[EXTRACTED]], 0705; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]706; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]707; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 [[VALUE]], i16 [[TMP2]]708; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32709; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536710; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]711; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4712; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1713; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0714; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]715; CHECK: atomicrmw.end:716; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16717; CHECK-NEXT: ret i16 [[EXTRACTED1]]718;719 %res = atomicrmw udec_wrap ptr %ptr, i16 %value seq_cst, align 4720 ret i16 %res721}722 723define half @test_atomicrmw_xchg_f16_global_system(ptr addrspace(1) %ptr, half %value) {724; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_system(725; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)726; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64727; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3728; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3729; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32730; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]731; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1732; CHECK-NEXT: [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16733; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32734; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]735; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4736; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]737; CHECK: atomicrmw.start:738; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]739; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]740; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]741; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4742; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1743; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0744; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]745; CHECK: atomicrmw.end:746; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]747; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16748; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to half749; CHECK-NEXT: ret half [[TMP9]]750;751 %res = atomicrmw xchg ptr addrspace(1) %ptr, half %value seq_cst752 ret half %res753}754 755define half @test_atomicrmw_xchg_f16_global_system_align4(ptr addrspace(1) %ptr, half %value) {756; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_system_align4(757; CHECK-NEXT: [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16758; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i32759; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4760; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]761; CHECK: atomicrmw.start:762; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]763; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -65536764; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]765; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] seq_cst seq_cst, align 4766; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1767; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0768; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]769; CHECK: atomicrmw.end:770; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16771; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to half772; CHECK-NEXT: ret half [[TMP7]]773;774 %res = atomicrmw xchg ptr addrspace(1) %ptr, half %value seq_cst, align 4775 ret half %res776}777 778define half @test_atomicrmw_xchg_f16_flat_system(ptr %ptr, half %value) {779; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_system(780; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)781; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64782; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3783; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3784; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32785; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]786; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1787; CHECK-NEXT: [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i16788; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32789; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]790; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4791; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]792; CHECK: atomicrmw.start:793; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]794; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]795; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]796; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4797; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1798; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0799; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]800; CHECK: atomicrmw.end:801; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]802; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16803; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to half804; CHECK-NEXT: ret half [[TMP9]]805;806 %res = atomicrmw xchg ptr %ptr, half %value seq_cst807 ret half %res808}809 810define half @test_atomicrmw_xchg_f16_flat_system_align4(ptr %ptr, half %value) {811; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_system_align4(812; CHECK-NEXT: [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i16813; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i32814; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[PTR:%.*]], align 4815; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]816; CHECK: atomicrmw.start:817; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]818; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -65536819; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]820; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[TMP5]] seq_cst seq_cst, align 4821; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1822; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0823; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]824; CHECK: atomicrmw.end:825; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16826; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to half827; CHECK-NEXT: ret half [[TMP7]]828;829 %res = atomicrmw xchg ptr %ptr, half %value seq_cst, align 4830 ret half %res831}832 833define bfloat @test_atomicrmw_xchg_bf16_flat_system(ptr %ptr, bfloat %value) {834; CHECK-LABEL: @test_atomicrmw_xchg_bf16_flat_system(835; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)836; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64837; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3838; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3839; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32840; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]841; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1842; CHECK-NEXT: [[TMP3:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16843; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i32844; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]845; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4846; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]847; CHECK: atomicrmw.start:848; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]849; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]850; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]851; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4852; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1853; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0854; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]855; CHECK: atomicrmw.end:856; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]857; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16858; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat859; CHECK-NEXT: ret bfloat [[TMP9]]860;861 %res = atomicrmw xchg ptr %ptr, bfloat %value seq_cst862 ret bfloat %res863}864 865define bfloat @test_atomicrmw_xchg_bf16_flat_system_align4(ptr %ptr, bfloat %value) {866; CHECK-LABEL: @test_atomicrmw_xchg_bf16_flat_system_align4(867; CHECK-NEXT: [[TMP1:%.*]] = bitcast bfloat [[VALUE:%.*]] to i16868; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i32869; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[PTR:%.*]], align 4870; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]871; CHECK: atomicrmw.start:872; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]873; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -65536874; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]875; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[TMP5]] seq_cst seq_cst, align 4876; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1877; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0878; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]879; CHECK: atomicrmw.end:880; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16881; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat882; CHECK-NEXT: ret bfloat [[TMP7]]883;884 %res = atomicrmw xchg ptr %ptr, bfloat %value seq_cst, align 4885 ret bfloat %res886}887