1754 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=atomic-expand %s | FileCheck --check-prefixes=CHECK,GCN %s3 4define i16 @test_atomicrmw_xchg_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {5; CHECK-LABEL: @test_atomicrmw_xchg_i16_global_agent(6; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)7; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i648; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 39; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 310; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i3211; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]12; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -113; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i3214; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]15; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 416; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]17; CHECK: atomicrmw.start:18; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]19; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]20; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]21; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] syncscope("agent") seq_cst seq_cst, align 422; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 123; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 024; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]25; CHECK: atomicrmw.end:26; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]27; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i1628; CHECK-NEXT: ret i16 [[EXTRACTED]]29;30 %res = atomicrmw xchg ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst31 ret i16 %res32}33 34define i16 @test_atomicrmw_xchg_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {35; CHECK-LABEL: @test_atomicrmw_xchg_i16_global_agent_align4(36; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i3237; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 438; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]39; CHECK: atomicrmw.start:40; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]41; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[LOADED]], -6553642; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]43; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP4]] syncscope("agent") seq_cst seq_cst, align 444; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 145; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 046; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]47; CHECK: atomicrmw.end:48; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i1649; CHECK-NEXT: ret i16 [[EXTRACTED]]50;51 %res = atomicrmw xchg ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 452 ret i16 %res53}54 55define i16 @test_atomicrmw_add_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {56; CHECK-LABEL: @test_atomicrmw_add_i16_global_agent(57; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)58; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i6459; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 360; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 361; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i3262; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]63; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -164; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i3265; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]66; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 467; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]68; CHECK: atomicrmw.start:69; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]70; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]71; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]72; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]73; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]74; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 475; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 176; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 077; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]78; CHECK: atomicrmw.end:79; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]80; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i1681; CHECK-NEXT: ret i16 [[EXTRACTED]]82;83 %res = atomicrmw add ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst84 ret i16 %res85}86 87define i16 @test_atomicrmw_add_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {88; CHECK-LABEL: @test_atomicrmw_add_i16_global_agent_align4(89; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i3290; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 491; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]92; CHECK: atomicrmw.start:93; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]94; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]95; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[NEW]], 6553596; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -6553697; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]98; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 499; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1100; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0101; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]102; CHECK: atomicrmw.end:103; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16104; CHECK-NEXT: ret i16 [[EXTRACTED]]105;106 %res = atomicrmw add ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4107 ret i16 %res108}109 110define i16 @test_atomicrmw_sub_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {111; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent(112; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)113; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64114; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3115; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3116; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32117; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]118; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1119; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32120; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]121; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4122; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]123; CHECK: atomicrmw.start:124; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]125; CHECK-NEXT: [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]126; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]127; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]128; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]129; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4130; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1131; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0132; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]133; CHECK: atomicrmw.end:134; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]135; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16136; CHECK-NEXT: ret i16 [[EXTRACTED]]137;138 %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst139 ret i16 %res140}141 142define i16 @test_atomicrmw_and_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {143; GCN-LABEL: @test_atomicrmw_and_i16_global_agent(144; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)145; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64146; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3147; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3148; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32149; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]150; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1151; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32152; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]153; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]154; GCN-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4155; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]156; GCN: atomicrmw.start:157; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]158; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]159; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4160; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1161; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0162; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]163; GCN: atomicrmw.end:164; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]165; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16166; GCN-NEXT: ret i16 [[EXTRACTED]]167;168; R600-LABEL: @test_atomicrmw_and_i16_global_agent(169; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)170; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64171; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3172; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3173; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32174; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]175; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1176; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32177; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]178; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]179; R600-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4180; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]181; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16182; R600-NEXT: ret i16 [[EXTRACTED]]183;184 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst185 ret i16 %res186}187 188define i16 @test_atomicrmw_and_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {189; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4(190; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32191; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536192; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4193; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]194; GCN: atomicrmw.start:195; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]196; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]197; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4198; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1199; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0200; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]201; GCN: atomicrmw.end:202; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16203; GCN-NEXT: ret i16 [[EXTRACTED]]204;205; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4(206; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32207; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536208; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4209; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16210; R600-NEXT: ret i16 [[EXTRACTED]]211;212 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4213 ret i16 %res214}215 216; Drop unknown metadata and noundef217define i16 @test_atomicrmw_and_i16_global_agent_drop_md(ptr addrspace(1) %ptr, i16 %value) {218; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_drop_md(219; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)220; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64221; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3222; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3223; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32224; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]225; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1226; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32227; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]228; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]229; GCN-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4230; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]231; GCN: atomicrmw.start:232; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]233; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]234; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4235; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1236; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0237; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]238; GCN: atomicrmw.end:239; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]240; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16241; GCN-NEXT: ret i16 [[EXTRACTED]]242;243; R600-LABEL: @test_atomicrmw_and_i16_global_agent_drop_md(244; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)245; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64246; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3247; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3248; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32249; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]250; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1251; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32252; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]253; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]254; R600-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4255; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]256; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16257; R600-NEXT: ret i16 [[EXTRACTED]]258;259 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !noundef !0, !some.unknown.md !0260 ret i16 %res261}262 263; Drop unknown metadata264define i16 @test_atomicrmw_and_i16_global_agent_align4_drop_md(ptr addrspace(1) %ptr, i16 %value) {265; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_drop_md(266; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32267; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536268; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4269; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]270; GCN: atomicrmw.start:271; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]272; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]273; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4274; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1275; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0276; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]277; GCN: atomicrmw.end:278; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16279; GCN-NEXT: ret i16 [[EXTRACTED]]280;281; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_drop_md(282; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32283; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536284; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4285; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16286; R600-NEXT: ret i16 [[EXTRACTED]]287;288 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noundef !0, !some.unknown.md !0289 ret i16 %res290}291 292; Drop noundef, preserve mmra293define i16 @test_atomicrmw_and_i16_global_agent_preserve_mmra(ptr addrspace(1) %ptr, i16 %value) {294; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_preserve_mmra(295; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)296; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64297; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3298; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3299; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32300; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]301; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1302; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32303; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]304; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]305; GCN-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4, !mmra [[META0:![0-9]+]]306; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]307; GCN: atomicrmw.start:308; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]309; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]310; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !mmra [[META0]]311; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1312; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0313; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]314; GCN: atomicrmw.end:315; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]316; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16317; GCN-NEXT: ret i16 [[EXTRACTED]]318;319; R600-LABEL: @test_atomicrmw_and_i16_global_agent_preserve_mmra(320; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)321; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64322; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3323; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3324; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32325; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]326; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1327; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32328; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]329; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]330; R600-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0:![0-9]+]]331; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]332; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16333; R600-NEXT: ret i16 [[EXTRACTED]]334;335 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !noundef !0, !mmra !1336 ret i16 %res337}338 339; Drop noundef, preserve mmra340define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(ptr addrspace(1) %ptr, i16 %value) {341; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(342; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32343; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536344; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4, !mmra [[META0]]345; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]346; GCN: atomicrmw.start:347; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]348; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]349; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !mmra [[META0]]350; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1351; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0352; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]353; GCN: atomicrmw.end:354; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16355; GCN-NEXT: ret i16 [[EXTRACTED]]356;357; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(358; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32359; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536360; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0]]361; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16362; R600-NEXT: ret i16 [[EXTRACTED]]363;364 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noundef !0, !mmra !1365 ret i16 %res366}367 368define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(ptr addrspace(1) %ptr, i16 %value) {369; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(370; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32371; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536372; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4373; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]374; GCN: atomicrmw.start:375; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]376; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]377; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4378; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1379; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0380; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]381; GCN: atomicrmw.end:382; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16383; GCN-NEXT: ret i16 [[EXTRACTED]]384;385; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(386; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32387; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536388; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !alias.scope [[META1:![0-9]+]]389; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16390; R600-NEXT: ret i16 [[EXTRACTED]]391;392 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !alias.scope !2393 ret i16 %res394}395 396define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(ptr addrspace(1) %ptr, i16 %value) {397; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(398; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32399; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536400; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4401; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]402; GCN: atomicrmw.start:403; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]404; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]405; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4406; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1407; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0408; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]409; GCN: atomicrmw.end:410; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16411; GCN-NEXT: ret i16 [[EXTRACTED]]412;413; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(414; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32415; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536416; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !noalias [[META1]]417; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16418; R600-NEXT: ret i16 [[EXTRACTED]]419;420 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noalias !2421 ret i16 %res422}423 424define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(ptr addrspace(1) %ptr, i16 %value) {425; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(426; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32427; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536428; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4429; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]430; GCN: atomicrmw.start:431; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]432; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]433; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4434; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1435; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0436; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]437; GCN: atomicrmw.end:438; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16439; GCN-NEXT: ret i16 [[EXTRACTED]]440;441; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(442; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32443; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536444; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]445; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16446; R600-NEXT: ret i16 [[EXTRACTED]]447;448 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !tbaa.struct !5449 ret i16 %res450}451 452define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(ptr addrspace(1) %ptr, i16 %value) {453; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(454; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32455; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536456; GCN-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4457; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]458; GCN: atomicrmw.start:459; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]460; GCN-NEXT: [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]461; GCN-NEXT: [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4462; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1463; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0464; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]465; GCN: atomicrmw.end:466; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16467; GCN-NEXT: ret i16 [[EXTRACTED]]468;469; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(470; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32471; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536472; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa [[TBAA5:![0-9]+]]473; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16474; R600-NEXT: ret i16 [[EXTRACTED]]475;476 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !tbaa !6477 ret i16 %res478}479 480define i16 @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {481; GCN-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(482; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)483; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64484; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3485; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3486; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32487; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]488; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1489; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32490; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]491; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]492; GCN-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META1:![0-9]+]]493; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]494; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16495; GCN-NEXT: ret i16 [[EXTRACTED]]496;497; R600-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(498; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)499; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64500; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3501; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3502; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32503; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]504; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1505; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32506; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]507; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]508; R600-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8:![0-9]+]]509; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]510; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16511; R600-NEXT: ret i16 [[EXTRACTED]]512;513 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0514 ret i16 %res515}516 517define i16 @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {518; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(519; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32520; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536521; GCN-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META1]]522; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16523; GCN-NEXT: ret i16 [[EXTRACTED]]524;525; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(526; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32527; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536528; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8]]529; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16530; R600-NEXT: ret i16 [[EXTRACTED]]531;532 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0533 ret i16 %res534}535 536define i16 @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {537; GCN-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(538; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)539; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64540; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3541; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3542; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32543; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]544; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1545; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32546; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]547; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]548; GCN-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META1]]549; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]550; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16551; GCN-NEXT: ret i16 [[EXTRACTED]]552;553; R600-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(554; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)555; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64556; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3557; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3558; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32559; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]560; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1561; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32562; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]563; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]564; R600-NEXT: [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]565; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]566; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16567; R600-NEXT: ret i16 [[EXTRACTED]]568;569 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0570 ret i16 %res571}572 573define i16 @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {574; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(575; GCN-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32576; GCN-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536577; GCN-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META1]]578; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16579; GCN-NEXT: ret i16 [[EXTRACTED]]580;581; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(582; R600-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32583; R600-NEXT: [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536584; R600-NEXT: [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]585; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16586; R600-NEXT: ret i16 [[EXTRACTED]]587;588 %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0589 ret i16 %res590}591 592define i16 @test_atomicrmw_nand_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {593; CHECK-LABEL: @test_atomicrmw_nand_i16_global_agent(594; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)595; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64596; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3597; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3598; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32599; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]600; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1601; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32602; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]603; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4604; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]605; CHECK: atomicrmw.start:606; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]607; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[VALOPERAND_SHIFTED]]608; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP5]], -1609; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[NEW]], [[MASK]]610; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[LOADED]], [[INV_MASK]]611; CHECK-NEXT: [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP6]]612; CHECK-NEXT: [[TMP9:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP8]] syncscope("agent") seq_cst seq_cst, align 4613; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP9]], 1614; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP9]], 0615; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]616; CHECK: atomicrmw.end:617; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]618; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16619; CHECK-NEXT: ret i16 [[EXTRACTED]]620;621 %res = atomicrmw nand ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst622 ret i16 %res623}624 625define i16 @test_atomicrmw_or_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {626; GCN-LABEL: @test_atomicrmw_or_i16_global_agent(627; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)628; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64629; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3630; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3631; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32632; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]633; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1634; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32635; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]636; GCN-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4637; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]638; GCN: atomicrmw.start:639; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]640; GCN-NEXT: [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]641; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4642; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1643; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0644; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]645; GCN: atomicrmw.end:646; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]647; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16648; GCN-NEXT: ret i16 [[EXTRACTED]]649;650; R600-LABEL: @test_atomicrmw_or_i16_global_agent(651; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)652; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64653; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3654; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3655; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32656; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]657; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1658; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32659; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]660; R600-NEXT: [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4661; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]662; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16663; R600-NEXT: ret i16 [[EXTRACTED]]664;665 %res = atomicrmw or ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst666 ret i16 %res667}668 669define i16 @test_atomicrmw_xor_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {670; GCN-LABEL: @test_atomicrmw_xor_i16_global_agent(671; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)672; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64673; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3674; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3675; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32676; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]677; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1678; GCN-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32679; GCN-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]680; GCN-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4681; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]682; GCN: atomicrmw.start:683; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]684; GCN-NEXT: [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]685; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4686; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1687; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0688; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]689; GCN: atomicrmw.end:690; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]691; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16692; GCN-NEXT: ret i16 [[EXTRACTED]]693;694; R600-LABEL: @test_atomicrmw_xor_i16_global_agent(695; R600-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)696; R600-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64697; R600-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3698; R600-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3699; R600-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32700; R600-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]701; R600-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1702; R600-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32703; R600-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]704; R600-NEXT: [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4705; R600-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]706; R600-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16707; R600-NEXT: ret i16 [[EXTRACTED]]708;709 %res = atomicrmw xor ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst710 ret i16 %res711}712 713define i16 @test_atomicrmw_max_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {714; CHECK-LABEL: @test_atomicrmw_max_i16_global_agent(715; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)716; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64717; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3718; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3719; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32720; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]721; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1722; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4723; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]724; CHECK: atomicrmw.start:725; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]726; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]727; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16728; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt i16 [[EXTRACTED]], [[VALUE:%.*]]729; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]730; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32731; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]732; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]733; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]734; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 4735; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1736; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0737; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]738; CHECK: atomicrmw.end:739; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]740; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16741; CHECK-NEXT: ret i16 [[EXTRACTED3]]742;743 %res = atomicrmw max ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst744 ret i16 %res745}746 747define i16 @test_atomicrmw_min_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {748; CHECK-LABEL: @test_atomicrmw_min_i16_global_agent(749; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)750; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64751; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3752; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3753; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32754; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]755; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1756; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4757; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]758; CHECK: atomicrmw.start:759; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]760; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]761; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16762; CHECK-NEXT: [[TMP4:%.*]] = icmp sle i16 [[EXTRACTED]], [[VALUE:%.*]]763; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]764; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32765; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]766; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]767; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]768; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 4769; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1770; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0771; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]772; CHECK: atomicrmw.end:773; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]774; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16775; CHECK-NEXT: ret i16 [[EXTRACTED3]]776;777 %res = atomicrmw min ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst778 ret i16 %res779}780 781define i16 @test_atomicrmw_umax_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {782; CHECK-LABEL: @test_atomicrmw_umax_i16_global_agent(783; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)784; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64785; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3786; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3787; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32788; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]789; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1790; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4791; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]792; CHECK: atomicrmw.start:793; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]794; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]795; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16796; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]797; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]798; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32799; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]800; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]801; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]802; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 4803; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1804; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0805; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]806; CHECK: atomicrmw.end:807; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]808; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16809; CHECK-NEXT: ret i16 [[EXTRACTED3]]810;811 %res = atomicrmw umax ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst812 ret i16 %res813}814 815define i16 @test_atomicrmw_umin_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {816; CHECK-LABEL: @test_atomicrmw_umin_i16_global_agent(817; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)818; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64819; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3820; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3821; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32822; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]823; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1824; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4825; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]826; CHECK: atomicrmw.start:827; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]828; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]829; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16830; CHECK-NEXT: [[TMP4:%.*]] = icmp ule i16 [[EXTRACTED]], [[VALUE:%.*]]831; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP4]], i16 [[EXTRACTED]], i16 [[VALUE]]832; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i32833; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]834; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]835; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]836; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 4837; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1838; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0839; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]840; CHECK: atomicrmw.end:841; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]842; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16843; CHECK-NEXT: ret i16 [[EXTRACTED3]]844;845 %res = atomicrmw umin ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst846 ret i16 %res847}848 849define i16 @test_cmpxchg_i16_global_agent(ptr addrspace(1) %out, i16 %in, i16 %old) {850; CHECK-LABEL: @test_cmpxchg_i16_global_agent(851; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr addrspace(1) [[OUT:%.*]], i64 4852; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[GEP]], i64 -4)853; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[GEP]] to i64854; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3855; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3856; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32857; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]858; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1859; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[IN:%.*]] to i32860; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]861; CHECK-NEXT: [[TMP5:%.*]] = zext i16 [[OLD:%.*]] to i32862; CHECK-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], [[SHIFTAMT]]863; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4864; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], [[INV_MASK]]865; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]866; CHECK: partword.cmpxchg.loop:867; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[TMP0:%.*]] ], [ [[TMP15:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]868; CHECK-NEXT: [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP4]]869; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP6]]870; CHECK-NEXT: [[TMP12:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[TMP11]], i32 [[TMP10]] seq_cst seq_cst, align 4871; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { i32, i1 } [[TMP12]], 0872; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP12]], 1873; CHECK-NEXT: br i1 [[TMP14]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]874; CHECK: partword.cmpxchg.failure:875; CHECK-NEXT: [[TMP15]] = and i32 [[TMP13]], [[INV_MASK]]876; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP9]], [[TMP15]]877; CHECK-NEXT: br i1 [[TMP16]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]878; CHECK: partword.cmpxchg.end:879; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP13]], [[SHIFTAMT]]880; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16881; CHECK-NEXT: [[TMP17:%.*]] = insertvalue { i16, i1 } poison, i16 [[EXTRACTED]], 0882; CHECK-NEXT: [[TMP18:%.*]] = insertvalue { i16, i1 } [[TMP17]], i1 [[TMP14]], 1883; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP18]], 0884; CHECK-NEXT: ret i16 [[EXTRACT]]885;886 %gep = getelementptr i16, ptr addrspace(1) %out, i64 4887 %res = cmpxchg ptr addrspace(1) %gep, i16 %old, i16 %in seq_cst seq_cst888 %extract = extractvalue {i16, i1} %res, 0889 ret i16 %extract890}891 892define i16 @test_cmpxchg_i16_global_agent_align4(ptr addrspace(1) %out, i16 %in, i16 %old) {893; CHECK-LABEL: @test_cmpxchg_i16_global_agent_align4(894; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr addrspace(1) [[OUT:%.*]], i64 4895; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[IN:%.*]] to i32896; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[OLD:%.*]] to i32897; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4898; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], -65536899; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]900; CHECK: partword.cmpxchg.loop:901; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[TMP11:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]902; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[TMP1]]903; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP5]], [[TMP2]]904; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[GEP]], i32 [[TMP7]], i32 [[TMP6]] seq_cst seq_cst, align 4905; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { i32, i1 } [[TMP8]], 0906; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1907; CHECK-NEXT: br i1 [[TMP10]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]908; CHECK: partword.cmpxchg.failure:909; CHECK-NEXT: [[TMP11]] = and i32 [[TMP9]], -65536910; CHECK-NEXT: [[TMP12:%.*]] = icmp ne i32 [[TMP5]], [[TMP11]]911; CHECK-NEXT: br i1 [[TMP12]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]912; CHECK: partword.cmpxchg.end:913; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP9]] to i16914; CHECK-NEXT: [[TMP13:%.*]] = insertvalue { i16, i1 } poison, i16 [[EXTRACTED]], 0915; CHECK-NEXT: [[TMP14:%.*]] = insertvalue { i16, i1 } [[TMP13]], i1 [[TMP10]], 1916; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP14]], 0917; CHECK-NEXT: ret i16 [[EXTRACT]]918;919 %gep = getelementptr i16, ptr addrspace(1) %out, i64 4920 %res = cmpxchg ptr addrspace(1) %gep, i16 %old, i16 %in seq_cst seq_cst, align 4921 %extract = extractvalue {i16, i1} %res, 0922 ret i16 %extract923}924 925define i16 @test_atomicrmw_xchg_i16_local(ptr addrspace(3) %ptr, i16 %value) {926; CHECK-LABEL: @test_atomicrmw_xchg_i16_local(927; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)928; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32929; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3930; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3931; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]932; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1933; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32934; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]935; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4936; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]937; CHECK: atomicrmw.start:938; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]939; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]940; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]941; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] seq_cst seq_cst, align 4942; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1943; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0944; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]945; CHECK: atomicrmw.end:946; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]947; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16948; CHECK-NEXT: ret i16 [[EXTRACTED]]949;950 %res = atomicrmw xchg ptr addrspace(3) %ptr, i16 %value seq_cst951 ret i16 %res952}953 954define i16 @test_cmpxchg_i16_local(ptr addrspace(3) %out, i16 %in, i16 %old) {955; CHECK-LABEL: @test_cmpxchg_i16_local(956; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr addrspace(3) [[OUT:%.*]], i64 4957; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[GEP]], i32 -4)958; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[GEP]] to i32959; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3960; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3961; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]962; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1963; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[IN:%.*]] to i32964; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP3]], [[TMP2]]965; CHECK-NEXT: [[TMP5:%.*]] = zext i16 [[OLD:%.*]] to i32966; CHECK-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], [[TMP2]]967; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4968; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], [[INV_MASK]]969; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]970; CHECK: partword.cmpxchg.loop:971; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[TMP0:%.*]] ], [ [[TMP15:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]972; CHECK-NEXT: [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP4]]973; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP6]]974; CHECK-NEXT: [[TMP12:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[TMP11]], i32 [[TMP10]] seq_cst seq_cst, align 4975; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { i32, i1 } [[TMP12]], 0976; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP12]], 1977; CHECK-NEXT: br i1 [[TMP14]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]978; CHECK: partword.cmpxchg.failure:979; CHECK-NEXT: [[TMP15]] = and i32 [[TMP13]], [[INV_MASK]]980; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP9]], [[TMP15]]981; CHECK-NEXT: br i1 [[TMP16]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]982; CHECK: partword.cmpxchg.end:983; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[TMP13]], [[TMP2]]984; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16985; CHECK-NEXT: [[TMP17:%.*]] = insertvalue { i16, i1 } poison, i16 [[EXTRACTED]], 0986; CHECK-NEXT: [[TMP18:%.*]] = insertvalue { i16, i1 } [[TMP17]], i1 [[TMP14]], 1987; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP18]], 0988; CHECK-NEXT: ret i16 [[EXTRACT]]989;990 %gep = getelementptr i16, ptr addrspace(3) %out, i64 4991 %res = cmpxchg ptr addrspace(3) %gep, i16 %old, i16 %in seq_cst seq_cst992 %extract = extractvalue {i16, i1} %res, 0993 ret i16 %extract994}995 996define i16 @test_atomicrmw_xor_i16_local_align4(ptr addrspace(3) %ptr, i16 %value) {997; CHECK-LABEL: @test_atomicrmw_xor_i16_local_align4(998; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32999; CHECK-NEXT: [[TMP2:%.*]] = atomicrmw xor ptr addrspace(3) [[PTR:%.*]], i32 [[TMP1]] seq_cst, align 41000; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i161001; CHECK-NEXT: ret i16 [[EXTRACTED]]1002;1003 %res = atomicrmw xor ptr addrspace(3) %ptr, i16 %value seq_cst, align 41004 ret i16 %res1005}1006 1007define i16 @test_atomicrmw_inc_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {1008; CHECK-LABEL: @test_atomicrmw_inc_i16_global_agent(1009; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1010; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641011; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31012; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31013; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321014; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1015; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11016; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41017; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1018; CHECK: atomicrmw.start:1019; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1020; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1021; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161022; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[EXTRACTED]], 11023; CHECK-NEXT: [[TMP5:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1024; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 0, i16 [[TMP4]]1025; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321026; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1027; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1028; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1029; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41030; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11031; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01032; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1033; CHECK: atomicrmw.end:1034; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1035; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161036; CHECK-NEXT: ret i16 [[EXTRACTED3]]1037;1038 %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst1039 ret i16 %res1040}1041 1042define i16 @test_atomicrmw_inc_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {1043; CHECK-LABEL: @test_atomicrmw_inc_i16_global_agent_align4(1044; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41045; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1046; CHECK: atomicrmw.start:1047; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1048; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161049; CHECK-NEXT: [[TMP2:%.*]] = add i16 [[EXTRACTED]], 11050; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1051; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP2]]1052; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321053; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361054; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1055; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41056; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11057; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01058; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1059; CHECK: atomicrmw.end:1060; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161061; CHECK-NEXT: ret i16 [[EXTRACTED1]]1062;1063 %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 41064 ret i16 %res1065}1066 1067define i16 @test_atomicrmw_inc_i16_local(ptr addrspace(3) %ptr, i16 %value) {1068; CHECK-LABEL: @test_atomicrmw_inc_i16_local(1069; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)1070; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i321071; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31072; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31073; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1074; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11075; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 41076; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1077; CHECK: atomicrmw.start:1078; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1079; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]1080; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161081; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[EXTRACTED]], 11082; CHECK-NEXT: [[TMP5:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1083; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 0, i16 [[TMP4]]1084; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321085; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]1086; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1087; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1088; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41089; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11090; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01091; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1092; CHECK: atomicrmw.end:1093; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1094; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161095; CHECK-NEXT: ret i16 [[EXTRACTED3]]1096;1097 %res = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i16 %value syncscope("agent") seq_cst1098 ret i16 %res1099}1100 1101define i16 @test_atomicrmw_inc_i16_local_align4(ptr addrspace(3) %ptr, i16 %value) {1102; CHECK-LABEL: @test_atomicrmw_inc_i16_local_align4(1103; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR:%.*]], align 41104; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1105; CHECK: atomicrmw.start:1106; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1107; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161108; CHECK-NEXT: [[TMP2:%.*]] = add i16 [[EXTRACTED]], 11109; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1110; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP2]]1111; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321112; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361113; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1114; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41115; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11116; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01117; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1118; CHECK: atomicrmw.end:1119; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161120; CHECK-NEXT: ret i16 [[EXTRACTED1]]1121;1122 %res = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i16 %value syncscope("agent") seq_cst, align 41123 ret i16 %res1124}1125 1126define i16 @test_atomicrmw_inc_i16_flat_agent(ptr %ptr, i16 %value) {1127; CHECK-LABEL: @test_atomicrmw_inc_i16_flat_agent(1128; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)1129; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i641130; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31131; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31132; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321133; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1134; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11135; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 41136; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1137; CHECK: atomicrmw.start:1138; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1139; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1140; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161141; CHECK-NEXT: [[TMP4:%.*]] = add i16 [[EXTRACTED]], 11142; CHECK-NEXT: [[TMP5:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1143; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 0, i16 [[TMP4]]1144; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321145; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1146; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1147; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1148; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41149; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11150; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01151; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1152; CHECK: atomicrmw.end:1153; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1154; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161155; CHECK-NEXT: ret i16 [[EXTRACTED3]]1156;1157 %res = atomicrmw uinc_wrap ptr %ptr, i16 %value syncscope("agent") seq_cst1158 ret i16 %res1159}1160 1161define i16 @test_atomicrmw_inc_i16_flat_agent_align4(ptr %ptr, i16 %value) {1162; CHECK-LABEL: @test_atomicrmw_inc_i16_flat_agent_align4(1163; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 41164; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1165; CHECK: atomicrmw.start:1166; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1167; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161168; CHECK-NEXT: [[TMP2:%.*]] = add i16 [[EXTRACTED]], 11169; CHECK-NEXT: [[TMP3:%.*]] = icmp uge i16 [[EXTRACTED]], [[VALUE:%.*]]1170; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i16 0, i16 [[TMP2]]1171; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321172; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361173; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1174; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41175; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11176; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01177; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1178; CHECK: atomicrmw.end:1179; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161180; CHECK-NEXT: ret i16 [[EXTRACTED1]]1181;1182 %res = atomicrmw uinc_wrap ptr %ptr, i16 %value syncscope("agent") seq_cst, align 41183 ret i16 %res1184}1185 1186define i16 @test_atomicrmw_dec_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {1187; CHECK-LABEL: @test_atomicrmw_dec_i16_global_agent(1188; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1189; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641190; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31191; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31192; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321193; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1194; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11195; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41196; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1197; CHECK: atomicrmw.start:1198; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1199; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1200; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161201; CHECK-NEXT: [[TMP4:%.*]] = sub i16 [[EXTRACTED]], 11202; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i16 [[EXTRACTED]], 01203; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1204; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]1205; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP7]], i16 [[VALUE]], i16 [[TMP4]]1206; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321207; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1208; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1209; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1210; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41211; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11212; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01213; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1214; CHECK: atomicrmw.end:1215; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1216; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161217; CHECK-NEXT: ret i16 [[EXTRACTED3]]1218;1219 %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst1220 ret i16 %res1221}1222 1223define i16 @test_atomicrmw_dec_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {1224; CHECK-LABEL: @test_atomicrmw_dec_i16_global_agent_align4(1225; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41226; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1227; CHECK: atomicrmw.start:1228; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1229; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161230; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[EXTRACTED]], 11231; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[EXTRACTED]], 01232; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1233; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]1234; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 [[VALUE]], i16 [[TMP2]]1235; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321236; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361237; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1238; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41239; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11240; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01241; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1242; CHECK: atomicrmw.end:1243; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161244; CHECK-NEXT: ret i16 [[EXTRACTED1]]1245;1246 %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 41247 ret i16 %res1248}1249 1250define i16 @test_atomicrmw_dec_i16_local(ptr addrspace(3) %ptr, i16 %value) {1251; CHECK-LABEL: @test_atomicrmw_dec_i16_local(1252; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)1253; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i321254; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31255; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31256; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1257; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11258; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 41259; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1260; CHECK: atomicrmw.start:1261; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1262; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]1263; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161264; CHECK-NEXT: [[TMP4:%.*]] = sub i16 [[EXTRACTED]], 11265; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i16 [[EXTRACTED]], 01266; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1267; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]1268; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP7]], i16 [[VALUE]], i16 [[TMP4]]1269; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321270; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]1271; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1272; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1273; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41274; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11275; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01276; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1277; CHECK: atomicrmw.end:1278; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1279; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161280; CHECK-NEXT: ret i16 [[EXTRACTED3]]1281;1282 %res = atomicrmw udec_wrap ptr addrspace(3) %ptr, i16 %value seq_cst1283 ret i16 %res1284}1285 1286define i16 @test_atomicrmw_dec_i16_local_align4(ptr addrspace(3) %ptr, i16 %value) {1287; CHECK-LABEL: @test_atomicrmw_dec_i16_local_align4(1288; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR:%.*]], align 41289; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1290; CHECK: atomicrmw.start:1291; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1292; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161293; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[EXTRACTED]], 11294; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[EXTRACTED]], 01295; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1296; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]1297; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 [[VALUE]], i16 [[TMP2]]1298; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321299; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361300; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1301; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41302; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11303; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01304; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1305; CHECK: atomicrmw.end:1306; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161307; CHECK-NEXT: ret i16 [[EXTRACTED1]]1308;1309 %res = atomicrmw udec_wrap ptr addrspace(3) %ptr, i16 %value seq_cst, align 41310 ret i16 %res1311}1312 1313define i16 @test_atomicrmw_dec_i16_flat_agent(ptr %ptr, i16 %value) {1314; CHECK-LABEL: @test_atomicrmw_dec_i16_flat_agent(1315; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)1316; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i641317; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31318; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31319; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321320; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1321; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11322; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 41323; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1324; CHECK: atomicrmw.start:1325; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1326; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1327; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161328; CHECK-NEXT: [[TMP4:%.*]] = sub i16 [[EXTRACTED]], 11329; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i16 [[EXTRACTED]], 01330; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1331; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]1332; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP7]], i16 [[VALUE]], i16 [[TMP4]]1333; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321334; CHECK-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1335; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1336; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1337; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41338; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11339; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01340; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1341; CHECK: atomicrmw.end:1342; CHECK-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1343; CHECK-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161344; CHECK-NEXT: ret i16 [[EXTRACTED3]]1345;1346 %res = atomicrmw udec_wrap ptr %ptr, i16 %value syncscope("agent") seq_cst1347 ret i16 %res1348}1349 1350define i16 @test_atomicrmw_dec_i16_flat_agent_align4(ptr %ptr, i16 %value) {1351; CHECK-LABEL: @test_atomicrmw_dec_i16_flat_agent_align4(1352; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 41353; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1354; CHECK: atomicrmw.start:1355; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1356; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161357; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[EXTRACTED]], 11358; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i16 [[EXTRACTED]], 01359; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i16 [[EXTRACTED]], [[VALUE:%.*]]1360; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]1361; CHECK-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i16 [[VALUE]], i16 [[TMP2]]1362; CHECK-NEXT: [[EXTENDED:%.*]] = zext i16 [[NEW]] to i321363; CHECK-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361364; CHECK-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1365; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") seq_cst seq_cst, align 41366; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11367; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01368; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1369; CHECK: atomicrmw.end:1370; CHECK-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161371; CHECK-NEXT: ret i16 [[EXTRACTED1]]1372;1373 %res = atomicrmw udec_wrap ptr %ptr, i16 %value syncscope("agent") seq_cst, align 41374 ret i16 %res1375}1376 1377define half @test_atomicrmw_xchg_f16_global_agent(ptr addrspace(1) %ptr, half %value) {1378; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_agent(1379; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1380; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641381; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31382; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31383; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321384; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1385; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11386; CHECK-NEXT: [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i161387; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i321388; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]1389; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41390; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1391; CHECK: atomicrmw.start:1392; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1393; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1394; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]1395; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41396; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11397; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01398; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1399; CHECK: atomicrmw.end:1400; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1401; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161402; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to half1403; CHECK-NEXT: ret half [[TMP9]]1404;1405 %res = atomicrmw xchg ptr addrspace(1) %ptr, half %value syncscope("agent") seq_cst1406 ret half %res1407}1408 1409define half @test_atomicrmw_xchg_f16_global_agent_align4(ptr addrspace(1) %ptr, half %value) {1410; CHECK-LABEL: @test_atomicrmw_xchg_f16_global_agent_align4(1411; CHECK-NEXT: [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i161412; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i321413; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41414; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1415; CHECK: atomicrmw.start:1416; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1417; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -655361418; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]1419; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 41420; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11421; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01422; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1423; CHECK: atomicrmw.end:1424; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i161425; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to half1426; CHECK-NEXT: ret half [[TMP7]]1427;1428 %res = atomicrmw xchg ptr addrspace(1) %ptr, half %value syncscope("agent") seq_cst, align 41429 ret half %res1430}1431 1432define half @test_atomicrmw_xchg_f16_flat_agent(ptr %ptr, half %value) {1433; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_agent(1434; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)1435; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i641436; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31437; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31438; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321439; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1440; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11441; CHECK-NEXT: [[TMP3:%.*]] = bitcast half [[VALUE:%.*]] to i161442; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i321443; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]1444; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 41445; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1446; CHECK: atomicrmw.start:1447; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1448; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1449; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]1450; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41451; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11452; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01453; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1454; CHECK: atomicrmw.end:1455; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1456; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161457; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to half1458; CHECK-NEXT: ret half [[TMP9]]1459;1460 %res = atomicrmw xchg ptr %ptr, half %value syncscope("agent") seq_cst1461 ret half %res1462}1463 1464define half @test_atomicrmw_xchg_f16_flat_agent_align4(ptr %ptr, half %value) {1465; CHECK-LABEL: @test_atomicrmw_xchg_f16_flat_agent_align4(1466; CHECK-NEXT: [[TMP1:%.*]] = bitcast half [[VALUE:%.*]] to i161467; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i321468; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[PTR:%.*]], align 41469; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1470; CHECK: atomicrmw.start:1471; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1472; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -655361473; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]1474; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 41475; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11476; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01477; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1478; CHECK: atomicrmw.end:1479; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i161480; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to half1481; CHECK-NEXT: ret half [[TMP7]]1482;1483 %res = atomicrmw xchg ptr %ptr, half %value syncscope("agent") seq_cst, align 41484 ret half %res1485}1486 1487define bfloat @test_atomicrmw_xchg_bf16_global_agent(ptr addrspace(1) %ptr, bfloat %value) {1488; CHECK-LABEL: @test_atomicrmw_xchg_bf16_global_agent(1489; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1490; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641491; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31492; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31493; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321494; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1495; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11496; CHECK-NEXT: [[TMP3:%.*]] = bitcast bfloat [[VALUE:%.*]] to i161497; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[TMP3]] to i321498; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]1499; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41500; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1501; CHECK: atomicrmw.start:1502; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1503; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1504; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[VALOPERAND_SHIFTED]]1505; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41506; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11507; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01508; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1509; CHECK: atomicrmw.end:1510; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1511; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161512; CHECK-NEXT: [[TMP9:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1513; CHECK-NEXT: ret bfloat [[TMP9]]1514;1515 %res = atomicrmw xchg ptr addrspace(1) %ptr, bfloat %value syncscope("agent") seq_cst1516 ret bfloat %res1517}1518 1519define bfloat @test_atomicrmw_xchg_bf16_global_agent_align4(ptr addrspace(1) %ptr, bfloat %value) {1520; CHECK-LABEL: @test_atomicrmw_xchg_bf16_global_agent_align4(1521; CHECK-NEXT: [[TMP1:%.*]] = bitcast bfloat [[VALUE:%.*]] to i161522; CHECK-NEXT: [[TMP2:%.*]] = zext i16 [[TMP1]] to i321523; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41524; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1525; CHECK: atomicrmw.start:1526; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1527; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -655361528; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP2]]1529; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 41530; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11531; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01532; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1533; CHECK: atomicrmw.end:1534; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i161535; CHECK-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1536; CHECK-NEXT: ret bfloat [[TMP7]]1537;1538 %res = atomicrmw xchg ptr addrspace(1) %ptr, bfloat %value syncscope("agent") seq_cst, align 41539 ret bfloat %res1540}1541 1542define i16 @test_atomicrmw_xchg_i16_buffer_fat_agent(ptr addrspace(7) %ptr, i16 %value) {1543; CHECK-LABEL: @test_atomicrmw_xchg_i16_buffer_fat_agent(1544; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)1545; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i321546; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31547; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31548; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1549; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11550; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i321551; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]1552; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 41553; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1554; CHECK: atomicrmw.start:1555; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1556; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1557; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]1558; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] syncscope("agent") seq_cst seq_cst, align 41559; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 11560; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 01561; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1562; CHECK: atomicrmw.end:1563; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1564; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161565; CHECK-NEXT: ret i16 [[EXTRACTED]]1566;1567 %res = atomicrmw xchg ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst1568 ret i16 %res1569}1570 1571define i16 @test_atomicrmw_xchg_i16_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i16 %value) {1572; CHECK-LABEL: @test_atomicrmw_xchg_i16_buffer_fat_agent_align4(1573; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i321574; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 41575; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1576; CHECK: atomicrmw.start:1577; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1578; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[LOADED]], -655361579; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]1580; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP4]] syncscope("agent") seq_cst seq_cst, align 41581; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 11582; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 01583; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1584; CHECK: atomicrmw.end:1585; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i161586; CHECK-NEXT: ret i16 [[EXTRACTED]]1587;1588 %res = atomicrmw xchg ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst, align 41589 ret i16 %res1590}1591 1592define i16 @test_atomicrmw_add_i16_buffer_fat_agent(ptr addrspace(7) %ptr, i16 %value) {1593; CHECK-LABEL: @test_atomicrmw_add_i16_buffer_fat_agent(1594; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)1595; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i321596; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31597; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31598; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1599; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11600; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i321601; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]1602; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 41603; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1604; CHECK: atomicrmw.start:1605; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1606; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]1607; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]1608; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1609; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]1610; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41611; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11612; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01613; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1614; CHECK: atomicrmw.end:1615; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1616; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161617; CHECK-NEXT: ret i16 [[EXTRACTED]]1618;1619 %res = atomicrmw add ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst1620 ret i16 %res1621}1622 1623define i16 @test_atomicrmw_add_i16_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i16 %value) {1624; CHECK-LABEL: @test_atomicrmw_add_i16_buffer_fat_agent_align4(1625; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i321626; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 41627; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1628; CHECK: atomicrmw.start:1629; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1630; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]1631; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[NEW]], 655351632; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -655361633; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]1634; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 41635; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11636; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01637; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1638; CHECK: atomicrmw.end:1639; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i161640; CHECK-NEXT: ret i16 [[EXTRACTED]]1641;1642 %res = atomicrmw add ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst, align 41643 ret i16 %res1644}1645 1646define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {1647; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_remote_memory(1648; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1649; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641650; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31651; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31652; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321653; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1654; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11655; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i321656; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]1657; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41658; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1659; CHECK: atomicrmw.start:1660; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1661; CHECK-NEXT: [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]1662; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]1663; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1664; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]1665; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41666; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11667; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01668; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1669; CHECK: atomicrmw.end:1670; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1671; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161672; CHECK-NEXT: ret i16 [[EXTRACTED]]1673;1674 %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01675 ret i16 %res1676}1677 1678define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {1679; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory(1680; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1681; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641682; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31683; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31684; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321685; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1686; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11687; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i321688; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]1689; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41690; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1691; CHECK: atomicrmw.start:1692; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1693; CHECK-NEXT: [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]1694; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]1695; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1696; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]1697; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41698; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11699; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01700; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1701; CHECK: atomicrmw.end:1702; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1703; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161704; CHECK-NEXT: ret i16 [[EXTRACTED]]1705;1706 %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01707 ret i16 %res1708}1709 1710define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {1711; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(1712; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1713; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641714; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31715; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31716; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321717; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1718; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11719; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i321720; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]1721; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41722; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]1723; CHECK: atomicrmw.start:1724; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1725; CHECK-NEXT: [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]1726; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]1727; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1728; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]1729; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 41730; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 11731; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 01732; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1733; CHECK: atomicrmw.end:1734; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1735; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161736; CHECK-NEXT: ret i16 [[EXTRACTED]]1737;1738 %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !01739 ret i16 %res1740}1741 1742!0 = !{}1743!1 = !{!"foo", !"bar"}1744!2 = !{!3}1745!3 = distinct !{!3, !4}1746!4 = distinct !{!4}1747!5 = !{i64 0, i64 4, !1, i64 8, i64 4}1748!6 = !{!7, !7, i64 0}1749!7 = !{!"omnipotent char", !8, i64 0}1750!8 = !{!"Simple C/C++ TBAA"}1751 1752;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:1753; GCN: {{.*}}1754