665 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=atomic-expand %s | FileCheck -check-prefix=GCN %s3; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=atomic-expand %s | FileCheck -check-prefix=GCN %s4 5define float @test_atomicrmw_fsub_f32_flat(ptr %ptr, float %value) {6; GCN-LABEL: @test_atomicrmw_fsub_f32_flat(7; GCN-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 48; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]9; GCN: atomicrmw.start:10; GCN-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]11; GCN-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]12; GCN-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3213; GCN-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3214; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 415; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 116; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 017; GCN-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float18; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]19; GCN: atomicrmw.end:20; GCN-NEXT: ret float [[TMP5]]21;22 %res = atomicrmw fsub ptr %ptr, float %value seq_cst23 ret float %res24}25 26define float @test_atomicrmw_fsub_f32_global(ptr addrspace(1) %ptr, float %value) {27; GCN-LABEL: @test_atomicrmw_fsub_f32_global(28; GCN-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 429; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]30; GCN: atomicrmw.start:31; GCN-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]32; GCN-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]33; GCN-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3234; GCN-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3235; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 436; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 137; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 038; GCN-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float39; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]40; GCN: atomicrmw.end:41; GCN-NEXT: ret float [[TMP5]]42;43 %res = atomicrmw fsub ptr addrspace(1) %ptr, float %value seq_cst44 ret float %res45}46 47define float @test_atomicrmw_fsub_f32_local(ptr addrspace(3) %ptr, float %value) {48; GCN-LABEL: @test_atomicrmw_fsub_f32_local(49; GCN-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 450; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]51; GCN: atomicrmw.start:52; GCN-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]53; GCN-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]54; GCN-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3255; GCN-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3256; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 457; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 158; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 059; GCN-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float60; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]61; GCN: atomicrmw.end:62; GCN-NEXT: ret float [[TMP5]]63;64 %res = atomicrmw fsub ptr addrspace(3) %ptr, float %value seq_cst65 ret float %res66}67 68define half @test_atomicrmw_fsub_f16_flat(ptr %ptr, half %value) {69; GCN-LABEL: @test_atomicrmw_fsub_f16_flat(70; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)71; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i6472; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 373; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 374; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i3275; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]76; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -177; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 478; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]79; GCN: atomicrmw.start:80; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]81; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]82; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i1683; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half84; GCN-NEXT: [[NEW:%.*]] = fsub half [[TMP4]], [[VALUE:%.*]]85; GCN-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i1686; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i3287; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]88; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]89; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]90; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 491; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 192; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 093; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]94; GCN: atomicrmw.end:95; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]96; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i1697; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half98; GCN-NEXT: ret half [[TMP7]]99;100 %res = atomicrmw fsub ptr %ptr, half %value seq_cst101 ret half %res102}103 104define half @test_atomicrmw_fsub_f16_global(ptr addrspace(1) %ptr, half %value) {105; GCN-LABEL: @test_atomicrmw_fsub_f16_global(106; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)107; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64108; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3109; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3110; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32111; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]112; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1113; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4114; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]115; GCN: atomicrmw.start:116; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]117; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]118; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16119; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half120; GCN-NEXT: [[NEW:%.*]] = fsub half [[TMP4]], [[VALUE:%.*]]121; GCN-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i16122; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32123; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]124; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]125; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]126; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4127; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1128; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0129; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]130; GCN: atomicrmw.end:131; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]132; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16133; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half134; GCN-NEXT: ret half [[TMP7]]135;136 %res = atomicrmw fsub ptr addrspace(1) %ptr, half %value seq_cst137 ret half %res138}139 140define half @test_atomicrmw_fsub_f16_global_align4(ptr addrspace(1) %ptr, half %value) {141; GCN-LABEL: @test_atomicrmw_fsub_f16_global_align4(142; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4143; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]144; GCN: atomicrmw.start:145; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]146; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16147; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to half148; GCN-NEXT: [[NEW:%.*]] = fsub half [[TMP2]], [[VALUE:%.*]]149; GCN-NEXT: [[TMP3:%.*]] = bitcast half [[NEW]] to i16150; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32151; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536152; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]153; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4154; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1155; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0156; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]157; GCN: atomicrmw.end:158; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16159; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to half160; GCN-NEXT: ret half [[TMP5]]161;162 %res = atomicrmw fsub ptr addrspace(1) %ptr, half %value seq_cst, align 4163 ret half %res164}165 166define half @test_atomicrmw_fsub_f16_local(ptr addrspace(3) %ptr, half %value) {167; GCN-LABEL: @test_atomicrmw_fsub_f16_local(168; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)169; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32170; GCN-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3171; GCN-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3172; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]173; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1174; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4175; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]176; GCN: atomicrmw.start:177; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]178; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]179; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16180; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half181; GCN-NEXT: [[NEW:%.*]] = fsub half [[TMP4]], [[VALUE:%.*]]182; GCN-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i16183; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32184; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]185; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]186; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]187; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4188; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1189; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0190; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]191; GCN: atomicrmw.end:192; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]193; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16194; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half195; GCN-NEXT: ret half [[TMP7]]196;197 %res = atomicrmw fsub ptr addrspace(3) %ptr, half %value seq_cst198 ret half %res199}200 201define double @test_atomicrmw_fsub_f64_flat__noprivate(ptr %ptr, double %value) {202; GCN-LABEL: @test_atomicrmw_fsub_f64_flat__noprivate(203; GCN-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8204; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]205; GCN: atomicrmw.start:206; GCN-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]207; GCN-NEXT: [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]208; GCN-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64209; GCN-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64210; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !noalias.addrspace [[META0:![0-9]+]]211; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1212; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0213; GCN-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double214; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]215; GCN: atomicrmw.end:216; GCN-NEXT: ret double [[TMP5]]217;218 %res = atomicrmw fsub ptr %ptr, double %value seq_cst, !noalias.addrspace !0219 ret double %res220}221 222define double @test_atomicrmw_fsub_f64_flat(ptr %ptr, double %value) {223; GCN-LABEL: @test_atomicrmw_fsub_f64_flat(224; GCN-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])225; GCN-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]226; GCN: atomicrmw.private:227; GCN-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)228; GCN-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8229; GCN-NEXT: [[NEW:%.*]] = fsub double [[LOADED_PRIVATE]], [[VALUE:%.*]]230; GCN-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8231; GCN-NEXT: br label [[ATOMICRMW_PHI:%.*]]232; GCN: atomicrmw.global:233; GCN-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8234; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]235; GCN: atomicrmw.start:236; GCN-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]237; GCN-NEXT: [[NEW2:%.*]] = fsub double [[LOADED]], [[VALUE]]238; GCN-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64239; GCN-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64240; GCN-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]241; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1242; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0243; GCN-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double244; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]245; GCN: atomicrmw.end1:246; GCN-NEXT: br label [[ATOMICRMW_PHI]]247; GCN: atomicrmw.phi:248; GCN-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]249; GCN-NEXT: br label [[ATOMICRMW_END:%.*]]250; GCN: atomicrmw.end:251; GCN-NEXT: ret double [[RES]]252;253 %res = atomicrmw fsub ptr %ptr, double %value seq_cst254 ret double %res255}256 257define double @test_atomicrmw_fsub_f64_global(ptr addrspace(1) %ptr, double %value) {258; GCN-LABEL: @test_atomicrmw_fsub_f64_global(259; GCN-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8260; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]261; GCN: atomicrmw.start:262; GCN-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]263; GCN-NEXT: [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]264; GCN-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64265; GCN-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64266; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8267; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1268; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0269; GCN-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double270; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]271; GCN: atomicrmw.end:272; GCN-NEXT: ret double [[TMP5]]273;274 %res = atomicrmw fsub ptr addrspace(1) %ptr, double %value seq_cst275 ret double %res276}277 278define double @test_atomicrmw_fsub_f64_local(ptr addrspace(3) %ptr, double %value) {279; GCN-LABEL: @test_atomicrmw_fsub_f64_local(280; GCN-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 8281; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]282; GCN: atomicrmw.start:283; GCN-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]284; GCN-NEXT: [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]285; GCN-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64286; GCN-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64287; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8288; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1289; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0290; GCN-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double291; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]292; GCN: atomicrmw.end:293; GCN-NEXT: ret double [[TMP5]]294;295 %res = atomicrmw fsub ptr addrspace(3) %ptr, double %value seq_cst296 ret double %res297}298 299define float @test_atomicrmw_fsub_f32_global_strictfp(ptr addrspace(1) %ptr, float %value) strictfp {300; GCN-LABEL: @test_atomicrmw_fsub_f32_global_strictfp(301; GCN-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4302; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]303; GCN: atomicrmw.start:304; GCN-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]305; GCN-NEXT: [[NEW:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[LOADED]], float [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR4:[0-9]+]]306; GCN-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32307; GCN-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32308; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4309; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1310; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0311; GCN-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float312; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]313; GCN: atomicrmw.end:314; GCN-NEXT: ret float [[TMP5]]315;316 %res = atomicrmw fsub ptr addrspace(1) %ptr, float %value seq_cst317 ret float %res318}319 320define bfloat @test_atomicrmw_fadd_bf16_local(ptr addrspace(3) %ptr, bfloat %value) {321; GCN-LABEL: @test_atomicrmw_fadd_bf16_local(322; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)323; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32324; GCN-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3325; GCN-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3326; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]327; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1328; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4329; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]330; GCN: atomicrmw.start:331; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]332; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]333; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16334; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat335; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]336; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16337; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32338; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]339; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]340; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]341; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4342; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1343; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0344; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]345; GCN: atomicrmw.end:346; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]347; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16348; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat349; GCN-NEXT: ret bfloat [[TMP7]]350;351 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic352 ret bfloat %res353}354 355define bfloat @test_atomicrmw_fadd_bf16_local_align4(ptr addrspace(3) %ptr, bfloat %value) {356; GCN-LABEL: @test_atomicrmw_fadd_bf16_local_align4(357; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR:%.*]], align 4358; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]359; GCN: atomicrmw.start:360; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]361; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16362; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat363; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]364; GCN-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16365; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32366; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536367; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]368; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4369; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1370; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0371; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]372; GCN: atomicrmw.end:373; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16374; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat375; GCN-NEXT: ret bfloat [[TMP5]]376;377 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic, align 4378 ret bfloat %res379}380 381define bfloat @test_atomicrmw_fadd_bf16_global_agent(ptr addrspace(1) %ptr, bfloat %value) {382; GCN-LABEL: @test_atomicrmw_fadd_bf16_global_agent(383; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)384; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64385; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3386; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3387; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32388; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]389; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1390; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4391; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]392; GCN: atomicrmw.start:393; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]394; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]395; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16396; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat397; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]398; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16399; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32400; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]401; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]402; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]403; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 4404; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1405; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0406; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]407; GCN: atomicrmw.end:408; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]409; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16410; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat411; GCN-NEXT: ret bfloat [[TMP7]]412;413 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value syncscope("agent") monotonic414 ret bfloat %res415}416 417define bfloat @test_atomicrmw_fadd_bf16_global_agent_align4(ptr addrspace(1) %ptr, bfloat %value) {418; GCN-LABEL: @test_atomicrmw_fadd_bf16_global_agent_align4(419; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4420; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]421; GCN: atomicrmw.start:422; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]423; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16424; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat425; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]426; GCN-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16427; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32428; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536429; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]430; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 4431; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1432; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0433; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]434; GCN: atomicrmw.end:435; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16436; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat437; GCN-NEXT: ret bfloat [[TMP5]]438;439 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value syncscope("agent") monotonic, align 4440 ret bfloat %res441}442 443define bfloat @test_atomicrmw_fadd_bf16_global_system(ptr addrspace(1) %ptr, bfloat %value) {444; GCN-LABEL: @test_atomicrmw_fadd_bf16_global_system(445; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)446; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64447; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3448; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3449; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32450; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]451; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1452; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4453; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]454; GCN: atomicrmw.start:455; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]456; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]457; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16458; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat459; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]460; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16461; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32462; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]463; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]464; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]465; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4466; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1467; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0468; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]469; GCN: atomicrmw.end:470; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]471; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16472; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat473; GCN-NEXT: ret bfloat [[TMP7]]474;475 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value monotonic476 ret bfloat %res477}478 479define bfloat @test_atomicrmw_fadd_bf16_global_system_align4(ptr addrspace(1) %ptr, bfloat %value) {480; GCN-LABEL: @test_atomicrmw_fadd_bf16_global_system_align4(481; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4482; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]483; GCN: atomicrmw.start:484; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]485; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16486; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat487; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]488; GCN-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16489; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32490; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536491; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]492; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4493; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1494; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0495; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]496; GCN: atomicrmw.end:497; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16498; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat499; GCN-NEXT: ret bfloat [[TMP5]]500;501 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value monotonic, align 4502 ret bfloat %res503}504 505define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bfloat %value) #2 {506; GCN-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp(507; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)508; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i32509; GCN-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3510; GCN-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3511; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]512; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1513; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 4514; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]515; GCN: atomicrmw.start:516; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]517; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]518; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16519; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat520; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]521; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16522; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32523; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]524; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]525; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]526; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4527; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1528; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0529; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]530; GCN: atomicrmw.end:531; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]532; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16533; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat534; GCN-NEXT: ret bfloat [[TMP7]]535;536 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic537 ret bfloat %res538}539 540define bfloat @test_atomicrmw_fadd_bf16_flat_agent(ptr %ptr, bfloat %value) {541; GCN-LABEL: @test_atomicrmw_fadd_bf16_flat_agent(542; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)543; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64544; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3545; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3546; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32547; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]548; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1549; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4550; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]551; GCN: atomicrmw.start:552; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]553; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]554; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16555; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat556; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]557; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16558; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32559; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]560; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]561; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]562; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 4563; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1564; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0565; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]566; GCN: atomicrmw.end:567; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]568; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16569; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat570; GCN-NEXT: ret bfloat [[TMP7]]571;572 %res = atomicrmw fadd ptr %ptr, bfloat %value syncscope("agent") monotonic573 ret bfloat %res574}575 576define bfloat @test_atomicrmw_fadd_bf16_flat_agent_align4(ptr %ptr, bfloat %value) {577; GCN-LABEL: @test_atomicrmw_fadd_bf16_flat_agent_align4(578; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4579; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]580; GCN: atomicrmw.start:581; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]582; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16583; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat584; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]585; GCN-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16586; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32587; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536588; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]589; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 4590; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1591; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0592; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]593; GCN: atomicrmw.end:594; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16595; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat596; GCN-NEXT: ret bfloat [[TMP5]]597;598 %res = atomicrmw fadd ptr %ptr, bfloat %value syncscope("agent") monotonic, align 4599 ret bfloat %res600}601 602define bfloat @test_atomicrmw_fadd_bf16_flat_system(ptr %ptr, bfloat %value) {603; GCN-LABEL: @test_atomicrmw_fadd_bf16_flat_system(604; GCN-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)605; GCN-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64606; GCN-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3607; GCN-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3608; GCN-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32609; GCN-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]610; GCN-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1611; GCN-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4612; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]613; GCN: atomicrmw.start:614; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]615; GCN-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]616; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16617; GCN-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat618; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]619; GCN-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i16620; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i32621; GCN-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]622; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]623; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]624; GCN-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4625; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1626; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0627; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]628; GCN: atomicrmw.end:629; GCN-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]630; GCN-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i16631; GCN-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat632; GCN-NEXT: ret bfloat [[TMP7]]633;634 %res = atomicrmw fadd ptr %ptr, bfloat %value monotonic635 ret bfloat %res636}637 638define bfloat @test_atomicrmw_fadd_bf16_flat_system_align4(ptr %ptr, bfloat %value) {639; GCN-LABEL: @test_atomicrmw_fadd_bf16_flat_system_align4(640; GCN-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4641; GCN-NEXT: br label [[ATOMICRMW_START:%.*]]642; GCN: atomicrmw.start:643; GCN-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]644; GCN-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i16645; GCN-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat646; GCN-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]647; GCN-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i16648; GCN-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i32649; GCN-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -65536650; GCN-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]651; GCN-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 4652; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1653; GCN-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0654; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]655; GCN: atomicrmw.end:656; GCN-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i16657; GCN-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat658; GCN-NEXT: ret bfloat [[TMP5]]659;660 %res = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 4661 ret bfloat %res662}663 664!0 = !{i32 5, i32 6}665