5231 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,CI %s3; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX9 %s4; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX908 %s5; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX90A %s6; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX942 %s7; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX11 %s8 9define void @test_atomicrmw_fadd_f32_global_no_use_unsafe(ptr addrspace(1) %ptr, float %value) #3 {10; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(11; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 412; CI-NEXT: br label [[ATOMICRMW_START:%.*]]13; CI: atomicrmw.start:14; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]15; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]16; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3217; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3218; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 419; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 120; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 021; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float22; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]23; CI: atomicrmw.end:24; CI-NEXT: ret void25;26; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(27; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 428; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]29; GFX9: atomicrmw.start:30; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]31; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]32; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3233; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3234; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 435; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 136; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 037; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float38; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]39; GFX9: atomicrmw.end:40; GFX9-NEXT: ret void41;42; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(43; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 444; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]45; GFX908: atomicrmw.start:46; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]47; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]48; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3249; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3250; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 451; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 152; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 053; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float54; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]55; GFX908: atomicrmw.end:56; GFX908-NEXT: ret void57;58; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(59; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 460; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]61; GFX90A: atomicrmw.start:62; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]63; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]64; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3265; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3266; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 467; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 168; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 069; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float70; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]71; GFX90A: atomicrmw.end:72; GFX90A-NEXT: ret void73;74; GFX942-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(75; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 476; GFX942-NEXT: ret void77;78; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe(79; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 480; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]81; GFX11: atomicrmw.start:82; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]83; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]84; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i3285; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i3286; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 487; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 188; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 089; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float90; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]91; GFX11: atomicrmw.end:92; GFX11-NEXT: ret void93;94 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value syncscope("wavefront") monotonic95 ret void96}97 98define void @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(ptr addrspace(7) %ptr, float %value) #3 {99; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(100; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4101; CI-NEXT: br label [[ATOMICRMW_START:%.*]]102; CI: atomicrmw.start:103; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]104; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]105; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32106; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32107; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4108; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1109; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0110; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float111; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]112; CI: atomicrmw.end:113; CI-NEXT: ret void114;115; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(116; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4117; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]118; GFX9: atomicrmw.start:119; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]120; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]121; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32122; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32123; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4124; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1125; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0126; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float127; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]128; GFX9: atomicrmw.end:129; GFX9-NEXT: ret void130;131; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(132; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4133; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]134; GFX908: atomicrmw.start:135; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]136; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]137; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32138; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32139; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4140; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1141; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0142; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float143; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]144; GFX908: atomicrmw.end:145; GFX908-NEXT: ret void146;147; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(148; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4149; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]150; GFX90A: atomicrmw.start:151; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]152; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]153; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32154; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32155; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4156; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1157; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0158; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float159; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]160; GFX90A: atomicrmw.end:161; GFX90A-NEXT: ret void162;163; GFX942-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(164; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4165; GFX942-NEXT: ret void166;167; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(168; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4169; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]170; GFX11: atomicrmw.start:171; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]172; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]173; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32174; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32175; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4176; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1177; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0178; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float179; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]180; GFX11: atomicrmw.end:181; GFX11-NEXT: ret void182;183 %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic184 ret void185}186 187define void @test_atomicrmw_fadd_f32_as999_no_use_unsafe(ptr addrspace(999) %ptr, float %value) #3 {188; CI-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(189; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4190; CI-NEXT: br label [[ATOMICRMW_START:%.*]]191; CI: atomicrmw.start:192; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]193; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]194; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32195; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32196; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4197; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1198; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0199; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float200; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]201; CI: atomicrmw.end:202; CI-NEXT: ret void203;204; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(205; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4206; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]207; GFX9: atomicrmw.start:208; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]209; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]210; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32211; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32212; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4213; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1214; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0215; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float216; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]217; GFX9: atomicrmw.end:218; GFX9-NEXT: ret void219;220; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(221; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4222; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]223; GFX908: atomicrmw.start:224; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]225; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]226; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32227; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32228; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4229; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1230; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0231; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float232; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]233; GFX908: atomicrmw.end:234; GFX908-NEXT: ret void235;236; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(237; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4238; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]239; GFX90A: atomicrmw.start:240; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]241; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]242; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32243; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32244; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4245; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1246; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0247; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float248; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]249; GFX90A: atomicrmw.end:250; GFX90A-NEXT: ret void251;252; GFX942-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(253; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4254; GFX942-NEXT: ret void255;256; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe(257; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4258; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]259; GFX11: atomicrmw.start:260; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]261; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]262; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32263; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32264; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4265; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1266; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0267; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float268; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]269; GFX11: atomicrmw.end:270; GFX11-NEXT: ret void271;272 %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic273 ret void274}275 276define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr, float %value) #3 {277; CI-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(278; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4279; CI-NEXT: br label [[ATOMICRMW_START:%.*]]280; CI: atomicrmw.start:281; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]282; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]283; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32284; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32285; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]286; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1287; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0288; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float289; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]290; CI: atomicrmw.end:291; CI-NEXT: ret float [[TMP5]]292;293; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(294; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4295; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]296; GFX9: atomicrmw.start:297; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]298; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]299; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32300; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32301; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]302; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1303; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0304; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float305; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]306; GFX9: atomicrmw.end:307; GFX9-NEXT: ret float [[TMP5]]308;309; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(310; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4311; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]312; GFX908: atomicrmw.start:313; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]314; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]315; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32316; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32317; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]318; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1319; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0320; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float321; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]322; GFX908: atomicrmw.end:323; GFX908-NEXT: ret float [[TMP5]]324;325; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(326; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]327; GFX90A-NEXT: ret float [[RES]]328;329; GFX942-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(330; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]331; GFX942-NEXT: ret float [[RES]]332;333; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_unsafe(334; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]335; GFX11-NEXT: ret float [[RES]]336;337 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0338 ret float %res339}340 341define float @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7) %ptr, float %value) #3 {342; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(343; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4344; CI-NEXT: br label [[ATOMICRMW_START:%.*]]345; CI: atomicrmw.start:346; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]347; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]348; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32349; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32350; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]351; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1352; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0353; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float354; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]355; CI: atomicrmw.end:356; CI-NEXT: ret float [[TMP5]]357;358; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(359; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4360; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]361; GFX9: atomicrmw.start:362; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]363; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]364; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32365; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32366; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]367; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1368; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0369; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float370; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]371; GFX9: atomicrmw.end:372; GFX9-NEXT: ret float [[TMP5]]373;374; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(375; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4376; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]377; GFX908: atomicrmw.start:378; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]379; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]380; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32381; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32382; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]383; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1384; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0385; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float386; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]387; GFX908: atomicrmw.end:388; GFX908-NEXT: ret float [[TMP5]]389;390; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(391; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]392; GFX90A-NEXT: ret float [[RES]]393;394; GFX942-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(395; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]396; GFX942-NEXT: ret float [[RES]]397;398; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(399; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]400; GFX11-NEXT: ret float [[RES]]401;402 %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0403 ret float %res404}405 406define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr addrspace(999) %ptr, float %value) #3 {407; CI-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(408; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4409; CI-NEXT: br label [[ATOMICRMW_START:%.*]]410; CI: atomicrmw.start:411; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]412; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]413; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32414; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32415; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]416; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1417; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0418; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float419; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]420; CI: atomicrmw.end:421; CI-NEXT: ret float [[TMP5]]422;423; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(424; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4425; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]426; GFX9: atomicrmw.start:427; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]428; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]429; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32430; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32431; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]432; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1433; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0434; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float435; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]436; GFX9: atomicrmw.end:437; GFX9-NEXT: ret float [[TMP5]]438;439; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(440; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4441; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]442; GFX908: atomicrmw.start:443; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]444; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]445; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32446; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32447; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]448; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1449; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0450; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float451; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]452; GFX908: atomicrmw.end:453; GFX908-NEXT: ret float [[TMP5]]454;455; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(456; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]457; GFX90A-NEXT: ret float [[RES]]458;459; GFX942-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(460; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]461; GFX942-NEXT: ret float [[RES]]462;463; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe(464; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]465; GFX11-NEXT: ret float [[RES]]466;467 %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0468 ret float %res469}470 471define double @test_atomicrmw_fadd_f64_global_unsafe(ptr addrspace(1) %ptr, double %value) #3 {472; CI-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(473; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8474; CI-NEXT: br label [[ATOMICRMW_START:%.*]]475; CI: atomicrmw.start:476; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]477; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]478; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64479; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64480; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]481; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1482; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0483; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double484; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]485; CI: atomicrmw.end:486; CI-NEXT: ret double [[TMP5]]487;488; GFX9-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(489; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8490; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]491; GFX9: atomicrmw.start:492; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]493; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]494; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64495; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64496; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]497; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1498; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0499; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double500; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]501; GFX9: atomicrmw.end:502; GFX9-NEXT: ret double [[TMP5]]503;504; GFX908-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(505; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8506; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]507; GFX908: atomicrmw.start:508; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]509; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]510; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64511; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64512; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]513; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1514; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0515; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double516; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]517; GFX908: atomicrmw.end:518; GFX908-NEXT: ret double [[TMP5]]519;520; GFX90A-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(521; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]522; GFX90A-NEXT: ret double [[RES]]523;524; GFX942-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(525; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]526; GFX942-NEXT: ret double [[RES]]527;528; GFX11-LABEL: @test_atomicrmw_fadd_f64_global_unsafe(529; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8530; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]531; GFX11: atomicrmw.start:532; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]533; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]534; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64535; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64536; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]537; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1538; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0539; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double540; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]541; GFX11: atomicrmw.end:542; GFX11-NEXT: ret double [[TMP5]]543;544 %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0545 ret double %res546}547 548define float @test_atomicrmw_fadd_f32_flat_unsafe(ptr %ptr, float %value) #3 {549; CI-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(550; CI-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4551; CI-NEXT: br label [[ATOMICRMW_START:%.*]]552; CI: atomicrmw.start:553; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]554; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]555; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32556; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32557; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]558; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1559; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0560; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float561; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]562; CI: atomicrmw.end:563; CI-NEXT: ret float [[TMP5]]564;565; GFX9-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(566; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4567; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]568; GFX9: atomicrmw.start:569; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]570; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]571; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32572; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32573; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]574; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1575; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0576; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float577; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]578; GFX9: atomicrmw.end:579; GFX9-NEXT: ret float [[TMP5]]580;581; GFX908-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(582; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4583; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]584; GFX908: atomicrmw.start:585; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]586; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]587; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32588; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32589; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]590; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1591; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0592; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float593; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]594; GFX908: atomicrmw.end:595; GFX908-NEXT: ret float [[TMP5]]596;597; GFX90A-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(598; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])599; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]600; GFX90A: atomicrmw.shared:601; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)602; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]603; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]604; GFX90A: atomicrmw.check.private:605; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])606; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]607; GFX90A: atomicrmw.private:608; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)609; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load float, ptr addrspace(5) [[TMP3]], align 4610; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED_PRIVATE]], [[VALUE]]611; GFX90A-NEXT: store float [[NEW]], ptr addrspace(5) [[TMP3]], align 4612; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]613; GFX90A: atomicrmw.global:614; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)615; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], float [[VALUE]] syncscope("wavefront") monotonic, align 4, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]616; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]617; GFX90A: atomicrmw.phi:618; GFX90A-NEXT: [[RES:%.*]] = phi float [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ]619; GFX90A-NEXT: br label [[ATOMICRMW_END:%.*]]620; GFX90A: atomicrmw.end:621; GFX90A-NEXT: ret float [[RES]]622;623; GFX942-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(624; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]625; GFX942-NEXT: ret float [[RES]]626;627; GFX11-LABEL: @test_atomicrmw_fadd_f32_flat_unsafe(628; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !noalias.addrspace [[META1:![0-9]+]], !amdgpu.no.fine.grained.memory [[META0]]629; GFX11-NEXT: ret float [[RES]]630;631 %res = atomicrmw fadd ptr %ptr, float %value syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0632 ret float %res633}634 635define double @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(ptr %ptr, double %value) #3 {636; CI-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(637; CI-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8638; CI-NEXT: br label [[ATOMICRMW_START:%.*]]639; CI: atomicrmw.start:640; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]641; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]642; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64643; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64644; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]645; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1646; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0647; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double648; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]649; CI: atomicrmw.end:650; CI-NEXT: ret double [[TMP5]]651;652; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(653; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8654; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]655; GFX9: atomicrmw.start:656; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]657; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]658; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64659; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64660; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]661; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1662; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0663; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double664; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]665; GFX9: atomicrmw.end:666; GFX9-NEXT: ret double [[TMP5]]667;668; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(669; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8670; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]671; GFX908: atomicrmw.start:672; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]673; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]674; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64675; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64676; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]677; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1678; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0679; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double680; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]681; GFX908: atomicrmw.end:682; GFX908-NEXT: ret double [[TMP5]]683;684; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(685; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]686; GFX90A-NEXT: ret double [[RES]]687;688; GFX942-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(689; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]690; GFX942-NEXT: ret double [[RES]]691;692; GFX11-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe__noprivate(693; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 8694; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]695; GFX11: atomicrmw.start:696; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]697; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]698; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64699; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64700; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]701; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1702; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0703; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double704; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]705; GFX11: atomicrmw.end:706; GFX11-NEXT: ret double [[TMP5]]707;708 %res = atomicrmw fadd ptr %ptr, double %value syncscope("wavefront") monotonic, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0709 ret double %res710}711 712define double @test_atomicrmw_fadd_f64_flat_unsafe(ptr %ptr, double %value) #3 {713; CI-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(714; CI-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])715; CI-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]716; CI: atomicrmw.private:717; CI-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)718; CI-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8719; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]720; CI-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8721; CI-NEXT: br label [[ATOMICRMW_PHI:%.*]]722; CI: atomicrmw.global:723; CI-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8724; CI-NEXT: br label [[ATOMICRMW_START:%.*]]725; CI: atomicrmw.start:726; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]727; CI-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]728; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64729; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64730; CI-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]731; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1732; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0733; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double734; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]735; CI: atomicrmw.end1:736; CI-NEXT: br label [[ATOMICRMW_PHI]]737; CI: atomicrmw.phi:738; CI-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]739; CI-NEXT: br label [[ATOMICRMW_END:%.*]]740; CI: atomicrmw.end:741; CI-NEXT: ret double [[RES]]742;743; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(744; GFX9-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])745; GFX9-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]746; GFX9: atomicrmw.private:747; GFX9-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)748; GFX9-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8749; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]750; GFX9-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8751; GFX9-NEXT: br label [[ATOMICRMW_PHI:%.*]]752; GFX9: atomicrmw.global:753; GFX9-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8754; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]755; GFX9: atomicrmw.start:756; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]757; GFX9-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]758; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64759; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64760; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]761; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1762; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0763; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double764; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]765; GFX9: atomicrmw.end1:766; GFX9-NEXT: br label [[ATOMICRMW_PHI]]767; GFX9: atomicrmw.phi:768; GFX9-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]769; GFX9-NEXT: br label [[ATOMICRMW_END:%.*]]770; GFX9: atomicrmw.end:771; GFX9-NEXT: ret double [[RES]]772;773; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(774; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])775; GFX908-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]776; GFX908: atomicrmw.private:777; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)778; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8779; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]780; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8781; GFX908-NEXT: br label [[ATOMICRMW_PHI:%.*]]782; GFX908: atomicrmw.global:783; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8784; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]785; GFX908: atomicrmw.start:786; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]787; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]788; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64789; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64790; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]791; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1792; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0793; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double794; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]795; GFX908: atomicrmw.end1:796; GFX908-NEXT: br label [[ATOMICRMW_PHI]]797; GFX908: atomicrmw.phi:798; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]799; GFX908-NEXT: br label [[ATOMICRMW_END:%.*]]800; GFX908: atomicrmw.end:801; GFX908-NEXT: ret double [[RES]]802;803; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(804; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])805; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]806; GFX90A: atomicrmw.shared:807; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)808; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]809; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]810; GFX90A: atomicrmw.check.private:811; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])812; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]813; GFX90A: atomicrmw.private:814; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)815; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8816; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]817; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8818; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]819; GFX90A: atomicrmw.global:820; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)821; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]822; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]823; GFX90A: atomicrmw.phi:824; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ]825; GFX90A-NEXT: br label [[ATOMICRMW_END:%.*]]826; GFX90A: atomicrmw.end:827; GFX90A-NEXT: ret double [[RES]]828;829; GFX942-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(830; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])831; GFX942-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]832; GFX942: atomicrmw.shared:833; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)834; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]835; GFX942-NEXT: br label [[ATOMICRMW_PHI:%.*]]836; GFX942: atomicrmw.check.private:837; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])838; GFX942-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]839; GFX942: atomicrmw.private:840; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)841; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8842; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]843; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8844; GFX942-NEXT: br label [[ATOMICRMW_PHI]]845; GFX942: atomicrmw.global:846; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)847; GFX942-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]848; GFX942-NEXT: br label [[ATOMICRMW_PHI]]849; GFX942: atomicrmw.phi:850; GFX942-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ]851; GFX942-NEXT: br label [[ATOMICRMW_END:%.*]]852; GFX942: atomicrmw.end:853; GFX942-NEXT: ret double [[RES]]854;855; GFX11-LABEL: @test_atomicrmw_fadd_f64_flat_unsafe(856; GFX11-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])857; GFX11-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]858; GFX11: atomicrmw.private:859; GFX11-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)860; GFX11-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8861; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]862; GFX11-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8863; GFX11-NEXT: br label [[ATOMICRMW_PHI:%.*]]864; GFX11: atomicrmw.global:865; GFX11-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8866; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]867; GFX11: atomicrmw.start:868; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]869; GFX11-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]870; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64871; GFX11-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64872; GFX11-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("wavefront") monotonic monotonic, align 8, !noalias.addrspace [[META1]], !amdgpu.no.fine.grained.memory [[META0]]873; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1874; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0875; GFX11-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double876; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]877; GFX11: atomicrmw.end1:878; GFX11-NEXT: br label [[ATOMICRMW_PHI]]879; GFX11: atomicrmw.phi:880; GFX11-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]881; GFX11-NEXT: br label [[ATOMICRMW_END:%.*]]882; GFX11: atomicrmw.end:883; GFX11-NEXT: ret double [[RES]]884;885 %res = atomicrmw fadd ptr %ptr, double %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0886 ret double %res887}888 889define float @test_atomicrmw_fadd_f32_flat(ptr %ptr, float %value) {890; ALL-LABEL: @test_atomicrmw_fadd_f32_flat(891; ALL-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 4892; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]893; ALL: atomicrmw.start:894; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]895; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]896; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32897; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32898; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4899; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1900; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0901; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float902; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]903; ALL: atomicrmw.end:904; ALL-NEXT: ret float [[TMP5]]905;906 %res = atomicrmw fadd ptr %ptr, float %value seq_cst907 ret float %res908}909 910define float @test_atomicrmw_fadd_f32_global(ptr addrspace(1) %ptr, float %value) {911; ALL-LABEL: @test_atomicrmw_fadd_f32_global(912; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4913; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]914; ALL: atomicrmw.start:915; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]916; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]917; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32918; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32919; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4920; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1921; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0922; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float923; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]924; ALL: atomicrmw.end:925; ALL-NEXT: ret float [[TMP5]]926;927 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value seq_cst928 ret float %res929}930 931define float @test_atomicrmw_fadd_f32_buffer_fat_ptr(ptr addrspace(7) %ptr, float %value) {932; ALL-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr(933; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4934; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]935; ALL: atomicrmw.start:936; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]937; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]938; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32939; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32940; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4941; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1942; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0943; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float944; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]945; ALL: atomicrmw.end:946; ALL-NEXT: ret float [[TMP5]]947;948 %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value seq_cst949 ret float %res950}951 952define float @test_atomicrmw_fadd_f32_as999(ptr addrspace(999) %ptr, float %value) {953; ALL-LABEL: @test_atomicrmw_fadd_f32_as999(954; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4955; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]956; ALL: atomicrmw.start:957; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]958; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]959; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32960; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32961; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4962; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1963; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0964; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float965; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]966; ALL: atomicrmw.end:967; ALL-NEXT: ret float [[TMP5]]968;969 %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value seq_cst970 ret float %res971}972 973define void @test_atomicrmw_fadd_f32_global_no_use_ieee(ptr addrspace(1) %ptr, float %value) {974; ALL-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee(975; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4976; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]977; ALL: atomicrmw.start:978; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]979; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]980; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32981; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32982; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4983; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1984; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0985; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float986; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]987; ALL: atomicrmw.end:988; ALL-NEXT: ret void989;990 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value seq_cst991 ret void992}993 994define void @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(ptr addrspace(1) %ptr, float %value) #4 {995; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(996; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4997; CI-NEXT: br label [[ATOMICRMW_START:%.*]]998; CI: atomicrmw.start:999; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1000; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1001; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321002; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321003; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1004; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11005; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01006; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1007; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1008; CI: atomicrmw.end:1009; CI-NEXT: ret void1010;1011; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(1012; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41013; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1014; GFX9: atomicrmw.start:1015; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1016; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1017; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321018; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321019; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1020; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11021; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01022; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1023; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1024; GFX9: atomicrmw.end:1025; GFX9-NEXT: ret void1026;1027; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(1028; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41029; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]1030; GFX908: atomicrmw.start:1031; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1032; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1033; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321034; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321035; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1036; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11037; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01038; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1039; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1040; GFX908: atomicrmw.end:1041; GFX908-NEXT: ret void1042;1043; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(1044; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41045; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]1046; GFX90A: atomicrmw.start:1047; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1048; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1049; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321050; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321051; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1052; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11053; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01054; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1055; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1056; GFX90A: atomicrmw.end:1057; GFX90A-NEXT: ret void1058;1059; GFX942-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(1060; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1061; GFX942-NEXT: ret void1062;1063; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_denorm_flush(1064; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]1065; GFX11-NEXT: ret void1066;1067 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value seq_cst, !amdgpu.no.fine.grained.memory !01068 ret void1069}1070 1071define float @test_atomicrmw_fadd_f32_local(ptr addrspace(3) %ptr, float %value) {1072; CI-LABEL: @test_atomicrmw_fadd_f32_local(1073; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 41074; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1075; CI: atomicrmw.start:1076; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1077; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1078; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321079; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321080; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 41081; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11082; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01083; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1084; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1085; CI: atomicrmw.end:1086; CI-NEXT: ret float [[TMP5]]1087;1088; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(1089; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41090; GFX9-NEXT: ret float [[RES]]1091;1092; GFX908-LABEL: @test_atomicrmw_fadd_f32_local(1093; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41094; GFX908-NEXT: ret float [[RES]]1095;1096; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local(1097; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41098; GFX90A-NEXT: ret float [[RES]]1099;1100; GFX942-LABEL: @test_atomicrmw_fadd_f32_local(1101; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41102; GFX942-NEXT: ret float [[RES]]1103;1104; GFX11-LABEL: @test_atomicrmw_fadd_f32_local(1105; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41106; GFX11-NEXT: ret float [[RES]]1107;1108 %res = atomicrmw fadd ptr addrspace(3) %ptr, float %value seq_cst1109 ret float %res1110}1111 1112define half @test_atomicrmw_fadd_f16_flat(ptr %ptr, half %value) {1113; ALL-LABEL: @test_atomicrmw_fadd_f16_flat(1114; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)1115; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i641116; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31117; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31118; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321119; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1120; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11121; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 41122; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1123; ALL: atomicrmw.start:1124; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1125; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1126; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161127; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half1128; ALL-NEXT: [[NEW:%.*]] = fadd half [[TMP4]], [[VALUE:%.*]]1129; ALL-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i161130; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321131; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1132; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1133; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1134; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41135; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11136; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01137; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1138; ALL: atomicrmw.end:1139; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1140; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161141; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half1142; ALL-NEXT: ret half [[TMP7]]1143;1144 %res = atomicrmw fadd ptr %ptr, half %value seq_cst1145 ret half %res1146}1147 1148define half @test_atomicrmw_fadd_f16_global(ptr addrspace(1) %ptr, half %value) {1149; ALL-LABEL: @test_atomicrmw_fadd_f16_global(1150; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1151; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641152; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31153; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31154; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321155; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1156; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11157; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41158; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1159; ALL: atomicrmw.start:1160; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1161; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1162; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161163; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half1164; ALL-NEXT: [[NEW:%.*]] = fadd half [[TMP4]], [[VALUE:%.*]]1165; ALL-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i161166; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321167; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1168; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1169; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1170; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41171; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11172; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01173; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1174; ALL: atomicrmw.end:1175; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1176; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161177; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half1178; ALL-NEXT: ret half [[TMP7]]1179;1180 %res = atomicrmw fadd ptr addrspace(1) %ptr, half %value seq_cst1181 ret half %res1182}1183 1184define half @test_atomicrmw_fadd_f16_global_align4(ptr addrspace(1) %ptr, half %value) {1185; ALL-LABEL: @test_atomicrmw_fadd_f16_global_align4(1186; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41187; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1188; ALL: atomicrmw.start:1189; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1190; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161191; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to half1192; ALL-NEXT: [[NEW:%.*]] = fadd half [[TMP2]], [[VALUE:%.*]]1193; ALL-NEXT: [[TMP3:%.*]] = bitcast half [[NEW]] to i161194; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i321195; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361196; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1197; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41198; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11199; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01200; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1201; ALL: atomicrmw.end:1202; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161203; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to half1204; ALL-NEXT: ret half [[TMP5]]1205;1206 %res = atomicrmw fadd ptr addrspace(1) %ptr, half %value seq_cst, align 41207 ret half %res1208}1209 1210define half @test_atomicrmw_fadd_f16_local(ptr addrspace(3) %ptr, half %value) {1211; ALL-LABEL: @test_atomicrmw_fadd_f16_local(1212; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)1213; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i321214; ALL-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31215; ALL-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31216; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1217; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11218; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 41219; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1220; ALL: atomicrmw.start:1221; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1222; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]1223; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161224; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to half1225; ALL-NEXT: [[NEW:%.*]] = fadd half [[TMP4]], [[VALUE:%.*]]1226; ALL-NEXT: [[TMP5:%.*]] = bitcast half [[NEW]] to i161227; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321228; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]1229; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1230; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1231; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 41232; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11233; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01234; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1235; ALL: atomicrmw.end:1236; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1237; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161238; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to half1239; ALL-NEXT: ret half [[TMP7]]1240;1241 %res = atomicrmw fadd ptr addrspace(3) %ptr, half %value seq_cst1242 ret half %res1243}1244 1245define double @test_atomicrmw_fadd_f64_flat(ptr %ptr, double %value) {1246; CI-LABEL: @test_atomicrmw_fadd_f64_flat(1247; CI-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])1248; CI-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1249; CI: atomicrmw.private:1250; CI-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1251; CI-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81252; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]1253; CI-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 81254; CI-NEXT: br label [[ATOMICRMW_PHI:%.*]]1255; CI: atomicrmw.global:1256; CI-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 81257; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1258; CI: atomicrmw.start:1259; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]1260; CI-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1261; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i641262; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641263; CI-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]1264; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11265; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01266; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1267; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1268; CI: atomicrmw.end1:1269; CI-NEXT: br label [[ATOMICRMW_PHI]]1270; CI: atomicrmw.phi:1271; CI-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]1272; CI-NEXT: br label [[ATOMICRMW_END:%.*]]1273; CI: atomicrmw.end:1274; CI-NEXT: ret double [[RES]]1275;1276; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat(1277; GFX9-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])1278; GFX9-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1279; GFX9: atomicrmw.private:1280; GFX9-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1281; GFX9-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81282; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]1283; GFX9-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 81284; GFX9-NEXT: br label [[ATOMICRMW_PHI:%.*]]1285; GFX9: atomicrmw.global:1286; GFX9-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 81287; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1288; GFX9: atomicrmw.start:1289; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]1290; GFX9-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1291; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i641292; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641293; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]1294; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11295; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01296; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1297; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1298; GFX9: atomicrmw.end1:1299; GFX9-NEXT: br label [[ATOMICRMW_PHI]]1300; GFX9: atomicrmw.phi:1301; GFX9-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]1302; GFX9-NEXT: br label [[ATOMICRMW_END:%.*]]1303; GFX9: atomicrmw.end:1304; GFX9-NEXT: ret double [[RES]]1305;1306; GFX908-LABEL: @test_atomicrmw_fadd_f64_flat(1307; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])1308; GFX908-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1309; GFX908: atomicrmw.private:1310; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1311; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81312; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]1313; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 81314; GFX908-NEXT: br label [[ATOMICRMW_PHI:%.*]]1315; GFX908: atomicrmw.global:1316; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 81317; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]1318; GFX908: atomicrmw.start:1319; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]1320; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1321; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i641322; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641323; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]1324; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11325; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01326; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1327; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1328; GFX908: atomicrmw.end1:1329; GFX908-NEXT: br label [[ATOMICRMW_PHI]]1330; GFX908: atomicrmw.phi:1331; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]1332; GFX908-NEXT: br label [[ATOMICRMW_END:%.*]]1333; GFX908: atomicrmw.end:1334; GFX908-NEXT: ret double [[RES]]1335;1336; GFX90A-LABEL: @test_atomicrmw_fadd_f64_flat(1337; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])1338; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]1339; GFX90A: atomicrmw.shared:1340; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)1341; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] seq_cst, align 81342; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]1343; GFX90A: atomicrmw.check.private:1344; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1345; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1346; GFX90A: atomicrmw.private:1347; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1348; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 81349; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]1350; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 81351; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]1352; GFX90A: atomicrmw.global:1353; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)1354; GFX90A-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 81355; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]1356; GFX90A: atomicrmw.start:1357; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]1358; GFX90A-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1359; GFX90A-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i641360; GFX90A-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i641361; GFX90A-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] seq_cst seq_cst, align 81362; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 11363; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 01364; GFX90A-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double1365; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1366; GFX90A: atomicrmw.end1:1367; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]1368; GFX90A: atomicrmw.phi:1369; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]1370; GFX90A-NEXT: br label [[ATOMICRMW_END:%.*]]1371; GFX90A: atomicrmw.end:1372; GFX90A-NEXT: ret double [[RES]]1373;1374; GFX942-LABEL: @test_atomicrmw_fadd_f64_flat(1375; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])1376; GFX942-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]1377; GFX942: atomicrmw.shared:1378; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)1379; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] seq_cst, align 81380; GFX942-NEXT: br label [[ATOMICRMW_PHI:%.*]]1381; GFX942: atomicrmw.check.private:1382; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1383; GFX942-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1384; GFX942: atomicrmw.private:1385; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1386; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 81387; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]1388; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 81389; GFX942-NEXT: br label [[ATOMICRMW_PHI]]1390; GFX942: atomicrmw.global:1391; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)1392; GFX942-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 81393; GFX942-NEXT: br label [[ATOMICRMW_START:%.*]]1394; GFX942: atomicrmw.start:1395; GFX942-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]1396; GFX942-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1397; GFX942-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i641398; GFX942-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i641399; GFX942-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] seq_cst seq_cst, align 81400; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 11401; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 01402; GFX942-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double1403; GFX942-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1404; GFX942: atomicrmw.end1:1405; GFX942-NEXT: br label [[ATOMICRMW_PHI]]1406; GFX942: atomicrmw.phi:1407; GFX942-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]1408; GFX942-NEXT: br label [[ATOMICRMW_END:%.*]]1409; GFX942: atomicrmw.end:1410; GFX942-NEXT: ret double [[RES]]1411;1412; GFX11-LABEL: @test_atomicrmw_fadd_f64_flat(1413; GFX11-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])1414; GFX11-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]1415; GFX11: atomicrmw.private:1416; GFX11-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1417; GFX11-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81418; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]1419; GFX11-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 81420; GFX11-NEXT: br label [[ATOMICRMW_PHI:%.*]]1421; GFX11: atomicrmw.global:1422; GFX11-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 81423; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]1424; GFX11: atomicrmw.start:1425; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]1426; GFX11-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]1427; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i641428; GFX11-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641429; GFX11-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1]]1430; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11431; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01432; GFX11-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1433; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]1434; GFX11: atomicrmw.end1:1435; GFX11-NEXT: br label [[ATOMICRMW_PHI]]1436; GFX11: atomicrmw.phi:1437; GFX11-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]1438; GFX11-NEXT: br label [[ATOMICRMW_END:%.*]]1439; GFX11: atomicrmw.end:1440; GFX11-NEXT: ret double [[RES]]1441;1442 %res = atomicrmw fadd ptr %ptr, double %value seq_cst1443 ret double %res1444}1445 1446define double @test_atomicrmw_fadd_f64_flat__noprivate(ptr %ptr, double %value) {1447; ALL-LABEL: @test_atomicrmw_fadd_f64_flat__noprivate(1448; ALL-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR:%.*]], align 81449; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1450; ALL: atomicrmw.start:1451; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1452; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1453; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641454; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641455; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8, !noalias.addrspace [[META1:![0-9]+]]1456; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11457; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01458; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1459; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1460; ALL: atomicrmw.end:1461; ALL-NEXT: ret double [[TMP5]]1462;1463 %res = atomicrmw fadd ptr %ptr, double %value seq_cst, !noalias.addrspace !11464 ret double %res1465}1466 1467define double @test_atomicrmw_fadd_f64_global(ptr addrspace(1) %ptr, double %value) {1468; ALL-LABEL: @test_atomicrmw_fadd_f64_global(1469; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 81470; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1471; ALL: atomicrmw.start:1472; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1473; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1474; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641475; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641476; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 81477; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11478; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01479; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1480; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1481; ALL: atomicrmw.end:1482; ALL-NEXT: ret double [[TMP5]]1483;1484 %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value seq_cst1485 ret double %res1486}1487 1488define double @test_atomicrmw_fadd_f64_local(ptr addrspace(3) %ptr, double %value) {1489; CI-LABEL: @test_atomicrmw_fadd_f64_local(1490; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 81491; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1492; CI: atomicrmw.start:1493; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1494; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1495; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641496; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641497; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 81498; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11499; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01500; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1501; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1502; CI: atomicrmw.end:1503; CI-NEXT: ret double [[TMP5]]1504;1505; GFX9-LABEL: @test_atomicrmw_fadd_f64_local(1506; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 81507; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1508; GFX9: atomicrmw.start:1509; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1510; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1511; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641512; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641513; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 81514; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11515; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01516; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1517; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1518; GFX9: atomicrmw.end:1519; GFX9-NEXT: ret double [[TMP5]]1520;1521; GFX908-LABEL: @test_atomicrmw_fadd_f64_local(1522; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 81523; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]1524; GFX908: atomicrmw.start:1525; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1526; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1527; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641528; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641529; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 81530; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11531; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01532; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1533; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1534; GFX908: atomicrmw.end:1535; GFX908-NEXT: ret double [[TMP5]]1536;1537; GFX90A-LABEL: @test_atomicrmw_fadd_f64_local(1538; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] seq_cst, align 81539; GFX90A-NEXT: ret double [[RES]]1540;1541; GFX942-LABEL: @test_atomicrmw_fadd_f64_local(1542; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] seq_cst, align 81543; GFX942-NEXT: ret double [[RES]]1544;1545; GFX11-LABEL: @test_atomicrmw_fadd_f64_local(1546; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 81547; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]1548; GFX11: atomicrmw.start:1549; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1550; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]1551; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641552; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641553; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 81554; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11555; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01556; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1557; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1558; GFX11: atomicrmw.end:1559; GFX11-NEXT: ret double [[TMP5]]1560;1561 %res = atomicrmw fadd ptr addrspace(3) %ptr, double %value seq_cst1562 ret double %res1563}1564 1565define float @test_atomicrmw_fadd_f32_global_agent(ptr addrspace(1) %ptr, float %value) {1566; CI-LABEL: @test_atomicrmw_fadd_f32_global_agent(1567; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41568; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1569; CI: atomicrmw.start:1570; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1571; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1572; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321573; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321574; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") monotonic monotonic, align 41575; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11576; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01577; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1578; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1579; CI: atomicrmw.end:1580; CI-NEXT: ret float [[TMP5]]1581;1582; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_agent(1583; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41584; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1585; GFX9: atomicrmw.start:1586; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1587; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1588; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321589; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321590; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") monotonic monotonic, align 41591; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11592; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01593; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1594; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1595; GFX9: atomicrmw.end:1596; GFX9-NEXT: ret float [[TMP5]]1597;1598; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_agent(1599; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41600; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]1601; GFX908: atomicrmw.start:1602; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1603; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1604; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321605; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321606; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") monotonic monotonic, align 41607; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11608; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01609; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1610; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1611; GFX908: atomicrmw.end:1612; GFX908-NEXT: ret float [[TMP5]]1613;1614; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_agent(1615; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41616; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]1617; GFX90A: atomicrmw.start:1618; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1619; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1620; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321621; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321622; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") monotonic monotonic, align 41623; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11624; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01625; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1626; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1627; GFX90A: atomicrmw.end:1628; GFX90A-NEXT: ret float [[TMP5]]1629;1630; GFX942-LABEL: @test_atomicrmw_fadd_f32_global_agent(1631; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("agent") monotonic, align 41632; GFX942-NEXT: ret float [[RES]]1633;1634; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_agent(1635; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41636; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]1637; GFX11: atomicrmw.start:1638; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1639; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1640; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321641; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321642; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") monotonic monotonic, align 41643; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11644; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01645; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1646; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1647; GFX11: atomicrmw.end:1648; GFX11-NEXT: ret float [[TMP5]]1649;1650 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value syncscope("agent") monotonic1651 ret float %res1652}1653 1654define float @test_atomicrmw_fadd_f32_global_one_as(ptr addrspace(1) %ptr, float %value) {1655; ALL-LABEL: @test_atomicrmw_fadd_f32_global_one_as(1656; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41657; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1658; ALL: atomicrmw.start:1659; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1660; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]1661; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321662; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321663; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("one-as") monotonic monotonic, align 41664; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11665; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01666; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1667; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1668; ALL: atomicrmw.end:1669; ALL-NEXT: ret float [[TMP5]]1670;1671 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value syncscope("one-as") monotonic1672 ret float %res1673}1674 1675define void @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(ptr addrspace(1) %ptr, float %value) #1 {1676; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1677; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41678; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1679; CI: atomicrmw.start:1680; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1681; CI-NEXT: [[NEW:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]1682; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321683; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321684; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1685; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11686; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01687; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1688; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1689; CI: atomicrmw.end:1690; CI-NEXT: ret void1691;1692; GFX9-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1693; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 41694; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1695; GFX9: atomicrmw.start:1696; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1697; GFX9-NEXT: [[NEW:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]1698; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321699; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321700; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1701; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11702; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01703; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1704; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1705; GFX9: atomicrmw.end:1706; GFX9-NEXT: ret void1707;1708; GFX908-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1709; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1710; GFX908-NEXT: ret void1711;1712; GFX90A-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1713; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1714; GFX90A-NEXT: ret void1715;1716; GFX942-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1717; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1718; GFX942-NEXT: ret void1719;1720; GFX11-LABEL: @test_atomicrmw_fadd_f32_global_no_use_unsafe_strictfp(1721; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META0]]1722; GFX11-NEXT: ret void1723;1724 %res = atomicrmw fadd ptr addrspace(1) %ptr, float %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !01725 ret void1726}1727 1728define double @test_atomicrmw_fadd_f64_global_unsafe_strictfp(ptr addrspace(1) %ptr, double %value) #1 {1729; CI-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1730; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 81731; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1732; CI: atomicrmw.start:1733; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1734; CI-NEXT: [[NEW:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[LOADED]], double [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]1735; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641736; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641737; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1738; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11739; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01740; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1741; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1742; CI: atomicrmw.end:1743; CI-NEXT: ret double [[TMP5]]1744;1745; GFX9-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1746; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 81747; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]1748; GFX9: atomicrmw.start:1749; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1750; GFX9-NEXT: [[NEW:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[LOADED]], double [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]1751; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641752; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641753; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1754; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11755; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01756; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1757; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1758; GFX9: atomicrmw.end:1759; GFX9-NEXT: ret double [[TMP5]]1760;1761; GFX908-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1762; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 81763; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]1764; GFX908: atomicrmw.start:1765; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1766; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[LOADED]], double [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]1767; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641768; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641769; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1770; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11771; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01772; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1773; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1774; GFX908: atomicrmw.end:1775; GFX908-NEXT: ret double [[TMP5]]1776;1777; GFX90A-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1778; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1779; GFX90A-NEXT: ret double [[RES]]1780;1781; GFX942-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1782; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VALUE:%.*]] syncscope("wavefront") monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1783; GFX942-NEXT: ret double [[RES]]1784;1785; GFX11-LABEL: @test_atomicrmw_fadd_f64_global_unsafe_strictfp(1786; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 81787; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]1788; GFX11: atomicrmw.start:1789; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1790; GFX11-NEXT: [[NEW:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[LOADED]], double [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8:[0-9]+]]1791; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641792; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641793; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 8, !amdgpu.no.fine.grained.memory [[META0]]1794; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11795; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01796; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1797; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1798; GFX11: atomicrmw.end:1799; GFX11-NEXT: ret double [[TMP5]]1800;1801 %res = atomicrmw fadd ptr addrspace(1) %ptr, double %value syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !01802 ret double %res1803}1804 1805define float @test_atomicrmw_fadd_f32_local_strictfp(ptr addrspace(3) %ptr, float %value) #2 {1806; CI-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1807; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 41808; CI-NEXT: br label [[ATOMICRMW_START:%.*]]1809; CI: atomicrmw.start:1810; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]1811; CI-NEXT: [[NEW:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[LOADED]], float [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]1812; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321813; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321814; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 41815; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11816; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01817; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1818; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1819; CI: atomicrmw.end:1820; CI-NEXT: ret float [[TMP5]]1821;1822; GFX9-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1823; GFX9-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41824; GFX9-NEXT: ret float [[RES]]1825;1826; GFX908-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1827; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41828; GFX908-NEXT: ret float [[RES]]1829;1830; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1831; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41832; GFX90A-NEXT: ret float [[RES]]1833;1834; GFX942-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1835; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41836; GFX942-NEXT: ret float [[RES]]1837;1838; GFX11-LABEL: @test_atomicrmw_fadd_f32_local_strictfp(1839; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 41840; GFX11-NEXT: ret float [[RES]]1841;1842 %res = atomicrmw fadd ptr addrspace(3) %ptr, float %value seq_cst1843 ret float %res1844}1845 1846define bfloat @test_atomicrmw_fadd_bf16_local(ptr addrspace(3) %ptr, bfloat %value) {1847; ALL-LABEL: @test_atomicrmw_fadd_bf16_local(1848; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4)1849; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i321850; ALL-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 31851; ALL-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 31852; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]1853; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11854; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 41855; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1856; ALL: atomicrmw.start:1857; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1858; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]1859; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161860; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1861; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]1862; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i161863; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321864; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]1865; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1866; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1867; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 41868; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11869; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01870; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1871; ALL: atomicrmw.end:1872; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]1873; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161874; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat1875; ALL-NEXT: ret bfloat [[TMP7]]1876;1877 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic1878 ret bfloat %res1879}1880 1881define bfloat @test_atomicrmw_fadd_bf16_local_align4(ptr addrspace(3) %ptr, bfloat %value) {1882; ALL-LABEL: @test_atomicrmw_fadd_bf16_local_align4(1883; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(3) [[PTR:%.*]], align 41884; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1885; ALL: atomicrmw.start:1886; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1887; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161888; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1889; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]1890; ALL-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i161891; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i321892; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361893; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1894; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 41895; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11896; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01897; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1898; ALL: atomicrmw.end:1899; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161900; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat1901; ALL-NEXT: ret bfloat [[TMP5]]1902;1903 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic, align 41904 ret bfloat %res1905}1906 1907define bfloat @test_atomicrmw_fadd_bf16_global_agent(ptr addrspace(1) %ptr, bfloat %value) {1908; ALL-LABEL: @test_atomicrmw_fadd_bf16_global_agent(1909; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1910; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641911; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31912; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31913; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321914; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1915; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11916; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41917; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1918; ALL: atomicrmw.start:1919; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1920; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1921; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161922; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1923; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]1924; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i161925; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321926; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1927; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1928; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1929; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 41930; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11931; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01932; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1933; ALL: atomicrmw.end:1934; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1935; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161936; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat1937; ALL-NEXT: ret bfloat [[TMP7]]1938;1939 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value syncscope("agent") monotonic1940 ret bfloat %res1941}1942 1943define bfloat @test_atomicrmw_fadd_bf16_global_agent_align4(ptr addrspace(1) %ptr, bfloat %value) {1944; ALL-LABEL: @test_atomicrmw_fadd_bf16_global_agent_align4(1945; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 41946; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1947; ALL: atomicrmw.start:1948; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1949; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i161950; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1951; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]1952; ALL-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i161953; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i321954; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655361955; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]1956; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 41957; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11958; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 01959; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1960; ALL: atomicrmw.end:1961; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i161962; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat1963; ALL-NEXT: ret bfloat [[TMP5]]1964;1965 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value syncscope("agent") monotonic, align 41966 ret bfloat %res1967}1968 1969define bfloat @test_atomicrmw_fadd_bf16_global_system(ptr addrspace(1) %ptr, bfloat %value) {1970; ALL-LABEL: @test_atomicrmw_fadd_bf16_global_system(1971; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)1972; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i641973; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 31974; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 31975; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i321976; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]1977; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -11978; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 41979; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]1980; ALL: atomicrmw.start:1981; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]1982; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]1983; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i161984; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat1985; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]1986; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i161987; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i321988; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]1989; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]1990; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]1991; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 41992; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 11993; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 01994; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]1995; ALL: atomicrmw.end:1996; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]1997; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i161998; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat1999; ALL-NEXT: ret bfloat [[TMP7]]2000;2001 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value monotonic2002 ret bfloat %res2003}2004 2005define bfloat @test_atomicrmw_fadd_bf16_global_system_align4(ptr addrspace(1) %ptr, bfloat %value) {2006; ALL-LABEL: @test_atomicrmw_fadd_bf16_global_system_align4(2007; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 42008; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2009; ALL: atomicrmw.start:2010; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2011; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i162012; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2013; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]2014; ALL-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i162015; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i322016; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655362017; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]2018; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 42019; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12020; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 02021; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2022; ALL: atomicrmw.end:2023; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i162024; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat2025; ALL-NEXT: ret bfloat [[TMP5]]2026;2027 %res = atomicrmw fadd ptr addrspace(1) %ptr, bfloat %value monotonic, align 42028 ret bfloat %res2029}2030 2031define bfloat @test_atomicrmw_fadd_bf16_local_strictfp(ptr addrspace(3) %ptr, bfloat %value) #2 {2032; ALL-LABEL: @test_atomicrmw_fadd_bf16_local_strictfp(2033; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3) [[PTR:%.*]], i32 -4) #[[ATTR8:[0-9]+]]2034; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(3) [[PTR]] to i322035; ALL-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 32036; ALL-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 32037; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]2038; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -12039; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(3) [[ALIGNEDADDR]], align 42040; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2041; ALL: atomicrmw.start:2042; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2043; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[TMP2]]2044; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i162045; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2046; ALL-NEXT: [[NEW:%.*]] = call bfloat @llvm.experimental.constrained.fadd.bf16(bfloat [[TMP4]], bfloat [[VALUE:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]2047; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i162048; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i322049; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[TMP2]]2050; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]2051; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]2052; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(3) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 42053; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 12054; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 02055; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2056; ALL: atomicrmw.end:2057; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]2058; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i162059; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat2060; ALL-NEXT: ret bfloat [[TMP7]]2061;2062 %res = atomicrmw fadd ptr addrspace(3) %ptr, bfloat %value monotonic2063 ret bfloat %res2064}2065 2066define bfloat @test_atomicrmw_fadd_bf16_flat_agent(ptr %ptr, bfloat %value) {2067; ALL-LABEL: @test_atomicrmw_fadd_bf16_flat_agent(2068; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)2069; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i642070; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 32071; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 32072; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i322073; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]2074; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -12075; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 42076; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2077; ALL: atomicrmw.start:2078; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2079; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]2080; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i162081; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2082; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]2083; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i162084; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i322085; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]2086; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]2087; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]2088; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 42089; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 12090; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 02091; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2092; ALL: atomicrmw.end:2093; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]2094; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i162095; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat2096; ALL-NEXT: ret bfloat [[TMP7]]2097;2098 %res = atomicrmw fadd ptr %ptr, bfloat %value syncscope("agent") monotonic2099 ret bfloat %res2100}2101 2102define bfloat @test_atomicrmw_fadd_bf16_flat_agent_align4(ptr %ptr, bfloat %value) {2103; ALL-LABEL: @test_atomicrmw_fadd_bf16_flat_agent_align4(2104; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 42105; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2106; ALL: atomicrmw.start:2107; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2108; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i162109; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2110; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]2111; ALL-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i162112; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i322113; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655362114; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]2115; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] syncscope("agent") monotonic monotonic, align 42116; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12117; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 02118; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2119; ALL: atomicrmw.end:2120; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i162121; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat2122; ALL-NEXT: ret bfloat [[TMP5]]2123;2124 %res = atomicrmw fadd ptr %ptr, bfloat %value syncscope("agent") monotonic, align 42125 ret bfloat %res2126}2127 2128define bfloat @test_atomicrmw_fadd_bf16_flat_system(ptr %ptr, bfloat %value) {2129; ALL-LABEL: @test_atomicrmw_fadd_bf16_flat_system(2130; ALL-NEXT: [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)2131; ALL-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i642132; ALL-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 32133; ALL-NEXT: [[TMP2:%.*]] = shl i64 [[PTRLSB]], 32134; ALL-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i322135; ALL-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]2136; ALL-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -12137; ALL-NEXT: [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 42138; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2139; ALL: atomicrmw.start:2140; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2141; ALL-NEXT: [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]2142; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i162143; ALL-NEXT: [[TMP4:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2144; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP4]], [[VALUE:%.*]]2145; ALL-NEXT: [[TMP5:%.*]] = bitcast bfloat [[NEW]] to i162146; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP5]] to i322147; ALL-NEXT: [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]2148; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]2149; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]2150; ALL-NEXT: [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 42151; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 12152; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 02153; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2154; ALL: atomicrmw.end:2155; ALL-NEXT: [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]2156; ALL-NEXT: [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i162157; ALL-NEXT: [[TMP7:%.*]] = bitcast i16 [[EXTRACTED3]] to bfloat2158; ALL-NEXT: ret bfloat [[TMP7]]2159;2160 %res = atomicrmw fadd ptr %ptr, bfloat %value monotonic2161 ret bfloat %res2162}2163 2164define bfloat @test_atomicrmw_fadd_bf16_flat_system_align4(ptr %ptr, bfloat %value) {2165; ALL-LABEL: @test_atomicrmw_fadd_bf16_flat_system_align4(2166; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 42167; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2168; ALL: atomicrmw.start:2169; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]2170; ALL-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i162171; ALL-NEXT: [[TMP2:%.*]] = bitcast i16 [[EXTRACTED]] to bfloat2172; ALL-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP2]], [[VALUE:%.*]]2173; ALL-NEXT: [[TMP3:%.*]] = bitcast bfloat [[NEW]] to i162174; ALL-NEXT: [[EXTENDED:%.*]] = zext i16 [[TMP3]] to i322175; ALL-NEXT: [[UNMASKED:%.*]] = and i32 [[LOADED]], -655362176; ALL-NEXT: [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]2177; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] monotonic monotonic, align 42178; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12179; ALL-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 02180; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2181; ALL: atomicrmw.end:2182; ALL-NEXT: [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i162183; ALL-NEXT: [[TMP5:%.*]] = bitcast i16 [[EXTRACTED1]] to bfloat2184; ALL-NEXT: ret bfloat [[TMP5]]2185;2186 %res = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 42187 ret bfloat %res2188}2189 2190define void @test_atomicrmw_fadd_f32_global_system_noret(ptr addrspace(1) %ptr, float %value) {2191; ALL-LABEL: @test_atomicrmw_fadd_f32_global_system_noret(2192; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42193; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2194; ALL: atomicrmw.start:2195; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2196; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2197; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322198; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322199; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42200; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12201; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02202; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2203; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2204; ALL: atomicrmw.end:2205; ALL-NEXT: ret void2206;2207 %unused = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic2208 ret void2209}2210 2211define float @test_atomicrmw_fadd_f32_global_system_ret(ptr addrspace(1) %ptr, float %value) {2212; ALL-LABEL: @test_atomicrmw_fadd_f32_global_system_ret(2213; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42214; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2215; ALL: atomicrmw.start:2216; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2217; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2218; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322219; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322220; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42221; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12222; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02223; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2224; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2225; ALL: atomicrmw.end:2226; ALL-NEXT: ret float [[TMP5]]2227;2228 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic2229 ret float %ret2230}2231 2232define void @test_atomicrmw_fadd_f32_global_system_noret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) {2233; ALL-LABEL: @test_atomicrmw_fadd_f32_global_system_noret__amdgpu_ignore_denormal_mode(2234; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42235; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2236; ALL: atomicrmw.start:2237; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2238; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2239; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322240; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322241; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42242; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12243; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02244; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2245; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2246; ALL: atomicrmw.end:2247; ALL-NEXT: ret void2248;2249 %unused = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02250 ret void2251}2252 2253define float @test_atomicrmw_fadd_f32_global_system_ret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) {2254; ALL-LABEL: @test_atomicrmw_fadd_f32_global_system_ret__amdgpu_ignore_denormal_mode(2255; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42256; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2257; ALL: atomicrmw.start:2258; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2259; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2260; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322261; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322262; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42263; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12264; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02265; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2266; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2267; ALL: atomicrmw.end:2268; ALL-NEXT: ret float [[TMP5]]2269;2270 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02271 ret float %ret2272}2273 2274define void @test_atomicrmw_fadd_f32_daz_global_system_noret(ptr addrspace(1) %ptr, float %value) #3 {2275; ALL-LABEL: @test_atomicrmw_fadd_f32_daz_global_system_noret(2276; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42277; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2278; ALL: atomicrmw.start:2279; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2280; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2281; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322282; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322283; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42284; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12285; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02286; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2287; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2288; ALL: atomicrmw.end:2289; ALL-NEXT: ret void2290;2291 %unused = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic2292 ret void2293}2294 2295define float @test_atomicrmw_fadd_f32_daz_global_system_ret(ptr addrspace(1) %ptr, float %value) #3 {2296; ALL-LABEL: @test_atomicrmw_fadd_f32_daz_global_system_ret(2297; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42298; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2299; ALL: atomicrmw.start:2300; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2301; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2302; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322303; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322304; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42305; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12306; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02307; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2308; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2309; ALL: atomicrmw.end:2310; ALL-NEXT: ret float [[TMP5]]2311;2312 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic2313 ret float %ret2314}2315 2316define void @test_atomicrmw_fadd_f32_daz_global_system_noret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) #3 {2317; ALL-LABEL: @test_atomicrmw_fadd_f32_daz_global_system_noret__amdgpu_ignore_denormal_mode(2318; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42319; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2320; ALL: atomicrmw.start:2321; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2322; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2323; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322324; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322325; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42326; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12327; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02328; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2329; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2330; ALL: atomicrmw.end:2331; ALL-NEXT: ret void2332;2333 %unused = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02334 ret void2335}2336 2337define float @test_atomicrmw_fadd_f32_daz_global_system_ret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) #3 {2338; ALL-LABEL: @test_atomicrmw_fadd_f32_daz_global_system_ret__amdgpu_ignore_denormal_mode(2339; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42340; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2341; ALL: atomicrmw.start:2342; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2343; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2344; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322345; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322346; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42347; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12348; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02349; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2350; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2351; ALL: atomicrmw.end:2352; ALL-NEXT: ret float [[TMP5]]2353;2354 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02355 ret float %ret2356}2357 2358define void @test_atomicrmw_fadd_f32_dyndenorm_global_system_noret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) #4 {2359; ALL-LABEL: @test_atomicrmw_fadd_f32_dyndenorm_global_system_noret__amdgpu_ignore_denormal_mode(2360; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42361; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2362; ALL: atomicrmw.start:2363; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2364; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2365; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322366; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322367; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42368; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12369; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02370; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2371; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2372; ALL: atomicrmw.end:2373; ALL-NEXT: ret void2374;2375 %unused = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02376 ret void2377}2378 2379define float @test_atomicrmw_fadd_f32_dyndenorm_global_system_ret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, float %value) #4 {2380; ALL-LABEL: @test_atomicrmw_fadd_f32_dyndenorm_global_system_ret__amdgpu_ignore_denormal_mode(2381; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 42382; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2383; ALL: atomicrmw.start:2384; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2385; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2386; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322387; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322388; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42389; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12390; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02391; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2392; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2393; ALL: atomicrmw.end:2394; ALL-NEXT: ret float [[TMP5]]2395;2396 %ret = atomicrmw fadd ptr addrspace(1) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02397 ret float %ret2398}2399 2400define void @test_atomicrmw_fadd_f32_local_noret(ptr addrspace(3) %ptr, float %value) {2401; CI-LABEL: @test_atomicrmw_fadd_f32_local_noret(2402; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 42403; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2404; CI: atomicrmw.start:2405; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2406; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2407; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322408; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322409; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42410; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12411; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02412; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2413; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2414; CI: atomicrmw.end:2415; CI-NEXT: ret void2416;2417; GFX9-LABEL: @test_atomicrmw_fadd_f32_local_noret(2418; GFX9-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42419; GFX9-NEXT: ret void2420;2421; GFX908-LABEL: @test_atomicrmw_fadd_f32_local_noret(2422; GFX908-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42423; GFX908-NEXT: ret void2424;2425; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local_noret(2426; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42427; GFX90A-NEXT: ret void2428;2429; GFX942-LABEL: @test_atomicrmw_fadd_f32_local_noret(2430; GFX942-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42431; GFX942-NEXT: ret void2432;2433; GFX11-LABEL: @test_atomicrmw_fadd_f32_local_noret(2434; GFX11-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42435; GFX11-NEXT: ret void2436;2437 %unused = atomicrmw fadd ptr addrspace(3) %ptr, float %value monotonic2438 ret void2439}2440 2441define float @test_atomicrmw_fadd_f32_local_ret(ptr addrspace(3) %ptr, float %value) {2442; CI-LABEL: @test_atomicrmw_fadd_f32_local_ret(2443; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 42444; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2445; CI: atomicrmw.start:2446; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2447; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2448; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322449; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322450; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42451; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12452; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02453; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2454; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2455; CI: atomicrmw.end:2456; CI-NEXT: ret float [[TMP5]]2457;2458; GFX9-LABEL: @test_atomicrmw_fadd_f32_local_ret(2459; GFX9-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42460; GFX9-NEXT: ret float [[RET]]2461;2462; GFX908-LABEL: @test_atomicrmw_fadd_f32_local_ret(2463; GFX908-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42464; GFX908-NEXT: ret float [[RET]]2465;2466; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local_ret(2467; GFX90A-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42468; GFX90A-NEXT: ret float [[RET]]2469;2470; GFX942-LABEL: @test_atomicrmw_fadd_f32_local_ret(2471; GFX942-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42472; GFX942-NEXT: ret float [[RET]]2473;2474; GFX11-LABEL: @test_atomicrmw_fadd_f32_local_ret(2475; GFX11-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 42476; GFX11-NEXT: ret float [[RET]]2477;2478 %ret = atomicrmw fadd ptr addrspace(3) %ptr, float %value monotonic2479 ret float %ret2480}2481 2482define void @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr, float %value) {2483; CI-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2484; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 42485; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2486; CI: atomicrmw.start:2487; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2488; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2489; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322490; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322491; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42492; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12493; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02494; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2495; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2496; CI: atomicrmw.end:2497; CI-NEXT: ret void2498;2499; GFX9-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2500; GFX9-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2501; GFX9-NEXT: ret void2502;2503; GFX908-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2504; GFX908-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2505; GFX908-NEXT: ret void2506;2507; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2508; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2509; GFX90A-NEXT: ret void2510;2511; GFX942-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2512; GFX942-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2513; GFX942-NEXT: ret void2514;2515; GFX11-LABEL: @test_atomicrmw_fadd_f32_local_noret__amdgpu_ignore_denormal_mode(2516; GFX11-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2517; GFX11-NEXT: ret void2518;2519 %unused = atomicrmw fadd ptr addrspace(3) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02520 ret void2521}2522 2523define float @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr, float %value) {2524; CI-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2525; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(3) [[PTR:%.*]], align 42526; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2527; CI: atomicrmw.start:2528; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2529; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2530; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322531; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322532; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42533; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12534; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02535; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2536; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2537; CI: atomicrmw.end:2538; CI-NEXT: ret float [[TMP5]]2539;2540; GFX9-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2541; GFX9-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2542; GFX9-NEXT: ret float [[RET]]2543;2544; GFX908-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2545; GFX908-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2546; GFX908-NEXT: ret float [[RET]]2547;2548; GFX90A-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2549; GFX90A-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2550; GFX90A-NEXT: ret float [[RET]]2551;2552; GFX942-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2553; GFX942-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2554; GFX942-NEXT: ret float [[RET]]2555;2556; GFX11-LABEL: @test_atomicrmw_fadd_f32_local_ret__amdgpu_ignore_denormal_mode(2557; GFX11-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], float [[VALUE:%.*]] monotonic, align 4, !amdgpu.ignore.denormal.mode [[META0]]2558; GFX11-NEXT: ret float [[RET]]2559;2560 %ret = atomicrmw fadd ptr addrspace(3) %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02561 ret float %ret2562}2563 2564define void @test_atomicrmw_fadd_f64_dyndenorm_global_system_noret(ptr addrspace(1) %ptr, double %value) #5 {2565; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_global_system_noret(2566; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 82567; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2568; ALL: atomicrmw.start:2569; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2570; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2571; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642572; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642573; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82574; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12575; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02576; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2577; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2578; ALL: atomicrmw.end:2579; ALL-NEXT: ret void2580;2581 %unused = atomicrmw fadd ptr addrspace(1) %ptr, double %value monotonic2582 ret void2583}2584 2585define double @test_atomicrmw_fadd_f64_dyndenorm_global_system_ret(ptr addrspace(1) %ptr, double %value) #5 {2586; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_global_system_ret(2587; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 82588; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2589; ALL: atomicrmw.start:2590; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2591; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2592; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642593; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642594; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82595; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12596; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02597; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2598; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2599; ALL: atomicrmw.end:2600; ALL-NEXT: ret double [[TMP5]]2601;2602 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %value monotonic2603 ret double %ret2604}2605 2606define void @test_atomicrmw_fadd_f64_dyndenorm_global_system_noret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) #5 {2607; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_global_system_noret__amdgpu_ignore_denormal_mode(2608; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 82609; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2610; ALL: atomicrmw.start:2611; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2612; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2613; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642614; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642615; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82616; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12617; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02618; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2619; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2620; ALL: atomicrmw.end:2621; ALL-NEXT: ret void2622;2623 %unused = atomicrmw fadd ptr addrspace(1) %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !02624 ret void2625}2626 2627define double @test_atomicrmw_fadd_f64_dyndenorm_global_system_ret__amdgpu_ignore_denormal_mode(ptr addrspace(1) %ptr, double %value) #5 {2628; ALL-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_global_system_ret__amdgpu_ignore_denormal_mode(2629; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 82630; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2631; ALL: atomicrmw.start:2632; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2633; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2634; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642635; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642636; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82637; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12638; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02639; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2640; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2641; ALL: atomicrmw.end:2642; ALL-NEXT: ret double [[TMP5]]2643;2644 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !02645 ret double %ret2646}2647 2648define void @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(ptr addrspace(3) %ptr, double %value) #5 {2649; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2650; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82651; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2652; CI: atomicrmw.start:2653; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2654; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2655; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642656; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642657; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82658; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12659; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02660; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2661; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2662; CI: atomicrmw.end:2663; CI-NEXT: ret void2664;2665; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2666; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82667; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]2668; GFX9: atomicrmw.start:2669; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2670; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2671; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642672; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642673; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82674; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12675; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02676; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2677; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2678; GFX9: atomicrmw.end:2679; GFX9-NEXT: ret void2680;2681; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2682; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82683; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]2684; GFX908: atomicrmw.start:2685; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2686; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2687; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642688; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642689; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82690; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12691; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02692; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2693; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2694; GFX908: atomicrmw.end:2695; GFX908-NEXT: ret void2696;2697; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2698; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 82699; GFX90A-NEXT: ret void2700;2701; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2702; GFX942-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 82703; GFX942-NEXT: ret void2704;2705; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret(2706; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82707; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]2708; GFX11: atomicrmw.start:2709; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2710; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2711; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642712; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642713; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82714; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12715; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02716; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2717; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2718; GFX11: atomicrmw.end:2719; GFX11-NEXT: ret void2720;2721 %unused = atomicrmw fadd ptr addrspace(3) %ptr, double %value monotonic2722 ret void2723}2724 2725define double @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(ptr addrspace(3) %ptr, double %value) #5 {2726; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2727; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82728; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2729; CI: atomicrmw.start:2730; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2731; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2732; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642733; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642734; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82735; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12736; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02737; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2738; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2739; CI: atomicrmw.end:2740; CI-NEXT: ret double [[TMP5]]2741;2742; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2743; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82744; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]2745; GFX9: atomicrmw.start:2746; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2747; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2748; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642749; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642750; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82751; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12752; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02753; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2754; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2755; GFX9: atomicrmw.end:2756; GFX9-NEXT: ret double [[TMP5]]2757;2758; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2759; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82760; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]2761; GFX908: atomicrmw.start:2762; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2763; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2764; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642765; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642766; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82767; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12768; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02769; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2770; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2771; GFX908: atomicrmw.end:2772; GFX908-NEXT: ret double [[TMP5]]2773;2774; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2775; GFX90A-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 82776; GFX90A-NEXT: ret double [[RET]]2777;2778; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2779; GFX942-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 82780; GFX942-NEXT: ret double [[RET]]2781;2782; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret(2783; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82784; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]2785; GFX11: atomicrmw.start:2786; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2787; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2788; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642789; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642790; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82791; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12792; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02793; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2794; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2795; GFX11: atomicrmw.end:2796; GFX11-NEXT: ret double [[TMP5]]2797;2798 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double %value monotonic2799 ret double %ret2800}2801 2802define void @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr, double %value) #5 {2803; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2804; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82805; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2806; CI: atomicrmw.start:2807; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2808; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2809; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642810; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642811; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82812; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12813; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02814; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2815; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2816; CI: atomicrmw.end:2817; CI-NEXT: ret void2818;2819; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2820; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82821; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]2822; GFX9: atomicrmw.start:2823; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2824; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2825; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642826; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642827; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82828; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12829; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02830; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2831; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2832; GFX9: atomicrmw.end:2833; GFX9-NEXT: ret void2834;2835; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2836; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82837; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]2838; GFX908: atomicrmw.start:2839; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2840; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2841; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642842; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642843; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82844; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12845; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02846; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2847; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2848; GFX908: atomicrmw.end:2849; GFX908-NEXT: ret void2850;2851; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2852; GFX90A-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]2853; GFX90A-NEXT: ret void2854;2855; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2856; GFX942-NEXT: [[UNUSED:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]2857; GFX942-NEXT: ret void2858;2859; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_noret__amdgpu_ignore_denormal_mode(2860; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82861; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]2862; GFX11: atomicrmw.start:2863; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2864; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2865; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642866; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642867; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82868; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12869; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02870; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2871; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2872; GFX11: atomicrmw.end:2873; GFX11-NEXT: ret void2874;2875 %unused = atomicrmw fadd ptr addrspace(3) %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !02876 ret void2877}2878 2879define double @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr, double %value) #5 {2880; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2881; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82882; CI-NEXT: br label [[ATOMICRMW_START:%.*]]2883; CI: atomicrmw.start:2884; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2885; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2886; CI-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642887; CI-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642888; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82889; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12890; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02891; CI-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2892; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2893; CI: atomicrmw.end:2894; CI-NEXT: ret double [[TMP5]]2895;2896; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2897; GFX9-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82898; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]2899; GFX9: atomicrmw.start:2900; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2901; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2902; GFX9-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642903; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642904; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82905; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12906; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02907; GFX9-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2908; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2909; GFX9: atomicrmw.end:2910; GFX9-NEXT: ret double [[TMP5]]2911;2912; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2913; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82914; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]2915; GFX908: atomicrmw.start:2916; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2917; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2918; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642919; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642920; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82921; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12922; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02923; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2924; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2925; GFX908: atomicrmw.end:2926; GFX908-NEXT: ret double [[TMP5]]2927;2928; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2929; GFX90A-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]2930; GFX90A-NEXT: ret double [[RET]]2931;2932; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2933; GFX942-NEXT: [[RET:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]2934; GFX942-NEXT: ret double [[RET]]2935;2936; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_local_system_ret__amdgpu_ignore_denormal_mode(2937; GFX11-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(3) [[PTR:%.*]], align 82938; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]2939; GFX11: atomicrmw.start:2940; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2941; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]2942; GFX11-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i642943; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i642944; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 82945; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 12946; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 02947; GFX11-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double2948; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2949; GFX11: atomicrmw.end:2950; GFX11-NEXT: ret double [[TMP5]]2951;2952 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !02953 ret double %ret2954}2955 2956define void @test_atomicrmw_fadd_f32_flat_system_noret__amdgpu_ignore_denormal_mode(ptr %ptr, float %value) {2957; ALL-LABEL: @test_atomicrmw_fadd_f32_flat_system_noret__amdgpu_ignore_denormal_mode(2958; ALL-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 42959; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2960; ALL: atomicrmw.start:2961; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2962; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2963; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322964; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322965; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42966; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12967; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02968; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2969; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2970; ALL: atomicrmw.end:2971; ALL-NEXT: ret void2972;2973 %unused = atomicrmw fadd ptr %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02974 ret void2975}2976 2977define float @test_atomicrmw_fadd_f32_flat_system_ret__amdgpu_ignore_denormal_mode(ptr %ptr, float %value) {2978; ALL-LABEL: @test_atomicrmw_fadd_f32_flat_system_ret__amdgpu_ignore_denormal_mode(2979; ALL-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR:%.*]], align 42980; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]2981; ALL: atomicrmw.start:2982; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]2983; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]2984; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i322985; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i322986; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 42987; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 12988; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 02989; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float2990; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]2991; ALL: atomicrmw.end:2992; ALL-NEXT: ret float [[TMP5]]2993;2994 %ret = atomicrmw fadd ptr %ptr, float %value monotonic, !amdgpu.ignore.denormal.mode !02995 ret float %ret2996}2997 2998define void @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(ptr %ptr, double %value) #5 {2999; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3000; CI-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3001; CI-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3002; CI: atomicrmw.private:3003; CI-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3004; CI-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83005; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3006; CI-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83007; CI-NEXT: br label [[ATOMICRMW_PHI:%.*]]3008; CI: atomicrmw.global:3009; CI-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83010; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3011; CI: atomicrmw.start:3012; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3013; CI-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3014; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643015; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643016; CI-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3017; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13018; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03019; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3020; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3021; CI: atomicrmw.end1:3022; CI-NEXT: br label [[ATOMICRMW_PHI]]3023; CI: atomicrmw.phi:3024; CI-NEXT: br label [[ATOMICRMW_END:%.*]]3025; CI: atomicrmw.end:3026; CI-NEXT: ret void3027;3028; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3029; GFX9-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3030; GFX9-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3031; GFX9: atomicrmw.private:3032; GFX9-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3033; GFX9-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83034; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3035; GFX9-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83036; GFX9-NEXT: br label [[ATOMICRMW_PHI:%.*]]3037; GFX9: atomicrmw.global:3038; GFX9-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83039; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3040; GFX9: atomicrmw.start:3041; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3042; GFX9-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3043; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643044; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643045; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3046; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13047; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03048; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3049; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3050; GFX9: atomicrmw.end1:3051; GFX9-NEXT: br label [[ATOMICRMW_PHI]]3052; GFX9: atomicrmw.phi:3053; GFX9-NEXT: br label [[ATOMICRMW_END:%.*]]3054; GFX9: atomicrmw.end:3055; GFX9-NEXT: ret void3056;3057; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3058; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3059; GFX908-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3060; GFX908: atomicrmw.private:3061; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3062; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83063; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3064; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83065; GFX908-NEXT: br label [[ATOMICRMW_PHI:%.*]]3066; GFX908: atomicrmw.global:3067; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83068; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3069; GFX908: atomicrmw.start:3070; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3071; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3072; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643073; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643074; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3075; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13076; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03077; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3078; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3079; GFX908: atomicrmw.end1:3080; GFX908-NEXT: br label [[ATOMICRMW_PHI]]3081; GFX908: atomicrmw.phi:3082; GFX908-NEXT: br label [[ATOMICRMW_END:%.*]]3083; GFX908: atomicrmw.end:3084; GFX908-NEXT: ret void3085;3086; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3087; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])3088; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]3089; GFX90A: atomicrmw.shared:3090; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)3091; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]3092; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]3093; GFX90A: atomicrmw.check.private:3094; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])3095; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3096; GFX90A: atomicrmw.private:3097; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3098; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 83099; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]3100; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 83101; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]3102; GFX90A: atomicrmw.global:3103; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)3104; GFX90A-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 83105; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3106; GFX90A: atomicrmw.start:3107; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]3108; GFX90A-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3109; GFX90A-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i643110; GFX90A-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i643111; GFX90A-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 83112; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 13113; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 03114; GFX90A-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double3115; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3116; GFX90A: atomicrmw.end1:3117; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]3118; GFX90A: atomicrmw.phi:3119; GFX90A-NEXT: br label [[ATOMICRMW_END:%.*]]3120; GFX90A: atomicrmw.end:3121; GFX90A-NEXT: ret void3122;3123; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3124; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])3125; GFX942-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]3126; GFX942: atomicrmw.shared:3127; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)3128; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]3129; GFX942-NEXT: br label [[ATOMICRMW_PHI:%.*]]3130; GFX942: atomicrmw.check.private:3131; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])3132; GFX942-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3133; GFX942: atomicrmw.private:3134; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3135; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 83136; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]3137; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 83138; GFX942-NEXT: br label [[ATOMICRMW_PHI]]3139; GFX942: atomicrmw.global:3140; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)3141; GFX942-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 83142; GFX942-NEXT: br label [[ATOMICRMW_START:%.*]]3143; GFX942: atomicrmw.start:3144; GFX942-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]3145; GFX942-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3146; GFX942-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i643147; GFX942-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i643148; GFX942-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 83149; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 13150; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 03151; GFX942-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double3152; GFX942-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3153; GFX942: atomicrmw.end1:3154; GFX942-NEXT: br label [[ATOMICRMW_PHI]]3155; GFX942: atomicrmw.phi:3156; GFX942-NEXT: br label [[ATOMICRMW_END:%.*]]3157; GFX942: atomicrmw.end:3158; GFX942-NEXT: ret void3159;3160; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_noret__amdgpu_ignore_denormal_mode(3161; GFX11-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3162; GFX11-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3163; GFX11: atomicrmw.private:3164; GFX11-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3165; GFX11-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83166; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3167; GFX11-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83168; GFX11-NEXT: br label [[ATOMICRMW_PHI:%.*]]3169; GFX11: atomicrmw.global:3170; GFX11-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83171; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3172; GFX11: atomicrmw.start:3173; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3174; GFX11-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3175; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643176; GFX11-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643177; GFX11-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3178; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13179; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03180; GFX11-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3181; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3182; GFX11: atomicrmw.end1:3183; GFX11-NEXT: br label [[ATOMICRMW_PHI]]3184; GFX11: atomicrmw.phi:3185; GFX11-NEXT: br label [[ATOMICRMW_END:%.*]]3186; GFX11: atomicrmw.end:3187; GFX11-NEXT: ret void3188;3189 %unused = atomicrmw fadd ptr %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !03190 ret void3191}3192 3193define double @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(ptr %ptr, double %value) #5 {3194; CI-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3195; CI-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3196; CI-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3197; CI: atomicrmw.private:3198; CI-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3199; CI-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83200; CI-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3201; CI-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83202; CI-NEXT: br label [[ATOMICRMW_PHI:%.*]]3203; CI: atomicrmw.global:3204; CI-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83205; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3206; CI: atomicrmw.start:3207; CI-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3208; CI-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3209; CI-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643210; CI-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643211; CI-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3212; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13213; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03214; CI-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3215; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3216; CI: atomicrmw.end1:3217; CI-NEXT: br label [[ATOMICRMW_PHI]]3218; CI: atomicrmw.phi:3219; CI-NEXT: [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]3220; CI-NEXT: br label [[ATOMICRMW_END:%.*]]3221; CI: atomicrmw.end:3222; CI-NEXT: ret double [[RET]]3223;3224; GFX9-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3225; GFX9-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3226; GFX9-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3227; GFX9: atomicrmw.private:3228; GFX9-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3229; GFX9-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83230; GFX9-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3231; GFX9-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83232; GFX9-NEXT: br label [[ATOMICRMW_PHI:%.*]]3233; GFX9: atomicrmw.global:3234; GFX9-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83235; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3236; GFX9: atomicrmw.start:3237; GFX9-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3238; GFX9-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3239; GFX9-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643240; GFX9-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643241; GFX9-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3242; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13243; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03244; GFX9-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3245; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3246; GFX9: atomicrmw.end1:3247; GFX9-NEXT: br label [[ATOMICRMW_PHI]]3248; GFX9: atomicrmw.phi:3249; GFX9-NEXT: [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]3250; GFX9-NEXT: br label [[ATOMICRMW_END:%.*]]3251; GFX9: atomicrmw.end:3252; GFX9-NEXT: ret double [[RET]]3253;3254; GFX908-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3255; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3256; GFX908-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3257; GFX908: atomicrmw.private:3258; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3259; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83260; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3261; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83262; GFX908-NEXT: br label [[ATOMICRMW_PHI:%.*]]3263; GFX908: atomicrmw.global:3264; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83265; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3266; GFX908: atomicrmw.start:3267; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3268; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3269; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643270; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643271; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3272; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13273; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03274; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3275; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3276; GFX908: atomicrmw.end1:3277; GFX908-NEXT: br label [[ATOMICRMW_PHI]]3278; GFX908: atomicrmw.phi:3279; GFX908-NEXT: [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]3280; GFX908-NEXT: br label [[ATOMICRMW_END:%.*]]3281; GFX908: atomicrmw.end:3282; GFX908-NEXT: ret double [[RET]]3283;3284; GFX90A-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3285; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])3286; GFX90A-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]3287; GFX90A: atomicrmw.shared:3288; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)3289; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]3290; GFX90A-NEXT: br label [[ATOMICRMW_PHI:%.*]]3291; GFX90A: atomicrmw.check.private:3292; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])3293; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3294; GFX90A: atomicrmw.private:3295; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3296; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 83297; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]3298; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 83299; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]3300; GFX90A: atomicrmw.global:3301; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)3302; GFX90A-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 83303; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3304; GFX90A: atomicrmw.start:3305; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]3306; GFX90A-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3307; GFX90A-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i643308; GFX90A-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i643309; GFX90A-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 83310; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 13311; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 03312; GFX90A-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double3313; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3314; GFX90A: atomicrmw.end1:3315; GFX90A-NEXT: br label [[ATOMICRMW_PHI]]3316; GFX90A: atomicrmw.phi:3317; GFX90A-NEXT: [[RET:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]3318; GFX90A-NEXT: br label [[ATOMICRMW_END:%.*]]3319; GFX90A: atomicrmw.end:3320; GFX90A-NEXT: ret double [[RET]]3321;3322; GFX942-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3323; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR:%.*]])3324; GFX942-NEXT: br i1 [[IS_SHARED]], label [[ATOMICRMW_SHARED:%.*]], label [[ATOMICRMW_CHECK_PRIVATE:%.*]]3325; GFX942: atomicrmw.shared:3326; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)3327; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], double [[VALUE:%.*]] monotonic, align 8, !amdgpu.ignore.denormal.mode [[META0]]3328; GFX942-NEXT: br label [[ATOMICRMW_PHI:%.*]]3329; GFX942: atomicrmw.check.private:3330; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])3331; GFX942-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3332; GFX942: atomicrmw.private:3333; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3334; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP3]], align 83335; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]3336; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP3]], align 83337; GFX942-NEXT: br label [[ATOMICRMW_PHI]]3338; GFX942: atomicrmw.global:3339; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)3340; GFX942-NEXT: [[TMP5:%.*]] = load double, ptr addrspace(1) [[TMP4]], align 83341; GFX942-NEXT: br label [[ATOMICRMW_START:%.*]]3342; GFX942: atomicrmw.start:3343; GFX942-NEXT: [[LOADED:%.*]] = phi double [ [[TMP5]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP9:%.*]], [[ATOMICRMW_START]] ]3344; GFX942-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3345; GFX942-NEXT: [[TMP6:%.*]] = bitcast double [[NEW2]] to i643346; GFX942-NEXT: [[TMP7:%.*]] = bitcast double [[LOADED]] to i643347; GFX942-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[TMP4]], i64 [[TMP7]], i64 [[TMP6]] monotonic monotonic, align 83348; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP8]], 13349; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP8]], 03350; GFX942-NEXT: [[TMP9]] = bitcast i64 [[NEWLOADED]] to double3351; GFX942-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3352; GFX942: atomicrmw.end1:3353; GFX942-NEXT: br label [[ATOMICRMW_PHI]]3354; GFX942: atomicrmw.phi:3355; GFX942-NEXT: [[RET:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP9]], [[ATOMICRMW_END1]] ]3356; GFX942-NEXT: br label [[ATOMICRMW_END:%.*]]3357; GFX942: atomicrmw.end:3358; GFX942-NEXT: ret double [[RET]]3359;3360; GFX11-LABEL: @test_atomicrmw_fadd_f64_dyndenorm_flat_system_ret__amdgpu_ignore_denormal_mode(3361; GFX11-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR:%.*]])3362; GFX11-NEXT: br i1 [[IS_PRIVATE]], label [[ATOMICRMW_PRIVATE:%.*]], label [[ATOMICRMW_GLOBAL:%.*]]3363; GFX11: atomicrmw.private:3364; GFX11-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)3365; GFX11-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 83366; GFX11-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE:%.*]]3367; GFX11-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 83368; GFX11-NEXT: br label [[ATOMICRMW_PHI:%.*]]3369; GFX11: atomicrmw.global:3370; GFX11-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 83371; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3372; GFX11: atomicrmw.start:3373; GFX11-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], [[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]3374; GFX11-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]3375; GFX11-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i643376; GFX11-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i643377; GFX11-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] monotonic monotonic, align 8, !noalias.addrspace [[META1]]3378; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 13379; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 03380; GFX11-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double3381; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END1:%.*]], label [[ATOMICRMW_START]]3382; GFX11: atomicrmw.end1:3383; GFX11-NEXT: br label [[ATOMICRMW_PHI]]3384; GFX11: atomicrmw.phi:3385; GFX11-NEXT: [[RET:%.*]] = phi double [ [[LOADED_PRIVATE]], [[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], [[ATOMICRMW_END1]] ]3386; GFX11-NEXT: br label [[ATOMICRMW_END:%.*]]3387; GFX11: atomicrmw.end:3388; GFX11-NEXT: ret double [[RET]]3389;3390 %ret = atomicrmw fadd ptr %ptr, double %value monotonic, !amdgpu.ignore.denormal.mode !03391 ret double %ret3392}3393 3394define void @test_atomicrmw_fadd_f32_region_noret(ptr addrspace(2) %ptr, float %value) {3395; ALL-LABEL: @test_atomicrmw_fadd_f32_region_noret(3396; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(2) [[PTR:%.*]], align 43397; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]3398; ALL: atomicrmw.start:3399; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3400; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]3401; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i323402; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i323403; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(2) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 43404; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13405; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03406; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float3407; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3408; ALL: atomicrmw.end:3409; ALL-NEXT: ret void3410;3411 %unused = atomicrmw fadd ptr addrspace(2) %ptr, float %value monotonic3412 ret void3413}3414 3415define float @test_atomicrmw_fadd_f32_region_ret(ptr addrspace(2) %ptr, float %value) {3416; ALL-LABEL: @test_atomicrmw_fadd_f32_region_ret(3417; ALL-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(2) [[PTR:%.*]], align 43418; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]3419; ALL: atomicrmw.start:3420; ALL-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3421; ALL-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]3422; ALL-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i323423; ALL-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i323424; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(2) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] monotonic monotonic, align 43425; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13426; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03427; ALL-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float3428; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3429; ALL: atomicrmw.end:3430; ALL-NEXT: ret float [[TMP5]]3431;3432 %ret = atomicrmw fadd ptr addrspace(2) %ptr, float %value monotonic3433 ret float %ret3434}3435 3436define void @test_atomicrmw_fadd_f64_region_noret(ptr addrspace(2) %ptr, double %value) {3437; ALL-LABEL: @test_atomicrmw_fadd_f64_region_noret(3438; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(2) [[PTR:%.*]], align 83439; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]3440; ALL: atomicrmw.start:3441; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3442; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]3443; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i643444; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i643445; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(2) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 83446; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 13447; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 03448; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double3449; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3450; ALL: atomicrmw.end:3451; ALL-NEXT: ret void3452;3453 %unused = atomicrmw fadd ptr addrspace(2) %ptr, double %value monotonic3454 ret void3455}3456 3457define double @test_atomicrmw_fadd_f64_region_ret(ptr addrspace(2) %ptr, double %value) {3458; ALL-LABEL: @test_atomicrmw_fadd_f64_region_ret(3459; ALL-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(2) [[PTR:%.*]], align 83460; ALL-NEXT: br label [[ATOMICRMW_START:%.*]]3461; ALL: atomicrmw.start:3462; ALL-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3463; ALL-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]3464; ALL-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i643465; ALL-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i643466; ALL-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(2) [[PTR]], i64 [[TMP3]], i64 [[TMP2]] monotonic monotonic, align 83467; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 13468; ALL-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 03469; ALL-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double3470; ALL-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3471; ALL: atomicrmw.end:3472; ALL-NEXT: ret double [[TMP5]]3473;3474 %ret = atomicrmw fadd ptr addrspace(2) %ptr, double %value monotonic3475 ret double %ret3476}3477 3478define <2 x half> @test_atomicrmw_fadd_v2f16_flat_agent(ptr %ptr, <2 x half> %value) {3479; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3480; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43481; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3482; CI: atomicrmw.start:3483; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3484; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3485; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323486; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323487; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43488; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13489; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03490; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3491; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3492; CI: atomicrmw.end:3493; CI-NEXT: ret <2 x half> [[TMP5]]3494;3495; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3496; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43497; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3498; GFX9: atomicrmw.start:3499; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3500; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3501; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323502; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323503; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43504; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13505; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03506; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3507; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3508; GFX9: atomicrmw.end:3509; GFX9-NEXT: ret <2 x half> [[TMP5]]3510;3511; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3512; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43513; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3514; GFX908: atomicrmw.start:3515; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3516; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3517; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323518; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323519; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43520; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13521; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03522; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3523; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3524; GFX908: atomicrmw.end:3525; GFX908-NEXT: ret <2 x half> [[TMP5]]3526;3527; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3528; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43529; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3530; GFX90A: atomicrmw.start:3531; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3532; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3533; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323534; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323535; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43536; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13537; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03538; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3539; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3540; GFX90A: atomicrmw.end:3541; GFX90A-NEXT: ret <2 x half> [[TMP5]]3542;3543; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3544; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43545; GFX942-NEXT: ret <2 x half> [[RES]]3546;3547; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent(3548; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43549; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3550; GFX11: atomicrmw.start:3551; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3552; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3553; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323554; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323555; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43556; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13557; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03558; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3559; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3560; GFX11: atomicrmw.end:3561; GFX11-NEXT: ret <2 x half> [[TMP5]]3562;3563 %res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst3564 ret <2 x half> %res3565}3566 3567define void @test_atomicrmw_fadd_v2f16_flat_agent_noret(ptr %ptr, <2 x half> %value) {3568; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3569; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43570; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3571; CI: atomicrmw.start:3572; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3573; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3574; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323575; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323576; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43577; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13578; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03579; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3580; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3581; CI: atomicrmw.end:3582; CI-NEXT: ret void3583;3584; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3585; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43586; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3587; GFX9: atomicrmw.start:3588; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3589; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3590; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323591; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323592; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43593; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13594; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03595; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3596; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3597; GFX9: atomicrmw.end:3598; GFX9-NEXT: ret void3599;3600; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3601; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43602; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3603; GFX908: atomicrmw.start:3604; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3605; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3606; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323607; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323608; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43609; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13610; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03611; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3612; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3613; GFX908: atomicrmw.end:3614; GFX908-NEXT: ret void3615;3616; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3617; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43618; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3619; GFX90A: atomicrmw.start:3620; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3621; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3622; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323623; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323624; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43625; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13626; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03627; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3628; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3629; GFX90A: atomicrmw.end:3630; GFX90A-NEXT: ret void3631;3632; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3633; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43634; GFX942-NEXT: ret void3635;3636; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret(3637; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 43638; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3639; GFX11: atomicrmw.start:3640; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3641; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3642; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323643; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323644; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43645; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13646; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03647; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3648; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3649; GFX11: atomicrmw.end:3650; GFX11-NEXT: ret void3651;3652 %res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst3653 ret void3654}3655 3656define <2 x half> @test_atomicrmw_fadd_v2f16_global_agent(ptr addrspace(1) %ptr, <2 x half> %value) {3657; CI-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3658; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43659; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3660; CI: atomicrmw.start:3661; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3662; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3663; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323664; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323665; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43666; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13667; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03668; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3669; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3670; CI: atomicrmw.end:3671; CI-NEXT: ret <2 x half> [[TMP5]]3672;3673; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3674; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43675; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3676; GFX9: atomicrmw.start:3677; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3678; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3679; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323680; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323681; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43682; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13683; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03684; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3685; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3686; GFX9: atomicrmw.end:3687; GFX9-NEXT: ret <2 x half> [[TMP5]]3688;3689; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3690; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43691; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3692; GFX908: atomicrmw.start:3693; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3694; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3695; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323696; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323697; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43698; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13699; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03700; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3701; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3702; GFX908: atomicrmw.end:3703; GFX908-NEXT: ret <2 x half> [[TMP5]]3704;3705; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3706; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43707; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3708; GFX90A: atomicrmw.start:3709; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3710; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3711; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323712; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323713; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43714; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13715; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03716; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3717; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3718; GFX90A: atomicrmw.end:3719; GFX90A-NEXT: ret <2 x half> [[TMP5]]3720;3721; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3722; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43723; GFX942-NEXT: ret <2 x half> [[RES]]3724;3725; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_global_agent(3726; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43727; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3728; GFX11: atomicrmw.start:3729; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3730; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3731; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323732; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323733; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43734; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13735; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03736; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3737; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3738; GFX11: atomicrmw.end:3739; GFX11-NEXT: ret <2 x half> [[TMP5]]3740;3741 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value syncscope("agent") seq_cst3742 ret <2 x half> %res3743}3744 3745define void @test_atomicrmw_fadd_v2f16_flat_global_noret(ptr addrspace(1) %ptr, <2 x half> %value) {3746; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3747; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43748; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3749; CI: atomicrmw.start:3750; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3751; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3752; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323753; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323754; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43755; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13756; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03757; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3758; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3759; CI: atomicrmw.end:3760; CI-NEXT: ret void3761;3762; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3763; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43764; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3765; GFX9: atomicrmw.start:3766; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3767; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3768; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323769; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323770; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43771; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13772; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03773; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3774; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3775; GFX9: atomicrmw.end:3776; GFX9-NEXT: ret void3777;3778; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3779; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43780; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3781; GFX908: atomicrmw.start:3782; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3783; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3784; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323785; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323786; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43787; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13788; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03789; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3790; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3791; GFX908: atomicrmw.end:3792; GFX908-NEXT: ret void3793;3794; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3795; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43796; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3797; GFX90A: atomicrmw.start:3798; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3799; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3800; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323801; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323802; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43803; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13804; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03805; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3806; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3807; GFX90A: atomicrmw.end:3808; GFX90A-NEXT: ret void3809;3810; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3811; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43812; GFX942-NEXT: ret void3813;3814; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_global_noret(3815; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 43816; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3817; GFX11: atomicrmw.start:3818; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3819; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3820; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323821; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323822; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43823; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13824; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03825; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3826; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3827; GFX11: atomicrmw.end:3828; GFX11-NEXT: ret void3829;3830 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value syncscope("agent") seq_cst3831 ret void3832}3833 3834define <2 x half> @test_atomicrmw_fadd_v2f16_local_agent(ptr addrspace(3) %ptr, <2 x half> %value) {3835; CI-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3836; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43837; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3838; CI: atomicrmw.start:3839; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3840; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3841; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323842; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323843; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43844; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13845; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03846; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3847; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3848; CI: atomicrmw.end:3849; CI-NEXT: ret <2 x half> [[TMP5]]3850;3851; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3852; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43853; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3854; GFX9: atomicrmw.start:3855; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3856; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3857; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323858; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323859; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43860; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13861; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03862; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3863; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3864; GFX9: atomicrmw.end:3865; GFX9-NEXT: ret <2 x half> [[TMP5]]3866;3867; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3868; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43869; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3870; GFX908: atomicrmw.start:3871; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3872; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3873; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323874; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323875; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43876; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13877; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03878; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3879; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3880; GFX908: atomicrmw.end:3881; GFX908-NEXT: ret <2 x half> [[TMP5]]3882;3883; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3884; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43885; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3886; GFX90A: atomicrmw.start:3887; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3888; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3889; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323890; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323891; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43892; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13893; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03894; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3895; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3896; GFX90A: atomicrmw.end:3897; GFX90A-NEXT: ret <2 x half> [[TMP5]]3898;3899; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3900; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43901; GFX942-NEXT: ret <2 x half> [[RES]]3902;3903; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_local_agent(3904; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43905; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3906; GFX11: atomicrmw.start:3907; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3908; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3909; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323910; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323911; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43912; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13913; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03914; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3915; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3916; GFX11: atomicrmw.end:3917; GFX11-NEXT: ret <2 x half> [[TMP5]]3918;3919 %res = atomicrmw fadd ptr addrspace(3) %ptr, <2 x half> %value syncscope("agent") seq_cst3920 ret <2 x half> %res3921}3922 3923define void @test_atomicrmw_fadd_v2f16_flat_local_noret(ptr addrspace(3) %ptr, <2 x half> %value) {3924; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3925; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43926; CI-NEXT: br label [[ATOMICRMW_START:%.*]]3927; CI: atomicrmw.start:3928; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3929; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3930; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323931; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323932; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43933; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13934; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03935; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3936; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3937; CI: atomicrmw.end:3938; CI-NEXT: ret void3939;3940; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3941; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43942; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]3943; GFX9: atomicrmw.start:3944; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3945; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3946; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323947; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323948; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43949; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13950; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03951; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3952; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3953; GFX9: atomicrmw.end:3954; GFX9-NEXT: ret void3955;3956; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3957; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43958; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]3959; GFX908: atomicrmw.start:3960; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3961; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3962; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323963; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323964; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43965; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13966; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03967; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3968; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3969; GFX908: atomicrmw.end:3970; GFX908-NEXT: ret void3971;3972; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3973; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43974; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]3975; GFX90A: atomicrmw.start:3976; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3977; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3978; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323979; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i323980; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 43981; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 13982; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 03983; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>3984; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]3985; GFX90A: atomicrmw.end:3986; GFX90A-NEXT: ret void3987;3988; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3989; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 43990; GFX942-NEXT: ret void3991;3992; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_local_noret(3993; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[PTR:%.*]], align 43994; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]3995; GFX11: atomicrmw.start:3996; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]3997; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]3998; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i323999; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324000; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44001; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14002; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04003; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4004; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4005; GFX11: atomicrmw.end:4006; GFX11-NEXT: ret void4007;4008 %res = atomicrmw fadd ptr addrspace(3) %ptr, <2 x half> %value syncscope("agent") seq_cst4009 ret void4010}4011 4012define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_flat_agent(ptr %ptr, <2 x bfloat> %value) {4013; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4014; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44015; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4016; CI: atomicrmw.start:4017; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4018; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4019; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324020; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324021; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44022; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14023; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04024; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4025; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4026; CI: atomicrmw.end:4027; CI-NEXT: ret <2 x bfloat> [[TMP5]]4028;4029; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4030; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44031; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4032; GFX9: atomicrmw.start:4033; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4034; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4035; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324036; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324037; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44038; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14039; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04040; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4041; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4042; GFX9: atomicrmw.end:4043; GFX9-NEXT: ret <2 x bfloat> [[TMP5]]4044;4045; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4046; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44047; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4048; GFX908: atomicrmw.start:4049; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4050; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4051; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324052; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324053; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44054; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14055; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04056; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4057; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4058; GFX908: atomicrmw.end:4059; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]4060;4061; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4062; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44063; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4064; GFX90A: atomicrmw.start:4065; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4066; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4067; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324068; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324069; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44070; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14071; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04072; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4073; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4074; GFX90A: atomicrmw.end:4075; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]4076;4077; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4078; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44079; GFX942-NEXT: ret <2 x bfloat> [[RES]]4080;4081; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent(4082; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44083; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4084; GFX11: atomicrmw.start:4085; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4086; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4087; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324088; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324089; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44090; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14091; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04092; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4093; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4094; GFX11: atomicrmw.end:4095; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]4096;4097 %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4098 ret <2 x bfloat> %res4099}4100 4101define void @test_atomicrmw_fadd_v2bf16_flat_agent_noret(ptr %ptr, <2 x bfloat> %value) {4102; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4103; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44104; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4105; CI: atomicrmw.start:4106; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4107; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4108; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324109; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324110; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44111; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14112; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04113; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4114; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4115; CI: atomicrmw.end:4116; CI-NEXT: ret void4117;4118; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4119; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44120; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4121; GFX9: atomicrmw.start:4122; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4123; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4124; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324125; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324126; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44127; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14128; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04129; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4130; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4131; GFX9: atomicrmw.end:4132; GFX9-NEXT: ret void4133;4134; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4135; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44136; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4137; GFX908: atomicrmw.start:4138; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4139; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4140; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324141; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324142; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44143; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14144; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04145; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4146; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4147; GFX908: atomicrmw.end:4148; GFX908-NEXT: ret void4149;4150; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4151; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44152; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4153; GFX90A: atomicrmw.start:4154; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4155; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4156; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324157; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324158; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44159; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14160; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04161; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4162; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4163; GFX90A: atomicrmw.end:4164; GFX90A-NEXT: ret void4165;4166; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4167; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44168; GFX942-NEXT: ret void4169;4170; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret(4171; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44172; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4173; GFX11: atomicrmw.start:4174; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4175; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4176; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324177; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324178; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44179; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14180; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04181; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4182; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4183; GFX11: atomicrmw.end:4184; GFX11-NEXT: ret void4185;4186 %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4187 ret void4188}4189 4190define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_global_agent(ptr addrspace(1) %ptr, <2 x bfloat> %value) {4191; CI-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4192; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44193; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4194; CI: atomicrmw.start:4195; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4196; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4197; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324198; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324199; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44200; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14201; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04202; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4203; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4204; CI: atomicrmw.end:4205; CI-NEXT: ret <2 x bfloat> [[TMP5]]4206;4207; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4208; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44209; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4210; GFX9: atomicrmw.start:4211; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4212; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4213; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324214; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324215; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44216; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14217; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04218; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4219; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4220; GFX9: atomicrmw.end:4221; GFX9-NEXT: ret <2 x bfloat> [[TMP5]]4222;4223; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4224; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44225; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4226; GFX908: atomicrmw.start:4227; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4228; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4229; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324230; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324231; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44232; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14233; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04234; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4235; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4236; GFX908: atomicrmw.end:4237; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]4238;4239; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4240; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44241; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4242; GFX90A: atomicrmw.start:4243; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4244; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4245; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324246; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324247; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44248; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14249; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04250; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4251; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4252; GFX90A: atomicrmw.end:4253; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]4254;4255; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4256; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44257; GFX942-NEXT: ret <2 x bfloat> [[RES]]4258;4259; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent(4260; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44261; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4262; GFX11: atomicrmw.start:4263; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4264; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4265; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324266; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324267; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44268; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14269; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04270; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4271; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4272; GFX11: atomicrmw.end:4273; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]4274;4275 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4276 ret <2 x bfloat> %res4277}4278 4279define void @test_atomicrmw_fadd_v2bf16_flat_global_noret(ptr addrspace(1) %ptr, <2 x bfloat> %value) {4280; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4281; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44282; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4283; CI: atomicrmw.start:4284; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4285; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4286; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324287; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324288; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44289; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14290; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04291; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4292; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4293; CI: atomicrmw.end:4294; CI-NEXT: ret void4295;4296; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4297; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44298; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4299; GFX9: atomicrmw.start:4300; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4301; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4302; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324303; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324304; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44305; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14306; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04307; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4308; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4309; GFX9: atomicrmw.end:4310; GFX9-NEXT: ret void4311;4312; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4313; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44314; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4315; GFX908: atomicrmw.start:4316; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4317; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4318; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324319; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324320; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44321; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14322; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04323; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4324; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4325; GFX908: atomicrmw.end:4326; GFX908-NEXT: ret void4327;4328; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4329; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44330; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4331; GFX90A: atomicrmw.start:4332; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4333; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4334; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324335; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324336; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44337; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14338; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04339; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4340; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4341; GFX90A: atomicrmw.end:4342; GFX90A-NEXT: ret void4343;4344; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4345; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44346; GFX942-NEXT: ret void4347;4348; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_global_noret(4349; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 44350; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4351; GFX11: atomicrmw.start:4352; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4353; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4354; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324355; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324356; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44357; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14358; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04359; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4360; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4361; GFX11: atomicrmw.end:4362; GFX11-NEXT: ret void4363;4364 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4365 ret void4366}4367 4368define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_local_agent(ptr addrspace(3) %ptr, <2 x bfloat> %value) {4369; CI-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4370; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44371; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4372; CI: atomicrmw.start:4373; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4374; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4375; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324376; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324377; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44378; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14379; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04380; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4381; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4382; CI: atomicrmw.end:4383; CI-NEXT: ret <2 x bfloat> [[TMP5]]4384;4385; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4386; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44387; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4388; GFX9: atomicrmw.start:4389; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4390; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4391; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324392; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324393; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44394; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14395; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04396; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4397; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4398; GFX9: atomicrmw.end:4399; GFX9-NEXT: ret <2 x bfloat> [[TMP5]]4400;4401; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4402; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44403; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4404; GFX908: atomicrmw.start:4405; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4406; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4407; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324408; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324409; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44410; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14411; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04412; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4413; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4414; GFX908: atomicrmw.end:4415; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]4416;4417; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4418; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44419; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4420; GFX90A: atomicrmw.start:4421; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4422; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4423; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324424; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324425; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44426; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14427; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04428; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4429; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4430; GFX90A: atomicrmw.end:4431; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]4432;4433; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4434; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44435; GFX942-NEXT: ret <2 x bfloat> [[RES]]4436;4437; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_local_agent(4438; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44439; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4440; GFX11: atomicrmw.start:4441; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4442; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4443; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324444; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324445; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44446; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14447; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04448; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4449; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4450; GFX11: atomicrmw.end:4451; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]4452;4453 %res = atomicrmw fadd ptr addrspace(3) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4454 ret <2 x bfloat> %res4455}4456 4457define void @test_atomicrmw_fadd_v2bf16_flat_local_noret(ptr addrspace(3) %ptr, <2 x bfloat> %value) {4458; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4459; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44460; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4461; CI: atomicrmw.start:4462; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4463; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4464; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324465; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324466; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44467; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14468; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04469; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4470; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4471; CI: atomicrmw.end:4472; CI-NEXT: ret void4473;4474; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4475; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44476; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4477; GFX9: atomicrmw.start:4478; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4479; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4480; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324481; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324482; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44483; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14484; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04485; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4486; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4487; GFX9: atomicrmw.end:4488; GFX9-NEXT: ret void4489;4490; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4491; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44492; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4493; GFX908: atomicrmw.start:4494; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4495; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4496; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324497; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324498; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44499; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14500; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04501; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4502; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4503; GFX908: atomicrmw.end:4504; GFX908-NEXT: ret void4505;4506; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4507; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44508; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4509; GFX90A: atomicrmw.start:4510; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4511; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4512; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324513; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324514; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44515; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14516; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04517; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4518; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4519; GFX90A: atomicrmw.end:4520; GFX90A-NEXT: ret void4521;4522; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4523; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(3) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44524; GFX942-NEXT: ret void4525;4526; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_local_noret(4527; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(3) [[PTR:%.*]], align 44528; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4529; GFX11: atomicrmw.start:4530; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4531; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4532; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324533; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324534; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(3) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44535; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14536; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04537; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4538; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4539; GFX11: atomicrmw.end:4540; GFX11-NEXT: ret void4541;4542 %res = atomicrmw fadd ptr addrspace(3) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst4543 ret void4544}4545 4546define <2 x half> @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(ptr %ptr, <2 x half> %value) {4547; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4548; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44549; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4550; CI: atomicrmw.start:4551; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4552; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4553; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324554; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324555; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44556; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14557; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04558; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4559; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4560; CI: atomicrmw.end:4561; CI-NEXT: ret <2 x half> [[TMP5]]4562;4563; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4564; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44565; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4566; GFX9: atomicrmw.start:4567; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4568; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4569; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324570; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324571; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44572; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14573; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04574; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4575; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4576; GFX9: atomicrmw.end:4577; GFX9-NEXT: ret <2 x half> [[TMP5]]4578;4579; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4580; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44581; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4582; GFX908: atomicrmw.start:4583; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4584; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4585; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324586; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324587; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44588; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14589; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04590; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4591; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4592; GFX908: atomicrmw.end:4593; GFX908-NEXT: ret <2 x half> [[TMP5]]4594;4595; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4596; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44597; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4598; GFX90A: atomicrmw.start:4599; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4600; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4601; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324602; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324603; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44604; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14605; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04606; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4607; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4608; GFX90A: atomicrmw.end:4609; GFX90A-NEXT: ret <2 x half> [[TMP5]]4610;4611; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4612; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44613; GFX942-NEXT: ret <2 x half> [[RES]]4614;4615; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent__unsafe(4616; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44617; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4618; GFX11: atomicrmw.start:4619; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4620; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4621; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324622; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324623; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44624; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14625; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04626; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4627; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4628; GFX11: atomicrmw.end:4629; GFX11-NEXT: ret <2 x half> [[TMP5]]4630;4631 %res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst4632 ret <2 x half> %res4633}4634 4635define void @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(ptr %ptr, <2 x half> %value) {4636; CI-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4637; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44638; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4639; CI: atomicrmw.start:4640; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4641; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4642; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324643; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324644; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44645; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14646; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04647; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4648; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4649; CI: atomicrmw.end:4650; CI-NEXT: ret void4651;4652; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4653; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44654; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4655; GFX9: atomicrmw.start:4656; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4657; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4658; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324659; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324660; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44661; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14662; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04663; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4664; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4665; GFX9: atomicrmw.end:4666; GFX9-NEXT: ret void4667;4668; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4669; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44670; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4671; GFX908: atomicrmw.start:4672; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4673; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4674; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324675; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324676; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44677; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14678; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04679; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4680; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4681; GFX908: atomicrmw.end:4682; GFX908-NEXT: ret void4683;4684; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4685; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44686; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4687; GFX90A: atomicrmw.start:4688; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4689; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4690; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324691; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324692; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44693; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14694; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04695; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4696; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4697; GFX90A: atomicrmw.end:4698; GFX90A-NEXT: ret void4699;4700; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4701; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 44702; GFX942-NEXT: ret void4703;4704; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_flat_agent_noret__unsafe(4705; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR:%.*]], align 44706; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4707; GFX11: atomicrmw.start:4708; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4709; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4710; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324711; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324712; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 44713; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14714; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04715; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4716; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4717; GFX11: atomicrmw.end:4718; GFX11-NEXT: ret void4719;4720 %res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst4721 ret void4722}4723 4724define <2 x half> @test_atomicrmw_fadd_v2f16_global_agent__unsafe(ptr addrspace(1) %ptr, <2 x half> %value) {4725; CI-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4726; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44727; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4728; CI: atomicrmw.start:4729; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4730; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4731; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324732; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324733; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4734; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14735; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04736; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4737; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4738; CI: atomicrmw.end:4739; CI-NEXT: ret <2 x half> [[TMP5]]4740;4741; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4742; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44743; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4744; GFX9: atomicrmw.start:4745; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4746; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4747; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324748; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324749; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4750; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14751; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04752; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4753; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4754; GFX9: atomicrmw.end:4755; GFX9-NEXT: ret <2 x half> [[TMP5]]4756;4757; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4758; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44759; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4760; GFX908: atomicrmw.start:4761; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4762; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4763; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324764; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324765; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4766; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14767; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04768; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4769; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4770; GFX908: atomicrmw.end:4771; GFX908-NEXT: ret <2 x half> [[TMP5]]4772;4773; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4774; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4775; GFX90A-NEXT: ret <2 x half> [[RES]]4776;4777; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4778; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4779; GFX942-NEXT: ret <2 x half> [[RES]]4780;4781; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_global_agent__unsafe(4782; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44783; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4784; GFX11: atomicrmw.start:4785; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4786; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4787; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324788; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324789; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4790; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14791; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04792; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4793; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4794; GFX11: atomicrmw.end:4795; GFX11-NEXT: ret <2 x half> [[TMP5]]4796;4797 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04798 ret <2 x half> %res4799}4800 4801define void @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(ptr addrspace(1) %ptr, <2 x half> %value) {4802; CI-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4803; CI-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44804; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4805; CI: atomicrmw.start:4806; CI-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4807; CI-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4808; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324809; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324810; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4811; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14812; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04813; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4814; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4815; CI: atomicrmw.end:4816; CI-NEXT: ret void4817;4818; GFX9-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4819; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44820; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4821; GFX9: atomicrmw.start:4822; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4823; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4824; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324825; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324826; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4827; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14828; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04829; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4830; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4831; GFX9: atomicrmw.end:4832; GFX9-NEXT: ret void4833;4834; GFX908-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4835; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4836; GFX908-NEXT: ret void4837;4838; GFX90A-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4839; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4840; GFX90A-NEXT: ret void4841;4842; GFX942-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4843; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x half> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4844; GFX942-NEXT: ret void4845;4846; GFX11-LABEL: @test_atomicrmw_fadd_v2f16_global_agent_noret__unsafe(4847; GFX11-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(1) [[PTR:%.*]], align 44848; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4849; GFX11: atomicrmw.start:4850; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4851; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE:%.*]]4852; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i324853; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i324854; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4855; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14856; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04857; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>4858; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4859; GFX11: atomicrmw.end:4860; GFX11-NEXT: ret void4861;4862 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04863 ret void4864}4865 4866define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(ptr %ptr, <2 x bfloat> %value) {4867; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4868; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44869; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4870; CI: atomicrmw.start:4871; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4872; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4873; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324874; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324875; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4876; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14877; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04878; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4879; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4880; CI: atomicrmw.end:4881; CI-NEXT: ret <2 x bfloat> [[TMP5]]4882;4883; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4884; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44885; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4886; GFX9: atomicrmw.start:4887; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4888; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4889; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324890; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324891; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4892; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14893; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04894; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4895; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4896; GFX9: atomicrmw.end:4897; GFX9-NEXT: ret <2 x bfloat> [[TMP5]]4898;4899; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4900; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44901; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4902; GFX908: atomicrmw.start:4903; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4904; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4905; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324906; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324907; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4908; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14909; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04910; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4911; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4912; GFX908: atomicrmw.end:4913; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]4914;4915; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4916; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44917; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]4918; GFX90A: atomicrmw.start:4919; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4920; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4921; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324922; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324923; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4924; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14925; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04926; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4927; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4928; GFX90A: atomicrmw.end:4929; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]4930;4931; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4932; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4933; GFX942-NEXT: ret <2 x bfloat> [[RES]]4934;4935; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent__unsafe(4936; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44937; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]4938; GFX11: atomicrmw.start:4939; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4940; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4941; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324942; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324943; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4944; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14945; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04946; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4947; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4948; GFX11: atomicrmw.end:4949; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]4950;4951 %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04952 ret <2 x bfloat> %res4953}4954 4955define void @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(ptr %ptr, <2 x bfloat> %value) {4956; CI-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(4957; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44958; CI-NEXT: br label [[ATOMICRMW_START:%.*]]4959; CI: atomicrmw.start:4960; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4961; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4962; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324963; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324964; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4965; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14966; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04967; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4968; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4969; CI: atomicrmw.end:4970; CI-NEXT: ret void4971;4972; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(4973; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44974; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]4975; GFX9: atomicrmw.start:4976; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4977; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4978; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324979; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324980; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4981; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14982; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04983; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>4984; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]4985; GFX9: atomicrmw.end:4986; GFX9-NEXT: ret void4987;4988; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(4989; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 44990; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]4991; GFX908: atomicrmw.start:4992; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]4993; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]4994; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i324995; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i324996; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]4997; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 14998; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 04999; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5000; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5001; GFX908: atomicrmw.end:5002; GFX908-NEXT: ret void5003;5004; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(5005; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 45006; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]5007; GFX90A: atomicrmw.start:5008; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5009; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5010; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325011; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325012; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5013; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15014; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05015; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5016; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5017; GFX90A: atomicrmw.end:5018; GFX90A-NEXT: ret void5019;5020; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(5021; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5022; GFX942-NEXT: ret void5023;5024; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_flat_agent_noret__unsafe(5025; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR:%.*]], align 45026; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]5027; GFX11: atomicrmw.start:5028; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5029; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5030; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325031; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325032; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5033; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15034; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05035; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5036; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5037; GFX11: atomicrmw.end:5038; GFX11-NEXT: ret void5039;5040 %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05041 ret void5042}5043 5044define <2 x bfloat> @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(ptr addrspace(1) %ptr, <2 x bfloat> %value) {5045; CI-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5046; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45047; CI-NEXT: br label [[ATOMICRMW_START:%.*]]5048; CI: atomicrmw.start:5049; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5050; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5051; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325052; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325053; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5054; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15055; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05056; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5057; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5058; CI: atomicrmw.end:5059; CI-NEXT: ret <2 x bfloat> [[TMP5]]5060;5061; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5062; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45063; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]5064; GFX9: atomicrmw.start:5065; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5066; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5067; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325068; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325069; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5070; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15071; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05072; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5073; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5074; GFX9: atomicrmw.end:5075; GFX9-NEXT: ret <2 x bfloat> [[TMP5]]5076;5077; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5078; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45079; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]5080; GFX908: atomicrmw.start:5081; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5082; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5083; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325084; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325085; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5086; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15087; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05088; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5089; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5090; GFX908: atomicrmw.end:5091; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]5092;5093; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5094; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45095; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]5096; GFX90A: atomicrmw.start:5097; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5098; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5099; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325100; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325101; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5102; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15103; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05104; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5105; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5106; GFX90A: atomicrmw.end:5107; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]5108;5109; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5110; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5111; GFX942-NEXT: ret <2 x bfloat> [[RES]]5112;5113; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent__unsafe(5114; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45115; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]5116; GFX11: atomicrmw.start:5117; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5118; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5119; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325120; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325121; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5122; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15123; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05124; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5125; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5126; GFX11: atomicrmw.end:5127; GFX11-NEXT: ret <2 x bfloat> [[TMP5]]5128;5129 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05130 ret <2 x bfloat> %res5131}5132 5133define void @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(ptr addrspace(1) %ptr, <2 x bfloat> %value) {5134; CI-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5135; CI-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45136; CI-NEXT: br label [[ATOMICRMW_START:%.*]]5137; CI: atomicrmw.start:5138; CI-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5139; CI-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5140; CI-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325141; CI-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325142; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5143; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15144; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05145; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5146; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5147; CI: atomicrmw.end:5148; CI-NEXT: ret void5149;5150; GFX9-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5151; GFX9-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45152; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]]5153; GFX9: atomicrmw.start:5154; GFX9-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5155; GFX9-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5156; GFX9-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325157; GFX9-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325158; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5159; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15160; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05161; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5162; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5163; GFX9: atomicrmw.end:5164; GFX9-NEXT: ret void5165;5166; GFX908-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5167; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45168; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]]5169; GFX908: atomicrmw.start:5170; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5171; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5172; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325173; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325174; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5175; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15176; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05177; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5178; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5179; GFX908: atomicrmw.end:5180; GFX908-NEXT: ret void5181;5182; GFX90A-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5183; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45184; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]]5185; GFX90A: atomicrmw.start:5186; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5187; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5188; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325189; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325190; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5191; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15192; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05193; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5194; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5195; GFX90A: atomicrmw.end:5196; GFX90A-NEXT: ret void5197;5198; GFX942-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5199; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5200; GFX942-NEXT: ret void5201;5202; GFX11-LABEL: @test_atomicrmw_fadd_v2bf16_global_agent_noret__unsafe(5203; GFX11-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr addrspace(1) [[PTR:%.*]], align 45204; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]]5205; GFX11: atomicrmw.start:5206; GFX11-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]5207; GFX11-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE:%.*]]5208; GFX11-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i325209; GFX11-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i325210; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]5211; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 15212; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 05213; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>5214; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]5215; GFX11: atomicrmw.end:5216; GFX11-NEXT: ret void5217;5218 %res = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05219 ret void5220}5221 5222attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }5223attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" }5224attributes #2 = { strictfp }5225attributes #3 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" }5226attributes #4 = { "denormal-fp-math-f32"="dynamic,dynamic" }5227attributes #5 = { "denormal-fp-math"="dynamic,dynamic" }5228 5229!0 = !{}5230!1 = !{i32 5, i32 6}5231