2186 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX7 %s3; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX900 %s4; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX908 %s5; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX90A %s6; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX942 %s7; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX942 %s8; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -passes=atomic-expand %s | FileCheck -check-prefixes=ALL,GFX12 %s9 10; --------------------------------------------------------------------11; Idempotent expansion cases without noalias.addrspace12; --------------------------------------------------------------------13 14define i64 @test_flat_atomicrmw_add_0_i64_agent(ptr %ptr) {15; ALL-LABEL: define i64 @test_flat_atomicrmw_add_0_i64_agent(16; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {17; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])18; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]19; ALL: [[ATOMICRMW_PRIVATE]]:20; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)21; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 822; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 023; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 824; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]25; ALL: [[ATOMICRMW_GLOBAL]]:26; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0:![0-9]+]], !amdgpu.no.fine.grained.memory [[META1:![0-9]+]]27; ALL-NEXT: br label %[[ATOMICRMW_PHI]]28; ALL: [[ATOMICRMW_PHI]]:29; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]30; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]31; ALL: [[ATOMICRMW_END]]:32; ALL-NEXT: ret i64 [[RES]]33;34 %res = atomicrmw add ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !035 ret i64 %res36}37 38define i64 @test_flat_atomicrmw_sub_0_i64_agent(ptr %ptr) {39; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_0_i64_agent(40; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {41; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])42; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]43; ALL: [[ATOMICRMW_PRIVATE]]:44; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)45; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 846; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 047; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 848; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]49; ALL: [[ATOMICRMW_GLOBAL]]:50; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]51; ALL-NEXT: br label %[[ATOMICRMW_PHI]]52; ALL: [[ATOMICRMW_PHI]]:53; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]54; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]55; ALL: [[ATOMICRMW_END]]:56; ALL-NEXT: ret i64 [[RES]]57;58 %res = atomicrmw sub ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !059 ret i64 %res60}61 62define i64 @test_flat_atomicrmw_or_0_i64_agent(ptr %ptr) {63; ALL-LABEL: define i64 @test_flat_atomicrmw_or_0_i64_agent(64; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {65; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])66; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]67; ALL: [[ATOMICRMW_PRIVATE]]:68; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)69; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 870; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 071; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 872; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]73; ALL: [[ATOMICRMW_GLOBAL]]:74; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]75; ALL-NEXT: br label %[[ATOMICRMW_PHI]]76; ALL: [[ATOMICRMW_PHI]]:77; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]78; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]79; ALL: [[ATOMICRMW_END]]:80; ALL-NEXT: ret i64 [[RES]]81;82 %res = atomicrmw or ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !083 ret i64 %res84}85 86define i64 @test_flat_atomicrmw_xor_0_i64_agent(ptr %ptr) {87; ALL-LABEL: define i64 @test_flat_atomicrmw_xor_0_i64_agent(88; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {89; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])90; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]91; ALL: [[ATOMICRMW_PRIVATE]]:92; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)93; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 894; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], 095; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 896; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]97; ALL: [[ATOMICRMW_GLOBAL]]:98; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]99; ALL-NEXT: br label %[[ATOMICRMW_PHI]]100; ALL: [[ATOMICRMW_PHI]]:101; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]102; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]103; ALL: [[ATOMICRMW_END]]:104; ALL-NEXT: ret i64 [[RES]]105;106 %res = atomicrmw xor ptr %ptr, i64 0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0107 ret i64 %res108}109 110; --------------------------------------------------------------------111; Idempotent expansion cases with noalias.addrspace112; --------------------------------------------------------------------113 114define i64 @test_flat_atomicrmw_add_0_i64_agent__noalias_addrspace_5(ptr %ptr) {115; ALL-LABEL: define i64 @test_flat_atomicrmw_add_0_i64_agent__noalias_addrspace_5(116; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {117; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]118; ALL-NEXT: ret i64 [[RES]]119;120 %res = atomicrmw add ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0121 ret i64 %res122}123 124define i64 @test_flat_atomicrmw_sub_0_i64_agent__noalias_addrspace_5(ptr %ptr) {125; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_0_i64_agent__noalias_addrspace_5(126; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {127; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]128; ALL-NEXT: ret i64 [[RES]]129;130 %res = atomicrmw sub ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0131 ret i64 %res132}133 134define i64 @test_flat_atomicrmw_or_0_i64_agent__noalias_addrspace_5(ptr %ptr) {135; ALL-LABEL: define i64 @test_flat_atomicrmw_or_0_i64_agent__noalias_addrspace_5(136; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {137; ALL-NEXT: [[RES:%.*]] = atomicrmw or ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]138; ALL-NEXT: ret i64 [[RES]]139;140 %res = atomicrmw or ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0141 ret i64 %res142}143 144define i64 @test_flat_atomicrmw_xor_0_i64_agent__noalias_addrspace_5(ptr %ptr) {145; ALL-LABEL: define i64 @test_flat_atomicrmw_xor_0_i64_agent__noalias_addrspace_5(146; ALL-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {147; ALL-NEXT: [[RES:%.*]] = atomicrmw xor ptr [[PTR]], i64 0 syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]148; ALL-NEXT: ret i64 [[RES]]149;150 %res = atomicrmw xor ptr %ptr, i64 0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0151 ret i64 %res152}153 154; --------------------------------------------------------------------155; General expansion for add156; --------------------------------------------------------------------157 158define i64 @test_flat_atomicrmw_add_i64_agent(ptr %ptr, i64 %value) {159; ALL-LABEL: define i64 @test_flat_atomicrmw_add_i64_agent(160; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {161; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])162; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]163; ALL: [[ATOMICRMW_PRIVATE]]:164; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)165; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8166; ALL-NEXT: [[NEW:%.*]] = add i64 [[LOADED_PRIVATE]], [[VALUE]]167; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8168; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]169; ALL: [[ATOMICRMW_GLOBAL]]:170; ALL-NEXT: [[TMP2:%.*]] = atomicrmw add ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]171; ALL-NEXT: br label %[[ATOMICRMW_PHI]]172; ALL: [[ATOMICRMW_PHI]]:173; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]174; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]175; ALL: [[ATOMICRMW_END]]:176; ALL-NEXT: ret i64 [[RES]]177;178 %res = atomicrmw add ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0179 ret i64 %res180}181 182define i64 @test_flat_atomicrmw_add_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {183; ALL-LABEL: define i64 @test_flat_atomicrmw_add_i64_agent__noalias_addrspace_5(184; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {185; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]186; ALL-NEXT: ret i64 [[RES]]187;188 %res = atomicrmw add ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0189 ret i64 %res190}191 192define i32 @test_flat_atomicrmw_add_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {193; ALL-LABEL: define i32 @test_flat_atomicrmw_add_i32_agent__noalias_addrspace_5(194; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {195; ALL-NEXT: [[RES:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]196; ALL-NEXT: ret i32 [[RES]]197;198 %res = atomicrmw add ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0199 ret i32 %res200}201 202; --------------------------------------------------------------------203; General expansion for xchg204; --------------------------------------------------------------------205 206define i64 @test_flat_atomicrmw_xchg_i64_agent(ptr %ptr, i64 %value) {207; ALL-LABEL: define i64 @test_flat_atomicrmw_xchg_i64_agent(208; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {209; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])210; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]211; ALL: [[ATOMICRMW_PRIVATE]]:212; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)213; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8214; ALL-NEXT: store i64 [[VALUE]], ptr addrspace(5) [[TMP1]], align 8215; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]216; ALL: [[ATOMICRMW_GLOBAL]]:217; ALL-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]218; ALL-NEXT: br label %[[ATOMICRMW_PHI]]219; ALL: [[ATOMICRMW_PHI]]:220; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]221; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]222; ALL: [[ATOMICRMW_END]]:223; ALL-NEXT: ret i64 [[RES]]224;225 %res = atomicrmw xchg ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0226 ret i64 %res227}228 229define i64 @test_flat_atomicrmw_xchg_i64_agent__noalias_xchgrspace_5(ptr %ptr, i64 %value) {230; ALL-LABEL: define i64 @test_flat_atomicrmw_xchg_i64_agent__noalias_xchgrspace_5(231; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {232; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]233; ALL-NEXT: ret i64 [[RES]]234;235 %res = atomicrmw xchg ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0236 ret i64 %res237}238 239define i32 @test_flat_atomicrmw_xchg_i32_agent__noalias_xchgrspace_5(ptr %ptr, i32 %value) {240; ALL-LABEL: define i32 @test_flat_atomicrmw_xchg_i32_agent__noalias_xchgrspace_5(241; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {242; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]243; ALL-NEXT: ret i32 [[RES]]244;245 %res = atomicrmw xchg ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0246 ret i32 %res247}248 249; --------------------------------------------------------------------250; General expansion for xchg (pointer type)251; --------------------------------------------------------------------252 253define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent(ptr %ptr, ptr addrspace(1) %value) {254; ALL-LABEL: define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent(255; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(1) [[VALUE:%.*]]) #[[ATTR0]] {256; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])257; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]258; ALL: [[ATOMICRMW_PRIVATE]]:259; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)260; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[TMP1]], align 8261; ALL-NEXT: store ptr addrspace(1) [[VALUE]], ptr addrspace(5) [[TMP1]], align 8262; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]263; ALL: [[ATOMICRMW_GLOBAL]]:264; ALL-NEXT: [[TMP2:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(1) [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]265; ALL-NEXT: br label %[[ATOMICRMW_PHI]]266; ALL: [[ATOMICRMW_PHI]]:267; ALL-NEXT: [[RES:%.*]] = phi ptr addrspace(1) [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]268; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]269; ALL: [[ATOMICRMW_END]]:270; ALL-NEXT: ret ptr addrspace(1) [[RES]]271;272 %res = atomicrmw xchg ptr %ptr, ptr addrspace(1) %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0273 ret ptr addrspace(1) %res274}275 276define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent__noalias_xchgrspace_5(ptr %ptr, ptr addrspace(1) %value) {277; ALL-LABEL: define ptr addrspace(1) @test_flat_atomicrmw_xchg_p1_agent__noalias_xchgrspace_5(278; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(1) [[VALUE:%.*]]) #[[ATTR0]] {279; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(1) [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]280; ALL-NEXT: ret ptr addrspace(1) [[RES]]281;282 %res = atomicrmw xchg ptr %ptr, ptr addrspace(1) %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0283 ret ptr addrspace(1) %res284}285 286define ptr addrspace(3) @test_flat_atomicrmw_xchg_p3_agent__noalias_xchgrspace_5(ptr %ptr, ptr addrspace(3) %value) {287; ALL-LABEL: define ptr addrspace(3) @test_flat_atomicrmw_xchg_p3_agent__noalias_xchgrspace_5(288; ALL-SAME: ptr [[PTR:%.*]], ptr addrspace(3) [[VALUE:%.*]]) #[[ATTR0]] {289; ALL-NEXT: [[RES:%.*]] = atomicrmw xchg ptr [[PTR]], ptr addrspace(3) [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]290; ALL-NEXT: ret ptr addrspace(3) [[RES]]291;292 %res = atomicrmw xchg ptr %ptr, ptr addrspace(3) %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0293 ret ptr addrspace(3) %res294}295 296; --------------------------------------------------------------------297; General expansion for and298; --------------------------------------------------------------------299 300define i64 @test_flat_atomicrmw_and_i64_agent(ptr %ptr, i64 %value) {301; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent(302; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {303; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])304; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]305; ALL: [[ATOMICRMW_PRIVATE]]:306; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)307; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8308; ALL-NEXT: [[NEW:%.*]] = and i64 [[LOADED_PRIVATE]], [[VALUE]]309; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8310; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]311; ALL: [[ATOMICRMW_GLOBAL]]:312; ALL-NEXT: [[TMP2:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]313; ALL-NEXT: br label %[[ATOMICRMW_PHI]]314; ALL: [[ATOMICRMW_PHI]]:315; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]316; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]317; ALL: [[ATOMICRMW_END]]:318; ALL-NEXT: ret i64 [[RES]]319;320 %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0321 ret i64 %res322}323 324define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {325; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5(326; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {327; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]328; ALL-NEXT: ret i64 [[RES]]329;330 %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0331 ret i64 %res332}333 334define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {335; GFX7-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(336; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {337; GFX7-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8338; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]339; GFX7: [[ATOMICRMW_START]]:340; GFX7-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]341; GFX7-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]342; GFX7-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]343; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1344; GFX7-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0345; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]346; GFX7: [[ATOMICRMW_END]]:347; GFX7-NEXT: ret i64 [[NEWLOADED]]348;349; GFX900-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(350; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {351; GFX900-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8352; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]353; GFX900: [[ATOMICRMW_START]]:354; GFX900-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]355; GFX900-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]356; GFX900-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]357; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1358; GFX900-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0359; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]360; GFX900: [[ATOMICRMW_END]]:361; GFX900-NEXT: ret i64 [[NEWLOADED]]362;363; GFX908-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(364; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {365; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8366; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]367; GFX908: [[ATOMICRMW_START]]:368; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]369; GFX908-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]370; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]371; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1372; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0373; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]374; GFX908: [[ATOMICRMW_END]]:375; GFX908-NEXT: ret i64 [[NEWLOADED]]376;377; GFX90A-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(378; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {379; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8380; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]381; GFX90A: [[ATOMICRMW_START]]:382; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]383; GFX90A-NEXT: [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]384; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]385; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1386; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0387; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]388; GFX90A: [[ATOMICRMW_END]]:389; GFX90A-NEXT: ret i64 [[NEWLOADED]]390;391; GFX940-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(392; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {393; GFX940-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]394; GFX940-NEXT: ret i64 [[RES]]395;396; GFX12-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(397; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {398; GFX12-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]399; GFX12-NEXT: ret i64 [[RES]]400;401 %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1402 ret i64 %res403}404 405define i32 @test_flat_atomicrmw_and_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {406; ALL-LABEL: define i32 @test_flat_atomicrmw_and_i32_agent__noalias_addrspace_5(407; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {408; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]409; ALL-NEXT: ret i32 [[RES]]410;411 %res = atomicrmw and ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0412 ret i32 %res413}414 415define i64 @test_flat_atomicrmw_and_i64_agent__mmra(ptr %ptr, i64 %value) {416; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__mmra(417; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {418; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])419; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]420; ALL: [[ATOMICRMW_PRIVATE]]:421; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)422; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8423; ALL-NEXT: [[NEW:%.*]] = and i64 [[LOADED_PRIVATE]], [[VALUE]]424; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8425; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]426; ALL: [[ATOMICRMW_GLOBAL]]:427; ALL-NEXT: [[TMP2:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2:![0-9]+]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]428; ALL-NEXT: br label %[[ATOMICRMW_PHI]]429; ALL: [[ATOMICRMW_PHI]]:430; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]431; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]432; ALL: [[ATOMICRMW_END]]:433; ALL-NEXT: ret i64 [[RES]]434;435 %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !mmra !4, !amdgpu.no.fine.grained.memory !0436 ret i64 %res437}438 439define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__mmra(ptr %ptr, i64 %value) {440; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__mmra(441; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {442; ALL-NEXT: [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]443; ALL-NEXT: ret i64 [[RES]]444;445 %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !mmra !4, !amdgpu.no.fine.grained.memory !0446 ret i64 %res447}448 449; --------------------------------------------------------------------450; General expansion for subb451; --------------------------------------------------------------------452 453define i64 @test_flat_atomicrmw_sub_i64_agent(ptr %ptr, i64 %value) {454; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent(455; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {456; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])457; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]458; ALL: [[ATOMICRMW_PRIVATE]]:459; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)460; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8461; ALL-NEXT: [[NEW:%.*]] = sub i64 [[LOADED_PRIVATE]], [[VALUE]]462; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8463; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]464; ALL: [[ATOMICRMW_GLOBAL]]:465; ALL-NEXT: [[TMP2:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]466; ALL-NEXT: br label %[[ATOMICRMW_PHI]]467; ALL: [[ATOMICRMW_PHI]]:468; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]469; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]470; ALL: [[ATOMICRMW_END]]:471; ALL-NEXT: ret i64 [[RES]]472;473 %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0474 ret i64 %res475}476 477define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {478; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5(479; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {480; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]481; ALL-NEXT: ret i64 [[RES]]482;483 %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0484 ret i64 %res485}486 487define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {488; GFX7-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(489; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {490; GFX7-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8491; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]492; GFX7: [[ATOMICRMW_START]]:493; GFX7-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]494; GFX7-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]495; GFX7-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]496; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1497; GFX7-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0498; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]499; GFX7: [[ATOMICRMW_END]]:500; GFX7-NEXT: ret i64 [[NEWLOADED]]501;502; GFX900-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(503; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {504; GFX900-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8505; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]506; GFX900: [[ATOMICRMW_START]]:507; GFX900-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]508; GFX900-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]509; GFX900-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]510; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1511; GFX900-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0512; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]513; GFX900: [[ATOMICRMW_END]]:514; GFX900-NEXT: ret i64 [[NEWLOADED]]515;516; GFX908-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(517; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {518; GFX908-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8519; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]520; GFX908: [[ATOMICRMW_START]]:521; GFX908-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]522; GFX908-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]523; GFX908-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]524; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1525; GFX908-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0526; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]527; GFX908: [[ATOMICRMW_END]]:528; GFX908-NEXT: ret i64 [[NEWLOADED]]529;530; GFX90A-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(531; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {532; GFX90A-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8533; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]534; GFX90A: [[ATOMICRMW_START]]:535; GFX90A-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]536; GFX90A-NEXT: [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]537; GFX90A-NEXT: [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]538; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1539; GFX90A-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0540; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]541; GFX90A: [[ATOMICRMW_END]]:542; GFX90A-NEXT: ret i64 [[NEWLOADED]]543;544; GFX940-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(545; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {546; GFX940-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]547; GFX940-NEXT: ret i64 [[RES]]548;549; GFX12-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(550; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {551; GFX12-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]552; GFX12-NEXT: ret i64 [[RES]]553;554 %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1555 ret i64 %res556}557 558define i32 @test_flat_atomicrmw_sub_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {559; ALL-LABEL: define i32 @test_flat_atomicrmw_sub_i32_agent__noalias_addrspace_5(560; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {561; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]562; ALL-NEXT: ret i32 [[RES]]563;564 %res = atomicrmw sub ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0565 ret i32 %res566}567 568define i64 @test_flat_atomicrmw_sub_i64_agent__mmra(ptr %ptr, i64 %value) {569; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__mmra(570; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {571; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])572; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]573; ALL: [[ATOMICRMW_PRIVATE]]:574; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)575; ALL-NEXT: [[LOADED_PRIVATE:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8576; ALL-NEXT: [[NEW:%.*]] = sub i64 [[LOADED_PRIVATE]], [[VALUE]]577; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 8578; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]579; ALL: [[ATOMICRMW_GLOBAL]]:580; ALL-NEXT: [[TMP2:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]581; ALL-NEXT: br label %[[ATOMICRMW_PHI]]582; ALL: [[ATOMICRMW_PHI]]:583; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ]584; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]585; ALL: [[ATOMICRMW_END]]:586; ALL-NEXT: ret i64 [[RES]]587;588 %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !mmra !4, !amdgpu.no.fine.grained.memory !0589 ret i64 %res590}591 592define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__mmra(ptr %ptr, i64 %value) {593; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__mmra(594; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {595; ALL-NEXT: [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !mmra [[META2]], !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]596; ALL-NEXT: ret i64 [[RES]]597;598 %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !mmra !4, !amdgpu.no.fine.grained.memory !0599 ret i64 %res600}601 602; --------------------------------------------------------------------603; General expansion for fadd604; --------------------------------------------------------------------605 606define double @test_flat_atomicrmw_fadd_f64_agent(ptr %ptr, double %value) {607; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(608; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {609; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])610; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]611; GFX7: [[ATOMICRMW_PRIVATE]]:612; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)613; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8614; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]615; GFX7-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8616; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]617; GFX7: [[ATOMICRMW_GLOBAL]]:618; GFX7-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8619; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]620; GFX7: [[ATOMICRMW_START]]:621; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]622; GFX7-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]623; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64624; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64625; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]626; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1627; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0628; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double629; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]630; GFX7: [[ATOMICRMW_END1]]:631; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]632; GFX7: [[ATOMICRMW_PHI]]:633; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]634; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]635; GFX7: [[ATOMICRMW_END]]:636; GFX7-NEXT: ret double [[RES]]637;638; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(639; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {640; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])641; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]642; GFX900: [[ATOMICRMW_PRIVATE]]:643; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)644; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8645; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]646; GFX900-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8647; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]648; GFX900: [[ATOMICRMW_GLOBAL]]:649; GFX900-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8650; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]651; GFX900: [[ATOMICRMW_START]]:652; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]653; GFX900-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]654; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64655; GFX900-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64656; GFX900-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]657; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1658; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0659; GFX900-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double660; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]661; GFX900: [[ATOMICRMW_END1]]:662; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]663; GFX900: [[ATOMICRMW_PHI]]:664; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]665; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]666; GFX900: [[ATOMICRMW_END]]:667; GFX900-NEXT: ret double [[RES]]668;669; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(670; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {671; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])672; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]673; GFX908: [[ATOMICRMW_PRIVATE]]:674; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)675; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8676; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]677; GFX908-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8678; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]679; GFX908: [[ATOMICRMW_GLOBAL]]:680; GFX908-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8681; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]682; GFX908: [[ATOMICRMW_START]]:683; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]684; GFX908-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]685; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64686; GFX908-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64687; GFX908-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]688; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1689; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0690; GFX908-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double691; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]692; GFX908: [[ATOMICRMW_END1]]:693; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]694; GFX908: [[ATOMICRMW_PHI]]:695; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]696; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]697; GFX908: [[ATOMICRMW_END]]:698; GFX908-NEXT: ret double [[RES]]699;700; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(701; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {702; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])703; GFX90A-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]704; GFX90A: [[ATOMICRMW_SHARED]]:705; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)706; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]707; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]708; GFX90A: [[ATOMICRMW_CHECK_PRIVATE]]:709; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])710; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]711; GFX90A: [[ATOMICRMW_PRIVATE]]:712; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)713; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8714; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]715; GFX90A-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8716; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]717; GFX90A: [[ATOMICRMW_GLOBAL]]:718; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)719; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]720; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]721; GFX90A: [[ATOMICRMW_PHI]]:722; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]723; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]724; GFX90A: [[ATOMICRMW_END]]:725; GFX90A-NEXT: ret double [[RES]]726;727; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(728; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {729; GFX942-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])730; GFX942-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]731; GFX942: [[ATOMICRMW_SHARED]]:732; GFX942-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)733; GFX942-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP3]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]734; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]735; GFX942: [[ATOMICRMW_CHECK_PRIVATE]]:736; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])737; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]738; GFX942: [[ATOMICRMW_PRIVATE]]:739; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)740; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8741; GFX942-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]742; GFX942-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8743; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]744; GFX942: [[ATOMICRMW_GLOBAL]]:745; GFX942-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)746; GFX942-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META1]]747; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]748; GFX942: [[ATOMICRMW_PHI]]:749; GFX942-NEXT: [[RES:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]750; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]751; GFX942: [[ATOMICRMW_END]]:752; GFX942-NEXT: ret double [[RES]]753;754; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent(755; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {756; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])757; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]758; GFX12: [[ATOMICRMW_PRIVATE]]:759; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)760; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 8761; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED_PRIVATE]], [[VALUE]]762; GFX12-NEXT: store double [[NEW]], ptr addrspace(5) [[TMP1]], align 8763; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]764; GFX12: [[ATOMICRMW_GLOBAL]]:765; GFX12-NEXT: [[TMP2:%.*]] = load double, ptr [[PTR]], align 8766; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]767; GFX12: [[ATOMICRMW_START]]:768; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP2]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]769; GFX12-NEXT: [[NEW2:%.*]] = fadd double [[LOADED]], [[VALUE]]770; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[NEW2]] to i64771; GFX12-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64772; GFX12-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]773; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1774; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0775; GFX12-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double776; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]777; GFX12: [[ATOMICRMW_END1]]:778; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]779; GFX12: [[ATOMICRMW_PHI]]:780; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP6]], %[[ATOMICRMW_END1]] ]781; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]782; GFX12: [[ATOMICRMW_END]]:783; GFX12-NEXT: ret double [[RES]]784;785 %res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0786 ret double %res787}788 789define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {790; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(791; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {792; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8793; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]794; GFX7: [[ATOMICRMW_START]]:795; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]796; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]797; GFX7-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64798; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64799; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]800; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1801; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0802; GFX7-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double803; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]804; GFX7: [[ATOMICRMW_END]]:805; GFX7-NEXT: ret double [[TMP5]]806;807; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(808; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {809; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8810; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]811; GFX900: [[ATOMICRMW_START]]:812; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]813; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]814; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64815; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64816; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]817; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1818; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0819; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double820; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]821; GFX900: [[ATOMICRMW_END]]:822; GFX900-NEXT: ret double [[TMP5]]823;824; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(825; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {826; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8827; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]828; GFX908: [[ATOMICRMW_START]]:829; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]830; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]831; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64832; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64833; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]834; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1835; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0836; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double837; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]838; GFX908: [[ATOMICRMW_END]]:839; GFX908-NEXT: ret double [[RES]]840;841; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(842; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {843; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]844; GFX90A-NEXT: ret double [[RES]]845;846; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(847; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {848; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]849; GFX942-NEXT: ret double [[RES]]850;851; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5(852; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {853; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8854; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]855; GFX12: [[ATOMICRMW_START]]:856; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]857; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]858; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64859; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64860; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]861; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1862; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0863; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double864; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]865; GFX12: [[ATOMICRMW_END]]:866; GFX12-NEXT: ret double [[TMP5]]867;868 %res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0869 ret double %res870}871 872define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {873; GFX7-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(874; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {875; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8876; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]877; GFX7: [[ATOMICRMW_START]]:878; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]879; GFX7-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]880; GFX7-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64881; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64882; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]883; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1884; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0885; GFX7-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double886; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]887; GFX7: [[ATOMICRMW_END]]:888; GFX7-NEXT: ret double [[TMP5]]889;890; GFX900-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(891; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {892; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8893; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]894; GFX900: [[ATOMICRMW_START]]:895; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]896; GFX900-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]897; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64898; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64899; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]900; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1901; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0902; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double903; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]904; GFX900: [[ATOMICRMW_END]]:905; GFX900-NEXT: ret double [[TMP5]]906;907; GFX908-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(908; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {909; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8910; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]911; GFX908: [[ATOMICRMW_START]]:912; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]913; GFX908-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]914; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64915; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64916; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]917; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1918; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0919; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double920; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]921; GFX908: [[ATOMICRMW_END]]:922; GFX908-NEXT: ret double [[TMP5]]923;924; GFX90A-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(925; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {926; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8927; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]928; GFX90A: [[ATOMICRMW_START]]:929; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]930; GFX90A-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]931; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64932; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64933; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]934; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1935; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0936; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double937; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]938; GFX90A: [[ATOMICRMW_END]]:939; GFX90A-NEXT: ret double [[TMP5]]940;941; GFX942-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(942; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {943; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]944; GFX942-NEXT: ret double [[RES]]945;946; GFX12-LABEL: define double @test_flat_atomicrmw_fadd_f64_agent__noalias_addrspace_5__maybe_fine_grained(947; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {948; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 8949; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]950; GFX12: [[ATOMICRMW_START]]:951; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]952; GFX12-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE]]953; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64954; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64955; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]956; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1957; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0958; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double959; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]960; GFX12: [[ATOMICRMW_END]]:961; GFX12-NEXT: ret double [[TMP5]]962;963 %res = atomicrmw fadd ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1964 ret double %res965}966 967define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {968; GFX7-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(969; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {970; GFX7-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4971; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]972; GFX7: [[ATOMICRMW_START]]:973; GFX7-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]974; GFX7-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]975; GFX7-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32976; GFX7-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32977; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]978; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1979; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0980; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float981; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]982; GFX7: [[ATOMICRMW_END]]:983; GFX7-NEXT: ret float [[TMP5]]984;985; GFX900-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(986; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {987; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 4988; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]989; GFX900: [[ATOMICRMW_START]]:990; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]991; GFX900-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]992; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32993; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32994; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]995; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1996; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0997; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float998; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]999; GFX900: [[ATOMICRMW_END]]:1000; GFX900-NEXT: ret float [[TMP5]]1001;1002; GFX908-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(1003; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1004; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41005; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1006; GFX908: [[ATOMICRMW_START]]:1007; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1008; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE]]1009; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321010; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321011; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1012; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11013; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01014; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1015; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1016; GFX908: [[ATOMICRMW_END]]:1017; GFX908-NEXT: ret float [[TMP5]]1018;1019; GFX90A-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(1020; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1021; GFX90A-NEXT: [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[PTR]])1022; GFX90A-NEXT: br i1 [[IS_SHARED]], label %[[ATOMICRMW_SHARED:.*]], label %[[ATOMICRMW_CHECK_PRIVATE:.*]]1023; GFX90A: [[ATOMICRMW_SHARED]]:1024; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(3)1025; GFX90A-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(3) [[TMP1]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]1026; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]1027; GFX90A: [[ATOMICRMW_CHECK_PRIVATE]]:1028; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1029; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1030; GFX90A: [[ATOMICRMW_PRIVATE]]:1031; GFX90A-NEXT: [[TMP3:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1032; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load float, ptr addrspace(5) [[TMP3]], align 41033; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED_PRIVATE]], [[VALUE]]1034; GFX90A-NEXT: store float [[NEW]], ptr addrspace(5) [[TMP3]], align 41035; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]1036; GFX90A: [[ATOMICRMW_GLOBAL]]:1037; GFX90A-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(1)1038; GFX90A-NEXT: [[TMP5:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP4]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]1039; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]1040; GFX90A: [[ATOMICRMW_PHI]]:1041; GFX90A-NEXT: [[LOADED_PHI:%.*]] = phi float [ [[TMP2]], %[[ATOMICRMW_SHARED]] ], [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP5]], %[[ATOMICRMW_GLOBAL]] ]1042; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]1043; GFX90A: [[ATOMICRMW_END]]:1044; GFX90A-NEXT: ret float [[LOADED_PHI]]1045;1046; GFX942-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(1047; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1048; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]1049; GFX942-NEXT: ret float [[RES]]1050;1051; GFX12-LABEL: define float @test_flat_atomicrmw_fadd_f32_agent__noalias_addrspace_5(1052; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1053; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]], !amdgpu.ignore.denormal.mode [[META1]]1054; GFX12-NEXT: ret float [[RES]]1055;1056 %res = atomicrmw fadd ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !01057 ret float %res1058}1059 1060define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(ptr %ptr, <2 x half> %value) {1061; GFX7-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1062; GFX7-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1063; GFX7-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 41064; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]1065; GFX7: [[ATOMICRMW_START]]:1066; GFX7-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1067; GFX7-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]1068; GFX7-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i321069; GFX7-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i321070; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1071; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11072; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01073; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>1074; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1075; GFX7: [[ATOMICRMW_END]]:1076; GFX7-NEXT: ret <2 x half> [[TMP5]]1077;1078; GFX900-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1079; GFX900-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1080; GFX900-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 41081; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1082; GFX900: [[ATOMICRMW_START]]:1083; GFX900-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1084; GFX900-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]1085; GFX900-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i321086; GFX900-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i321087; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1088; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11089; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01090; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>1091; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1092; GFX900: [[ATOMICRMW_END]]:1093; GFX900-NEXT: ret <2 x half> [[TMP5]]1094;1095; GFX908-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1096; GFX908-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1097; GFX908-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 41098; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1099; GFX908: [[ATOMICRMW_START]]:1100; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1101; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]1102; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i321103; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i321104; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1105; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11106; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01107; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>1108; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1109; GFX908: [[ATOMICRMW_END]]:1110; GFX908-NEXT: ret <2 x half> [[TMP5]]1111;1112; GFX90A-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1113; GFX90A-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1114; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[PTR]], align 41115; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]1116; GFX90A: [[ATOMICRMW_START]]:1117; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x half> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1118; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x half> [[LOADED]], [[VALUE]]1119; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x half> [[NEW]] to i321120; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x half> [[LOADED]] to i321121; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1122; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11123; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01124; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x half>1125; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1126; GFX90A: [[ATOMICRMW_END]]:1127; GFX90A-NEXT: ret <2 x half> [[TMP5]]1128;1129; GFX942-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1130; GFX942-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1131; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x half> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1132; GFX942-NEXT: ret <2 x half> [[RES]]1133;1134; GFX12-LABEL: define <2 x half> @test_flat_atomicrmw_fadd_v2f16_agent__noalias_addrspace_5(1135; GFX12-SAME: ptr [[PTR:%.*]], <2 x half> [[VALUE:%.*]]) #[[ATTR0]] {1136; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x half> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1137; GFX12-NEXT: ret <2 x half> [[RES]]1138;1139 %res = atomicrmw fadd ptr %ptr, <2 x half> %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01140 ret <2 x half> %res1141}1142 1143define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(ptr %ptr, <2 x bfloat> %value) {1144; GFX7-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1145; GFX7-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1146; GFX7-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 41147; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]1148; GFX7: [[ATOMICRMW_START]]:1149; GFX7-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1150; GFX7-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]1151; GFX7-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i321152; GFX7-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i321153; GFX7-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1154; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11155; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01156; GFX7-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>1157; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1158; GFX7: [[ATOMICRMW_END]]:1159; GFX7-NEXT: ret <2 x bfloat> [[TMP5]]1160;1161; GFX900-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1162; GFX900-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1163; GFX900-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 41164; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1165; GFX900: [[ATOMICRMW_START]]:1166; GFX900-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1167; GFX900-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]1168; GFX900-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i321169; GFX900-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i321170; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1171; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11172; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01173; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>1174; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1175; GFX900: [[ATOMICRMW_END]]:1176; GFX900-NEXT: ret <2 x bfloat> [[TMP5]]1177;1178; GFX908-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1179; GFX908-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1180; GFX908-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 41181; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1182; GFX908: [[ATOMICRMW_START]]:1183; GFX908-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1184; GFX908-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]1185; GFX908-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i321186; GFX908-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i321187; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1188; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11189; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01190; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>1191; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1192; GFX908: [[ATOMICRMW_END]]:1193; GFX908-NEXT: ret <2 x bfloat> [[TMP5]]1194;1195; GFX90A-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1196; GFX90A-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1197; GFX90A-NEXT: [[TMP1:%.*]] = load <2 x bfloat>, ptr [[PTR]], align 41198; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]1199; GFX90A: [[ATOMICRMW_START]]:1200; GFX90A-NEXT: [[LOADED:%.*]] = phi <2 x bfloat> [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1201; GFX90A-NEXT: [[NEW:%.*]] = fadd <2 x bfloat> [[LOADED]], [[VALUE]]1202; GFX90A-NEXT: [[TMP2:%.*]] = bitcast <2 x bfloat> [[NEW]] to i321203; GFX90A-NEXT: [[TMP3:%.*]] = bitcast <2 x bfloat> [[LOADED]] to i321204; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1205; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11206; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01207; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to <2 x bfloat>1208; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1209; GFX90A: [[ATOMICRMW_END]]:1210; GFX90A-NEXT: ret <2 x bfloat> [[TMP5]]1211;1212; GFX942-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1213; GFX942-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1214; GFX942-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x bfloat> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1215; GFX942-NEXT: ret <2 x bfloat> [[RES]]1216;1217; GFX12-LABEL: define <2 x bfloat> @test_flat_atomicrmw_fadd_v2bf16_agent__noalias_addrspace_5(1218; GFX12-SAME: ptr [[PTR:%.*]], <2 x bfloat> [[VALUE:%.*]]) #[[ATTR0]] {1219; GFX12-NEXT: [[RES:%.*]] = atomicrmw fadd ptr [[PTR]], <2 x bfloat> [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1220; GFX12-NEXT: ret <2 x bfloat> [[RES]]1221;1222 %res = atomicrmw fadd ptr %ptr, <2 x bfloat> %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01223 ret <2 x bfloat> %res1224}1225 1226; --------------------------------------------------------------------1227; General expansion for fmin1228; --------------------------------------------------------------------1229 1230define double @test_flat_atomicrmw_fmin_f64_agent(ptr %ptr, double %value) {1231; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1232; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1233; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1234; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1235; GFX7: [[ATOMICRMW_PRIVATE]]:1236; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1237; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81238; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1239; GFX7-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81240; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]1241; GFX7: [[ATOMICRMW_GLOBAL]]:1242; GFX7-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1243; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]1244; GFX7: [[ATOMICRMW_PHI]]:1245; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1246; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]1247; GFX7: [[ATOMICRMW_END]]:1248; GFX7-NEXT: ret double [[RES]]1249;1250; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1251; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1252; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1253; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1254; GFX900: [[ATOMICRMW_PRIVATE]]:1255; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1256; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81257; GFX900-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1258; GFX900-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81259; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]1260; GFX900: [[ATOMICRMW_GLOBAL]]:1261; GFX900-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81262; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1263; GFX900: [[ATOMICRMW_START]]:1264; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1265; GFX900-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1266; GFX900-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641267; GFX900-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641268; GFX900-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1269; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11270; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01271; GFX900-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1272; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1273; GFX900: [[ATOMICRMW_END1]]:1274; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]1275; GFX900: [[ATOMICRMW_PHI]]:1276; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1277; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]1278; GFX900: [[ATOMICRMW_END]]:1279; GFX900-NEXT: ret double [[RES]]1280;1281; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1282; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1283; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1284; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1285; GFX908: [[ATOMICRMW_PRIVATE]]:1286; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1287; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81288; GFX908-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1289; GFX908-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81290; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]1291; GFX908: [[ATOMICRMW_GLOBAL]]:1292; GFX908-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81293; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1294; GFX908: [[ATOMICRMW_START]]:1295; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1296; GFX908-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1297; GFX908-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641298; GFX908-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641299; GFX908-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1300; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11301; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01302; GFX908-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1303; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1304; GFX908: [[ATOMICRMW_END1]]:1305; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]1306; GFX908: [[ATOMICRMW_PHI]]:1307; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1308; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]1309; GFX908: [[ATOMICRMW_END]]:1310; GFX908-NEXT: ret double [[RES]]1311;1312; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1313; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1314; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1315; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1316; GFX90A: [[ATOMICRMW_PRIVATE]]:1317; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1318; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81319; GFX90A-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1320; GFX90A-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81321; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]1322; GFX90A: [[ATOMICRMW_GLOBAL]]:1323; GFX90A-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1324; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]1325; GFX90A: [[ATOMICRMW_PHI]]:1326; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1327; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]1328; GFX90A: [[ATOMICRMW_END]]:1329; GFX90A-NEXT: ret double [[RES]]1330;1331; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1332; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1333; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1334; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1335; GFX942: [[ATOMICRMW_PRIVATE]]:1336; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1337; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81338; GFX942-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1339; GFX942-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81340; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]1341; GFX942: [[ATOMICRMW_GLOBAL]]:1342; GFX942-NEXT: [[TMP3:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1343; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]1344; GFX942: [[ATOMICRMW_PHI]]:1345; GFX942-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1346; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]1347; GFX942: [[ATOMICRMW_END]]:1348; GFX942-NEXT: ret double [[RES]]1349;1350; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent(1351; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1352; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1353; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1354; GFX12: [[ATOMICRMW_PRIVATE]]:1355; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1356; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81357; GFX12-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1358; GFX12-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81359; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]1360; GFX12: [[ATOMICRMW_GLOBAL]]:1361; GFX12-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81362; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1363; GFX12: [[ATOMICRMW_START]]:1364; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1365; GFX12-NEXT: [[TMP4:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1366; GFX12-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641367; GFX12-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641368; GFX12-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1369; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11370; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01371; GFX12-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1372; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1373; GFX12: [[ATOMICRMW_END1]]:1374; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]1375; GFX12: [[ATOMICRMW_PHI]]:1376; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1377; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]1378; GFX12: [[ATOMICRMW_END]]:1379; GFX12-NEXT: ret double [[RES]]1380;1381 %res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01382 ret double %res1383}1384 1385define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {1386; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1387; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1388; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1389; GFX7-NEXT: ret double [[RES]]1390;1391; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1392; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1393; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81394; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1395; GFX900: [[ATOMICRMW_START]]:1396; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1397; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1398; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641399; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641400; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1401; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11402; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01403; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1404; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1405; GFX900: [[ATOMICRMW_END]]:1406; GFX900-NEXT: ret double [[TMP5]]1407;1408; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1409; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1410; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81411; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1412; GFX908: [[ATOMICRMW_START]]:1413; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]1414; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1415; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641416; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641417; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1418; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11419; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01420; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double1421; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1422; GFX908: [[ATOMICRMW_END]]:1423; GFX908-NEXT: ret double [[RES]]1424;1425; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1426; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1427; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1428; GFX90A-NEXT: ret double [[RES]]1429;1430; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1431; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1432; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1433; GFX942-NEXT: ret double [[RES]]1434;1435; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5(1436; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1437; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81438; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1439; GFX12: [[ATOMICRMW_START]]:1440; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1441; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1442; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641443; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641444; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1445; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11446; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01447; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1448; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1449; GFX12: [[ATOMICRMW_END]]:1450; GFX12-NEXT: ret double [[TMP5]]1451;1452 %res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01453 ret double %res1454}1455 1456define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {1457; GFX7-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1458; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1459; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81460; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]1461; GFX7: [[ATOMICRMW_START]]:1462; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]1463; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1464; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i641465; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641466; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1467; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11468; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01469; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1470; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1471; GFX7: [[ATOMICRMW_END]]:1472; GFX7-NEXT: ret double [[TMP6]]1473;1474; GFX900-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1475; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1476; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81477; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1478; GFX900: [[ATOMICRMW_START]]:1479; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1480; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1481; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641482; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641483; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1484; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11485; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01486; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1487; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1488; GFX900: [[ATOMICRMW_END]]:1489; GFX900-NEXT: ret double [[TMP5]]1490;1491; GFX908-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1492; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1493; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81494; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1495; GFX908: [[ATOMICRMW_START]]:1496; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1497; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1498; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641499; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641500; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1501; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11502; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01503; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1504; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1505; GFX908: [[ATOMICRMW_END]]:1506; GFX908-NEXT: ret double [[TMP5]]1507;1508; GFX90A-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1509; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1510; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81511; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]1512; GFX90A: [[ATOMICRMW_START]]:1513; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1514; GFX90A-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1515; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641516; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641517; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1518; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11519; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01520; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1521; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1522; GFX90A: [[ATOMICRMW_END]]:1523; GFX90A-NEXT: ret double [[TMP5]]1524;1525; GFX942-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1526; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1527; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]1528; GFX942-NEXT: ret double [[RES]]1529;1530; GFX12-LABEL: define double @test_flat_atomicrmw_fmin_f64_agent__noalias_addrspace_5__maybe_fine_grained(1531; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1532; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81533; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1534; GFX12: [[ATOMICRMW_START]]:1535; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1536; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE]])1537; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641538; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641539; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1540; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11541; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01542; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1543; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1544; GFX12: [[ATOMICRMW_END]]:1545; GFX12-NEXT: ret double [[TMP5]]1546;1547 %res = atomicrmw fmin ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !11548 ret double %res1549}1550 1551define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {1552; GFX7-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1553; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1554; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1555; GFX7-NEXT: ret float [[RES]]1556;1557; GFX900-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1558; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1559; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41560; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1561; GFX900: [[ATOMICRMW_START]]:1562; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1563; GFX900-NEXT: [[NEW:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])1564; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321565; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321566; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1567; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11568; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01569; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1570; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1571; GFX900: [[ATOMICRMW_END]]:1572; GFX900-NEXT: ret float [[TMP5]]1573;1574; GFX908-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1575; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1576; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41577; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1578; GFX908: [[ATOMICRMW_START]]:1579; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1580; GFX908-NEXT: [[NEW:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])1581; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321582; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321583; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1584; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11585; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01586; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1587; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1588; GFX908: [[ATOMICRMW_END]]:1589; GFX908-NEXT: ret float [[TMP5]]1590;1591; GFX90A-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1592; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1593; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41594; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]1595; GFX90A: [[ATOMICRMW_START]]:1596; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[LOADED_PHI:%.*]], %[[ATOMICRMW_START]] ]1597; GFX90A-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])1598; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i321599; GFX90A-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i321600; GFX90A-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1601; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 11602; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 01603; GFX90A-NEXT: [[LOADED_PHI]] = bitcast i32 [[NEWLOADED]] to float1604; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1605; GFX90A: [[ATOMICRMW_END]]:1606; GFX90A-NEXT: ret float [[LOADED_PHI]]1607;1608; GFX942-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1609; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1610; GFX942-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41611; GFX942-NEXT: br label %[[ATOMICRMW_START:.*]]1612; GFX942: [[ATOMICRMW_START]]:1613; GFX942-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]1614; GFX942-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE]])1615; GFX942-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i321616; GFX942-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i321617; GFX942-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1618; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 11619; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 01620; GFX942-NEXT: [[RES]] = bitcast i32 [[NEWLOADED]] to float1621; GFX942-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1622; GFX942: [[ATOMICRMW_END]]:1623; GFX942-NEXT: ret float [[RES]]1624;1625; GFX12-LABEL: define float @test_flat_atomicrmw_fmin_f32_agent__noalias_addrspace_5(1626; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1627; GFX12-NEXT: [[RES:%.*]] = atomicrmw fmin ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1628; GFX12-NEXT: ret float [[RES]]1629;1630 %res = atomicrmw fmin ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01631 ret float %res1632}1633 1634; --------------------------------------------------------------------1635; General expansion for fmax1636; --------------------------------------------------------------------1637 1638define double @test_flat_atomicrmw_fmax_f64_agent(ptr %ptr, double %value) {1639; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1640; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1641; GFX7-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1642; GFX7-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1643; GFX7: [[ATOMICRMW_PRIVATE]]:1644; GFX7-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1645; GFX7-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81646; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1647; GFX7-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81648; GFX7-NEXT: br label %[[ATOMICRMW_PHI:.*]]1649; GFX7: [[ATOMICRMW_GLOBAL]]:1650; GFX7-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1651; GFX7-NEXT: br label %[[ATOMICRMW_PHI]]1652; GFX7: [[ATOMICRMW_PHI]]:1653; GFX7-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1654; GFX7-NEXT: br label %[[ATOMICRMW_END:.*]]1655; GFX7: [[ATOMICRMW_END]]:1656; GFX7-NEXT: ret double [[RES]]1657;1658; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1659; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1660; GFX900-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1661; GFX900-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1662; GFX900: [[ATOMICRMW_PRIVATE]]:1663; GFX900-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1664; GFX900-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81665; GFX900-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1666; GFX900-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81667; GFX900-NEXT: br label %[[ATOMICRMW_PHI:.*]]1668; GFX900: [[ATOMICRMW_GLOBAL]]:1669; GFX900-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81670; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1671; GFX900: [[ATOMICRMW_START]]:1672; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1673; GFX900-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1674; GFX900-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641675; GFX900-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641676; GFX900-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1677; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11678; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01679; GFX900-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1680; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1681; GFX900: [[ATOMICRMW_END1]]:1682; GFX900-NEXT: br label %[[ATOMICRMW_PHI]]1683; GFX900: [[ATOMICRMW_PHI]]:1684; GFX900-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1685; GFX900-NEXT: br label %[[ATOMICRMW_END:.*]]1686; GFX900: [[ATOMICRMW_END]]:1687; GFX900-NEXT: ret double [[RES]]1688;1689; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1690; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1691; GFX908-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1692; GFX908-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1693; GFX908: [[ATOMICRMW_PRIVATE]]:1694; GFX908-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1695; GFX908-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81696; GFX908-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1697; GFX908-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81698; GFX908-NEXT: br label %[[ATOMICRMW_PHI:.*]]1699; GFX908: [[ATOMICRMW_GLOBAL]]:1700; GFX908-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81701; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1702; GFX908: [[ATOMICRMW_START]]:1703; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1704; GFX908-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1705; GFX908-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641706; GFX908-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641707; GFX908-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1708; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11709; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01710; GFX908-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1711; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1712; GFX908: [[ATOMICRMW_END1]]:1713; GFX908-NEXT: br label %[[ATOMICRMW_PHI]]1714; GFX908: [[ATOMICRMW_PHI]]:1715; GFX908-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1716; GFX908-NEXT: br label %[[ATOMICRMW_END:.*]]1717; GFX908: [[ATOMICRMW_END]]:1718; GFX908-NEXT: ret double [[RES]]1719;1720; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1721; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1722; GFX90A-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1723; GFX90A-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1724; GFX90A: [[ATOMICRMW_PRIVATE]]:1725; GFX90A-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1726; GFX90A-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81727; GFX90A-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1728; GFX90A-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81729; GFX90A-NEXT: br label %[[ATOMICRMW_PHI:.*]]1730; GFX90A: [[ATOMICRMW_GLOBAL]]:1731; GFX90A-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1732; GFX90A-NEXT: br label %[[ATOMICRMW_PHI]]1733; GFX90A: [[ATOMICRMW_PHI]]:1734; GFX90A-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1735; GFX90A-NEXT: br label %[[ATOMICRMW_END:.*]]1736; GFX90A: [[ATOMICRMW_END]]:1737; GFX90A-NEXT: ret double [[RES]]1738;1739; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1740; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1741; GFX942-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1742; GFX942-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1743; GFX942: [[ATOMICRMW_PRIVATE]]:1744; GFX942-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1745; GFX942-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81746; GFX942-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1747; GFX942-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81748; GFX942-NEXT: br label %[[ATOMICRMW_PHI:.*]]1749; GFX942: [[ATOMICRMW_GLOBAL]]:1750; GFX942-NEXT: [[TMP3:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1751; GFX942-NEXT: br label %[[ATOMICRMW_PHI]]1752; GFX942: [[ATOMICRMW_PHI]]:1753; GFX942-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ]1754; GFX942-NEXT: br label %[[ATOMICRMW_END:.*]]1755; GFX942: [[ATOMICRMW_END]]:1756; GFX942-NEXT: ret double [[RES]]1757;1758; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent(1759; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1760; GFX12-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])1761; GFX12-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]1762; GFX12: [[ATOMICRMW_PRIVATE]]:1763; GFX12-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)1764; GFX12-NEXT: [[LOADED_PRIVATE:%.*]] = load double, ptr addrspace(5) [[TMP1]], align 81765; GFX12-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED_PRIVATE]], double [[VALUE]])1766; GFX12-NEXT: store double [[TMP2]], ptr addrspace(5) [[TMP1]], align 81767; GFX12-NEXT: br label %[[ATOMICRMW_PHI:.*]]1768; GFX12: [[ATOMICRMW_GLOBAL]]:1769; GFX12-NEXT: [[TMP3:%.*]] = load double, ptr [[PTR]], align 81770; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1771; GFX12: [[ATOMICRMW_START]]:1772; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[TMP8:%.*]], %[[ATOMICRMW_START]] ]1773; GFX12-NEXT: [[TMP4:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1774; GFX12-NEXT: [[TMP5:%.*]] = bitcast double [[TMP4]] to i641775; GFX12-NEXT: [[TMP6:%.*]] = bitcast double [[LOADED]] to i641776; GFX12-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP6]], i64 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1777; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP7]], 11778; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP7]], 01779; GFX12-NEXT: [[TMP8]] = bitcast i64 [[NEWLOADED]] to double1780; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]1781; GFX12: [[ATOMICRMW_END1]]:1782; GFX12-NEXT: br label %[[ATOMICRMW_PHI]]1783; GFX12: [[ATOMICRMW_PHI]]:1784; GFX12-NEXT: [[RES:%.*]] = phi double [ [[LOADED_PRIVATE]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP8]], %[[ATOMICRMW_END1]] ]1785; GFX12-NEXT: br label %[[ATOMICRMW_END:.*]]1786; GFX12: [[ATOMICRMW_END]]:1787; GFX12-NEXT: ret double [[RES]]1788;1789 %res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01790 ret double %res1791}1792 1793define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(ptr %ptr, double %value) {1794; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1795; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1796; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1797; GFX7-NEXT: ret double [[RES]]1798;1799; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1800; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1801; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81802; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1803; GFX900: [[ATOMICRMW_START]]:1804; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1805; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1806; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641807; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641808; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1809; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11810; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01811; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1812; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1813; GFX900: [[ATOMICRMW_END]]:1814; GFX900-NEXT: ret double [[TMP5]]1815;1816; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1817; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1818; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81819; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1820; GFX908: [[ATOMICRMW_START]]:1821; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]1822; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1823; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641824; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641825; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1826; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11827; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01828; GFX908-NEXT: [[RES]] = bitcast i64 [[NEWLOADED]] to double1829; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1830; GFX908: [[ATOMICRMW_END]]:1831; GFX908-NEXT: ret double [[RES]]1832;1833; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1834; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1835; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1836; GFX90A-NEXT: ret double [[RES]]1837;1838; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1839; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1840; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1841; GFX942-NEXT: ret double [[RES]]1842;1843; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5(1844; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1845; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81846; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1847; GFX12: [[ATOMICRMW_START]]:1848; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1849; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1850; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641851; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641852; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1853; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11854; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01855; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1856; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1857; GFX12: [[ATOMICRMW_END]]:1858; GFX12-NEXT: ret double [[TMP5]]1859;1860 %res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01861 ret double %res1862}1863 1864define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, double %value) {1865; GFX7-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1866; GFX7-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1867; GFX7-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81868; GFX7-NEXT: br label %[[ATOMICRMW_START:.*]]1869; GFX7: [[ATOMICRMW_START]]:1870; GFX7-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], %[[ATOMICRMW_START]] ]1871; GFX7-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1872; GFX7-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i641873; GFX7-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i641874; GFX7-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1875; GFX7-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 11876; GFX7-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 01877; GFX7-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double1878; GFX7-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1879; GFX7: [[ATOMICRMW_END]]:1880; GFX7-NEXT: ret double [[TMP6]]1881;1882; GFX900-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1883; GFX900-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1884; GFX900-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81885; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1886; GFX900: [[ATOMICRMW_START]]:1887; GFX900-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1888; GFX900-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1889; GFX900-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641890; GFX900-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641891; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1892; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11893; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01894; GFX900-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1895; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1896; GFX900: [[ATOMICRMW_END]]:1897; GFX900-NEXT: ret double [[TMP5]]1898;1899; GFX908-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1900; GFX908-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1901; GFX908-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81902; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1903; GFX908: [[ATOMICRMW_START]]:1904; GFX908-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1905; GFX908-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1906; GFX908-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641907; GFX908-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641908; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1909; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11910; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01911; GFX908-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1912; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1913; GFX908: [[ATOMICRMW_END]]:1914; GFX908-NEXT: ret double [[TMP5]]1915;1916; GFX90A-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1917; GFX90A-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1918; GFX90A-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81919; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]1920; GFX90A: [[ATOMICRMW_START]]:1921; GFX90A-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1922; GFX90A-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1923; GFX90A-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641924; GFX90A-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641925; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1926; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11927; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01928; GFX90A-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1929; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1930; GFX90A: [[ATOMICRMW_END]]:1931; GFX90A-NEXT: ret double [[TMP5]]1932;1933; GFX942-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1934; GFX942-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1935; GFX942-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], double [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]1936; GFX942-NEXT: ret double [[RES]]1937;1938; GFX12-LABEL: define double @test_flat_atomicrmw_fmax_f64_agent__noalias_addrspace_5__maybe_fine_grained(1939; GFX12-SAME: ptr [[PTR:%.*]], double [[VALUE:%.*]]) #[[ATTR0]] {1940; GFX12-NEXT: [[TMP1:%.*]] = load double, ptr [[PTR]], align 81941; GFX12-NEXT: br label %[[ATOMICRMW_START:.*]]1942; GFX12: [[ATOMICRMW_START]]:1943; GFX12-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1944; GFX12-NEXT: [[NEW:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE]])1945; GFX12-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i641946; GFX12-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i641947; GFX12-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]1948; GFX12-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 11949; GFX12-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 01950; GFX12-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double1951; GFX12-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1952; GFX12: [[ATOMICRMW_END]]:1953; GFX12-NEXT: ret double [[TMP5]]1954;1955 %res = atomicrmw fmax ptr %ptr, double %value syncscope("agent") seq_cst, !noalias.addrspace !11956 ret double %res1957}1958 1959define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(ptr %ptr, float %value) {1960; GFX7-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(1961; GFX7-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1962; GFX7-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1963; GFX7-NEXT: ret float [[RES]]1964;1965; GFX900-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(1966; GFX900-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1967; GFX900-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41968; GFX900-NEXT: br label %[[ATOMICRMW_START:.*]]1969; GFX900: [[ATOMICRMW_START]]:1970; GFX900-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1971; GFX900-NEXT: [[NEW:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])1972; GFX900-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321973; GFX900-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321974; GFX900-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1975; GFX900-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11976; GFX900-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01977; GFX900-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1978; GFX900-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1979; GFX900: [[ATOMICRMW_END]]:1980; GFX900-NEXT: ret float [[TMP5]]1981;1982; GFX908-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(1983; GFX908-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {1984; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 41985; GFX908-NEXT: br label %[[ATOMICRMW_START:.*]]1986; GFX908: [[ATOMICRMW_START]]:1987; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], %[[ATOMICRMW_START]] ]1988; GFX908-NEXT: [[NEW:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])1989; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i321990; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i321991; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]1992; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 11993; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 01994; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float1995; GFX908-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]1996; GFX908: [[ATOMICRMW_END]]:1997; GFX908-NEXT: ret float [[TMP5]]1998;1999; GFX90A-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(2000; GFX90A-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {2001; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 42002; GFX90A-NEXT: br label %[[ATOMICRMW_START:.*]]2003; GFX90A: [[ATOMICRMW_START]]:2004; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[LOADED_PHI:%.*]], %[[ATOMICRMW_START]] ]2005; GFX90A-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])2006; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i322007; GFX90A-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i322008; GFX90A-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2009; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 12010; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 02011; GFX90A-NEXT: [[LOADED_PHI]] = bitcast i32 [[NEWLOADED]] to float2012; GFX90A-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]2013; GFX90A: [[ATOMICRMW_END]]:2014; GFX90A-NEXT: ret float [[LOADED_PHI]]2015;2016; GFX942-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(2017; GFX942-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {2018; GFX942-NEXT: [[TMP1:%.*]] = load float, ptr [[PTR]], align 42019; GFX942-NEXT: br label %[[ATOMICRMW_START:.*]]2020; GFX942: [[ATOMICRMW_START]]:2021; GFX942-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]2022; GFX942-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE]])2023; GFX942-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i322024; GFX942-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i322025; GFX942-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2026; GFX942-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 12027; GFX942-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 02028; GFX942-NEXT: [[RES]] = bitcast i32 [[NEWLOADED]] to float2029; GFX942-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]2030; GFX942: [[ATOMICRMW_END]]:2031; GFX942-NEXT: ret float [[RES]]2032;2033; GFX12-LABEL: define float @test_flat_atomicrmw_fmax_f32_agent__noalias_addrspace_5(2034; GFX12-SAME: ptr [[PTR:%.*]], float [[VALUE:%.*]]) #[[ATTR0]] {2035; GFX12-NEXT: [[RES:%.*]] = atomicrmw fmax ptr [[PTR]], float [[VALUE]] syncscope("agent") seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2036; GFX12-NEXT: ret float [[RES]]2037;2038 %res = atomicrmw fmax ptr %ptr, float %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02039 ret float %res2040}2041 2042; --------------------------------------------------------------------2043; General expansion for nand2044; --------------------------------------------------------------------2045 2046define i64 @test_flat_atomicrmw_nand_i64_agent(ptr %ptr, i64 %value) {2047; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent(2048; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {2049; ALL-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])2050; ALL-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]2051; ALL: [[ATOMICRMW_PRIVATE]]:2052; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)2053; ALL-NEXT: [[LOADED:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 82054; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]2055; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -12056; ALL-NEXT: store i64 [[NEW]], ptr addrspace(5) [[TMP1]], align 82057; ALL-NEXT: br label %[[ATOMICRMW_PHI:.*]]2058; ALL: [[ATOMICRMW_GLOBAL]]:2059; ALL-NEXT: [[TMP3:%.*]] = load i64, ptr [[PTR]], align 82060; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]2061; ALL: [[ATOMICRMW_START]]:2062; ALL-NEXT: [[LOADED1:%.*]] = phi i64 [ [[TMP3]], %[[ATOMICRMW_GLOBAL]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]2063; ALL-NEXT: [[TMP4:%.*]] = and i64 [[LOADED1]], [[VALUE]]2064; ALL-NEXT: [[NEW2:%.*]] = xor i64 [[TMP4]], -12065; ALL-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED1]], i64 [[NEW2]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2066; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 12067; ALL-NEXT: [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP5]], 02068; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END1:.*]], label %[[ATOMICRMW_START]]2069; ALL: [[ATOMICRMW_END1]]:2070; ALL-NEXT: br label %[[ATOMICRMW_PHI]]2071; ALL: [[ATOMICRMW_PHI]]:2072; ALL-NEXT: [[RES:%.*]] = phi i64 [ [[LOADED]], %[[ATOMICRMW_PRIVATE]] ], [ [[NEWLOADED]], %[[ATOMICRMW_END1]] ]2073; ALL-NEXT: br label %[[ATOMICRMW_END:.*]]2074; ALL: [[ATOMICRMW_END]]:2075; ALL-NEXT: ret i64 [[RES]]2076;2077 %res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02078 ret i64 %res2079}2080 2081define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5(ptr %ptr, i64 %value) {2082; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5(2083; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {2084; ALL-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 82085; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]2086; ALL: [[ATOMICRMW_START]]:2087; ALL-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]2088; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]2089; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -12090; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2091; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 12092; ALL-NEXT: [[RES]] = extractvalue { i64, i1 } [[TMP3]], 02093; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]2094; ALL: [[ATOMICRMW_END]]:2095; ALL-NEXT: ret i64 [[RES]]2096;2097 %res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02098 ret i64 %res2099}2100 2101define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {2102; ALL-LABEL: define i64 @test_flat_atomicrmw_nand_i64_agent__noalias_addrspace_5__maybe_fine_grained(2103; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {2104; ALL-NEXT: [[TMP1:%.*]] = load i64, ptr [[PTR]], align 82105; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]2106; ALL: [[ATOMICRMW_START]]:2107; ALL-NEXT: [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]2108; ALL-NEXT: [[TMP2:%.*]] = and i64 [[LOADED]], [[VALUE]]2109; ALL-NEXT: [[NEW:%.*]] = xor i64 [[TMP2]], -12110; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]2111; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 12112; ALL-NEXT: [[RES]] = extractvalue { i64, i1 } [[TMP3]], 02113; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]2114; ALL: [[ATOMICRMW_END]]:2115; ALL-NEXT: ret i64 [[RES]]2116;2117 %res = atomicrmw nand ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !12118 ret i64 %res2119}2120 2121 2122define i32 @test_flat_atomicrmw_nand_i32_agent__noalias_addrspace_5(ptr %ptr, i32 %value) {2123; ALL-LABEL: define i32 @test_flat_atomicrmw_nand_i32_agent__noalias_addrspace_5(2124; ALL-SAME: ptr [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {2125; ALL-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR]], align 42126; ALL-NEXT: br label %[[ATOMICRMW_START:.*]]2127; ALL: [[ATOMICRMW_START]]:2128; ALL-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], %[[ATOMICRMW_START]] ]2129; ALL-NEXT: [[TMP2:%.*]] = and i32 [[LOADED]], [[VALUE]]2130; ALL-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -12131; ALL-NEXT: [[TMP3:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !noalias.addrspace [[META0]], !amdgpu.no.fine.grained.memory [[META1]]2132; ALL-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 12133; ALL-NEXT: [[RES]] = extractvalue { i32, i1 } [[TMP3]], 02134; ALL-NEXT: br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]2135; ALL: [[ATOMICRMW_END]]:2136; ALL-NEXT: ret i32 [[RES]]2137;2138 %res = atomicrmw nand ptr %ptr, i32 %value syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02139 ret i32 %res2140}2141 2142!0 = !{}2143!1 = !{i32 5, i32 6}2144!2 = !{!"foo", !"bar"}2145!3 = !{!"bux", !"baz"}2146!4 = !{!2, !3}2147!5 = !{}2148 2149;.2150; GFX7: [[META0]] = !{i32 5, i32 6}2151; GFX7: [[META1]] = !{}2152; GFX7: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2153; GFX7: [[META3]] = !{!"foo", !"bar"}2154; GFX7: [[META4]] = !{!"bux", !"baz"}2155;.2156; GFX900: [[META0]] = !{i32 5, i32 6}2157; GFX900: [[META1]] = !{}2158; GFX900: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2159; GFX900: [[META3]] = !{!"foo", !"bar"}2160; GFX900: [[META4]] = !{!"bux", !"baz"}2161;.2162; GFX908: [[META0]] = !{i32 5, i32 6}2163; GFX908: [[META1]] = !{}2164; GFX908: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2165; GFX908: [[META3]] = !{!"foo", !"bar"}2166; GFX908: [[META4]] = !{!"bux", !"baz"}2167;.2168; GFX90A: [[META0]] = !{i32 5, i32 6}2169; GFX90A: [[META1]] = !{}2170; GFX90A: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2171; GFX90A: [[META3]] = !{!"foo", !"bar"}2172; GFX90A: [[META4]] = !{!"bux", !"baz"}2173;.2174; GFX942: [[META0]] = !{i32 5, i32 6}2175; GFX942: [[META1]] = !{}2176; GFX942: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2177; GFX942: [[META3]] = !{!"foo", !"bar"}2178; GFX942: [[META4]] = !{!"bux", !"baz"}2179;.2180; GFX12: [[META0]] = !{i32 5, i32 6}2181; GFX12: [[META1]] = !{}2182; GFX12: [[META2]] = !{[[META3:![0-9]+]], [[META4:![0-9]+]]}2183; GFX12: [[META3]] = !{!"foo", !"bar"}2184; GFX12: [[META4]] = !{!"bux", !"baz"}2185;.2186