21352 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefixes=CHECK,GFX90A %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefixes=CHECK,GFX950 %s4 5;---------------------------------------------------------------------6; xchg i32 cases7;---------------------------------------------------------------------8 9; Input and result use AGPR10define void @flat_atomic_xchg_i32_ret_a_a(ptr %ptr) #0 {11; GFX90A-LABEL: flat_atomic_xchg_i32_ret_a_a:12; GFX90A: ; %bb.0:13; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14; GFX90A-NEXT: ;;#ASMSTART15; GFX90A-NEXT: ; def a016; GFX90A-NEXT: ;;#ASMEND17; GFX90A-NEXT: v_accvgpr_read_b32 v2, a018; GFX90A-NEXT: buffer_wbl219; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc20; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21; GFX90A-NEXT: buffer_invl222; GFX90A-NEXT: buffer_wbinvl1_vol23; GFX90A-NEXT: v_accvgpr_write_b32 a0, v024; GFX90A-NEXT: ;;#ASMSTART25; GFX90A-NEXT: ; use a026; GFX90A-NEXT: ;;#ASMEND27; GFX90A-NEXT: s_setpc_b64 s[30:31]28;29; GFX950-LABEL: flat_atomic_xchg_i32_ret_a_a:30; GFX950: ; %bb.0:31; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)32; GFX950-NEXT: ;;#ASMSTART33; GFX950-NEXT: ; def a034; GFX950-NEXT: ;;#ASMEND35; GFX950-NEXT: s_nop 036; GFX950-NEXT: v_accvgpr_read_b32 v2, a037; GFX950-NEXT: buffer_wbl2 sc0 sc138; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc139; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)40; GFX950-NEXT: buffer_inv sc0 sc141; GFX950-NEXT: v_accvgpr_write_b32 a0, v042; GFX950-NEXT: ;;#ASMSTART43; GFX950-NEXT: ; use a044; GFX950-NEXT: ;;#ASMEND45; GFX950-NEXT: s_setpc_b64 s[30:31]46 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1047 %data = call i32 asm "; def $0", "=a"()48 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst49 call void asm "; use $0", "a"(i32 %result)50 ret void51}52 53; Input is AGPR, result used as VGPR.54define void @flat_atomic_xchg_i32_ret_a_v(ptr %ptr) #0 {55; GFX90A-LABEL: flat_atomic_xchg_i32_ret_a_v:56; GFX90A: ; %bb.0:57; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)58; GFX90A-NEXT: ;;#ASMSTART59; GFX90A-NEXT: ; def a060; GFX90A-NEXT: ;;#ASMEND61; GFX90A-NEXT: v_accvgpr_read_b32 v2, a062; GFX90A-NEXT: buffer_wbl263; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc64; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)65; GFX90A-NEXT: buffer_invl266; GFX90A-NEXT: buffer_wbinvl1_vol67; GFX90A-NEXT: ;;#ASMSTART68; GFX90A-NEXT: ; use v069; GFX90A-NEXT: ;;#ASMEND70; GFX90A-NEXT: s_setpc_b64 s[30:31]71;72; GFX950-LABEL: flat_atomic_xchg_i32_ret_a_v:73; GFX950: ; %bb.0:74; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)75; GFX950-NEXT: ;;#ASMSTART76; GFX950-NEXT: ; def a077; GFX950-NEXT: ;;#ASMEND78; GFX950-NEXT: s_nop 079; GFX950-NEXT: v_accvgpr_read_b32 v2, a080; GFX950-NEXT: buffer_wbl2 sc0 sc181; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc182; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)83; GFX950-NEXT: buffer_inv sc0 sc184; GFX950-NEXT: ;;#ASMSTART85; GFX950-NEXT: ; use v086; GFX950-NEXT: ;;#ASMEND87; GFX950-NEXT: s_setpc_b64 s[30:31]88 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1089 %data = call i32 asm "; def $0", "=a"()90 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst91 call void asm "; use $0", "v"(i32 %result)92 ret void93}94 95; Input is VGPR, result used as AGPR96define void @flat_atomic_xchg_i32_ret_v_a(ptr %ptr) #0 {97; GFX90A-LABEL: flat_atomic_xchg_i32_ret_v_a:98; GFX90A: ; %bb.0:99; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)100; GFX90A-NEXT: ;;#ASMSTART101; GFX90A-NEXT: ; def v2102; GFX90A-NEXT: ;;#ASMEND103; GFX90A-NEXT: buffer_wbl2104; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc105; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)106; GFX90A-NEXT: buffer_invl2107; GFX90A-NEXT: buffer_wbinvl1_vol108; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0109; GFX90A-NEXT: ;;#ASMSTART110; GFX90A-NEXT: ; use a0111; GFX90A-NEXT: ;;#ASMEND112; GFX90A-NEXT: s_setpc_b64 s[30:31]113;114; GFX950-LABEL: flat_atomic_xchg_i32_ret_v_a:115; GFX950: ; %bb.0:116; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)117; GFX950-NEXT: ;;#ASMSTART118; GFX950-NEXT: ; def v2119; GFX950-NEXT: ;;#ASMEND120; GFX950-NEXT: buffer_wbl2 sc0 sc1121; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1122; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)123; GFX950-NEXT: buffer_inv sc0 sc1124; GFX950-NEXT: v_accvgpr_write_b32 a0, v0125; GFX950-NEXT: ;;#ASMSTART126; GFX950-NEXT: ; use a0127; GFX950-NEXT: ;;#ASMEND128; GFX950-NEXT: s_setpc_b64 s[30:31]129 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10130 %data = call i32 asm "; def $0", "=v"()131 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst132 call void asm "; use $0", "a"(i32 %result)133 ret void134}135 136; Input is AV, result also used as AV137define void @flat_atomic_xchg_i32_ret_av_av(ptr %ptr) #0 {138; GFX90A-LABEL: flat_atomic_xchg_i32_ret_av_av:139; GFX90A: ; %bb.0:140; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)141; GFX90A-NEXT: ;;#ASMSTART142; GFX90A-NEXT: ; def v2143; GFX90A-NEXT: ;;#ASMEND144; GFX90A-NEXT: buffer_wbl2145; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc146; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)147; GFX90A-NEXT: buffer_invl2148; GFX90A-NEXT: buffer_wbinvl1_vol149; GFX90A-NEXT: ;;#ASMSTART150; GFX90A-NEXT: ; use v0151; GFX90A-NEXT: ;;#ASMEND152; GFX90A-NEXT: s_setpc_b64 s[30:31]153;154; GFX950-LABEL: flat_atomic_xchg_i32_ret_av_av:155; GFX950: ; %bb.0:156; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)157; GFX950-NEXT: ;;#ASMSTART158; GFX950-NEXT: ; def v2159; GFX950-NEXT: ;;#ASMEND160; GFX950-NEXT: buffer_wbl2 sc0 sc1161; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1162; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)163; GFX950-NEXT: buffer_inv sc0 sc1164; GFX950-NEXT: ;;#ASMSTART165; GFX950-NEXT: ; use v0166; GFX950-NEXT: ;;#ASMEND167; GFX950-NEXT: s_setpc_b64 s[30:31]168 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10169 %data = call i32 asm "; def $0", "=^VA"()170 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst171 call void asm "; use $0", "^VA"(i32 %result)172 ret void173}174 175; Input is AV, used as v176define void @flat_atomic_xchg_i32_ret_av_v(ptr %ptr) #0 {177; GFX90A-LABEL: flat_atomic_xchg_i32_ret_av_v:178; GFX90A: ; %bb.0:179; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)180; GFX90A-NEXT: ;;#ASMSTART181; GFX90A-NEXT: ; def v2182; GFX90A-NEXT: ;;#ASMEND183; GFX90A-NEXT: buffer_wbl2184; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc185; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)186; GFX90A-NEXT: buffer_invl2187; GFX90A-NEXT: buffer_wbinvl1_vol188; GFX90A-NEXT: ;;#ASMSTART189; GFX90A-NEXT: ; use v0190; GFX90A-NEXT: ;;#ASMEND191; GFX90A-NEXT: s_setpc_b64 s[30:31]192;193; GFX950-LABEL: flat_atomic_xchg_i32_ret_av_v:194; GFX950: ; %bb.0:195; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)196; GFX950-NEXT: ;;#ASMSTART197; GFX950-NEXT: ; def v2198; GFX950-NEXT: ;;#ASMEND199; GFX950-NEXT: buffer_wbl2 sc0 sc1200; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1201; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)202; GFX950-NEXT: buffer_inv sc0 sc1203; GFX950-NEXT: ;;#ASMSTART204; GFX950-NEXT: ; use v0205; GFX950-NEXT: ;;#ASMEND206; GFX950-NEXT: s_setpc_b64 s[30:31]207 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10208 %data = call i32 asm "; def $0", "=^VA"()209 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst210 call void asm "; use $0", "v"(i32 %result)211 ret void212}213 214; Input is AV, used as a215define void @flat_atomic_xchg_i32_ret_av_a(ptr %ptr) #0 {216; GFX90A-LABEL: flat_atomic_xchg_i32_ret_av_a:217; GFX90A: ; %bb.0:218; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)219; GFX90A-NEXT: ;;#ASMSTART220; GFX90A-NEXT: ; def v2221; GFX90A-NEXT: ;;#ASMEND222; GFX90A-NEXT: buffer_wbl2223; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc224; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)225; GFX90A-NEXT: buffer_invl2226; GFX90A-NEXT: buffer_wbinvl1_vol227; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0228; GFX90A-NEXT: ;;#ASMSTART229; GFX90A-NEXT: ; use a0230; GFX90A-NEXT: ;;#ASMEND231; GFX90A-NEXT: s_setpc_b64 s[30:31]232;233; GFX950-LABEL: flat_atomic_xchg_i32_ret_av_a:234; GFX950: ; %bb.0:235; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)236; GFX950-NEXT: ;;#ASMSTART237; GFX950-NEXT: ; def v2238; GFX950-NEXT: ;;#ASMEND239; GFX950-NEXT: buffer_wbl2 sc0 sc1240; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1241; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)242; GFX950-NEXT: buffer_inv sc0 sc1243; GFX950-NEXT: v_accvgpr_write_b32 a0, v0244; GFX950-NEXT: ;;#ASMSTART245; GFX950-NEXT: ; use a0246; GFX950-NEXT: ;;#ASMEND247; GFX950-NEXT: s_setpc_b64 s[30:31]248 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10249 %data = call i32 asm "; def $0", "=^VA"()250 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst251 call void asm "; use $0", "a"(i32 %result)252 ret void253}254 255; Input is a, result used as AV256define void @flat_atomic_xchg_i32_ret_a_av(ptr %ptr) #0 {257; GFX90A-LABEL: flat_atomic_xchg_i32_ret_a_av:258; GFX90A: ; %bb.0:259; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)260; GFX90A-NEXT: ;;#ASMSTART261; GFX90A-NEXT: ; def a0262; GFX90A-NEXT: ;;#ASMEND263; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0264; GFX90A-NEXT: buffer_wbl2265; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc266; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)267; GFX90A-NEXT: buffer_invl2268; GFX90A-NEXT: buffer_wbinvl1_vol269; GFX90A-NEXT: ;;#ASMSTART270; GFX90A-NEXT: ; use v0271; GFX90A-NEXT: ;;#ASMEND272; GFX90A-NEXT: s_setpc_b64 s[30:31]273;274; GFX950-LABEL: flat_atomic_xchg_i32_ret_a_av:275; GFX950: ; %bb.0:276; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)277; GFX950-NEXT: ;;#ASMSTART278; GFX950-NEXT: ; def a0279; GFX950-NEXT: ;;#ASMEND280; GFX950-NEXT: s_nop 0281; GFX950-NEXT: v_accvgpr_read_b32 v2, a0282; GFX950-NEXT: buffer_wbl2 sc0 sc1283; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1284; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)285; GFX950-NEXT: buffer_inv sc0 sc1286; GFX950-NEXT: ;;#ASMSTART287; GFX950-NEXT: ; use v0288; GFX950-NEXT: ;;#ASMEND289; GFX950-NEXT: s_setpc_b64 s[30:31]290 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10291 %data = call i32 asm "; def $0", "=a"()292 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst293 call void asm "; use $0", "^VA"(i32 %result)294 ret void295}296 297; Input is v, result used as AV298define void @flat_atomic_xchg_i32_ret_v_av(ptr %ptr) #0 {299; GFX90A-LABEL: flat_atomic_xchg_i32_ret_v_av:300; GFX90A: ; %bb.0:301; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)302; GFX90A-NEXT: ;;#ASMSTART303; GFX90A-NEXT: ; def v2304; GFX90A-NEXT: ;;#ASMEND305; GFX90A-NEXT: buffer_wbl2306; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc307; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)308; GFX90A-NEXT: buffer_invl2309; GFX90A-NEXT: buffer_wbinvl1_vol310; GFX90A-NEXT: ;;#ASMSTART311; GFX90A-NEXT: ; use v0312; GFX90A-NEXT: ;;#ASMEND313; GFX90A-NEXT: s_setpc_b64 s[30:31]314;315; GFX950-LABEL: flat_atomic_xchg_i32_ret_v_av:316; GFX950: ; %bb.0:317; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)318; GFX950-NEXT: ;;#ASMSTART319; GFX950-NEXT: ; def v2320; GFX950-NEXT: ;;#ASMEND321; GFX950-NEXT: buffer_wbl2 sc0 sc1322; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1323; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)324; GFX950-NEXT: buffer_inv sc0 sc1325; GFX950-NEXT: ;;#ASMSTART326; GFX950-NEXT: ; use v0327; GFX950-NEXT: ;;#ASMEND328; GFX950-NEXT: s_setpc_b64 s[30:31]329 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10330 %data = call i32 asm "; def $0", "=v"()331 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst332 call void asm "; use $0", "^VA"(i32 %result)333 ret void334}335 336define void @flat_atomic_xchg_i32_ret_av_av_no_agprs(ptr %ptr) #0 {337; GFX90A-LABEL: flat_atomic_xchg_i32_ret_av_av_no_agprs:338; GFX90A: ; %bb.0:339; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)340; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0341; GFX90A-NEXT: v_accvgpr_write_b32 a3, v40 ; Reload Reuse342; GFX90A-NEXT: v_accvgpr_write_b32 a4, v41 ; Reload Reuse343; GFX90A-NEXT: v_accvgpr_write_b32 a5, v42 ; Reload Reuse344; GFX90A-NEXT: v_accvgpr_write_b32 a6, v43 ; Reload Reuse345; GFX90A-NEXT: v_accvgpr_write_b32 a7, v44 ; Reload Reuse346; GFX90A-NEXT: v_accvgpr_write_b32 a8, v45 ; Reload Reuse347; GFX90A-NEXT: v_accvgpr_write_b32 a9, v46 ; Reload Reuse348; GFX90A-NEXT: v_accvgpr_write_b32 a10, v47 ; Reload Reuse349; GFX90A-NEXT: v_accvgpr_write_b32 a11, v56 ; Reload Reuse350; GFX90A-NEXT: v_accvgpr_write_b32 a12, v57 ; Reload Reuse351; GFX90A-NEXT: v_accvgpr_write_b32 a13, v58 ; Reload Reuse352; GFX90A-NEXT: v_accvgpr_write_b32 a14, v59 ; Reload Reuse353; GFX90A-NEXT: v_accvgpr_write_b32 a15, v60 ; Reload Reuse354; GFX90A-NEXT: v_accvgpr_write_b32 a16, v61 ; Reload Reuse355; GFX90A-NEXT: v_accvgpr_write_b32 a17, v62 ; Reload Reuse356; GFX90A-NEXT: v_accvgpr_write_b32 a18, v63 ; Reload Reuse357; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1358; GFX90A-NEXT: ;;#ASMSTART359; GFX90A-NEXT: ; def v[0:31]360; GFX90A-NEXT: ;;#ASMEND361; GFX90A-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill362; GFX90A-NEXT: s_nop 0363; GFX90A-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill364; GFX90A-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill365; GFX90A-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill366; GFX90A-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill367; GFX90A-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill368; GFX90A-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill369; GFX90A-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill370; GFX90A-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill371; GFX90A-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill372; GFX90A-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill373; GFX90A-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill374; GFX90A-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill375; GFX90A-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill376; GFX90A-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill377; GFX90A-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill378; GFX90A-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill379; GFX90A-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill380; GFX90A-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill381; GFX90A-NEXT: ;;#ASMSTART382; GFX90A-NEXT: ; def a2383; GFX90A-NEXT: ;;#ASMEND384; GFX90A-NEXT: v_accvgpr_write_b32 a19, v31 ; Reload Reuse385; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0386; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1387; GFX90A-NEXT: v_accvgpr_read_b32 v2, a2388; GFX90A-NEXT: buffer_wbl2389; GFX90A-NEXT: s_waitcnt vmcnt(0)390; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc391; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)392; GFX90A-NEXT: buffer_invl2393; GFX90A-NEXT: buffer_wbinvl1_vol394; GFX90A-NEXT: v_accvgpr_write_b32 a31, v19 ; Reload Reuse395; GFX90A-NEXT: v_accvgpr_write_b32 a30, v20 ; Reload Reuse396; GFX90A-NEXT: v_accvgpr_write_b32 a29, v21 ; Reload Reuse397; GFX90A-NEXT: v_accvgpr_write_b32 a28, v22 ; Reload Reuse398; GFX90A-NEXT: v_accvgpr_write_b32 a27, v23 ; Reload Reuse399; GFX90A-NEXT: v_accvgpr_write_b32 a26, v24 ; Reload Reuse400; GFX90A-NEXT: v_accvgpr_write_b32 a25, v25 ; Reload Reuse401; GFX90A-NEXT: v_accvgpr_write_b32 a24, v26 ; Reload Reuse402; GFX90A-NEXT: v_accvgpr_write_b32 a23, v27 ; Reload Reuse403; GFX90A-NEXT: v_accvgpr_write_b32 a22, v28 ; Reload Reuse404; GFX90A-NEXT: v_accvgpr_write_b32 a21, v29 ; Reload Reuse405; GFX90A-NEXT: v_accvgpr_write_b32 a20, v30 ; Reload Reuse406; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0407; GFX90A-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload408; GFX90A-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload409; GFX90A-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload410; GFX90A-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload411; GFX90A-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload412; GFX90A-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload413; GFX90A-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload414; GFX90A-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload415; GFX90A-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload416; GFX90A-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload417; GFX90A-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload418; GFX90A-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload419; GFX90A-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload420; GFX90A-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload421; GFX90A-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload422; GFX90A-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload423; GFX90A-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload424; GFX90A-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload425; GFX90A-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload426; GFX90A-NEXT: v_accvgpr_read_b32 v19, a31 ; Reload Reuse427; GFX90A-NEXT: v_accvgpr_read_b32 v20, a30 ; Reload Reuse428; GFX90A-NEXT: v_accvgpr_read_b32 v21, a29 ; Reload Reuse429; GFX90A-NEXT: v_accvgpr_read_b32 v22, a28 ; Reload Reuse430; GFX90A-NEXT: v_accvgpr_read_b32 v23, a27 ; Reload Reuse431; GFX90A-NEXT: v_accvgpr_read_b32 v24, a26 ; Reload Reuse432; GFX90A-NEXT: v_accvgpr_read_b32 v25, a25 ; Reload Reuse433; GFX90A-NEXT: v_accvgpr_read_b32 v26, a24 ; Reload Reuse434; GFX90A-NEXT: v_accvgpr_read_b32 v27, a23 ; Reload Reuse435; GFX90A-NEXT: v_accvgpr_read_b32 v28, a22 ; Reload Reuse436; GFX90A-NEXT: v_accvgpr_read_b32 v29, a21 ; Reload Reuse437; GFX90A-NEXT: v_accvgpr_read_b32 v30, a20 ; Reload Reuse438; GFX90A-NEXT: ;;#ASMSTART439; GFX90A-NEXT: ; use a0440; GFX90A-NEXT: ;;#ASMEND441; GFX90A-NEXT: s_waitcnt vmcnt(0)442; GFX90A-NEXT: v_accvgpr_read_b32 v31, a19 ; Reload Reuse443; GFX90A-NEXT: ;;#ASMSTART444; GFX90A-NEXT: ; use v[0:31]445; GFX90A-NEXT: ;;#ASMEND446; GFX90A-NEXT: v_accvgpr_read_b32 v63, a18 ; Reload Reuse447; GFX90A-NEXT: v_accvgpr_read_b32 v62, a17 ; Reload Reuse448; GFX90A-NEXT: v_accvgpr_read_b32 v61, a16 ; Reload Reuse449; GFX90A-NEXT: v_accvgpr_read_b32 v60, a15 ; Reload Reuse450; GFX90A-NEXT: v_accvgpr_read_b32 v59, a14 ; Reload Reuse451; GFX90A-NEXT: v_accvgpr_read_b32 v58, a13 ; Reload Reuse452; GFX90A-NEXT: v_accvgpr_read_b32 v57, a12 ; Reload Reuse453; GFX90A-NEXT: v_accvgpr_read_b32 v56, a11 ; Reload Reuse454; GFX90A-NEXT: v_accvgpr_read_b32 v47, a10 ; Reload Reuse455; GFX90A-NEXT: v_accvgpr_read_b32 v46, a9 ; Reload Reuse456; GFX90A-NEXT: v_accvgpr_read_b32 v45, a8 ; Reload Reuse457; GFX90A-NEXT: v_accvgpr_read_b32 v44, a7 ; Reload Reuse458; GFX90A-NEXT: v_accvgpr_read_b32 v43, a6 ; Reload Reuse459; GFX90A-NEXT: v_accvgpr_read_b32 v42, a5 ; Reload Reuse460; GFX90A-NEXT: v_accvgpr_read_b32 v41, a4 ; Reload Reuse461; GFX90A-NEXT: v_accvgpr_read_b32 v40, a3 ; Reload Reuse462; GFX90A-NEXT: s_setpc_b64 s[30:31]463;464; GFX950-LABEL: flat_atomic_xchg_i32_ret_av_av_no_agprs:465; GFX950: ; %bb.0:466; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)467; GFX950-NEXT: v_accvgpr_write_b32 a0, v0468; GFX950-NEXT: v_accvgpr_write_b32 a3, v40 ; Reload Reuse469; GFX950-NEXT: v_accvgpr_write_b32 a4, v41 ; Reload Reuse470; GFX950-NEXT: v_accvgpr_write_b32 a5, v42 ; Reload Reuse471; GFX950-NEXT: v_accvgpr_write_b32 a6, v43 ; Reload Reuse472; GFX950-NEXT: v_accvgpr_write_b32 a7, v44 ; Reload Reuse473; GFX950-NEXT: v_accvgpr_write_b32 a8, v45 ; Reload Reuse474; GFX950-NEXT: v_accvgpr_write_b32 a9, v46 ; Reload Reuse475; GFX950-NEXT: v_accvgpr_write_b32 a10, v47 ; Reload Reuse476; GFX950-NEXT: v_accvgpr_write_b32 a11, v56 ; Reload Reuse477; GFX950-NEXT: v_accvgpr_write_b32 a12, v57 ; Reload Reuse478; GFX950-NEXT: v_accvgpr_write_b32 a13, v58 ; Reload Reuse479; GFX950-NEXT: v_accvgpr_write_b32 a14, v59 ; Reload Reuse480; GFX950-NEXT: v_accvgpr_write_b32 a15, v60 ; Reload Reuse481; GFX950-NEXT: v_accvgpr_write_b32 a16, v61 ; Reload Reuse482; GFX950-NEXT: v_accvgpr_write_b32 a17, v62 ; Reload Reuse483; GFX950-NEXT: v_accvgpr_write_b32 a18, v63 ; Reload Reuse484; GFX950-NEXT: v_accvgpr_write_b32 a1, v1485; GFX950-NEXT: ;;#ASMSTART486; GFX950-NEXT: ; def v[0:31]487; GFX950-NEXT: ;;#ASMEND488; GFX950-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; 16-byte Folded Spill489; GFX950-NEXT: s_nop 0490; GFX950-NEXT: scratch_store_dwordx4 off, v[4:7], s32 offset:16 ; 16-byte Folded Spill491; GFX950-NEXT: scratch_store_dwordx4 off, v[8:11], s32 offset:32 ; 16-byte Folded Spill492; GFX950-NEXT: scratch_store_dwordx4 off, v[12:15], s32 offset:48 ; 16-byte Folded Spill493; GFX950-NEXT: ;;#ASMSTART494; GFX950-NEXT: ; def a2495; GFX950-NEXT: ;;#ASMEND496; GFX950-NEXT: v_accvgpr_write_b32 a19, v31 ; Reload Reuse497; GFX950-NEXT: v_accvgpr_write_b32 a20, v30 ; Reload Reuse498; GFX950-NEXT: v_accvgpr_write_b32 a21, v29 ; Reload Reuse499; GFX950-NEXT: v_accvgpr_write_b32 a22, v28 ; Reload Reuse500; GFX950-NEXT: v_accvgpr_read_b32 v0, a0501; GFX950-NEXT: scratch_store_dwordx3 off, v[16:18], s32 offset:64 ; 12-byte Folded Spill502; GFX950-NEXT: v_accvgpr_read_b32 v1, a1503; GFX950-NEXT: v_accvgpr_read_b32 v2, a2504; GFX950-NEXT: buffer_wbl2 sc0 sc1505; GFX950-NEXT: s_waitcnt vmcnt(0)506; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc0 sc1507; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)508; GFX950-NEXT: buffer_inv sc0 sc1509; GFX950-NEXT: v_accvgpr_write_b32 a31, v19 ; Reload Reuse510; GFX950-NEXT: v_accvgpr_write_b32 a27, v23 ; Reload Reuse511; GFX950-NEXT: v_accvgpr_write_b32 a28, v22 ; Reload Reuse512; GFX950-NEXT: v_accvgpr_write_b32 a29, v21 ; Reload Reuse513; GFX950-NEXT: v_accvgpr_write_b32 a30, v20 ; Reload Reuse514; GFX950-NEXT: v_accvgpr_write_b32 a23, v27 ; Reload Reuse515; GFX950-NEXT: v_accvgpr_write_b32 a24, v26 ; Reload Reuse516; GFX950-NEXT: v_accvgpr_write_b32 a25, v25 ; Reload Reuse517; GFX950-NEXT: v_accvgpr_write_b32 a26, v24 ; Reload Reuse518; GFX950-NEXT: v_accvgpr_write_b32 a0, v0519; GFX950-NEXT: scratch_load_dwordx4 v[0:3], off, s32 ; 16-byte Folded Reload520; GFX950-NEXT: scratch_load_dwordx4 v[4:7], off, s32 offset:16 ; 16-byte Folded Reload521; GFX950-NEXT: scratch_load_dwordx4 v[8:11], off, s32 offset:32 ; 16-byte Folded Reload522; GFX950-NEXT: scratch_load_dwordx4 v[12:15], off, s32 offset:48 ; 16-byte Folded Reload523; GFX950-NEXT: v_accvgpr_read_b32 v19, a31 ; Reload Reuse524; GFX950-NEXT: scratch_load_dwordx3 v[16:18], off, s32 offset:64 ; 12-byte Folded Reload525; GFX950-NEXT: v_accvgpr_read_b32 v23, a27 ; Reload Reuse526; GFX950-NEXT: v_accvgpr_read_b32 v22, a28 ; Reload Reuse527; GFX950-NEXT: v_accvgpr_read_b32 v21, a29 ; Reload Reuse528; GFX950-NEXT: v_accvgpr_read_b32 v20, a30 ; Reload Reuse529; GFX950-NEXT: v_accvgpr_read_b32 v27, a23 ; Reload Reuse530; GFX950-NEXT: v_accvgpr_read_b32 v26, a24 ; Reload Reuse531; GFX950-NEXT: v_accvgpr_read_b32 v25, a25 ; Reload Reuse532; GFX950-NEXT: v_accvgpr_read_b32 v24, a26 ; Reload Reuse533; GFX950-NEXT: ;;#ASMSTART534; GFX950-NEXT: ; use a0535; GFX950-NEXT: ;;#ASMEND536; GFX950-NEXT: s_waitcnt vmcnt(0)537; GFX950-NEXT: v_accvgpr_read_b32 v31, a19 ; Reload Reuse538; GFX950-NEXT: v_accvgpr_read_b32 v30, a20 ; Reload Reuse539; GFX950-NEXT: v_accvgpr_read_b32 v29, a21 ; Reload Reuse540; GFX950-NEXT: v_accvgpr_read_b32 v28, a22 ; Reload Reuse541; GFX950-NEXT: ;;#ASMSTART542; GFX950-NEXT: ; use v[0:31]543; GFX950-NEXT: ;;#ASMEND544; GFX950-NEXT: v_accvgpr_read_b32 v63, a18 ; Reload Reuse545; GFX950-NEXT: v_accvgpr_read_b32 v62, a17 ; Reload Reuse546; GFX950-NEXT: v_accvgpr_read_b32 v61, a16 ; Reload Reuse547; GFX950-NEXT: v_accvgpr_read_b32 v60, a15 ; Reload Reuse548; GFX950-NEXT: v_accvgpr_read_b32 v59, a14 ; Reload Reuse549; GFX950-NEXT: v_accvgpr_read_b32 v58, a13 ; Reload Reuse550; GFX950-NEXT: v_accvgpr_read_b32 v57, a12 ; Reload Reuse551; GFX950-NEXT: v_accvgpr_read_b32 v56, a11 ; Reload Reuse552; GFX950-NEXT: v_accvgpr_read_b32 v47, a10 ; Reload Reuse553; GFX950-NEXT: v_accvgpr_read_b32 v46, a9 ; Reload Reuse554; GFX950-NEXT: v_accvgpr_read_b32 v45, a8 ; Reload Reuse555; GFX950-NEXT: v_accvgpr_read_b32 v44, a7 ; Reload Reuse556; GFX950-NEXT: v_accvgpr_read_b32 v43, a6 ; Reload Reuse557; GFX950-NEXT: v_accvgpr_read_b32 v42, a5 ; Reload Reuse558; GFX950-NEXT: v_accvgpr_read_b32 v41, a4 ; Reload Reuse559; GFX950-NEXT: v_accvgpr_read_b32 v40, a3 ; Reload Reuse560; GFX950-NEXT: s_setpc_b64 s[30:31]561 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10562 %data = call i32 asm "; def $0", "=^VA"()563 %vgpr.def = call { <32 x i32>, <32 x i32> } asm sideeffect "; def $0", "=${v[0:31]},=${v[32:63]}"()564 %vgpr.0 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 0565 %vgpr.1 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 1566 %result = atomicrmw xchg ptr %gep.0, i32 %data seq_cst567 call void asm sideeffect "; use $0", "{v[0:31]},{v[32:63]}"(<32 x i32> %vgpr.0, <32 x i32> %vgpr.1)568 call void asm "; use $0", "^VA"(i32 %result)569 ret void570}571 572define void @flat_atomic_xchg_i32_noret_a(ptr %ptr) #0 {573; GFX90A-LABEL: flat_atomic_xchg_i32_noret_a:574; GFX90A: ; %bb.0:575; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)576; GFX90A-NEXT: ;;#ASMSTART577; GFX90A-NEXT: ; def a0578; GFX90A-NEXT: ;;#ASMEND579; GFX90A-NEXT: buffer_wbl2580; GFX90A-NEXT: flat_atomic_swap v[0:1], a0 offset:40581; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)582; GFX90A-NEXT: buffer_invl2583; GFX90A-NEXT: buffer_wbinvl1_vol584; GFX90A-NEXT: s_setpc_b64 s[30:31]585;586; GFX950-LABEL: flat_atomic_xchg_i32_noret_a:587; GFX950: ; %bb.0:588; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)589; GFX950-NEXT: ;;#ASMSTART590; GFX950-NEXT: ; def a0591; GFX950-NEXT: ;;#ASMEND592; GFX950-NEXT: buffer_wbl2 sc0 sc1593; GFX950-NEXT: flat_atomic_swap v[0:1], a0 offset:40 sc1594; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)595; GFX950-NEXT: buffer_inv sc0 sc1596; GFX950-NEXT: s_setpc_b64 s[30:31]597 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10598 %data = call i32 asm "; def $0", "=a"()599 %unused = atomicrmw xchg ptr %gep.0, i32 %data seq_cst600 ret void601}602 603define void @flat_atomic_xchg_i32_noret_av(ptr %ptr) #0 {604; GFX90A-LABEL: flat_atomic_xchg_i32_noret_av:605; GFX90A: ; %bb.0:606; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)607; GFX90A-NEXT: ;;#ASMSTART608; GFX90A-NEXT: ; def v2609; GFX90A-NEXT: ;;#ASMEND610; GFX90A-NEXT: buffer_wbl2611; GFX90A-NEXT: flat_atomic_swap v[0:1], v2 offset:40612; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)613; GFX90A-NEXT: buffer_invl2614; GFX90A-NEXT: buffer_wbinvl1_vol615; GFX90A-NEXT: s_setpc_b64 s[30:31]616;617; GFX950-LABEL: flat_atomic_xchg_i32_noret_av:618; GFX950: ; %bb.0:619; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)620; GFX950-NEXT: ;;#ASMSTART621; GFX950-NEXT: ; def v2622; GFX950-NEXT: ;;#ASMEND623; GFX950-NEXT: buffer_wbl2 sc0 sc1624; GFX950-NEXT: flat_atomic_swap v[0:1], v2 offset:40 sc1625; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)626; GFX950-NEXT: buffer_inv sc0 sc1627; GFX950-NEXT: s_setpc_b64 s[30:31]628 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10629 %data = call i32 asm "; def $0", "=^VA"()630 %unused = atomicrmw xchg ptr %gep.0, i32 %data seq_cst631 ret void632}633 634;---------------------------------------------------------------------635; xchg i64 cases636;---------------------------------------------------------------------637 638; Input and result use AGPR639define void @flat_atomic_xchg_i64_ret_a_a(ptr %ptr) #0 {640; GFX90A-LABEL: flat_atomic_xchg_i64_ret_a_a:641; GFX90A: ; %bb.0:642; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)643; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v0644; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base645; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc646; GFX90A-NEXT: ;;#ASMSTART647; GFX90A-NEXT: ; def a[2:3]648; GFX90A-NEXT: ;;#ASMEND649; GFX90A-NEXT: v_accvgpr_read_b32 v2, a2650; GFX90A-NEXT: v_accvgpr_read_b32 v3, a3651; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1652; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1653; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc654; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]655; GFX90A-NEXT: s_cbranch_execz .LBB11_2656; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global657; GFX90A-NEXT: buffer_wbl2658; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc659; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)660; GFX90A-NEXT: buffer_invl2661; GFX90A-NEXT: buffer_wbinvl1_vol662; GFX90A-NEXT: ; implicit-def: $agpr2_agpr3663; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0664; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1665; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1666; GFX90A-NEXT: .LBB11_2: ; %Flow667; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]668; GFX90A-NEXT: s_cbranch_execz .LBB11_4669; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private670; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]671; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc672; GFX90A-NEXT: buffer_load_dword a0, v0, s[0:3], 0 offen673; GFX90A-NEXT: buffer_load_dword a1, v0, s[0:3], 0 offen offset:4674; GFX90A-NEXT: s_nop 0675; GFX90A-NEXT: buffer_store_dword a2, v0, s[0:3], 0 offen676; GFX90A-NEXT: buffer_store_dword a3, v0, s[0:3], 0 offen offset:4677; GFX90A-NEXT: .LBB11_4: ; %atomicrmw.phi678; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]679; GFX90A-NEXT: s_waitcnt vmcnt(2)680; GFX90A-NEXT: ;;#ASMSTART681; GFX90A-NEXT: ; use a[0:1]682; GFX90A-NEXT: ;;#ASMEND683; GFX90A-NEXT: s_waitcnt vmcnt(0)684; GFX90A-NEXT: s_setpc_b64 s[30:31]685;686; GFX950-LABEL: flat_atomic_xchg_i64_ret_a_a:687; GFX950: ; %bb.0:688; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)689; GFX950-NEXT: s_mov_b64 s[2:3], 0x50690; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base691; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]692; GFX950-NEXT: ;;#ASMSTART693; GFX950-NEXT: ; def a[2:3]694; GFX950-NEXT: ;;#ASMEND695; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1696; GFX950-NEXT: v_accvgpr_read_b32 v2, a2697; GFX950-NEXT: v_accvgpr_read_b32 v3, a3698; GFX950-NEXT: ; implicit-def: $agpr0_agpr1699; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc700; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]701; GFX950-NEXT: s_cbranch_execz .LBB11_2702; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global703; GFX950-NEXT: buffer_wbl2 sc0 sc1704; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] sc0 sc1705; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)706; GFX950-NEXT: buffer_inv sc0 sc1707; GFX950-NEXT: ; implicit-def: $agpr2_agpr3708; GFX950-NEXT: v_accvgpr_write_b32 a0, v0709; GFX950-NEXT: v_accvgpr_write_b32 a1, v1710; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1711; GFX950-NEXT: .LBB11_2: ; %Flow712; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]713; GFX950-NEXT: s_cbranch_execz .LBB11_4714; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private715; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]716; GFX950-NEXT: s_nop 1717; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc718; GFX950-NEXT: scratch_load_dwordx2 a[0:1], v0, off719; GFX950-NEXT: s_nop 0720; GFX950-NEXT: scratch_store_dwordx2 v0, a[2:3], off721; GFX950-NEXT: .LBB11_4: ; %atomicrmw.phi722; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]723; GFX950-NEXT: s_waitcnt vmcnt(1)724; GFX950-NEXT: ;;#ASMSTART725; GFX950-NEXT: ; use a[0:1]726; GFX950-NEXT: ;;#ASMEND727; GFX950-NEXT: s_waitcnt vmcnt(0)728; GFX950-NEXT: s_setpc_b64 s[30:31]729 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10730 %data = call i64 asm "; def $0", "=a"()731 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst732 call void asm "; use $0", "a"(i64 %result)733 ret void734}735 736; Input is AGPR, result used as VGPR.737define void @flat_atomic_xchg_i64_ret_a_v(ptr %ptr) #0 {738; GFX90A-LABEL: flat_atomic_xchg_i64_ret_a_v:739; GFX90A: ; %bb.0:740; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)741; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0742; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base743; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc744; GFX90A-NEXT: ;;#ASMSTART745; GFX90A-NEXT: ; def a[0:1]746; GFX90A-NEXT: ;;#ASMEND747; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1748; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0749; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3750; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1751; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc752; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]753; GFX90A-NEXT: s_cbranch_execz .LBB12_2754; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global755; GFX90A-NEXT: buffer_wbl2756; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc757; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)758; GFX90A-NEXT: buffer_invl2759; GFX90A-NEXT: buffer_wbinvl1_vol760; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3761; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1762; GFX90A-NEXT: .LBB12_2: ; %Flow763; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]764; GFX90A-NEXT: s_cbranch_execz .LBB12_4765; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private766; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]767; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc768; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen769; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4770; GFX90A-NEXT: s_nop 0771; GFX90A-NEXT: buffer_store_dword a0, v2, s[0:3], 0 offen772; GFX90A-NEXT: buffer_store_dword a1, v2, s[0:3], 0 offen offset:4773; GFX90A-NEXT: .LBB12_4: ; %atomicrmw.phi774; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]775; GFX90A-NEXT: s_waitcnt vmcnt(2)776; GFX90A-NEXT: ;;#ASMSTART777; GFX90A-NEXT: ; use v[0:1]778; GFX90A-NEXT: ;;#ASMEND779; GFX90A-NEXT: s_waitcnt vmcnt(0)780; GFX90A-NEXT: s_setpc_b64 s[30:31]781;782; GFX950-LABEL: flat_atomic_xchg_i64_ret_a_v:783; GFX950: ; %bb.0:784; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)785; GFX950-NEXT: s_mov_b64 s[2:3], 0x50786; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base787; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]788; GFX950-NEXT: ;;#ASMSTART789; GFX950-NEXT: ; def a[0:1]790; GFX950-NEXT: ;;#ASMEND791; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3792; GFX950-NEXT: v_accvgpr_read_b32 v5, a1793; GFX950-NEXT: v_accvgpr_read_b32 v4, a0794; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1795; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc796; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]797; GFX950-NEXT: s_cbranch_execz .LBB12_2798; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global799; GFX950-NEXT: buffer_wbl2 sc0 sc1800; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0 sc1801; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)802; GFX950-NEXT: buffer_inv sc0 sc1803; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3804; GFX950-NEXT: ; implicit-def: $agpr0_agpr1805; GFX950-NEXT: .LBB12_2: ; %Flow806; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]807; GFX950-NEXT: s_cbranch_execz .LBB12_4808; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private809; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]810; GFX950-NEXT: s_nop 1811; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc812; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off813; GFX950-NEXT: s_nop 0814; GFX950-NEXT: scratch_store_dwordx2 v2, a[0:1], off815; GFX950-NEXT: .LBB12_4: ; %atomicrmw.phi816; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]817; GFX950-NEXT: s_waitcnt vmcnt(1)818; GFX950-NEXT: ;;#ASMSTART819; GFX950-NEXT: ; use v[0:1]820; GFX950-NEXT: ;;#ASMEND821; GFX950-NEXT: s_waitcnt vmcnt(0)822; GFX950-NEXT: s_setpc_b64 s[30:31]823 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10824 %data = call i64 asm "; def $0", "=a"()825 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst826 call void asm "; use $0", "v"(i64 %result)827 ret void828}829 830; Input is VGPR, result used as AGPR831define void @flat_atomic_xchg_i64_ret_v_a(ptr %ptr) #0 {832; GFX90A-LABEL: flat_atomic_xchg_i64_ret_v_a:833; GFX90A: ; %bb.0:834; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)835; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v0836; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base837; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc838; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1839; GFX90A-NEXT: ;;#ASMSTART840; GFX90A-NEXT: ; def v[2:3]841; GFX90A-NEXT: ;;#ASMEND842; GFX90A-NEXT: ; implicit-def: $agpr0_agpr1843; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc844; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]845; GFX90A-NEXT: s_cbranch_execz .LBB13_2846; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global847; GFX90A-NEXT: buffer_wbl2848; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc849; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)850; GFX90A-NEXT: buffer_invl2851; GFX90A-NEXT: buffer_wbinvl1_vol852; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3853; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0854; GFX90A-NEXT: v_accvgpr_write_b32 a1, v1855; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1856; GFX90A-NEXT: .LBB13_2: ; %Flow857; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]858; GFX90A-NEXT: s_cbranch_execz .LBB13_4859; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private860; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]861; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc862; GFX90A-NEXT: buffer_load_dword a0, v0, s[0:3], 0 offen863; GFX90A-NEXT: buffer_load_dword a1, v0, s[0:3], 0 offen offset:4864; GFX90A-NEXT: s_nop 0865; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen866; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:4867; GFX90A-NEXT: .LBB13_4: ; %atomicrmw.phi868; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]869; GFX90A-NEXT: s_waitcnt vmcnt(2)870; GFX90A-NEXT: ;;#ASMSTART871; GFX90A-NEXT: ; use a[0:1]872; GFX90A-NEXT: ;;#ASMEND873; GFX90A-NEXT: s_waitcnt vmcnt(0)874; GFX90A-NEXT: s_setpc_b64 s[30:31]875;876; GFX950-LABEL: flat_atomic_xchg_i64_ret_v_a:877; GFX950: ; %bb.0:878; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)879; GFX950-NEXT: s_mov_b64 s[2:3], 0x50880; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base881; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]882; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v1883; GFX950-NEXT: ;;#ASMSTART884; GFX950-NEXT: ; def v[2:3]885; GFX950-NEXT: ;;#ASMEND886; GFX950-NEXT: ; implicit-def: $agpr0_agpr1887; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc888; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]889; GFX950-NEXT: s_cbranch_execz .LBB13_2890; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global891; GFX950-NEXT: buffer_wbl2 sc0 sc1892; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] sc0 sc1893; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)894; GFX950-NEXT: buffer_inv sc0 sc1895; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3896; GFX950-NEXT: v_accvgpr_write_b32 a0, v0897; GFX950-NEXT: v_accvgpr_write_b32 a1, v1898; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1899; GFX950-NEXT: .LBB13_2: ; %Flow900; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]901; GFX950-NEXT: s_cbranch_execz .LBB13_4902; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private903; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]904; GFX950-NEXT: s_nop 1905; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc906; GFX950-NEXT: scratch_load_dwordx2 a[0:1], v0, off907; GFX950-NEXT: s_nop 0908; GFX950-NEXT: scratch_store_dwordx2 v0, v[2:3], off909; GFX950-NEXT: .LBB13_4: ; %atomicrmw.phi910; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]911; GFX950-NEXT: s_waitcnt vmcnt(1)912; GFX950-NEXT: ;;#ASMSTART913; GFX950-NEXT: ; use a[0:1]914; GFX950-NEXT: ;;#ASMEND915; GFX950-NEXT: s_waitcnt vmcnt(0)916; GFX950-NEXT: s_setpc_b64 s[30:31]917 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10918 %data = call i64 asm "; def $0", "=v"()919 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst920 call void asm "; use $0", "a"(i64 %result)921 ret void922}923 924; Input is AV, result also used as AV925define void @flat_atomic_xchg_i64_ret_av_av(ptr %ptr) #0 {926; GFX90A-LABEL: flat_atomic_xchg_i64_ret_av_av:927; GFX90A: ; %bb.0:928; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)929; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v0930; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base931; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc932; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v3933; GFX90A-NEXT: ;;#ASMSTART934; GFX90A-NEXT: ; def v[4:5]935; GFX90A-NEXT: ;;#ASMEND936; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1937; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc938; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]939; GFX90A-NEXT: s_cbranch_execz .LBB14_2940; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global941; GFX90A-NEXT: buffer_wbl2942; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc943; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)944; GFX90A-NEXT: buffer_invl2945; GFX90A-NEXT: buffer_wbinvl1_vol946; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3947; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5948; GFX90A-NEXT: .LBB14_2: ; %Flow949; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]950; GFX90A-NEXT: s_cbranch_execz .LBB14_4951; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private952; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]953; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc954; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen955; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4956; GFX90A-NEXT: s_nop 0957; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen958; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:4959; GFX90A-NEXT: .LBB14_4: ; %atomicrmw.phi960; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]961; GFX90A-NEXT: s_waitcnt vmcnt(2)962; GFX90A-NEXT: ;;#ASMSTART963; GFX90A-NEXT: ; use v[0:1]964; GFX90A-NEXT: ;;#ASMEND965; GFX90A-NEXT: s_waitcnt vmcnt(0)966; GFX90A-NEXT: s_setpc_b64 s[30:31]967;968; GFX950-LABEL: flat_atomic_xchg_i64_ret_av_av:969; GFX950: ; %bb.0:970; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)971; GFX950-NEXT: s_mov_b64 s[2:3], 0x50972; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base973; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]974; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3975; GFX950-NEXT: ;;#ASMSTART976; GFX950-NEXT: ; def v[4:5]977; GFX950-NEXT: ;;#ASMEND978; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1979; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc980; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]981; GFX950-NEXT: s_cbranch_execz .LBB14_2982; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global983; GFX950-NEXT: buffer_wbl2 sc0 sc1984; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0 sc1985; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)986; GFX950-NEXT: buffer_inv sc0 sc1987; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3988; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr5989; GFX950-NEXT: .LBB14_2: ; %Flow990; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]991; GFX950-NEXT: s_cbranch_execz .LBB14_4992; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private993; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]994; GFX950-NEXT: s_nop 1995; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc996; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off997; GFX950-NEXT: s_nop 0998; GFX950-NEXT: scratch_store_dwordx2 v2, v[4:5], off999; GFX950-NEXT: .LBB14_4: ; %atomicrmw.phi1000; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1001; GFX950-NEXT: s_waitcnt vmcnt(1)1002; GFX950-NEXT: ;;#ASMSTART1003; GFX950-NEXT: ; use v[0:1]1004; GFX950-NEXT: ;;#ASMEND1005; GFX950-NEXT: s_waitcnt vmcnt(0)1006; GFX950-NEXT: s_setpc_b64 s[30:31]1007 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101008 %data = call i64 asm "; def $0", "=^VA"()1009 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst1010 call void asm "; use $0", "^VA"(i64 %result)1011 ret void1012}1013 1014; Input is AV, used as v1015define void @flat_atomic_xchg_i64_ret_av_v(ptr %ptr) #0 {1016; GFX90A-LABEL: flat_atomic_xchg_i64_ret_av_v:1017; GFX90A: ; %bb.0:1018; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1019; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v01020; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1021; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc1022; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v31023; GFX90A-NEXT: ;;#ASMSTART1024; GFX90A-NEXT: ; def v[4:5]1025; GFX90A-NEXT: ;;#ASMEND1026; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11027; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1028; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1029; GFX90A-NEXT: s_cbranch_execz .LBB15_21030; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global1031; GFX90A-NEXT: buffer_wbl21032; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc1033; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1034; GFX90A-NEXT: buffer_invl21035; GFX90A-NEXT: buffer_wbinvl1_vol1036; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr31037; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr51038; GFX90A-NEXT: .LBB15_2: ; %Flow1039; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1040; GFX90A-NEXT: s_cbranch_execz .LBB15_41041; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private1042; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1043; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1044; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen1045; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:41046; GFX90A-NEXT: s_nop 01047; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen1048; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:41049; GFX90A-NEXT: .LBB15_4: ; %atomicrmw.phi1050; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1051; GFX90A-NEXT: s_waitcnt vmcnt(2)1052; GFX90A-NEXT: ;;#ASMSTART1053; GFX90A-NEXT: ; use v[0:1]1054; GFX90A-NEXT: ;;#ASMEND1055; GFX90A-NEXT: s_waitcnt vmcnt(0)1056; GFX90A-NEXT: s_setpc_b64 s[30:31]1057;1058; GFX950-LABEL: flat_atomic_xchg_i64_ret_av_v:1059; GFX950: ; %bb.0:1060; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1061; GFX950-NEXT: s_mov_b64 s[2:3], 0x501062; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1063; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]1064; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v31065; GFX950-NEXT: ;;#ASMSTART1066; GFX950-NEXT: ; def v[4:5]1067; GFX950-NEXT: ;;#ASMEND1068; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11069; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1070; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1071; GFX950-NEXT: s_cbranch_execz .LBB15_21072; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global1073; GFX950-NEXT: buffer_wbl2 sc0 sc11074; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0 sc11075; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1076; GFX950-NEXT: buffer_inv sc0 sc11077; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr31078; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr51079; GFX950-NEXT: .LBB15_2: ; %Flow1080; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1081; GFX950-NEXT: s_cbranch_execz .LBB15_41082; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private1083; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1084; GFX950-NEXT: s_nop 11085; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1086; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off1087; GFX950-NEXT: s_nop 01088; GFX950-NEXT: scratch_store_dwordx2 v2, v[4:5], off1089; GFX950-NEXT: .LBB15_4: ; %atomicrmw.phi1090; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1091; GFX950-NEXT: s_waitcnt vmcnt(1)1092; GFX950-NEXT: ;;#ASMSTART1093; GFX950-NEXT: ; use v[0:1]1094; GFX950-NEXT: ;;#ASMEND1095; GFX950-NEXT: s_waitcnt vmcnt(0)1096; GFX950-NEXT: s_setpc_b64 s[30:31]1097 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101098 %data = call i64 asm "; def $0", "=^VA"()1099 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst1100 call void asm "; use $0", "v"(i64 %result)1101 ret void1102}1103 1104; Input is AV, used as a1105define void @flat_atomic_xchg_i64_ret_av_a(ptr %ptr) #0 {1106; GFX90A-LABEL: flat_atomic_xchg_i64_ret_av_a:1107; GFX90A: ; %bb.0:1108; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1109; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v01110; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1111; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc1112; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v11113; GFX90A-NEXT: ;;#ASMSTART1114; GFX90A-NEXT: ; def v[2:3]1115; GFX90A-NEXT: ;;#ASMEND1116; GFX90A-NEXT: ; implicit-def: $agpr0_agpr11117; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1118; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1119; GFX90A-NEXT: s_cbranch_execz .LBB16_21120; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global1121; GFX90A-NEXT: buffer_wbl21122; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc1123; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1124; GFX90A-NEXT: buffer_invl21125; GFX90A-NEXT: buffer_wbinvl1_vol1126; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr31127; GFX90A-NEXT: v_accvgpr_write_b32 a0, v01128; GFX90A-NEXT: v_accvgpr_write_b32 a1, v11129; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11130; GFX90A-NEXT: .LBB16_2: ; %Flow1131; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1132; GFX90A-NEXT: s_cbranch_execz .LBB16_41133; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private1134; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1135; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1136; GFX90A-NEXT: buffer_load_dword a0, v0, s[0:3], 0 offen1137; GFX90A-NEXT: buffer_load_dword a1, v0, s[0:3], 0 offen offset:41138; GFX90A-NEXT: s_nop 01139; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen1140; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:41141; GFX90A-NEXT: .LBB16_4: ; %atomicrmw.phi1142; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1143; GFX90A-NEXT: s_waitcnt vmcnt(2)1144; GFX90A-NEXT: ;;#ASMSTART1145; GFX90A-NEXT: ; use a[0:1]1146; GFX90A-NEXT: ;;#ASMEND1147; GFX90A-NEXT: s_waitcnt vmcnt(0)1148; GFX90A-NEXT: s_setpc_b64 s[30:31]1149;1150; GFX950-LABEL: flat_atomic_xchg_i64_ret_av_a:1151; GFX950: ; %bb.0:1152; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1153; GFX950-NEXT: s_mov_b64 s[2:3], 0x501154; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1155; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]1156; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11157; GFX950-NEXT: ;;#ASMSTART1158; GFX950-NEXT: ; def v[2:3]1159; GFX950-NEXT: ;;#ASMEND1160; GFX950-NEXT: ; implicit-def: $agpr0_agpr11161; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1162; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1163; GFX950-NEXT: s_cbranch_execz .LBB16_21164; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global1165; GFX950-NEXT: buffer_wbl2 sc0 sc11166; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] sc0 sc11167; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1168; GFX950-NEXT: buffer_inv sc0 sc11169; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr31170; GFX950-NEXT: v_accvgpr_write_b32 a0, v01171; GFX950-NEXT: v_accvgpr_write_b32 a1, v11172; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11173; GFX950-NEXT: .LBB16_2: ; %Flow1174; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1175; GFX950-NEXT: s_cbranch_execz .LBB16_41176; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private1177; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1178; GFX950-NEXT: s_nop 11179; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1180; GFX950-NEXT: scratch_load_dwordx2 a[0:1], v0, off1181; GFX950-NEXT: s_nop 01182; GFX950-NEXT: scratch_store_dwordx2 v0, v[2:3], off1183; GFX950-NEXT: .LBB16_4: ; %atomicrmw.phi1184; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1185; GFX950-NEXT: s_waitcnt vmcnt(1)1186; GFX950-NEXT: ;;#ASMSTART1187; GFX950-NEXT: ; use a[0:1]1188; GFX950-NEXT: ;;#ASMEND1189; GFX950-NEXT: s_waitcnt vmcnt(0)1190; GFX950-NEXT: s_setpc_b64 s[30:31]1191 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101192 %data = call i64 asm "; def $0", "=^VA"()1193 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst1194 call void asm "; use $0", "a"(i64 %result)1195 ret void1196}1197 1198; Input is a, result used as AV1199define void @flat_atomic_xchg_i64_ret_a_av(ptr %ptr) #0 {1200; GFX90A-LABEL: flat_atomic_xchg_i64_ret_a_av:1201; GFX90A: ; %bb.0:1202; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1203; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v01204; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1205; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc1206; GFX90A-NEXT: ;;#ASMSTART1207; GFX90A-NEXT: ; def a[0:1]1208; GFX90A-NEXT: ;;#ASMEND1209; GFX90A-NEXT: v_accvgpr_read_b32 v5, a11210; GFX90A-NEXT: v_accvgpr_read_b32 v4, a01211; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v31212; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11213; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1214; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1215; GFX90A-NEXT: s_cbranch_execz .LBB17_21216; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global1217; GFX90A-NEXT: buffer_wbl21218; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc1219; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1220; GFX90A-NEXT: buffer_invl21221; GFX90A-NEXT: buffer_wbinvl1_vol1222; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr31223; GFX90A-NEXT: ; implicit-def: $agpr0_agpr11224; GFX90A-NEXT: .LBB17_2: ; %Flow1225; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1226; GFX90A-NEXT: s_cbranch_execz .LBB17_41227; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private1228; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1229; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1230; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen1231; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:41232; GFX90A-NEXT: s_nop 01233; GFX90A-NEXT: buffer_store_dword a0, v2, s[0:3], 0 offen1234; GFX90A-NEXT: buffer_store_dword a1, v2, s[0:3], 0 offen offset:41235; GFX90A-NEXT: .LBB17_4: ; %atomicrmw.phi1236; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1237; GFX90A-NEXT: s_waitcnt vmcnt(2)1238; GFX90A-NEXT: ;;#ASMSTART1239; GFX90A-NEXT: ; use v[0:1]1240; GFX90A-NEXT: ;;#ASMEND1241; GFX90A-NEXT: s_waitcnt vmcnt(0)1242; GFX90A-NEXT: s_setpc_b64 s[30:31]1243;1244; GFX950-LABEL: flat_atomic_xchg_i64_ret_a_av:1245; GFX950: ; %bb.0:1246; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1247; GFX950-NEXT: s_mov_b64 s[2:3], 0x501248; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1249; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]1250; GFX950-NEXT: ;;#ASMSTART1251; GFX950-NEXT: ; def a[0:1]1252; GFX950-NEXT: ;;#ASMEND1253; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v31254; GFX950-NEXT: v_accvgpr_read_b32 v5, a11255; GFX950-NEXT: v_accvgpr_read_b32 v4, a01256; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11257; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1258; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1259; GFX950-NEXT: s_cbranch_execz .LBB17_21260; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global1261; GFX950-NEXT: buffer_wbl2 sc0 sc11262; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0 sc11263; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1264; GFX950-NEXT: buffer_inv sc0 sc11265; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr31266; GFX950-NEXT: ; implicit-def: $agpr0_agpr11267; GFX950-NEXT: .LBB17_2: ; %Flow1268; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1269; GFX950-NEXT: s_cbranch_execz .LBB17_41270; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private1271; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1272; GFX950-NEXT: s_nop 11273; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1274; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off1275; GFX950-NEXT: s_nop 01276; GFX950-NEXT: scratch_store_dwordx2 v2, a[0:1], off1277; GFX950-NEXT: .LBB17_4: ; %atomicrmw.phi1278; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1279; GFX950-NEXT: s_waitcnt vmcnt(1)1280; GFX950-NEXT: ;;#ASMSTART1281; GFX950-NEXT: ; use v[0:1]1282; GFX950-NEXT: ;;#ASMEND1283; GFX950-NEXT: s_waitcnt vmcnt(0)1284; GFX950-NEXT: s_setpc_b64 s[30:31]1285 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101286 %data = call i64 asm "; def $0", "=a"()1287 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst1288 call void asm "; use $0", "^VA"(i64 %result)1289 ret void1290}1291 1292; Input is v, result used as AV1293define void @flat_atomic_xchg_i64_ret_v_av(ptr %ptr) #0 {1294; GFX90A-LABEL: flat_atomic_xchg_i64_ret_v_av:1295; GFX90A: ; %bb.0:1296; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1297; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v01298; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1299; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc1300; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v31301; GFX90A-NEXT: ;;#ASMSTART1302; GFX90A-NEXT: ; def v[4:5]1303; GFX90A-NEXT: ;;#ASMEND1304; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11305; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1306; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1307; GFX90A-NEXT: s_cbranch_execz .LBB18_21308; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global1309; GFX90A-NEXT: buffer_wbl21310; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] glc1311; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1312; GFX90A-NEXT: buffer_invl21313; GFX90A-NEXT: buffer_wbinvl1_vol1314; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr31315; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr51316; GFX90A-NEXT: .LBB18_2: ; %Flow1317; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1318; GFX90A-NEXT: s_cbranch_execz .LBB18_41319; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private1320; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1321; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1322; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen1323; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:41324; GFX90A-NEXT: s_nop 01325; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen1326; GFX90A-NEXT: buffer_store_dword v5, v2, s[0:3], 0 offen offset:41327; GFX90A-NEXT: .LBB18_4: ; %atomicrmw.phi1328; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1329; GFX90A-NEXT: s_waitcnt vmcnt(2)1330; GFX90A-NEXT: ;;#ASMSTART1331; GFX90A-NEXT: ; use v[0:1]1332; GFX90A-NEXT: ;;#ASMEND1333; GFX90A-NEXT: s_waitcnt vmcnt(0)1334; GFX90A-NEXT: s_setpc_b64 s[30:31]1335;1336; GFX950-LABEL: flat_atomic_xchg_i64_ret_v_av:1337; GFX950: ; %bb.0:1338; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1339; GFX950-NEXT: s_mov_b64 s[2:3], 0x501340; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1341; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]1342; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v31343; GFX950-NEXT: ;;#ASMSTART1344; GFX950-NEXT: ; def v[4:5]1345; GFX950-NEXT: ;;#ASMEND1346; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11347; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1348; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1349; GFX950-NEXT: s_cbranch_execz .LBB18_21350; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global1351; GFX950-NEXT: buffer_wbl2 sc0 sc11352; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0 sc11353; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1354; GFX950-NEXT: buffer_inv sc0 sc11355; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr31356; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr51357; GFX950-NEXT: .LBB18_2: ; %Flow1358; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1359; GFX950-NEXT: s_cbranch_execz .LBB18_41360; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private1361; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1362; GFX950-NEXT: s_nop 11363; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc1364; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off1365; GFX950-NEXT: s_nop 01366; GFX950-NEXT: scratch_store_dwordx2 v2, v[4:5], off1367; GFX950-NEXT: .LBB18_4: ; %atomicrmw.phi1368; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1369; GFX950-NEXT: s_waitcnt vmcnt(1)1370; GFX950-NEXT: ;;#ASMSTART1371; GFX950-NEXT: ; use v[0:1]1372; GFX950-NEXT: ;;#ASMEND1373; GFX950-NEXT: s_waitcnt vmcnt(0)1374; GFX950-NEXT: s_setpc_b64 s[30:31]1375 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101376 %data = call i64 asm "; def $0", "=v"()1377 %result = atomicrmw xchg ptr %gep.0, i64 %data seq_cst1378 call void asm "; use $0", "^VA"(i64 %result)1379 ret void1380}1381 1382define void @flat_atomic_xchg_i64_noret_a(ptr %ptr) #0 {1383; GFX90A-LABEL: flat_atomic_xchg_i64_noret_a:1384; GFX90A: ; %bb.0:1385; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1386; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1387; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v11388; GFX90A-NEXT: ;;#ASMSTART1389; GFX90A-NEXT: ; def a[0:1]1390; GFX90A-NEXT: ;;#ASMEND1391; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1392; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1393; GFX90A-NEXT: s_cbranch_execnz .LBB19_31394; GFX90A-NEXT: ; %bb.1: ; %Flow1395; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1396; GFX90A-NEXT: s_cbranch_execnz .LBB19_41397; GFX90A-NEXT: .LBB19_2: ; %atomicrmw.phi1398; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1399; GFX90A-NEXT: s_setpc_b64 s[30:31]1400; GFX90A-NEXT: .LBB19_3: ; %atomicrmw.global1401; GFX90A-NEXT: buffer_wbl21402; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], a[0:1]1403; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1404; GFX90A-NEXT: buffer_invl21405; GFX90A-NEXT: buffer_wbinvl1_vol1406; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11407; GFX90A-NEXT: ; implicit-def: $agpr0_agpr11408; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1409; GFX90A-NEXT: s_cbranch_execz .LBB19_21410; GFX90A-NEXT: .LBB19_4: ; %atomicrmw.private1411; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1412; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1413; GFX90A-NEXT: buffer_store_dword a1, v0, s[0:3], 0 offen offset:41414; GFX90A-NEXT: buffer_store_dword a0, v0, s[0:3], 0 offen1415; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1416; GFX90A-NEXT: s_waitcnt vmcnt(0)1417; GFX90A-NEXT: s_setpc_b64 s[30:31]1418;1419; GFX950-LABEL: flat_atomic_xchg_i64_noret_a:1420; GFX950: ; %bb.0:1421; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1422; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1423; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11424; GFX950-NEXT: ;;#ASMSTART1425; GFX950-NEXT: ; def a[0:1]1426; GFX950-NEXT: ;;#ASMEND1427; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1428; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1429; GFX950-NEXT: s_cbranch_execnz .LBB19_31430; GFX950-NEXT: ; %bb.1: ; %Flow1431; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1432; GFX950-NEXT: s_cbranch_execnz .LBB19_41433; GFX950-NEXT: .LBB19_2: ; %atomicrmw.phi1434; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1435; GFX950-NEXT: s_setpc_b64 s[30:31]1436; GFX950-NEXT: .LBB19_3: ; %atomicrmw.global1437; GFX950-NEXT: buffer_wbl2 sc0 sc11438; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], a[0:1] sc11439; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1440; GFX950-NEXT: buffer_inv sc0 sc11441; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11442; GFX950-NEXT: ; implicit-def: $agpr0_agpr11443; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1444; GFX950-NEXT: s_cbranch_execz .LBB19_21445; GFX950-NEXT: .LBB19_4: ; %atomicrmw.private1446; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1447; GFX950-NEXT: s_nop 11448; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1449; GFX950-NEXT: scratch_store_dwordx2 v0, a[0:1], off1450; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1451; GFX950-NEXT: s_waitcnt vmcnt(0)1452; GFX950-NEXT: s_setpc_b64 s[30:31]1453 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101454 %data = call i64 asm "; def $0", "=a"()1455 %unused = atomicrmw xchg ptr %ptr, i64 %data seq_cst1456 ret void1457}1458 1459define void @flat_atomic_xchg_i64_noret_av(ptr %ptr) #0 {1460; GFX90A-LABEL: flat_atomic_xchg_i64_noret_av:1461; GFX90A: ; %bb.0:1462; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1463; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base1464; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v11465; GFX90A-NEXT: ;;#ASMSTART1466; GFX90A-NEXT: ; def v[2:3]1467; GFX90A-NEXT: ;;#ASMEND1468; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc1469; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1470; GFX90A-NEXT: s_cbranch_execnz .LBB20_31471; GFX90A-NEXT: ; %bb.1: ; %Flow1472; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1473; GFX90A-NEXT: s_cbranch_execnz .LBB20_41474; GFX90A-NEXT: .LBB20_2: ; %atomicrmw.phi1475; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1476; GFX90A-NEXT: s_setpc_b64 s[30:31]1477; GFX90A-NEXT: .LBB20_3: ; %atomicrmw.global1478; GFX90A-NEXT: buffer_wbl21479; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]1480; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1481; GFX90A-NEXT: buffer_invl21482; GFX90A-NEXT: buffer_wbinvl1_vol1483; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr11484; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]1485; GFX90A-NEXT: s_cbranch_execz .LBB20_21486; GFX90A-NEXT: .LBB20_4: ; %atomicrmw.private1487; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1488; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1489; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:41490; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen1491; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1492; GFX90A-NEXT: s_waitcnt vmcnt(0)1493; GFX90A-NEXT: s_setpc_b64 s[30:31]1494;1495; GFX950-LABEL: flat_atomic_xchg_i64_noret_av:1496; GFX950: ; %bb.0:1497; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1498; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base1499; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11500; GFX950-NEXT: ;;#ASMSTART1501; GFX950-NEXT: ; def v[2:3]1502; GFX950-NEXT: ;;#ASMEND1503; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc1504; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1505; GFX950-NEXT: s_cbranch_execnz .LBB20_31506; GFX950-NEXT: ; %bb.1: ; %Flow1507; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1508; GFX950-NEXT: s_cbranch_execnz .LBB20_41509; GFX950-NEXT: .LBB20_2: ; %atomicrmw.phi1510; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1511; GFX950-NEXT: s_setpc_b64 s[30:31]1512; GFX950-NEXT: .LBB20_3: ; %atomicrmw.global1513; GFX950-NEXT: buffer_wbl2 sc0 sc11514; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] sc11515; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1516; GFX950-NEXT: buffer_inv sc0 sc11517; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr11518; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1519; GFX950-NEXT: s_cbranch_execz .LBB20_21520; GFX950-NEXT: .LBB20_4: ; %atomicrmw.private1521; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1522; GFX950-NEXT: s_nop 11523; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1524; GFX950-NEXT: scratch_store_dwordx2 v0, v[2:3], off1525; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1526; GFX950-NEXT: s_waitcnt vmcnt(0)1527; GFX950-NEXT: s_setpc_b64 s[30:31]1528 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 101529 %data = call i64 asm "; def $0", "=^VA"()1530 %unused = atomicrmw xchg ptr %ptr, i64 %data seq_cst1531 ret void1532}1533 1534;---------------------------------------------------------------------1535; xor i32 cases with cmpxchg expansion1536;---------------------------------------------------------------------1537 1538; Input and result use AGPR1539define void @flat_atomic_xor_expansion_i32_ret_a_a(ptr %ptr) #0 {1540; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_a_a:1541; GFX90A: ; %bb.0:1542; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1543; GFX90A-NEXT: flat_load_dword v3, v[0:1]1544; GFX90A-NEXT: ;;#ASMSTART1545; GFX90A-NEXT: ; def a01546; GFX90A-NEXT: ;;#ASMEND1547; GFX90A-NEXT: v_accvgpr_read_b32 v4, a01548; GFX90A-NEXT: s_mov_b64 s[4:5], 01549; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start1550; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11551; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1552; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41553; GFX90A-NEXT: buffer_wbl21554; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1555; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1556; GFX90A-NEXT: buffer_invl21557; GFX90A-NEXT: buffer_wbinvl1_vol1558; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31559; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1560; GFX90A-NEXT: v_mov_b32_e32 v3, v21561; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1562; GFX90A-NEXT: s_cbranch_execnz .LBB21_11563; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1564; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1565; GFX90A-NEXT: v_accvgpr_write_b32 a0, v21566; GFX90A-NEXT: ;;#ASMSTART1567; GFX90A-NEXT: ; use a01568; GFX90A-NEXT: ;;#ASMEND1569; GFX90A-NEXT: s_setpc_b64 s[30:31]1570;1571; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_a_a:1572; GFX950: ; %bb.0:1573; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1574; GFX950-NEXT: flat_load_dword v3, v[0:1]1575; GFX950-NEXT: ;;#ASMSTART1576; GFX950-NEXT: ; def a01577; GFX950-NEXT: ;;#ASMEND1578; GFX950-NEXT: s_mov_b64 s[0:1], 01579; GFX950-NEXT: v_accvgpr_read_b32 v4, a01580; GFX950-NEXT: .LBB21_1: ; %atomicrmw.start1581; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11582; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1583; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41584; GFX950-NEXT: buffer_wbl2 sc0 sc11585; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11586; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1587; GFX950-NEXT: buffer_inv sc0 sc11588; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31589; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1590; GFX950-NEXT: v_mov_b32_e32 v3, v21591; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1592; GFX950-NEXT: s_cbranch_execnz .LBB21_11593; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1594; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1595; GFX950-NEXT: v_accvgpr_write_b32 a0, v21596; GFX950-NEXT: ;;#ASMSTART1597; GFX950-NEXT: ; use a01598; GFX950-NEXT: ;;#ASMEND1599; GFX950-NEXT: s_setpc_b64 s[30:31]1600 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101601 %data = call i32 asm "; def $0", "=a"()1602 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1603 call void asm "; use $0", "a"(i32 %result)1604 ret void1605}1606 1607; Input is AGPR, result used as VGPR.1608define void @flat_atomic_xor_expansion_i32_ret_a_v(ptr %ptr) #0 {1609; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_a_v:1610; GFX90A: ; %bb.0:1611; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1612; GFX90A-NEXT: flat_load_dword v3, v[0:1]1613; GFX90A-NEXT: ;;#ASMSTART1614; GFX90A-NEXT: ; def a01615; GFX90A-NEXT: ;;#ASMEND1616; GFX90A-NEXT: v_accvgpr_read_b32 v4, a01617; GFX90A-NEXT: s_mov_b64 s[4:5], 01618; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start1619; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11620; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1621; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41622; GFX90A-NEXT: buffer_wbl21623; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1624; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1625; GFX90A-NEXT: buffer_invl21626; GFX90A-NEXT: buffer_wbinvl1_vol1627; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31628; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1629; GFX90A-NEXT: v_mov_b32_e32 v3, v21630; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1631; GFX90A-NEXT: s_cbranch_execnz .LBB22_11632; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1633; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1634; GFX90A-NEXT: ;;#ASMSTART1635; GFX90A-NEXT: ; use v21636; GFX90A-NEXT: ;;#ASMEND1637; GFX90A-NEXT: s_setpc_b64 s[30:31]1638;1639; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_a_v:1640; GFX950: ; %bb.0:1641; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1642; GFX950-NEXT: flat_load_dword v3, v[0:1]1643; GFX950-NEXT: ;;#ASMSTART1644; GFX950-NEXT: ; def a01645; GFX950-NEXT: ;;#ASMEND1646; GFX950-NEXT: s_mov_b64 s[0:1], 01647; GFX950-NEXT: v_accvgpr_read_b32 v4, a01648; GFX950-NEXT: .LBB22_1: ; %atomicrmw.start1649; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11650; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1651; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41652; GFX950-NEXT: buffer_wbl2 sc0 sc11653; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11654; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1655; GFX950-NEXT: buffer_inv sc0 sc11656; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31657; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1658; GFX950-NEXT: v_mov_b32_e32 v3, v21659; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1660; GFX950-NEXT: s_cbranch_execnz .LBB22_11661; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1662; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1663; GFX950-NEXT: ;;#ASMSTART1664; GFX950-NEXT: ; use v21665; GFX950-NEXT: ;;#ASMEND1666; GFX950-NEXT: s_setpc_b64 s[30:31]1667 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101668 %data = call i32 asm "; def $0", "=a"()1669 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1670 call void asm "; use $0", "v"(i32 %result)1671 ret void1672}1673 1674; Input is VGPR, result used as AGPR1675define void @flat_atomic_xor_expansion_i32_ret_v_a(ptr %ptr) #0 {1676; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_v_a:1677; GFX90A: ; %bb.0:1678; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1679; GFX90A-NEXT: flat_load_dword v3, v[0:1]1680; GFX90A-NEXT: s_mov_b64 s[4:5], 01681; GFX90A-NEXT: ;;#ASMSTART1682; GFX90A-NEXT: ; def v41683; GFX90A-NEXT: ;;#ASMEND1684; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start1685; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11686; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1687; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41688; GFX90A-NEXT: buffer_wbl21689; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1690; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1691; GFX90A-NEXT: buffer_invl21692; GFX90A-NEXT: buffer_wbinvl1_vol1693; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31694; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1695; GFX90A-NEXT: v_mov_b32_e32 v3, v21696; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1697; GFX90A-NEXT: s_cbranch_execnz .LBB23_11698; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1699; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1700; GFX90A-NEXT: v_accvgpr_write_b32 a0, v21701; GFX90A-NEXT: ;;#ASMSTART1702; GFX90A-NEXT: ; use a01703; GFX90A-NEXT: ;;#ASMEND1704; GFX90A-NEXT: s_setpc_b64 s[30:31]1705;1706; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_v_a:1707; GFX950: ; %bb.0:1708; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1709; GFX950-NEXT: flat_load_dword v3, v[0:1]1710; GFX950-NEXT: s_mov_b64 s[0:1], 01711; GFX950-NEXT: ;;#ASMSTART1712; GFX950-NEXT: ; def v41713; GFX950-NEXT: ;;#ASMEND1714; GFX950-NEXT: .LBB23_1: ; %atomicrmw.start1715; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11716; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1717; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41718; GFX950-NEXT: buffer_wbl2 sc0 sc11719; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11720; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1721; GFX950-NEXT: buffer_inv sc0 sc11722; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31723; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1724; GFX950-NEXT: v_mov_b32_e32 v3, v21725; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1726; GFX950-NEXT: s_cbranch_execnz .LBB23_11727; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1728; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1729; GFX950-NEXT: v_accvgpr_write_b32 a0, v21730; GFX950-NEXT: ;;#ASMSTART1731; GFX950-NEXT: ; use a01732; GFX950-NEXT: ;;#ASMEND1733; GFX950-NEXT: s_setpc_b64 s[30:31]1734 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101735 %data = call i32 asm "; def $0", "=v"()1736 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1737 call void asm "; use $0", "a"(i32 %result)1738 ret void1739}1740 1741; Input is AV, result also used as AV1742define void @flat_atomic_xor_expansion_i32_ret_av_av(ptr %ptr) #0 {1743; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_av_av:1744; GFX90A: ; %bb.0:1745; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1746; GFX90A-NEXT: flat_load_dword v3, v[0:1]1747; GFX90A-NEXT: s_mov_b64 s[4:5], 01748; GFX90A-NEXT: ;;#ASMSTART1749; GFX90A-NEXT: ; def v41750; GFX90A-NEXT: ;;#ASMEND1751; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start1752; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11753; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1754; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41755; GFX90A-NEXT: buffer_wbl21756; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1757; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1758; GFX90A-NEXT: buffer_invl21759; GFX90A-NEXT: buffer_wbinvl1_vol1760; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31761; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1762; GFX90A-NEXT: v_mov_b32_e32 v3, v21763; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1764; GFX90A-NEXT: s_cbranch_execnz .LBB24_11765; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1766; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1767; GFX90A-NEXT: ;;#ASMSTART1768; GFX90A-NEXT: ; use v21769; GFX90A-NEXT: ;;#ASMEND1770; GFX90A-NEXT: s_setpc_b64 s[30:31]1771;1772; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_av_av:1773; GFX950: ; %bb.0:1774; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1775; GFX950-NEXT: flat_load_dword v3, v[0:1]1776; GFX950-NEXT: s_mov_b64 s[0:1], 01777; GFX950-NEXT: ;;#ASMSTART1778; GFX950-NEXT: ; def v41779; GFX950-NEXT: ;;#ASMEND1780; GFX950-NEXT: .LBB24_1: ; %atomicrmw.start1781; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11782; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1783; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41784; GFX950-NEXT: buffer_wbl2 sc0 sc11785; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11786; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1787; GFX950-NEXT: buffer_inv sc0 sc11788; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31789; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1790; GFX950-NEXT: v_mov_b32_e32 v3, v21791; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1792; GFX950-NEXT: s_cbranch_execnz .LBB24_11793; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1794; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1795; GFX950-NEXT: ;;#ASMSTART1796; GFX950-NEXT: ; use v21797; GFX950-NEXT: ;;#ASMEND1798; GFX950-NEXT: s_setpc_b64 s[30:31]1799 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101800 %data = call i32 asm "; def $0", "=^VA"()1801 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1802 call void asm "; use $0", "^VA"(i32 %result)1803 ret void1804}1805 1806; Input is AV, used as v1807define void @flat_atomic_xor_expansion_i32_ret_av_v(ptr %ptr) #0 {1808; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_av_v:1809; GFX90A: ; %bb.0:1810; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1811; GFX90A-NEXT: flat_load_dword v3, v[0:1]1812; GFX90A-NEXT: s_mov_b64 s[4:5], 01813; GFX90A-NEXT: ;;#ASMSTART1814; GFX90A-NEXT: ; def v41815; GFX90A-NEXT: ;;#ASMEND1816; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start1817; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11818; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1819; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41820; GFX90A-NEXT: buffer_wbl21821; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1822; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1823; GFX90A-NEXT: buffer_invl21824; GFX90A-NEXT: buffer_wbinvl1_vol1825; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31826; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1827; GFX90A-NEXT: v_mov_b32_e32 v3, v21828; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1829; GFX90A-NEXT: s_cbranch_execnz .LBB25_11830; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1831; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1832; GFX90A-NEXT: ;;#ASMSTART1833; GFX90A-NEXT: ; use v21834; GFX90A-NEXT: ;;#ASMEND1835; GFX90A-NEXT: s_setpc_b64 s[30:31]1836;1837; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_av_v:1838; GFX950: ; %bb.0:1839; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1840; GFX950-NEXT: flat_load_dword v3, v[0:1]1841; GFX950-NEXT: s_mov_b64 s[0:1], 01842; GFX950-NEXT: ;;#ASMSTART1843; GFX950-NEXT: ; def v41844; GFX950-NEXT: ;;#ASMEND1845; GFX950-NEXT: .LBB25_1: ; %atomicrmw.start1846; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11847; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1848; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41849; GFX950-NEXT: buffer_wbl2 sc0 sc11850; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11851; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1852; GFX950-NEXT: buffer_inv sc0 sc11853; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31854; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1855; GFX950-NEXT: v_mov_b32_e32 v3, v21856; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1857; GFX950-NEXT: s_cbranch_execnz .LBB25_11858; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1859; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1860; GFX950-NEXT: ;;#ASMSTART1861; GFX950-NEXT: ; use v21862; GFX950-NEXT: ;;#ASMEND1863; GFX950-NEXT: s_setpc_b64 s[30:31]1864 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101865 %data = call i32 asm "; def $0", "=^VA"()1866 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1867 call void asm "; use $0", "v"(i32 %result)1868 ret void1869}1870 1871; Input is AV, used as a1872define void @flat_atomic_xor_expansion_i32_ret_av_a(ptr %ptr) #0 {1873; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_av_a:1874; GFX90A: ; %bb.0:1875; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1876; GFX90A-NEXT: flat_load_dword v3, v[0:1]1877; GFX90A-NEXT: s_mov_b64 s[4:5], 01878; GFX90A-NEXT: ;;#ASMSTART1879; GFX90A-NEXT: ; def v41880; GFX90A-NEXT: ;;#ASMEND1881; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start1882; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11883; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1884; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41885; GFX90A-NEXT: buffer_wbl21886; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1887; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1888; GFX90A-NEXT: buffer_invl21889; GFX90A-NEXT: buffer_wbinvl1_vol1890; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31891; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1892; GFX90A-NEXT: v_mov_b32_e32 v3, v21893; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1894; GFX90A-NEXT: s_cbranch_execnz .LBB26_11895; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1896; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1897; GFX90A-NEXT: v_accvgpr_write_b32 a0, v21898; GFX90A-NEXT: ;;#ASMSTART1899; GFX90A-NEXT: ; use a01900; GFX90A-NEXT: ;;#ASMEND1901; GFX90A-NEXT: s_setpc_b64 s[30:31]1902;1903; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_av_a:1904; GFX950: ; %bb.0:1905; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1906; GFX950-NEXT: flat_load_dword v3, v[0:1]1907; GFX950-NEXT: s_mov_b64 s[0:1], 01908; GFX950-NEXT: ;;#ASMSTART1909; GFX950-NEXT: ; def v41910; GFX950-NEXT: ;;#ASMEND1911; GFX950-NEXT: .LBB26_1: ; %atomicrmw.start1912; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11913; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1914; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41915; GFX950-NEXT: buffer_wbl2 sc0 sc11916; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11917; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1918; GFX950-NEXT: buffer_inv sc0 sc11919; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31920; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1921; GFX950-NEXT: v_mov_b32_e32 v3, v21922; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1923; GFX950-NEXT: s_cbranch_execnz .LBB26_11924; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1925; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1926; GFX950-NEXT: v_accvgpr_write_b32 a0, v21927; GFX950-NEXT: ;;#ASMSTART1928; GFX950-NEXT: ; use a01929; GFX950-NEXT: ;;#ASMEND1930; GFX950-NEXT: s_setpc_b64 s[30:31]1931 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101932 %data = call i32 asm "; def $0", "=^VA"()1933 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst1934 call void asm "; use $0", "a"(i32 %result)1935 ret void1936}1937 1938; Input is a, result used as AV1939define void @flat_atomic_xor_expansion_i32_ret_a_av(ptr %ptr) #0 {1940; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_a_av:1941; GFX90A: ; %bb.0:1942; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1943; GFX90A-NEXT: flat_load_dword v3, v[0:1]1944; GFX90A-NEXT: ;;#ASMSTART1945; GFX90A-NEXT: ; def a01946; GFX90A-NEXT: ;;#ASMEND1947; GFX90A-NEXT: v_accvgpr_read_b32 v4, a01948; GFX90A-NEXT: s_mov_b64 s[4:5], 01949; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start1950; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11951; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1952; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v41953; GFX90A-NEXT: buffer_wbl21954; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc1955; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1956; GFX90A-NEXT: buffer_invl21957; GFX90A-NEXT: buffer_wbinvl1_vol1958; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31959; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1960; GFX90A-NEXT: v_mov_b32_e32 v3, v21961; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1962; GFX90A-NEXT: s_cbranch_execnz .LBB27_11963; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1964; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1965; GFX90A-NEXT: ;;#ASMSTART1966; GFX90A-NEXT: ; use v21967; GFX90A-NEXT: ;;#ASMEND1968; GFX90A-NEXT: s_setpc_b64 s[30:31]1969;1970; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_a_av:1971; GFX950: ; %bb.0:1972; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1973; GFX950-NEXT: flat_load_dword v3, v[0:1]1974; GFX950-NEXT: ;;#ASMSTART1975; GFX950-NEXT: ; def a01976; GFX950-NEXT: ;;#ASMEND1977; GFX950-NEXT: s_mov_b64 s[0:1], 01978; GFX950-NEXT: v_accvgpr_read_b32 v4, a01979; GFX950-NEXT: .LBB27_1: ; %atomicrmw.start1980; GFX950-NEXT: ; =>This Inner Loop Header: Depth=11981; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1982; GFX950-NEXT: v_xor_b32_e32 v2, v3, v41983; GFX950-NEXT: buffer_wbl2 sc0 sc11984; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc11985; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1986; GFX950-NEXT: buffer_inv sc0 sc11987; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v31988; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1989; GFX950-NEXT: v_mov_b32_e32 v3, v21990; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]1991; GFX950-NEXT: s_cbranch_execnz .LBB27_11992; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end1993; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]1994; GFX950-NEXT: ;;#ASMSTART1995; GFX950-NEXT: ; use v21996; GFX950-NEXT: ;;#ASMEND1997; GFX950-NEXT: s_setpc_b64 s[30:31]1998 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 101999 %data = call i32 asm "; def $0", "=a"()2000 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst2001 call void asm "; use $0", "^VA"(i32 %result)2002 ret void2003}2004 2005; Input is v, result used as AV2006define void @flat_atomic_xor_expansion_i32_ret_v_av(ptr %ptr) #0 {2007; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_v_av:2008; GFX90A: ; %bb.0:2009; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2010; GFX90A-NEXT: flat_load_dword v3, v[0:1]2011; GFX90A-NEXT: s_mov_b64 s[4:5], 02012; GFX90A-NEXT: ;;#ASMSTART2013; GFX90A-NEXT: ; def v42014; GFX90A-NEXT: ;;#ASMEND2015; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start2016; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12017; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2018; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v42019; GFX90A-NEXT: buffer_wbl22020; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc2021; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2022; GFX90A-NEXT: buffer_invl22023; GFX90A-NEXT: buffer_wbinvl1_vol2024; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32025; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2026; GFX90A-NEXT: v_mov_b32_e32 v3, v22027; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2028; GFX90A-NEXT: s_cbranch_execnz .LBB28_12029; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2030; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2031; GFX90A-NEXT: ;;#ASMSTART2032; GFX90A-NEXT: ; use v22033; GFX90A-NEXT: ;;#ASMEND2034; GFX90A-NEXT: s_setpc_b64 s[30:31]2035;2036; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_v_av:2037; GFX950: ; %bb.0:2038; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2039; GFX950-NEXT: flat_load_dword v3, v[0:1]2040; GFX950-NEXT: s_mov_b64 s[0:1], 02041; GFX950-NEXT: ;;#ASMSTART2042; GFX950-NEXT: ; def v42043; GFX950-NEXT: ;;#ASMEND2044; GFX950-NEXT: .LBB28_1: ; %atomicrmw.start2045; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12046; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2047; GFX950-NEXT: v_xor_b32_e32 v2, v3, v42048; GFX950-NEXT: buffer_wbl2 sc0 sc12049; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc12050; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2051; GFX950-NEXT: buffer_inv sc0 sc12052; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32053; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2054; GFX950-NEXT: v_mov_b32_e32 v3, v22055; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]2056; GFX950-NEXT: s_cbranch_execnz .LBB28_12057; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end2058; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2059; GFX950-NEXT: ;;#ASMSTART2060; GFX950-NEXT: ; use v22061; GFX950-NEXT: ;;#ASMEND2062; GFX950-NEXT: s_setpc_b64 s[30:31]2063 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 102064 %data = call i32 asm "; def $0", "=v"()2065 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst2066 call void asm "; use $0", "^VA"(i32 %result)2067 ret void2068}2069 2070define void @flat_atomic_xor_expansion_i32_ret_av_av_no_agprs(ptr %ptr) #0 {2071; GFX90A-LABEL: flat_atomic_xor_expansion_i32_ret_av_av_no_agprs:2072; GFX90A: ; %bb.0:2073; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2074; GFX90A-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill2075; GFX90A-NEXT: buffer_store_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill2076; GFX90A-NEXT: buffer_store_dword v42, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill2077; GFX90A-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill2078; GFX90A-NEXT: buffer_store_dword v44, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill2079; GFX90A-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill2080; GFX90A-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill2081; GFX90A-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill2082; GFX90A-NEXT: buffer_store_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill2083; GFX90A-NEXT: buffer_store_dword v57, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill2084; GFX90A-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill2085; GFX90A-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill2086; GFX90A-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill2087; GFX90A-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill2088; GFX90A-NEXT: buffer_store_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill2089; GFX90A-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill2090; GFX90A-NEXT: buffer_store_dword a32, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill2091; GFX90A-NEXT: buffer_store_dword a33, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill2092; GFX90A-NEXT: buffer_store_dword a34, off, s[0:3], s32 ; 4-byte Folded Spill2093; GFX90A-NEXT: v_accvgpr_write_b32 a33, v12094; GFX90A-NEXT: v_accvgpr_write_b32 a32, v02095; GFX90A-NEXT: ;;#ASMSTART2096; GFX90A-NEXT: ; def v[0:31]2097; GFX90A-NEXT: ;;#ASMEND2098; GFX90A-NEXT: v_accvgpr_write_b32 a0, v02099; GFX90A-NEXT: v_accvgpr_write_b32 a1, v12100; GFX90A-NEXT: v_accvgpr_write_b32 a2, v22101; GFX90A-NEXT: v_accvgpr_write_b32 a3, v32102; GFX90A-NEXT: v_accvgpr_write_b32 a4, v42103; GFX90A-NEXT: v_accvgpr_write_b32 a5, v52104; GFX90A-NEXT: v_accvgpr_write_b32 a6, v62105; GFX90A-NEXT: v_accvgpr_write_b32 a7, v72106; GFX90A-NEXT: v_accvgpr_write_b32 a8, v82107; GFX90A-NEXT: v_accvgpr_write_b32 a9, v92108; GFX90A-NEXT: v_accvgpr_write_b32 a10, v102109; GFX90A-NEXT: v_accvgpr_write_b32 a11, v112110; GFX90A-NEXT: v_accvgpr_write_b32 a12, v122111; GFX90A-NEXT: v_accvgpr_write_b32 a13, v132112; GFX90A-NEXT: v_accvgpr_write_b32 a14, v142113; GFX90A-NEXT: v_accvgpr_write_b32 a15, v152114; GFX90A-NEXT: v_accvgpr_write_b32 a16, v162115; GFX90A-NEXT: v_accvgpr_write_b32 a17, v172116; GFX90A-NEXT: v_accvgpr_write_b32 a18, v182117; GFX90A-NEXT: v_accvgpr_write_b32 a19, v192118; GFX90A-NEXT: v_accvgpr_write_b32 a20, v202119; GFX90A-NEXT: v_accvgpr_write_b32 a21, v212120; GFX90A-NEXT: v_accvgpr_write_b32 a22, v222121; GFX90A-NEXT: v_accvgpr_write_b32 a23, v232122; GFX90A-NEXT: v_accvgpr_write_b32 a24, v242123; GFX90A-NEXT: v_accvgpr_write_b32 a25, v252124; GFX90A-NEXT: v_accvgpr_write_b32 a26, v262125; GFX90A-NEXT: v_accvgpr_write_b32 a27, v272126; GFX90A-NEXT: v_accvgpr_write_b32 a28, v282127; GFX90A-NEXT: v_accvgpr_write_b32 a29, v292128; GFX90A-NEXT: v_accvgpr_write_b32 a30, v302129; GFX90A-NEXT: v_accvgpr_write_b32 a31, v312130; GFX90A-NEXT: v_accvgpr_read_b32 v2, a322131; GFX90A-NEXT: v_accvgpr_read_b32 v3, a332132; GFX90A-NEXT: flat_load_dword v1, v[2:3]2133; GFX90A-NEXT: ;;#ASMSTART2134; GFX90A-NEXT: ; def a342135; GFX90A-NEXT: ;;#ASMEND2136; GFX90A-NEXT: s_mov_b64 s[4:5], 02137; GFX90A-NEXT: v_accvgpr_read_b32 v4, a342138; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start2139; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12140; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2141; GFX90A-NEXT: v_xor_b32_e32 v0, v1, v42142; GFX90A-NEXT: buffer_wbl22143; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc2144; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2145; GFX90A-NEXT: buffer_invl22146; GFX90A-NEXT: buffer_wbinvl1_vol2147; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v12148; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2149; GFX90A-NEXT: v_mov_b32_e32 v1, v02150; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2151; GFX90A-NEXT: s_cbranch_execnz .LBB29_12152; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2153; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2154; GFX90A-NEXT: v_accvgpr_write_b32 a32, v02155; GFX90A-NEXT: v_accvgpr_read_b32 v0, a02156; GFX90A-NEXT: v_accvgpr_read_b32 v1, a12157; GFX90A-NEXT: v_accvgpr_read_b32 v2, a22158; GFX90A-NEXT: v_accvgpr_read_b32 v3, a32159; GFX90A-NEXT: v_accvgpr_read_b32 v4, a42160; GFX90A-NEXT: v_accvgpr_read_b32 v5, a52161; GFX90A-NEXT: v_accvgpr_read_b32 v6, a62162; GFX90A-NEXT: v_accvgpr_read_b32 v7, a72163; GFX90A-NEXT: v_accvgpr_read_b32 v8, a82164; GFX90A-NEXT: v_accvgpr_read_b32 v9, a92165; GFX90A-NEXT: v_accvgpr_read_b32 v10, a102166; GFX90A-NEXT: v_accvgpr_read_b32 v11, a112167; GFX90A-NEXT: v_accvgpr_read_b32 v12, a122168; GFX90A-NEXT: v_accvgpr_read_b32 v13, a132169; GFX90A-NEXT: v_accvgpr_read_b32 v14, a142170; GFX90A-NEXT: v_accvgpr_read_b32 v15, a152171; GFX90A-NEXT: v_accvgpr_read_b32 v16, a162172; GFX90A-NEXT: v_accvgpr_read_b32 v17, a172173; GFX90A-NEXT: v_accvgpr_read_b32 v18, a182174; GFX90A-NEXT: v_accvgpr_read_b32 v19, a192175; GFX90A-NEXT: v_accvgpr_read_b32 v20, a202176; GFX90A-NEXT: v_accvgpr_read_b32 v21, a212177; GFX90A-NEXT: v_accvgpr_read_b32 v22, a222178; GFX90A-NEXT: v_accvgpr_read_b32 v23, a232179; GFX90A-NEXT: v_accvgpr_read_b32 v24, a242180; GFX90A-NEXT: v_accvgpr_read_b32 v25, a252181; GFX90A-NEXT: v_accvgpr_read_b32 v26, a262182; GFX90A-NEXT: v_accvgpr_read_b32 v27, a272183; GFX90A-NEXT: v_accvgpr_read_b32 v28, a282184; GFX90A-NEXT: v_accvgpr_read_b32 v29, a292185; GFX90A-NEXT: v_accvgpr_read_b32 v30, a302186; GFX90A-NEXT: v_accvgpr_read_b32 v31, a312187; GFX90A-NEXT: ;;#ASMSTART2188; GFX90A-NEXT: ; use v[0:31]2189; GFX90A-NEXT: ;;#ASMEND2190; GFX90A-NEXT: ;;#ASMSTART2191; GFX90A-NEXT: ; use a322192; GFX90A-NEXT: ;;#ASMEND2193; GFX90A-NEXT: buffer_load_dword a34, off, s[0:3], s32 ; 4-byte Folded Reload2194; GFX90A-NEXT: buffer_load_dword a33, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload2195; GFX90A-NEXT: buffer_load_dword a32, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload2196; GFX90A-NEXT: buffer_load_dword v63, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload2197; GFX90A-NEXT: buffer_load_dword v62, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload2198; GFX90A-NEXT: buffer_load_dword v61, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload2199; GFX90A-NEXT: buffer_load_dword v60, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload2200; GFX90A-NEXT: buffer_load_dword v59, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload2201; GFX90A-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload2202; GFX90A-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload2203; GFX90A-NEXT: buffer_load_dword v56, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload2204; GFX90A-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload2205; GFX90A-NEXT: buffer_load_dword v46, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload2206; GFX90A-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload2207; GFX90A-NEXT: buffer_load_dword v44, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload2208; GFX90A-NEXT: buffer_load_dword v43, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload2209; GFX90A-NEXT: buffer_load_dword v42, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload2210; GFX90A-NEXT: buffer_load_dword v41, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload2211; GFX90A-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload2212; GFX90A-NEXT: s_waitcnt vmcnt(0)2213; GFX90A-NEXT: s_setpc_b64 s[30:31]2214;2215; GFX950-LABEL: flat_atomic_xor_expansion_i32_ret_av_av_no_agprs:2216; GFX950: ; %bb.0:2217; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2218; GFX950-NEXT: scratch_store_dword off, v40, s32 offset:72 ; 4-byte Folded Spill2219; GFX950-NEXT: scratch_store_dword off, v41, s32 offset:68 ; 4-byte Folded Spill2220; GFX950-NEXT: scratch_store_dword off, v42, s32 offset:64 ; 4-byte Folded Spill2221; GFX950-NEXT: scratch_store_dword off, v43, s32 offset:60 ; 4-byte Folded Spill2222; GFX950-NEXT: scratch_store_dword off, v44, s32 offset:56 ; 4-byte Folded Spill2223; GFX950-NEXT: scratch_store_dword off, v45, s32 offset:52 ; 4-byte Folded Spill2224; GFX950-NEXT: scratch_store_dword off, v46, s32 offset:48 ; 4-byte Folded Spill2225; GFX950-NEXT: scratch_store_dword off, v47, s32 offset:44 ; 4-byte Folded Spill2226; GFX950-NEXT: scratch_store_dword off, v56, s32 offset:40 ; 4-byte Folded Spill2227; GFX950-NEXT: scratch_store_dword off, v57, s32 offset:36 ; 4-byte Folded Spill2228; GFX950-NEXT: scratch_store_dword off, v58, s32 offset:32 ; 4-byte Folded Spill2229; GFX950-NEXT: scratch_store_dword off, v59, s32 offset:28 ; 4-byte Folded Spill2230; GFX950-NEXT: scratch_store_dword off, v60, s32 offset:24 ; 4-byte Folded Spill2231; GFX950-NEXT: scratch_store_dword off, v61, s32 offset:20 ; 4-byte Folded Spill2232; GFX950-NEXT: scratch_store_dword off, v62, s32 offset:16 ; 4-byte Folded Spill2233; GFX950-NEXT: scratch_store_dword off, v63, s32 offset:12 ; 4-byte Folded Spill2234; GFX950-NEXT: scratch_store_dword off, a32, s32 offset:8 ; 4-byte Folded Spill2235; GFX950-NEXT: scratch_store_dword off, a33, s32 offset:4 ; 4-byte Folded Spill2236; GFX950-NEXT: scratch_store_dword off, a34, s32 ; 4-byte Folded Spill2237; GFX950-NEXT: v_accvgpr_write_b32 a33, v12238; GFX950-NEXT: v_accvgpr_write_b32 a32, v02239; GFX950-NEXT: ;;#ASMSTART2240; GFX950-NEXT: ; def v[0:31]2241; GFX950-NEXT: ;;#ASMEND2242; GFX950-NEXT: ;;#ASMSTART2243; GFX950-NEXT: ; def a342244; GFX950-NEXT: ;;#ASMEND2245; GFX950-NEXT: s_mov_b64 s[0:1], 02246; GFX950-NEXT: v_accvgpr_write_b32 a0, v02247; GFX950-NEXT: v_accvgpr_write_b32 a1, v12248; GFX950-NEXT: v_accvgpr_write_b32 a2, v22249; GFX950-NEXT: v_accvgpr_write_b32 a3, v32250; GFX950-NEXT: v_accvgpr_write_b32 a4, v42251; GFX950-NEXT: v_accvgpr_write_b32 a5, v52252; GFX950-NEXT: v_accvgpr_write_b32 a6, v62253; GFX950-NEXT: v_accvgpr_write_b32 a7, v72254; GFX950-NEXT: v_accvgpr_write_b32 a8, v82255; GFX950-NEXT: v_accvgpr_write_b32 a9, v92256; GFX950-NEXT: v_accvgpr_write_b32 a10, v102257; GFX950-NEXT: v_accvgpr_write_b32 a11, v112258; GFX950-NEXT: v_accvgpr_write_b32 a12, v122259; GFX950-NEXT: v_accvgpr_write_b32 a13, v132260; GFX950-NEXT: v_accvgpr_write_b32 a14, v142261; GFX950-NEXT: v_accvgpr_write_b32 a15, v152262; GFX950-NEXT: v_accvgpr_write_b32 a16, v162263; GFX950-NEXT: v_accvgpr_write_b32 a17, v172264; GFX950-NEXT: v_accvgpr_write_b32 a18, v182265; GFX950-NEXT: v_accvgpr_write_b32 a19, v192266; GFX950-NEXT: v_accvgpr_write_b32 a20, v202267; GFX950-NEXT: v_accvgpr_write_b32 a21, v212268; GFX950-NEXT: v_accvgpr_write_b32 a22, v222269; GFX950-NEXT: v_accvgpr_write_b32 a23, v232270; GFX950-NEXT: v_accvgpr_write_b32 a24, v242271; GFX950-NEXT: v_accvgpr_write_b32 a25, v252272; GFX950-NEXT: v_accvgpr_write_b32 a26, v262273; GFX950-NEXT: v_accvgpr_write_b32 a27, v272274; GFX950-NEXT: v_accvgpr_write_b32 a28, v282275; GFX950-NEXT: v_accvgpr_write_b32 a29, v292276; GFX950-NEXT: v_accvgpr_write_b32 a30, v302277; GFX950-NEXT: v_accvgpr_write_b32 a31, v312278; GFX950-NEXT: v_accvgpr_read_b32 v2, a322279; GFX950-NEXT: v_accvgpr_read_b32 v3, a332280; GFX950-NEXT: flat_load_dword v1, v[2:3]2281; GFX950-NEXT: v_accvgpr_read_b32 v4, a342282; GFX950-NEXT: .LBB29_1: ; %atomicrmw.start2283; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12284; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2285; GFX950-NEXT: v_xor_b32_e32 v0, v1, v42286; GFX950-NEXT: buffer_wbl2 sc0 sc12287; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] sc0 sc12288; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2289; GFX950-NEXT: buffer_inv sc0 sc12290; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v12291; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2292; GFX950-NEXT: v_mov_b32_e32 v1, v02293; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]2294; GFX950-NEXT: s_cbranch_execnz .LBB29_12295; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end2296; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2297; GFX950-NEXT: v_accvgpr_write_b32 a32, v02298; GFX950-NEXT: v_accvgpr_read_b32 v0, a02299; GFX950-NEXT: v_accvgpr_read_b32 v1, a12300; GFX950-NEXT: v_accvgpr_read_b32 v2, a22301; GFX950-NEXT: v_accvgpr_read_b32 v3, a32302; GFX950-NEXT: v_accvgpr_read_b32 v4, a42303; GFX950-NEXT: v_accvgpr_read_b32 v5, a52304; GFX950-NEXT: v_accvgpr_read_b32 v6, a62305; GFX950-NEXT: v_accvgpr_read_b32 v7, a72306; GFX950-NEXT: v_accvgpr_read_b32 v8, a82307; GFX950-NEXT: v_accvgpr_read_b32 v9, a92308; GFX950-NEXT: v_accvgpr_read_b32 v10, a102309; GFX950-NEXT: v_accvgpr_read_b32 v11, a112310; GFX950-NEXT: v_accvgpr_read_b32 v12, a122311; GFX950-NEXT: v_accvgpr_read_b32 v13, a132312; GFX950-NEXT: v_accvgpr_read_b32 v14, a142313; GFX950-NEXT: v_accvgpr_read_b32 v15, a152314; GFX950-NEXT: v_accvgpr_read_b32 v16, a162315; GFX950-NEXT: v_accvgpr_read_b32 v17, a172316; GFX950-NEXT: v_accvgpr_read_b32 v18, a182317; GFX950-NEXT: v_accvgpr_read_b32 v19, a192318; GFX950-NEXT: v_accvgpr_read_b32 v20, a202319; GFX950-NEXT: v_accvgpr_read_b32 v21, a212320; GFX950-NEXT: v_accvgpr_read_b32 v22, a222321; GFX950-NEXT: v_accvgpr_read_b32 v23, a232322; GFX950-NEXT: v_accvgpr_read_b32 v24, a242323; GFX950-NEXT: v_accvgpr_read_b32 v25, a252324; GFX950-NEXT: v_accvgpr_read_b32 v26, a262325; GFX950-NEXT: v_accvgpr_read_b32 v27, a272326; GFX950-NEXT: v_accvgpr_read_b32 v28, a282327; GFX950-NEXT: v_accvgpr_read_b32 v29, a292328; GFX950-NEXT: v_accvgpr_read_b32 v30, a302329; GFX950-NEXT: v_accvgpr_read_b32 v31, a312330; GFX950-NEXT: ;;#ASMSTART2331; GFX950-NEXT: ; use v[0:31]2332; GFX950-NEXT: ;;#ASMEND2333; GFX950-NEXT: ;;#ASMSTART2334; GFX950-NEXT: ; use a322335; GFX950-NEXT: ;;#ASMEND2336; GFX950-NEXT: scratch_load_dword a34, off, s32 ; 4-byte Folded Reload2337; GFX950-NEXT: scratch_load_dword a33, off, s32 offset:4 ; 4-byte Folded Reload2338; GFX950-NEXT: scratch_load_dword a32, off, s32 offset:8 ; 4-byte Folded Reload2339; GFX950-NEXT: scratch_load_dword v63, off, s32 offset:12 ; 4-byte Folded Reload2340; GFX950-NEXT: scratch_load_dword v62, off, s32 offset:16 ; 4-byte Folded Reload2341; GFX950-NEXT: scratch_load_dword v61, off, s32 offset:20 ; 4-byte Folded Reload2342; GFX950-NEXT: scratch_load_dword v60, off, s32 offset:24 ; 4-byte Folded Reload2343; GFX950-NEXT: scratch_load_dword v59, off, s32 offset:28 ; 4-byte Folded Reload2344; GFX950-NEXT: scratch_load_dword v58, off, s32 offset:32 ; 4-byte Folded Reload2345; GFX950-NEXT: scratch_load_dword v57, off, s32 offset:36 ; 4-byte Folded Reload2346; GFX950-NEXT: scratch_load_dword v56, off, s32 offset:40 ; 4-byte Folded Reload2347; GFX950-NEXT: scratch_load_dword v47, off, s32 offset:44 ; 4-byte Folded Reload2348; GFX950-NEXT: scratch_load_dword v46, off, s32 offset:48 ; 4-byte Folded Reload2349; GFX950-NEXT: scratch_load_dword v45, off, s32 offset:52 ; 4-byte Folded Reload2350; GFX950-NEXT: scratch_load_dword v44, off, s32 offset:56 ; 4-byte Folded Reload2351; GFX950-NEXT: scratch_load_dword v43, off, s32 offset:60 ; 4-byte Folded Reload2352; GFX950-NEXT: scratch_load_dword v42, off, s32 offset:64 ; 4-byte Folded Reload2353; GFX950-NEXT: scratch_load_dword v41, off, s32 offset:68 ; 4-byte Folded Reload2354; GFX950-NEXT: scratch_load_dword v40, off, s32 offset:72 ; 4-byte Folded Reload2355; GFX950-NEXT: s_waitcnt vmcnt(0)2356; GFX950-NEXT: s_setpc_b64 s[30:31]2357 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 102358 %data = call i32 asm "; def $0", "=^VA"()2359 %vgpr.def = call { <32 x i32>, <32 x i32> } asm sideeffect "; def $0", "=${v[0:31]},=${v[32:63]}"()2360 %vgpr.0 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 02361 %vgpr.1 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 12362 %result = atomicrmw xor ptr %ptr, i32 %data seq_cst2363 call void asm sideeffect "; use $0", "{v[0:31]},{v[32:63]}"(<32 x i32> %vgpr.0, <32 x i32> %vgpr.1)2364 call void asm "; use $0", "^VA"(i32 %result)2365 ret void2366}2367 2368define void @flat_atomic_xor_expansion_i32_noret_a(ptr %ptr) #0 {2369; GFX90A-LABEL: flat_atomic_xor_expansion_i32_noret_a:2370; GFX90A: ; %bb.0:2371; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2372; GFX90A-NEXT: flat_load_dword v3, v[0:1]2373; GFX90A-NEXT: ;;#ASMSTART2374; GFX90A-NEXT: ; def a02375; GFX90A-NEXT: ;;#ASMEND2376; GFX90A-NEXT: v_accvgpr_read_b32 v4, a02377; GFX90A-NEXT: s_mov_b64 s[4:5], 02378; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start2379; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12380; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2381; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v42382; GFX90A-NEXT: buffer_wbl22383; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc2384; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2385; GFX90A-NEXT: buffer_invl22386; GFX90A-NEXT: buffer_wbinvl1_vol2387; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32388; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2389; GFX90A-NEXT: v_mov_b32_e32 v3, v22390; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2391; GFX90A-NEXT: s_cbranch_execnz .LBB30_12392; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2393; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2394; GFX90A-NEXT: s_setpc_b64 s[30:31]2395;2396; GFX950-LABEL: flat_atomic_xor_expansion_i32_noret_a:2397; GFX950: ; %bb.0:2398; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2399; GFX950-NEXT: flat_load_dword v3, v[0:1]2400; GFX950-NEXT: ;;#ASMSTART2401; GFX950-NEXT: ; def a02402; GFX950-NEXT: ;;#ASMEND2403; GFX950-NEXT: s_mov_b64 s[0:1], 02404; GFX950-NEXT: v_accvgpr_read_b32 v4, a02405; GFX950-NEXT: .LBB30_1: ; %atomicrmw.start2406; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12407; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2408; GFX950-NEXT: v_xor_b32_e32 v2, v3, v42409; GFX950-NEXT: buffer_wbl2 sc0 sc12410; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc12411; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2412; GFX950-NEXT: buffer_inv sc0 sc12413; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32414; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2415; GFX950-NEXT: v_mov_b32_e32 v3, v22416; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]2417; GFX950-NEXT: s_cbranch_execnz .LBB30_12418; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end2419; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2420; GFX950-NEXT: s_setpc_b64 s[30:31]2421 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 102422 %data = call i32 asm "; def $0", "=a"()2423 %unused = atomicrmw xor ptr %ptr, i32 %data seq_cst2424 ret void2425}2426 2427define void @flat_atomic_xor_expansion_i32_noret_av(ptr %ptr) #0 {2428; GFX90A-LABEL: flat_atomic_xor_expansion_i32_noret_av:2429; GFX90A: ; %bb.0:2430; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2431; GFX90A-NEXT: flat_load_dword v3, v[0:1]2432; GFX90A-NEXT: s_mov_b64 s[4:5], 02433; GFX90A-NEXT: ;;#ASMSTART2434; GFX90A-NEXT: ; def v42435; GFX90A-NEXT: ;;#ASMEND2436; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start2437; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12438; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2439; GFX90A-NEXT: v_xor_b32_e32 v2, v3, v42440; GFX90A-NEXT: buffer_wbl22441; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc2442; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2443; GFX90A-NEXT: buffer_invl22444; GFX90A-NEXT: buffer_wbinvl1_vol2445; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32446; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2447; GFX90A-NEXT: v_mov_b32_e32 v3, v22448; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2449; GFX90A-NEXT: s_cbranch_execnz .LBB31_12450; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2451; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2452; GFX90A-NEXT: s_setpc_b64 s[30:31]2453;2454; GFX950-LABEL: flat_atomic_xor_expansion_i32_noret_av:2455; GFX950: ; %bb.0:2456; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2457; GFX950-NEXT: flat_load_dword v3, v[0:1]2458; GFX950-NEXT: s_mov_b64 s[0:1], 02459; GFX950-NEXT: ;;#ASMSTART2460; GFX950-NEXT: ; def v42461; GFX950-NEXT: ;;#ASMEND2462; GFX950-NEXT: .LBB31_1: ; %atomicrmw.start2463; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12464; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2465; GFX950-NEXT: v_xor_b32_e32 v2, v3, v42466; GFX950-NEXT: buffer_wbl2 sc0 sc12467; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc12468; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2469; GFX950-NEXT: buffer_inv sc0 sc12470; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v32471; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2472; GFX950-NEXT: v_mov_b32_e32 v3, v22473; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]2474; GFX950-NEXT: s_cbranch_execnz .LBB31_12475; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end2476; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2477; GFX950-NEXT: s_setpc_b64 s[30:31]2478 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 102479 %data = call i32 asm "; def $0", "=^VA"()2480 %unused = atomicrmw xor ptr %ptr, i32 %data seq_cst2481 ret void2482}2483 2484;---------------------------------------------------------------------2485; xor i64 cases with cmpxchg expansion2486;---------------------------------------------------------------------2487 2488; Input and result use AGPR2489define void @flat_atomic_xor_expansion_i64_ret_a_a(ptr %ptr) #0 {2490; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_a_a:2491; GFX90A: ; %bb.0:2492; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2493; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base2494; GFX90A-NEXT: ;;#ASMSTART2495; GFX90A-NEXT: ; def a[0:1]2496; GFX90A-NEXT: ;;#ASMEND2497; GFX90A-NEXT: v_accvgpr_read_b32 v7, a12498; GFX90A-NEXT: v_accvgpr_read_b32 v6, a02499; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v12500; GFX90A-NEXT: ; implicit-def: $agpr0_agpr12501; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc2502; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2503; GFX90A-NEXT: s_cbranch_execz .LBB32_42504; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global2505; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2506; GFX90A-NEXT: s_mov_b64 s[6:7], 02507; GFX90A-NEXT: .LBB32_2: ; %atomicrmw.start2508; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12509; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2510; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v72511; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v62512; GFX90A-NEXT: buffer_wbl22513; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc2514; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2515; GFX90A-NEXT: buffer_invl22516; GFX90A-NEXT: buffer_wbinvl1_vol2517; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2518; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2519; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]2520; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]2521; GFX90A-NEXT: s_cbranch_execnz .LBB32_22522; GFX90A-NEXT: ; %bb.3: ; %Flow2523; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]2524; GFX90A-NEXT: v_accvgpr_write_b32 a0, v22525; GFX90A-NEXT: v_accvgpr_write_b32 a1, v32526; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr12527; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr72528; GFX90A-NEXT: .LBB32_4: ; %Flow32529; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]2530; GFX90A-NEXT: s_cbranch_execz .LBB32_62531; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private2532; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2533; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc2534; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen2535; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:42536; GFX90A-NEXT: s_waitcnt vmcnt(1)2537; GFX90A-NEXT: v_accvgpr_write_b32 a0, v12538; GFX90A-NEXT: s_waitcnt vmcnt(0)2539; GFX90A-NEXT: v_accvgpr_write_b32 a1, v22540; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v62541; GFX90A-NEXT: v_xor_b32_e32 v2, v2, v72542; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen2543; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:42544; GFX90A-NEXT: .LBB32_6: ; %atomicrmw.phi2545; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2546; GFX90A-NEXT: ;;#ASMSTART2547; GFX90A-NEXT: ; use a[0:1]2548; GFX90A-NEXT: ;;#ASMEND2549; GFX90A-NEXT: s_waitcnt vmcnt(0)2550; GFX90A-NEXT: s_setpc_b64 s[30:31]2551;2552; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_a_a:2553; GFX950: ; %bb.0:2554; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2555; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base2556; GFX950-NEXT: ;;#ASMSTART2557; GFX950-NEXT: ; def a[0:1]2558; GFX950-NEXT: ;;#ASMEND2559; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12560; GFX950-NEXT: v_accvgpr_read_b32 v7, a12561; GFX950-NEXT: v_accvgpr_read_b32 v6, a02562; GFX950-NEXT: ; implicit-def: $agpr0_agpr12563; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc2564; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2565; GFX950-NEXT: s_cbranch_execz .LBB32_42566; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global2567; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2568; GFX950-NEXT: s_mov_b64 s[2:3], 02569; GFX950-NEXT: .LBB32_2: ; %atomicrmw.start2570; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12571; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2572; GFX950-NEXT: v_xor_b32_e32 v3, v5, v72573; GFX950-NEXT: v_xor_b32_e32 v2, v4, v62574; GFX950-NEXT: buffer_wbl2 sc0 sc12575; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc12576; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2577; GFX950-NEXT: buffer_inv sc0 sc12578; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2579; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]2580; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]2581; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]2582; GFX950-NEXT: s_cbranch_execnz .LBB32_22583; GFX950-NEXT: ; %bb.3: ; %Flow2584; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]2585; GFX950-NEXT: v_accvgpr_write_b32 a0, v22586; GFX950-NEXT: v_accvgpr_write_b32 a1, v32587; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr12588; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr72589; GFX950-NEXT: .LBB32_4: ; %Flow32590; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2591; GFX950-NEXT: s_cbranch_execz .LBB32_62592; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private2593; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2594; GFX950-NEXT: s_nop 12595; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2596; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off2597; GFX950-NEXT: s_waitcnt vmcnt(0)2598; GFX950-NEXT: v_accvgpr_write_b32 a0, v02599; GFX950-NEXT: v_xor_b32_e32 v3, v1, v72600; GFX950-NEXT: v_xor_b32_e32 v2, v0, v62601; GFX950-NEXT: v_accvgpr_write_b32 a1, v12602; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off2603; GFX950-NEXT: .LBB32_6: ; %atomicrmw.phi2604; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2605; GFX950-NEXT: ;;#ASMSTART2606; GFX950-NEXT: ; use a[0:1]2607; GFX950-NEXT: ;;#ASMEND2608; GFX950-NEXT: s_waitcnt vmcnt(0)2609; GFX950-NEXT: s_setpc_b64 s[30:31]2610 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 102611 %data = call i64 asm "; def $0", "=a"()2612 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst2613 call void asm "; use $0", "a"(i64 %result)2614 ret void2615}2616 2617; Input is AGPR, result used as VGPR.2618define void @flat_atomic_xor_expansion_i64_ret_a_v(ptr %ptr) #0 {2619; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_a_v:2620; GFX90A: ; %bb.0:2621; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2622; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base2623; GFX90A-NEXT: ;;#ASMSTART2624; GFX90A-NEXT: ; def a[0:1]2625; GFX90A-NEXT: ;;#ASMEND2626; GFX90A-NEXT: v_accvgpr_read_b32 v7, a12627; GFX90A-NEXT: v_accvgpr_read_b32 v6, a02628; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v12629; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr32630; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc2631; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2632; GFX90A-NEXT: s_cbranch_execz .LBB33_42633; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global2634; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2635; GFX90A-NEXT: s_mov_b64 s[6:7], 02636; GFX90A-NEXT: .LBB33_2: ; %atomicrmw.start2637; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12638; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2639; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v72640; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v62641; GFX90A-NEXT: buffer_wbl22642; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc2643; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2644; GFX90A-NEXT: buffer_invl22645; GFX90A-NEXT: buffer_wbinvl1_vol2646; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2647; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2648; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]2649; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]2650; GFX90A-NEXT: s_cbranch_execnz .LBB33_22651; GFX90A-NEXT: ; %bb.3: ; %Flow2652; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]2653; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr12654; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr72655; GFX90A-NEXT: .LBB33_4: ; %Flow32656; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]2657; GFX90A-NEXT: s_cbranch_execz .LBB33_62658; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private2659; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2660; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc2661; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:42662; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen2663; GFX90A-NEXT: s_waitcnt vmcnt(1)2664; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v72665; GFX90A-NEXT: s_waitcnt vmcnt(0)2666; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v62667; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen2668; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:42669; GFX90A-NEXT: .LBB33_6: ; %atomicrmw.phi2670; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2671; GFX90A-NEXT: ;;#ASMSTART2672; GFX90A-NEXT: ; use v[2:3]2673; GFX90A-NEXT: ;;#ASMEND2674; GFX90A-NEXT: s_waitcnt vmcnt(0)2675; GFX90A-NEXT: s_setpc_b64 s[30:31]2676;2677; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_a_v:2678; GFX950: ; %bb.0:2679; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2680; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base2681; GFX950-NEXT: ;;#ASMSTART2682; GFX950-NEXT: ; def a[0:1]2683; GFX950-NEXT: ;;#ASMEND2684; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12685; GFX950-NEXT: v_accvgpr_read_b32 v7, a12686; GFX950-NEXT: v_accvgpr_read_b32 v6, a02687; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr32688; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc2689; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2690; GFX950-NEXT: s_cbranch_execz .LBB33_42691; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global2692; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2693; GFX950-NEXT: s_mov_b64 s[2:3], 02694; GFX950-NEXT: .LBB33_2: ; %atomicrmw.start2695; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12696; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2697; GFX950-NEXT: v_xor_b32_e32 v3, v5, v72698; GFX950-NEXT: v_xor_b32_e32 v2, v4, v62699; GFX950-NEXT: buffer_wbl2 sc0 sc12700; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc12701; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2702; GFX950-NEXT: buffer_inv sc0 sc12703; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2704; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]2705; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]2706; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]2707; GFX950-NEXT: s_cbranch_execnz .LBB33_22708; GFX950-NEXT: ; %bb.3: ; %Flow2709; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]2710; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr12711; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr72712; GFX950-NEXT: .LBB33_4: ; %Flow32713; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2714; GFX950-NEXT: s_cbranch_execz .LBB33_62715; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private2716; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2717; GFX950-NEXT: s_nop 12718; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2719; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off2720; GFX950-NEXT: s_waitcnt vmcnt(0)2721; GFX950-NEXT: v_xor_b32_e32 v1, v3, v72722; GFX950-NEXT: v_xor_b32_e32 v0, v2, v62723; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off2724; GFX950-NEXT: .LBB33_6: ; %atomicrmw.phi2725; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2726; GFX950-NEXT: ;;#ASMSTART2727; GFX950-NEXT: ; use v[2:3]2728; GFX950-NEXT: ;;#ASMEND2729; GFX950-NEXT: s_waitcnt vmcnt(0)2730; GFX950-NEXT: s_setpc_b64 s[30:31]2731 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 102732 %data = call i64 asm "; def $0", "=a"()2733 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst2734 call void asm "; use $0", "v"(i64 %result)2735 ret void2736}2737 2738; Input is VGPR, result used as AGPR2739define void @flat_atomic_xor_expansion_i64_ret_v_a(ptr %ptr) #0 {2740; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_v_a:2741; GFX90A: ; %bb.0:2742; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2743; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base2744; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v12745; GFX90A-NEXT: ;;#ASMSTART2746; GFX90A-NEXT: ; def v[6:7]2747; GFX90A-NEXT: ;;#ASMEND2748; GFX90A-NEXT: ; implicit-def: $agpr0_agpr12749; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc2750; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2751; GFX90A-NEXT: s_cbranch_execz .LBB34_42752; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global2753; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2754; GFX90A-NEXT: s_mov_b64 s[6:7], 02755; GFX90A-NEXT: .LBB34_2: ; %atomicrmw.start2756; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12757; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2758; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v72759; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v62760; GFX90A-NEXT: buffer_wbl22761; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc2762; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2763; GFX90A-NEXT: buffer_invl22764; GFX90A-NEXT: buffer_wbinvl1_vol2765; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2766; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2767; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]2768; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]2769; GFX90A-NEXT: s_cbranch_execnz .LBB34_22770; GFX90A-NEXT: ; %bb.3: ; %Flow2771; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]2772; GFX90A-NEXT: v_accvgpr_write_b32 a0, v22773; GFX90A-NEXT: v_accvgpr_write_b32 a1, v32774; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr12775; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr72776; GFX90A-NEXT: .LBB34_4: ; %Flow32777; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]2778; GFX90A-NEXT: s_cbranch_execz .LBB34_62779; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private2780; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2781; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc2782; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen2783; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:42784; GFX90A-NEXT: s_waitcnt vmcnt(1)2785; GFX90A-NEXT: v_accvgpr_write_b32 a0, v12786; GFX90A-NEXT: s_waitcnt vmcnt(0)2787; GFX90A-NEXT: v_accvgpr_write_b32 a1, v22788; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v62789; GFX90A-NEXT: v_xor_b32_e32 v2, v2, v72790; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen2791; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:42792; GFX90A-NEXT: .LBB34_6: ; %atomicrmw.phi2793; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2794; GFX90A-NEXT: ;;#ASMSTART2795; GFX90A-NEXT: ; use a[0:1]2796; GFX90A-NEXT: ;;#ASMEND2797; GFX90A-NEXT: s_waitcnt vmcnt(0)2798; GFX90A-NEXT: s_setpc_b64 s[30:31]2799;2800; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_v_a:2801; GFX950: ; %bb.0:2802; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2803; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base2804; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12805; GFX950-NEXT: ;;#ASMSTART2806; GFX950-NEXT: ; def v[6:7]2807; GFX950-NEXT: ;;#ASMEND2808; GFX950-NEXT: ; implicit-def: $agpr0_agpr12809; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc2810; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2811; GFX950-NEXT: s_cbranch_execz .LBB34_42812; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global2813; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2814; GFX950-NEXT: s_mov_b64 s[2:3], 02815; GFX950-NEXT: .LBB34_2: ; %atomicrmw.start2816; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12817; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2818; GFX950-NEXT: v_xor_b32_e32 v3, v5, v72819; GFX950-NEXT: v_xor_b32_e32 v2, v4, v62820; GFX950-NEXT: buffer_wbl2 sc0 sc12821; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc12822; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2823; GFX950-NEXT: buffer_inv sc0 sc12824; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2825; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]2826; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]2827; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]2828; GFX950-NEXT: s_cbranch_execnz .LBB34_22829; GFX950-NEXT: ; %bb.3: ; %Flow2830; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]2831; GFX950-NEXT: v_accvgpr_write_b32 a0, v22832; GFX950-NEXT: v_accvgpr_write_b32 a1, v32833; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr12834; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr72835; GFX950-NEXT: .LBB34_4: ; %Flow32836; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2837; GFX950-NEXT: s_cbranch_execz .LBB34_62838; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private2839; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2840; GFX950-NEXT: s_nop 12841; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2842; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off2843; GFX950-NEXT: s_waitcnt vmcnt(0)2844; GFX950-NEXT: v_accvgpr_write_b32 a0, v02845; GFX950-NEXT: v_xor_b32_e32 v3, v1, v72846; GFX950-NEXT: v_xor_b32_e32 v2, v0, v62847; GFX950-NEXT: v_accvgpr_write_b32 a1, v12848; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off2849; GFX950-NEXT: .LBB34_6: ; %atomicrmw.phi2850; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2851; GFX950-NEXT: ;;#ASMSTART2852; GFX950-NEXT: ; use a[0:1]2853; GFX950-NEXT: ;;#ASMEND2854; GFX950-NEXT: s_waitcnt vmcnt(0)2855; GFX950-NEXT: s_setpc_b64 s[30:31]2856 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 102857 %data = call i64 asm "; def $0", "=v"()2858 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst2859 call void asm "; use $0", "a"(i64 %result)2860 ret void2861}2862 2863; Input is AV, result also used as AV2864define void @flat_atomic_xor_expansion_i64_ret_av_av(ptr %ptr) #0 {2865; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_av_av:2866; GFX90A: ; %bb.0:2867; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2868; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base2869; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v12870; GFX90A-NEXT: ;;#ASMSTART2871; GFX90A-NEXT: ; def v[6:7]2872; GFX90A-NEXT: ;;#ASMEND2873; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr32874; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc2875; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2876; GFX90A-NEXT: s_cbranch_execz .LBB35_42877; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global2878; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2879; GFX90A-NEXT: s_mov_b64 s[6:7], 02880; GFX90A-NEXT: .LBB35_2: ; %atomicrmw.start2881; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12882; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2883; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v72884; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v62885; GFX90A-NEXT: buffer_wbl22886; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc2887; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2888; GFX90A-NEXT: buffer_invl22889; GFX90A-NEXT: buffer_wbinvl1_vol2890; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2891; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2892; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]2893; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]2894; GFX90A-NEXT: s_cbranch_execnz .LBB35_22895; GFX90A-NEXT: ; %bb.3: ; %Flow2896; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]2897; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr12898; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr72899; GFX90A-NEXT: .LBB35_4: ; %Flow32900; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]2901; GFX90A-NEXT: s_cbranch_execz .LBB35_62902; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private2903; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2904; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc2905; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:42906; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen2907; GFX90A-NEXT: s_waitcnt vmcnt(1)2908; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v72909; GFX90A-NEXT: s_waitcnt vmcnt(0)2910; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v62911; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen2912; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:42913; GFX90A-NEXT: .LBB35_6: ; %atomicrmw.phi2914; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2915; GFX90A-NEXT: ;;#ASMSTART2916; GFX90A-NEXT: ; use v[2:3]2917; GFX90A-NEXT: ;;#ASMEND2918; GFX90A-NEXT: s_waitcnt vmcnt(0)2919; GFX90A-NEXT: s_setpc_b64 s[30:31]2920;2921; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_av_av:2922; GFX950: ; %bb.0:2923; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2924; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base2925; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12926; GFX950-NEXT: ;;#ASMSTART2927; GFX950-NEXT: ; def v[6:7]2928; GFX950-NEXT: ;;#ASMEND2929; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr32930; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc2931; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2932; GFX950-NEXT: s_cbranch_execz .LBB35_42933; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global2934; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2935; GFX950-NEXT: s_mov_b64 s[2:3], 02936; GFX950-NEXT: .LBB35_2: ; %atomicrmw.start2937; GFX950-NEXT: ; =>This Inner Loop Header: Depth=12938; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2939; GFX950-NEXT: v_xor_b32_e32 v3, v5, v72940; GFX950-NEXT: v_xor_b32_e32 v2, v4, v62941; GFX950-NEXT: buffer_wbl2 sc0 sc12942; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc12943; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2944; GFX950-NEXT: buffer_inv sc0 sc12945; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]2946; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]2947; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]2948; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]2949; GFX950-NEXT: s_cbranch_execnz .LBB35_22950; GFX950-NEXT: ; %bb.3: ; %Flow2951; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]2952; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr12953; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr72954; GFX950-NEXT: .LBB35_4: ; %Flow32955; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2956; GFX950-NEXT: s_cbranch_execz .LBB35_62957; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private2958; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2959; GFX950-NEXT: s_nop 12960; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2961; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off2962; GFX950-NEXT: s_waitcnt vmcnt(0)2963; GFX950-NEXT: v_xor_b32_e32 v1, v3, v72964; GFX950-NEXT: v_xor_b32_e32 v0, v2, v62965; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off2966; GFX950-NEXT: .LBB35_6: ; %atomicrmw.phi2967; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]2968; GFX950-NEXT: ;;#ASMSTART2969; GFX950-NEXT: ; use v[2:3]2970; GFX950-NEXT: ;;#ASMEND2971; GFX950-NEXT: s_waitcnt vmcnt(0)2972; GFX950-NEXT: s_setpc_b64 s[30:31]2973 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 102974 %data = call i64 asm "; def $0", "=^VA"()2975 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst2976 call void asm "; use $0", "^VA"(i64 %result)2977 ret void2978}2979 2980; Input is AV, used as v2981define void @flat_atomic_xor_expansion_i64_ret_av_v(ptr %ptr) #0 {2982; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_av_v:2983; GFX90A: ; %bb.0:2984; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2985; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base2986; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v12987; GFX90A-NEXT: ;;#ASMSTART2988; GFX90A-NEXT: ; def v[6:7]2989; GFX90A-NEXT: ;;#ASMEND2990; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr32991; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc2992; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2993; GFX90A-NEXT: s_cbranch_execz .LBB36_42994; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global2995; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2996; GFX90A-NEXT: s_mov_b64 s[6:7], 02997; GFX90A-NEXT: .LBB36_2: ; %atomicrmw.start2998; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12999; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3000; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73001; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63002; GFX90A-NEXT: buffer_wbl23003; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3004; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3005; GFX90A-NEXT: buffer_invl23006; GFX90A-NEXT: buffer_wbinvl1_vol3007; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3008; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3009; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3010; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3011; GFX90A-NEXT: s_cbranch_execnz .LBB36_23012; GFX90A-NEXT: ; %bb.3: ; %Flow3013; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3014; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13015; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73016; GFX90A-NEXT: .LBB36_4: ; %Flow33017; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3018; GFX90A-NEXT: s_cbranch_execz .LBB36_63019; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private3020; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3021; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3022; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:43023; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen3024; GFX90A-NEXT: s_waitcnt vmcnt(1)3025; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v73026; GFX90A-NEXT: s_waitcnt vmcnt(0)3027; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v63028; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen3029; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:43030; GFX90A-NEXT: .LBB36_6: ; %atomicrmw.phi3031; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3032; GFX90A-NEXT: ;;#ASMSTART3033; GFX90A-NEXT: ; use v[2:3]3034; GFX90A-NEXT: ;;#ASMEND3035; GFX90A-NEXT: s_waitcnt vmcnt(0)3036; GFX90A-NEXT: s_setpc_b64 s[30:31]3037;3038; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_av_v:3039; GFX950: ; %bb.0:3040; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3041; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3042; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13043; GFX950-NEXT: ;;#ASMSTART3044; GFX950-NEXT: ; def v[6:7]3045; GFX950-NEXT: ;;#ASMEND3046; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr33047; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3048; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3049; GFX950-NEXT: s_cbranch_execz .LBB36_43050; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global3051; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3052; GFX950-NEXT: s_mov_b64 s[2:3], 03053; GFX950-NEXT: .LBB36_2: ; %atomicrmw.start3054; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13055; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3056; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73057; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63058; GFX950-NEXT: buffer_wbl2 sc0 sc13059; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13060; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3061; GFX950-NEXT: buffer_inv sc0 sc13062; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3063; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3064; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3065; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3066; GFX950-NEXT: s_cbranch_execnz .LBB36_23067; GFX950-NEXT: ; %bb.3: ; %Flow3068; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3069; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13070; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73071; GFX950-NEXT: .LBB36_4: ; %Flow33072; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3073; GFX950-NEXT: s_cbranch_execz .LBB36_63074; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private3075; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3076; GFX950-NEXT: s_nop 13077; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3078; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off3079; GFX950-NEXT: s_waitcnt vmcnt(0)3080; GFX950-NEXT: v_xor_b32_e32 v1, v3, v73081; GFX950-NEXT: v_xor_b32_e32 v0, v2, v63082; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off3083; GFX950-NEXT: .LBB36_6: ; %atomicrmw.phi3084; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3085; GFX950-NEXT: ;;#ASMSTART3086; GFX950-NEXT: ; use v[2:3]3087; GFX950-NEXT: ;;#ASMEND3088; GFX950-NEXT: s_waitcnt vmcnt(0)3089; GFX950-NEXT: s_setpc_b64 s[30:31]3090 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103091 %data = call i64 asm "; def $0", "=^VA"()3092 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst3093 call void asm "; use $0", "v"(i64 %result)3094 ret void3095}3096 3097; Input is AV, used as a3098define void @flat_atomic_xor_expansion_i64_ret_av_a(ptr %ptr) #0 {3099; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_av_a:3100; GFX90A: ; %bb.0:3101; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3102; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base3103; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v13104; GFX90A-NEXT: ;;#ASMSTART3105; GFX90A-NEXT: ; def v[6:7]3106; GFX90A-NEXT: ;;#ASMEND3107; GFX90A-NEXT: ; implicit-def: $agpr0_agpr13108; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc3109; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]3110; GFX90A-NEXT: s_cbranch_execz .LBB37_43111; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global3112; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3113; GFX90A-NEXT: s_mov_b64 s[6:7], 03114; GFX90A-NEXT: .LBB37_2: ; %atomicrmw.start3115; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13116; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3117; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73118; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63119; GFX90A-NEXT: buffer_wbl23120; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3121; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3122; GFX90A-NEXT: buffer_invl23123; GFX90A-NEXT: buffer_wbinvl1_vol3124; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3125; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3126; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3127; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3128; GFX90A-NEXT: s_cbranch_execnz .LBB37_23129; GFX90A-NEXT: ; %bb.3: ; %Flow3130; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3131; GFX90A-NEXT: v_accvgpr_write_b32 a0, v23132; GFX90A-NEXT: v_accvgpr_write_b32 a1, v33133; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13134; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73135; GFX90A-NEXT: .LBB37_4: ; %Flow33136; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3137; GFX90A-NEXT: s_cbranch_execz .LBB37_63138; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private3139; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3140; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3141; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen3142; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:43143; GFX90A-NEXT: s_waitcnt vmcnt(1)3144; GFX90A-NEXT: v_accvgpr_write_b32 a0, v13145; GFX90A-NEXT: s_waitcnt vmcnt(0)3146; GFX90A-NEXT: v_accvgpr_write_b32 a1, v23147; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v63148; GFX90A-NEXT: v_xor_b32_e32 v2, v2, v73149; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen3150; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:43151; GFX90A-NEXT: .LBB37_6: ; %atomicrmw.phi3152; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3153; GFX90A-NEXT: ;;#ASMSTART3154; GFX90A-NEXT: ; use a[0:1]3155; GFX90A-NEXT: ;;#ASMEND3156; GFX90A-NEXT: s_waitcnt vmcnt(0)3157; GFX90A-NEXT: s_setpc_b64 s[30:31]3158;3159; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_av_a:3160; GFX950: ; %bb.0:3161; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3162; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3163; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13164; GFX950-NEXT: ;;#ASMSTART3165; GFX950-NEXT: ; def v[6:7]3166; GFX950-NEXT: ;;#ASMEND3167; GFX950-NEXT: ; implicit-def: $agpr0_agpr13168; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3169; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3170; GFX950-NEXT: s_cbranch_execz .LBB37_43171; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global3172; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3173; GFX950-NEXT: s_mov_b64 s[2:3], 03174; GFX950-NEXT: .LBB37_2: ; %atomicrmw.start3175; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13176; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3177; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73178; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63179; GFX950-NEXT: buffer_wbl2 sc0 sc13180; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13181; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3182; GFX950-NEXT: buffer_inv sc0 sc13183; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3184; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3185; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3186; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3187; GFX950-NEXT: s_cbranch_execnz .LBB37_23188; GFX950-NEXT: ; %bb.3: ; %Flow3189; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3190; GFX950-NEXT: v_accvgpr_write_b32 a0, v23191; GFX950-NEXT: v_accvgpr_write_b32 a1, v33192; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13193; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73194; GFX950-NEXT: .LBB37_4: ; %Flow33195; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3196; GFX950-NEXT: s_cbranch_execz .LBB37_63197; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private3198; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3199; GFX950-NEXT: s_nop 13200; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3201; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off3202; GFX950-NEXT: s_waitcnt vmcnt(0)3203; GFX950-NEXT: v_accvgpr_write_b32 a0, v03204; GFX950-NEXT: v_xor_b32_e32 v3, v1, v73205; GFX950-NEXT: v_xor_b32_e32 v2, v0, v63206; GFX950-NEXT: v_accvgpr_write_b32 a1, v13207; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off3208; GFX950-NEXT: .LBB37_6: ; %atomicrmw.phi3209; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3210; GFX950-NEXT: ;;#ASMSTART3211; GFX950-NEXT: ; use a[0:1]3212; GFX950-NEXT: ;;#ASMEND3213; GFX950-NEXT: s_waitcnt vmcnt(0)3214; GFX950-NEXT: s_setpc_b64 s[30:31]3215 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103216 %data = call i64 asm "; def $0", "=^VA"()3217 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst3218 call void asm "; use $0", "a"(i64 %result)3219 ret void3220}3221 3222; Input is a, result used as AV3223define void @flat_atomic_xor_expansion_i64_ret_a_av(ptr %ptr) #0 {3224; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_a_av:3225; GFX90A: ; %bb.0:3226; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3227; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base3228; GFX90A-NEXT: ;;#ASMSTART3229; GFX90A-NEXT: ; def a[0:1]3230; GFX90A-NEXT: ;;#ASMEND3231; GFX90A-NEXT: v_accvgpr_read_b32 v7, a13232; GFX90A-NEXT: v_accvgpr_read_b32 v6, a03233; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v13234; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr33235; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc3236; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]3237; GFX90A-NEXT: s_cbranch_execz .LBB38_43238; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global3239; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3240; GFX90A-NEXT: s_mov_b64 s[6:7], 03241; GFX90A-NEXT: .LBB38_2: ; %atomicrmw.start3242; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13243; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3244; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73245; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63246; GFX90A-NEXT: buffer_wbl23247; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3248; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3249; GFX90A-NEXT: buffer_invl23250; GFX90A-NEXT: buffer_wbinvl1_vol3251; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3252; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3253; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3254; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3255; GFX90A-NEXT: s_cbranch_execnz .LBB38_23256; GFX90A-NEXT: ; %bb.3: ; %Flow3257; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3258; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13259; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73260; GFX90A-NEXT: .LBB38_4: ; %Flow33261; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3262; GFX90A-NEXT: s_cbranch_execz .LBB38_63263; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private3264; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3265; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3266; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:43267; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen3268; GFX90A-NEXT: s_waitcnt vmcnt(1)3269; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v73270; GFX90A-NEXT: s_waitcnt vmcnt(0)3271; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v63272; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen3273; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:43274; GFX90A-NEXT: .LBB38_6: ; %atomicrmw.phi3275; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3276; GFX90A-NEXT: ;;#ASMSTART3277; GFX90A-NEXT: ; use v[2:3]3278; GFX90A-NEXT: ;;#ASMEND3279; GFX90A-NEXT: s_waitcnt vmcnt(0)3280; GFX90A-NEXT: s_setpc_b64 s[30:31]3281;3282; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_a_av:3283; GFX950: ; %bb.0:3284; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3285; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3286; GFX950-NEXT: ;;#ASMSTART3287; GFX950-NEXT: ; def a[0:1]3288; GFX950-NEXT: ;;#ASMEND3289; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13290; GFX950-NEXT: v_accvgpr_read_b32 v7, a13291; GFX950-NEXT: v_accvgpr_read_b32 v6, a03292; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr33293; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3294; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3295; GFX950-NEXT: s_cbranch_execz .LBB38_43296; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global3297; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3298; GFX950-NEXT: s_mov_b64 s[2:3], 03299; GFX950-NEXT: .LBB38_2: ; %atomicrmw.start3300; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13301; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3302; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73303; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63304; GFX950-NEXT: buffer_wbl2 sc0 sc13305; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13306; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3307; GFX950-NEXT: buffer_inv sc0 sc13308; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3309; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3310; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3311; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3312; GFX950-NEXT: s_cbranch_execnz .LBB38_23313; GFX950-NEXT: ; %bb.3: ; %Flow3314; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3315; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13316; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73317; GFX950-NEXT: .LBB38_4: ; %Flow33318; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3319; GFX950-NEXT: s_cbranch_execz .LBB38_63320; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private3321; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3322; GFX950-NEXT: s_nop 13323; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3324; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off3325; GFX950-NEXT: s_waitcnt vmcnt(0)3326; GFX950-NEXT: v_xor_b32_e32 v1, v3, v73327; GFX950-NEXT: v_xor_b32_e32 v0, v2, v63328; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off3329; GFX950-NEXT: .LBB38_6: ; %atomicrmw.phi3330; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3331; GFX950-NEXT: ;;#ASMSTART3332; GFX950-NEXT: ; use v[2:3]3333; GFX950-NEXT: ;;#ASMEND3334; GFX950-NEXT: s_waitcnt vmcnt(0)3335; GFX950-NEXT: s_setpc_b64 s[30:31]3336 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103337 %data = call i64 asm "; def $0", "=a"()3338 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst3339 call void asm "; use $0", "^VA"(i64 %result)3340 ret void3341}3342 3343; Input is v, result used as AV3344define void @flat_atomic_xor_expansion_i64_ret_v_av(ptr %ptr) #0 {3345; GFX90A-LABEL: flat_atomic_xor_expansion_i64_ret_v_av:3346; GFX90A: ; %bb.0:3347; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3348; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base3349; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v13350; GFX90A-NEXT: ;;#ASMSTART3351; GFX90A-NEXT: ; def v[6:7]3352; GFX90A-NEXT: ;;#ASMEND3353; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr33354; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc3355; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]3356; GFX90A-NEXT: s_cbranch_execz .LBB39_43357; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global3358; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3359; GFX90A-NEXT: s_mov_b64 s[6:7], 03360; GFX90A-NEXT: .LBB39_2: ; %atomicrmw.start3361; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13362; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3363; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73364; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63365; GFX90A-NEXT: buffer_wbl23366; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3367; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3368; GFX90A-NEXT: buffer_invl23369; GFX90A-NEXT: buffer_wbinvl1_vol3370; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3371; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3372; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3373; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3374; GFX90A-NEXT: s_cbranch_execnz .LBB39_23375; GFX90A-NEXT: ; %bb.3: ; %Flow3376; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3377; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13378; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73379; GFX90A-NEXT: .LBB39_4: ; %Flow33380; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3381; GFX90A-NEXT: s_cbranch_execz .LBB39_63382; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private3383; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3384; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3385; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:43386; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen3387; GFX90A-NEXT: s_waitcnt vmcnt(1)3388; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v73389; GFX90A-NEXT: s_waitcnt vmcnt(0)3390; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v63391; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen3392; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:43393; GFX90A-NEXT: .LBB39_6: ; %atomicrmw.phi3394; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3395; GFX90A-NEXT: ;;#ASMSTART3396; GFX90A-NEXT: ; use v[2:3]3397; GFX90A-NEXT: ;;#ASMEND3398; GFX90A-NEXT: s_waitcnt vmcnt(0)3399; GFX90A-NEXT: s_setpc_b64 s[30:31]3400;3401; GFX950-LABEL: flat_atomic_xor_expansion_i64_ret_v_av:3402; GFX950: ; %bb.0:3403; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3404; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3405; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13406; GFX950-NEXT: ;;#ASMSTART3407; GFX950-NEXT: ; def v[6:7]3408; GFX950-NEXT: ;;#ASMEND3409; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr33410; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3411; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3412; GFX950-NEXT: s_cbranch_execz .LBB39_43413; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global3414; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3415; GFX950-NEXT: s_mov_b64 s[2:3], 03416; GFX950-NEXT: .LBB39_2: ; %atomicrmw.start3417; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13418; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3419; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73420; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63421; GFX950-NEXT: buffer_wbl2 sc0 sc13422; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13423; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3424; GFX950-NEXT: buffer_inv sc0 sc13425; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3426; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3427; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3428; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3429; GFX950-NEXT: s_cbranch_execnz .LBB39_23430; GFX950-NEXT: ; %bb.3: ; %Flow3431; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3432; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13433; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73434; GFX950-NEXT: .LBB39_4: ; %Flow33435; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3436; GFX950-NEXT: s_cbranch_execz .LBB39_63437; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private3438; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3439; GFX950-NEXT: s_nop 13440; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3441; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off3442; GFX950-NEXT: s_waitcnt vmcnt(0)3443; GFX950-NEXT: v_xor_b32_e32 v1, v3, v73444; GFX950-NEXT: v_xor_b32_e32 v0, v2, v63445; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off3446; GFX950-NEXT: .LBB39_6: ; %atomicrmw.phi3447; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3448; GFX950-NEXT: ;;#ASMSTART3449; GFX950-NEXT: ; use v[2:3]3450; GFX950-NEXT: ;;#ASMEND3451; GFX950-NEXT: s_waitcnt vmcnt(0)3452; GFX950-NEXT: s_setpc_b64 s[30:31]3453 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103454 %data = call i64 asm "; def $0", "=v"()3455 %result = atomicrmw xor ptr %ptr, i64 %data seq_cst3456 call void asm "; use $0", "^VA"(i64 %result)3457 ret void3458}3459 3460define void @flat_atomic_xor_expansion_i64_noret_a(ptr %ptr) #0 {3461; GFX90A-LABEL: flat_atomic_xor_expansion_i64_noret_a:3462; GFX90A: ; %bb.0:3463; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3464; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base3465; GFX90A-NEXT: ;;#ASMSTART3466; GFX90A-NEXT: ; def a[0:1]3467; GFX90A-NEXT: ;;#ASMEND3468; GFX90A-NEXT: v_accvgpr_read_b32 v7, a13469; GFX90A-NEXT: v_accvgpr_read_b32 v6, a03470; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v13471; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc3472; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]3473; GFX90A-NEXT: s_cbranch_execnz .LBB40_33474; GFX90A-NEXT: ; %bb.1: ; %Flow33475; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3476; GFX90A-NEXT: s_cbranch_execnz .LBB40_63477; GFX90A-NEXT: .LBB40_2: ; %atomicrmw.phi3478; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3479; GFX90A-NEXT: s_setpc_b64 s[30:31]3480; GFX90A-NEXT: .LBB40_3: ; %atomicrmw.global3481; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3482; GFX90A-NEXT: s_mov_b64 s[6:7], 03483; GFX90A-NEXT: .LBB40_4: ; %atomicrmw.start3484; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13485; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3486; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73487; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63488; GFX90A-NEXT: buffer_wbl23489; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3490; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3491; GFX90A-NEXT: buffer_invl23492; GFX90A-NEXT: buffer_wbinvl1_vol3493; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3494; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3495; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3496; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3497; GFX90A-NEXT: s_cbranch_execnz .LBB40_43498; GFX90A-NEXT: ; %bb.5: ; %Flow3499; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3500; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13501; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73502; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3503; GFX90A-NEXT: s_cbranch_execz .LBB40_23504; GFX90A-NEXT: .LBB40_6: ; %atomicrmw.private3505; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3506; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3507; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:43508; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen3509; GFX90A-NEXT: s_waitcnt vmcnt(1)3510; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v73511; GFX90A-NEXT: s_waitcnt vmcnt(0)3512; GFX90A-NEXT: v_xor_b32_e32 v2, v2, v63513; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen3514; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:43515; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3516; GFX90A-NEXT: s_waitcnt vmcnt(0)3517; GFX90A-NEXT: s_setpc_b64 s[30:31]3518;3519; GFX950-LABEL: flat_atomic_xor_expansion_i64_noret_a:3520; GFX950: ; %bb.0:3521; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3522; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3523; GFX950-NEXT: ;;#ASMSTART3524; GFX950-NEXT: ; def a[0:1]3525; GFX950-NEXT: ;;#ASMEND3526; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13527; GFX950-NEXT: v_accvgpr_read_b32 v7, a13528; GFX950-NEXT: v_accvgpr_read_b32 v6, a03529; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3530; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3531; GFX950-NEXT: s_cbranch_execnz .LBB40_33532; GFX950-NEXT: ; %bb.1: ; %Flow33533; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3534; GFX950-NEXT: s_cbranch_execnz .LBB40_63535; GFX950-NEXT: .LBB40_2: ; %atomicrmw.phi3536; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3537; GFX950-NEXT: s_setpc_b64 s[30:31]3538; GFX950-NEXT: .LBB40_3: ; %atomicrmw.global3539; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3540; GFX950-NEXT: s_mov_b64 s[2:3], 03541; GFX950-NEXT: .LBB40_4: ; %atomicrmw.start3542; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13543; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3544; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73545; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63546; GFX950-NEXT: buffer_wbl2 sc0 sc13547; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13548; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3549; GFX950-NEXT: buffer_inv sc0 sc13550; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3551; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3552; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3553; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3554; GFX950-NEXT: s_cbranch_execnz .LBB40_43555; GFX950-NEXT: ; %bb.5: ; %Flow3556; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3557; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13558; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73559; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3560; GFX950-NEXT: s_cbranch_execz .LBB40_23561; GFX950-NEXT: .LBB40_6: ; %atomicrmw.private3562; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3563; GFX950-NEXT: s_nop 13564; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc3565; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off3566; GFX950-NEXT: s_waitcnt vmcnt(0)3567; GFX950-NEXT: v_xor_b32_e32 v1, v1, v73568; GFX950-NEXT: v_xor_b32_e32 v0, v0, v63569; GFX950-NEXT: scratch_store_dwordx2 v2, v[0:1], off3570; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3571; GFX950-NEXT: s_waitcnt vmcnt(0)3572; GFX950-NEXT: s_setpc_b64 s[30:31]3573 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103574 %data = call i64 asm "; def $0", "=a"()3575 %unused = atomicrmw xor ptr %ptr, i64 %data seq_cst3576 ret void3577}3578 3579define void @flat_atomic_xor_expansion_i64_noret_av(ptr %ptr) #0 {3580; GFX90A-LABEL: flat_atomic_xor_expansion_i64_noret_av:3581; GFX90A: ; %bb.0:3582; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3583; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base3584; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v13585; GFX90A-NEXT: ;;#ASMSTART3586; GFX90A-NEXT: ; def v[6:7]3587; GFX90A-NEXT: ;;#ASMEND3588; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc3589; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]3590; GFX90A-NEXT: s_cbranch_execnz .LBB41_33591; GFX90A-NEXT: ; %bb.1: ; %Flow33592; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3593; GFX90A-NEXT: s_cbranch_execnz .LBB41_63594; GFX90A-NEXT: .LBB41_2: ; %atomicrmw.phi3595; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3596; GFX90A-NEXT: s_setpc_b64 s[30:31]3597; GFX90A-NEXT: .LBB41_3: ; %atomicrmw.global3598; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3599; GFX90A-NEXT: s_mov_b64 s[6:7], 03600; GFX90A-NEXT: .LBB41_4: ; %atomicrmw.start3601; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13602; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3603; GFX90A-NEXT: v_xor_b32_e32 v3, v5, v73604; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v63605; GFX90A-NEXT: buffer_wbl23606; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc3607; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3608; GFX90A-NEXT: buffer_invl23609; GFX90A-NEXT: buffer_wbinvl1_vol3610; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3611; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]3612; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3613; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]3614; GFX90A-NEXT: s_cbranch_execnz .LBB41_43615; GFX90A-NEXT: ; %bb.5: ; %Flow3616; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]3617; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr13618; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr73619; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]3620; GFX90A-NEXT: s_cbranch_execz .LBB41_23621; GFX90A-NEXT: .LBB41_6: ; %atomicrmw.private3622; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3623; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc3624; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:43625; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen3626; GFX90A-NEXT: s_waitcnt vmcnt(1)3627; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v73628; GFX90A-NEXT: s_waitcnt vmcnt(0)3629; GFX90A-NEXT: v_xor_b32_e32 v2, v2, v63630; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen3631; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:43632; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3633; GFX90A-NEXT: s_waitcnt vmcnt(0)3634; GFX90A-NEXT: s_setpc_b64 s[30:31]3635;3636; GFX950-LABEL: flat_atomic_xor_expansion_i64_noret_av:3637; GFX950: ; %bb.0:3638; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3639; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base3640; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13641; GFX950-NEXT: ;;#ASMSTART3642; GFX950-NEXT: ; def v[6:7]3643; GFX950-NEXT: ;;#ASMEND3644; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc3645; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3646; GFX950-NEXT: s_cbranch_execnz .LBB41_33647; GFX950-NEXT: ; %bb.1: ; %Flow33648; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3649; GFX950-NEXT: s_cbranch_execnz .LBB41_63650; GFX950-NEXT: .LBB41_2: ; %atomicrmw.phi3651; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3652; GFX950-NEXT: s_setpc_b64 s[30:31]3653; GFX950-NEXT: .LBB41_3: ; %atomicrmw.global3654; GFX950-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3655; GFX950-NEXT: s_mov_b64 s[2:3], 03656; GFX950-NEXT: .LBB41_4: ; %atomicrmw.start3657; GFX950-NEXT: ; =>This Inner Loop Header: Depth=13658; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3659; GFX950-NEXT: v_xor_b32_e32 v3, v5, v73660; GFX950-NEXT: v_xor_b32_e32 v2, v4, v63661; GFX950-NEXT: buffer_wbl2 sc0 sc13662; GFX950-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] sc0 sc13663; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3664; GFX950-NEXT: buffer_inv sc0 sc13665; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]3666; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]3667; GFX950-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3668; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]3669; GFX950-NEXT: s_cbranch_execnz .LBB41_43670; GFX950-NEXT: ; %bb.5: ; %Flow3671; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]3672; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr13673; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr73674; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3675; GFX950-NEXT: s_cbranch_execz .LBB41_23676; GFX950-NEXT: .LBB41_6: ; %atomicrmw.private3677; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3678; GFX950-NEXT: s_nop 13679; GFX950-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc3680; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v2, off3681; GFX950-NEXT: s_waitcnt vmcnt(0)3682; GFX950-NEXT: v_xor_b32_e32 v1, v1, v73683; GFX950-NEXT: v_xor_b32_e32 v0, v0, v63684; GFX950-NEXT: scratch_store_dwordx2 v2, v[0:1], off3685; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]3686; GFX950-NEXT: s_waitcnt vmcnt(0)3687; GFX950-NEXT: s_setpc_b64 s[30:31]3688 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 103689 %data = call i64 asm "; def $0", "=^VA"()3690 %unused = atomicrmw xor ptr %ptr, i64 %data seq_cst3691 ret void3692}3693 3694;---------------------------------------------------------------------3695; xor i32 cases with instruction3696;---------------------------------------------------------------------3697 3698; Input and result use AGPR3699define void @flat_atomic_xor_i32_ret_a_a(ptr %ptr) #0 {3700; GFX90A-LABEL: flat_atomic_xor_i32_ret_a_a:3701; GFX90A: ; %bb.0:3702; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3703; GFX90A-NEXT: ;;#ASMSTART3704; GFX90A-NEXT: ; def a03705; GFX90A-NEXT: ;;#ASMEND3706; GFX90A-NEXT: v_accvgpr_read_b32 v2, a03707; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3708; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3709; GFX90A-NEXT: buffer_wbinvl1_vol3710; GFX90A-NEXT: v_accvgpr_write_b32 a0, v03711; GFX90A-NEXT: ;;#ASMSTART3712; GFX90A-NEXT: ; use a03713; GFX90A-NEXT: ;;#ASMEND3714; GFX90A-NEXT: s_setpc_b64 s[30:31]3715;3716; GFX950-LABEL: flat_atomic_xor_i32_ret_a_a:3717; GFX950: ; %bb.0:3718; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3719; GFX950-NEXT: ;;#ASMSTART3720; GFX950-NEXT: ; def a03721; GFX950-NEXT: ;;#ASMEND3722; GFX950-NEXT: s_nop 03723; GFX950-NEXT: v_accvgpr_read_b32 v2, a03724; GFX950-NEXT: buffer_wbl2 sc13725; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03726; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3727; GFX950-NEXT: buffer_inv sc13728; GFX950-NEXT: v_accvgpr_write_b32 a0, v03729; GFX950-NEXT: ;;#ASMSTART3730; GFX950-NEXT: ; use a03731; GFX950-NEXT: ;;#ASMEND3732; GFX950-NEXT: s_setpc_b64 s[30:31]3733 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103734 %data = call i32 asm "; def $0", "=a"()3735 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.fine.grained.memory !03736 call void asm "; use $0", "a"(i32 %result)3737 ret void3738}3739 3740; Input is AGPR, result used as VGPR.3741define void @flat_atomic_xor_i32_ret_a_v(ptr %ptr) #0 {3742; GFX90A-LABEL: flat_atomic_xor_i32_ret_a_v:3743; GFX90A: ; %bb.0:3744; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3745; GFX90A-NEXT: ;;#ASMSTART3746; GFX90A-NEXT: ; def a03747; GFX90A-NEXT: ;;#ASMEND3748; GFX90A-NEXT: v_accvgpr_read_b32 v2, a03749; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3750; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3751; GFX90A-NEXT: buffer_wbinvl1_vol3752; GFX90A-NEXT: ;;#ASMSTART3753; GFX90A-NEXT: ; use v03754; GFX90A-NEXT: ;;#ASMEND3755; GFX90A-NEXT: s_setpc_b64 s[30:31]3756;3757; GFX950-LABEL: flat_atomic_xor_i32_ret_a_v:3758; GFX950: ; %bb.0:3759; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3760; GFX950-NEXT: ;;#ASMSTART3761; GFX950-NEXT: ; def a03762; GFX950-NEXT: ;;#ASMEND3763; GFX950-NEXT: s_nop 03764; GFX950-NEXT: v_accvgpr_read_b32 v2, a03765; GFX950-NEXT: buffer_wbl2 sc13766; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03767; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3768; GFX950-NEXT: buffer_inv sc13769; GFX950-NEXT: ;;#ASMSTART3770; GFX950-NEXT: ; use v03771; GFX950-NEXT: ;;#ASMEND3772; GFX950-NEXT: s_setpc_b64 s[30:31]3773 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103774 %data = call i32 asm "; def $0", "=a"()3775 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03776 call void asm "; use $0", "v"(i32 %result)3777 ret void3778}3779 3780; Input is VGPR, result used as AGPR3781define void @flat_atomic_xor_i32_ret_v_a(ptr %ptr) #0 {3782; GFX90A-LABEL: flat_atomic_xor_i32_ret_v_a:3783; GFX90A: ; %bb.0:3784; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3785; GFX90A-NEXT: ;;#ASMSTART3786; GFX90A-NEXT: ; def v23787; GFX90A-NEXT: ;;#ASMEND3788; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3789; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3790; GFX90A-NEXT: buffer_wbinvl1_vol3791; GFX90A-NEXT: v_accvgpr_write_b32 a0, v03792; GFX90A-NEXT: ;;#ASMSTART3793; GFX90A-NEXT: ; use a03794; GFX90A-NEXT: ;;#ASMEND3795; GFX90A-NEXT: s_setpc_b64 s[30:31]3796;3797; GFX950-LABEL: flat_atomic_xor_i32_ret_v_a:3798; GFX950: ; %bb.0:3799; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3800; GFX950-NEXT: ;;#ASMSTART3801; GFX950-NEXT: ; def v23802; GFX950-NEXT: ;;#ASMEND3803; GFX950-NEXT: buffer_wbl2 sc13804; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03805; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3806; GFX950-NEXT: buffer_inv sc13807; GFX950-NEXT: v_accvgpr_write_b32 a0, v03808; GFX950-NEXT: ;;#ASMSTART3809; GFX950-NEXT: ; use a03810; GFX950-NEXT: ;;#ASMEND3811; GFX950-NEXT: s_setpc_b64 s[30:31]3812 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103813 %data = call i32 asm "; def $0", "=v"()3814 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03815 call void asm "; use $0", "a"(i32 %result)3816 ret void3817}3818 3819; Input is AV, result also used as AV3820define void @flat_atomic_xor_i32_ret_av_av(ptr %ptr) #0 {3821; GFX90A-LABEL: flat_atomic_xor_i32_ret_av_av:3822; GFX90A: ; %bb.0:3823; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3824; GFX90A-NEXT: ;;#ASMSTART3825; GFX90A-NEXT: ; def v23826; GFX90A-NEXT: ;;#ASMEND3827; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3828; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3829; GFX90A-NEXT: buffer_wbinvl1_vol3830; GFX90A-NEXT: ;;#ASMSTART3831; GFX90A-NEXT: ; use v03832; GFX90A-NEXT: ;;#ASMEND3833; GFX90A-NEXT: s_setpc_b64 s[30:31]3834;3835; GFX950-LABEL: flat_atomic_xor_i32_ret_av_av:3836; GFX950: ; %bb.0:3837; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3838; GFX950-NEXT: ;;#ASMSTART3839; GFX950-NEXT: ; def v23840; GFX950-NEXT: ;;#ASMEND3841; GFX950-NEXT: buffer_wbl2 sc13842; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03843; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3844; GFX950-NEXT: buffer_inv sc13845; GFX950-NEXT: ;;#ASMSTART3846; GFX950-NEXT: ; use v03847; GFX950-NEXT: ;;#ASMEND3848; GFX950-NEXT: s_setpc_b64 s[30:31]3849 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103850 %data = call i32 asm "; def $0", "=^VA"()3851 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03852 call void asm "; use $0", "^VA"(i32 %result)3853 ret void3854}3855 3856; Input is AV, used as v3857define void @flat_atomic_xor_i32_ret_av_v(ptr %ptr) #0 {3858; GFX90A-LABEL: flat_atomic_xor_i32_ret_av_v:3859; GFX90A: ; %bb.0:3860; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3861; GFX90A-NEXT: ;;#ASMSTART3862; GFX90A-NEXT: ; def v23863; GFX90A-NEXT: ;;#ASMEND3864; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3865; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3866; GFX90A-NEXT: buffer_wbinvl1_vol3867; GFX90A-NEXT: ;;#ASMSTART3868; GFX90A-NEXT: ; use v03869; GFX90A-NEXT: ;;#ASMEND3870; GFX90A-NEXT: s_setpc_b64 s[30:31]3871;3872; GFX950-LABEL: flat_atomic_xor_i32_ret_av_v:3873; GFX950: ; %bb.0:3874; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3875; GFX950-NEXT: ;;#ASMSTART3876; GFX950-NEXT: ; def v23877; GFX950-NEXT: ;;#ASMEND3878; GFX950-NEXT: buffer_wbl2 sc13879; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03880; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3881; GFX950-NEXT: buffer_inv sc13882; GFX950-NEXT: ;;#ASMSTART3883; GFX950-NEXT: ; use v03884; GFX950-NEXT: ;;#ASMEND3885; GFX950-NEXT: s_setpc_b64 s[30:31]3886 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103887 %data = call i32 asm "; def $0", "=^VA"()3888 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03889 call void asm "; use $0", "v"(i32 %result)3890 ret void3891}3892 3893; Input is AV, used as a3894define void @flat_atomic_xor_i32_ret_av_a(ptr %ptr) #0 {3895; GFX90A-LABEL: flat_atomic_xor_i32_ret_av_a:3896; GFX90A: ; %bb.0:3897; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3898; GFX90A-NEXT: ;;#ASMSTART3899; GFX90A-NEXT: ; def v23900; GFX90A-NEXT: ;;#ASMEND3901; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3902; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3903; GFX90A-NEXT: buffer_wbinvl1_vol3904; GFX90A-NEXT: v_accvgpr_write_b32 a0, v03905; GFX90A-NEXT: ;;#ASMSTART3906; GFX90A-NEXT: ; use a03907; GFX90A-NEXT: ;;#ASMEND3908; GFX90A-NEXT: s_setpc_b64 s[30:31]3909;3910; GFX950-LABEL: flat_atomic_xor_i32_ret_av_a:3911; GFX950: ; %bb.0:3912; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3913; GFX950-NEXT: ;;#ASMSTART3914; GFX950-NEXT: ; def v23915; GFX950-NEXT: ;;#ASMEND3916; GFX950-NEXT: buffer_wbl2 sc13917; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03918; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3919; GFX950-NEXT: buffer_inv sc13920; GFX950-NEXT: v_accvgpr_write_b32 a0, v03921; GFX950-NEXT: ;;#ASMSTART3922; GFX950-NEXT: ; use a03923; GFX950-NEXT: ;;#ASMEND3924; GFX950-NEXT: s_setpc_b64 s[30:31]3925 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103926 %data = call i32 asm "; def $0", "=^VA"()3927 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03928 call void asm "; use $0", "a"(i32 %result)3929 ret void3930}3931 3932; Input is a, result used as AV3933define void @flat_atomic_xor_i32_ret_a_av(ptr %ptr) #0 {3934; GFX90A-LABEL: flat_atomic_xor_i32_ret_a_av:3935; GFX90A: ; %bb.0:3936; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3937; GFX90A-NEXT: ;;#ASMSTART3938; GFX90A-NEXT: ; def a03939; GFX90A-NEXT: ;;#ASMEND3940; GFX90A-NEXT: v_accvgpr_read_b32 v2, a03941; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3942; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3943; GFX90A-NEXT: buffer_wbinvl1_vol3944; GFX90A-NEXT: ;;#ASMSTART3945; GFX90A-NEXT: ; use v03946; GFX90A-NEXT: ;;#ASMEND3947; GFX90A-NEXT: s_setpc_b64 s[30:31]3948;3949; GFX950-LABEL: flat_atomic_xor_i32_ret_a_av:3950; GFX950: ; %bb.0:3951; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3952; GFX950-NEXT: ;;#ASMSTART3953; GFX950-NEXT: ; def a03954; GFX950-NEXT: ;;#ASMEND3955; GFX950-NEXT: s_nop 03956; GFX950-NEXT: v_accvgpr_read_b32 v2, a03957; GFX950-NEXT: buffer_wbl2 sc13958; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03959; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3960; GFX950-NEXT: buffer_inv sc13961; GFX950-NEXT: ;;#ASMSTART3962; GFX950-NEXT: ; use v03963; GFX950-NEXT: ;;#ASMEND3964; GFX950-NEXT: s_setpc_b64 s[30:31]3965 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 103966 %data = call i32 asm "; def $0", "=a"()3967 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03968 call void asm "; use $0", "^VA"(i32 %result)3969 ret void3970}3971 3972; Input is v, result used as AV3973define void @flat_atomic_xor_i32_ret_v_av(ptr %ptr) #0 {3974; GFX90A-LABEL: flat_atomic_xor_i32_ret_v_av:3975; GFX90A: ; %bb.0:3976; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3977; GFX90A-NEXT: ;;#ASMSTART3978; GFX90A-NEXT: ; def v23979; GFX90A-NEXT: ;;#ASMEND3980; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc3981; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3982; GFX90A-NEXT: buffer_wbinvl1_vol3983; GFX90A-NEXT: ;;#ASMSTART3984; GFX90A-NEXT: ; use v03985; GFX90A-NEXT: ;;#ASMEND3986; GFX90A-NEXT: s_setpc_b64 s[30:31]3987;3988; GFX950-LABEL: flat_atomic_xor_i32_ret_v_av:3989; GFX950: ; %bb.0:3990; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3991; GFX950-NEXT: ;;#ASMSTART3992; GFX950-NEXT: ; def v23993; GFX950-NEXT: ;;#ASMEND3994; GFX950-NEXT: buffer_wbl2 sc13995; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc03996; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3997; GFX950-NEXT: buffer_inv sc13998; GFX950-NEXT: ;;#ASMSTART3999; GFX950-NEXT: ; use v04000; GFX950-NEXT: ;;#ASMEND4001; GFX950-NEXT: s_setpc_b64 s[30:31]4002 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 104003 %data = call i32 asm "; def $0", "=v"()4004 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04005 call void asm "; use $0", "^VA"(i32 %result)4006 ret void4007}4008 4009define void @flat_atomic_xor_i32_ret_av_av_no_agprs(ptr %ptr) #0 {4010; GFX90A-LABEL: flat_atomic_xor_i32_ret_av_av_no_agprs:4011; GFX90A: ; %bb.0:4012; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4013; GFX90A-NEXT: v_accvgpr_write_b32 a0, v04014; GFX90A-NEXT: v_accvgpr_write_b32 a3, v40 ; Reload Reuse4015; GFX90A-NEXT: v_accvgpr_write_b32 a4, v41 ; Reload Reuse4016; GFX90A-NEXT: v_accvgpr_write_b32 a5, v42 ; Reload Reuse4017; GFX90A-NEXT: v_accvgpr_write_b32 a6, v43 ; Reload Reuse4018; GFX90A-NEXT: v_accvgpr_write_b32 a7, v44 ; Reload Reuse4019; GFX90A-NEXT: v_accvgpr_write_b32 a8, v45 ; Reload Reuse4020; GFX90A-NEXT: v_accvgpr_write_b32 a9, v46 ; Reload Reuse4021; GFX90A-NEXT: v_accvgpr_write_b32 a10, v47 ; Reload Reuse4022; GFX90A-NEXT: v_accvgpr_write_b32 a11, v56 ; Reload Reuse4023; GFX90A-NEXT: v_accvgpr_write_b32 a12, v57 ; Reload Reuse4024; GFX90A-NEXT: v_accvgpr_write_b32 a13, v58 ; Reload Reuse4025; GFX90A-NEXT: v_accvgpr_write_b32 a14, v59 ; Reload Reuse4026; GFX90A-NEXT: v_accvgpr_write_b32 a15, v60 ; Reload Reuse4027; GFX90A-NEXT: v_accvgpr_write_b32 a16, v61 ; Reload Reuse4028; GFX90A-NEXT: v_accvgpr_write_b32 a17, v62 ; Reload Reuse4029; GFX90A-NEXT: v_accvgpr_write_b32 a18, v63 ; Reload Reuse4030; GFX90A-NEXT: v_accvgpr_write_b32 a1, v14031; GFX90A-NEXT: ;;#ASMSTART4032; GFX90A-NEXT: ; def v[0:31]4033; GFX90A-NEXT: ;;#ASMEND4034; GFX90A-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; 4-byte Folded Spill4035; GFX90A-NEXT: s_nop 04036; GFX90A-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill4037; GFX90A-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill4038; GFX90A-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill4039; GFX90A-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill4040; GFX90A-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill4041; GFX90A-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill4042; GFX90A-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill4043; GFX90A-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill4044; GFX90A-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill4045; GFX90A-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill4046; GFX90A-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill4047; GFX90A-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill4048; GFX90A-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill4049; GFX90A-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill4050; GFX90A-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill4051; GFX90A-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill4052; GFX90A-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill4053; GFX90A-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill4054; GFX90A-NEXT: ;;#ASMSTART4055; GFX90A-NEXT: ; def a24056; GFX90A-NEXT: ;;#ASMEND4057; GFX90A-NEXT: v_accvgpr_write_b32 a19, v31 ; Reload Reuse4058; GFX90A-NEXT: v_accvgpr_read_b32 v0, a04059; GFX90A-NEXT: v_accvgpr_read_b32 v1, a14060; GFX90A-NEXT: v_accvgpr_read_b32 v2, a24061; GFX90A-NEXT: s_waitcnt vmcnt(0)4062; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 glc4063; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4064; GFX90A-NEXT: buffer_wbinvl1_vol4065; GFX90A-NEXT: v_accvgpr_write_b32 a31, v19 ; Reload Reuse4066; GFX90A-NEXT: v_accvgpr_write_b32 a30, v20 ; Reload Reuse4067; GFX90A-NEXT: v_accvgpr_write_b32 a29, v21 ; Reload Reuse4068; GFX90A-NEXT: v_accvgpr_write_b32 a28, v22 ; Reload Reuse4069; GFX90A-NEXT: v_accvgpr_write_b32 a27, v23 ; Reload Reuse4070; GFX90A-NEXT: v_accvgpr_write_b32 a26, v24 ; Reload Reuse4071; GFX90A-NEXT: v_accvgpr_write_b32 a25, v25 ; Reload Reuse4072; GFX90A-NEXT: v_accvgpr_write_b32 a24, v26 ; Reload Reuse4073; GFX90A-NEXT: v_accvgpr_write_b32 a23, v27 ; Reload Reuse4074; GFX90A-NEXT: v_accvgpr_write_b32 a22, v28 ; Reload Reuse4075; GFX90A-NEXT: v_accvgpr_write_b32 a21, v29 ; Reload Reuse4076; GFX90A-NEXT: v_accvgpr_write_b32 a20, v30 ; Reload Reuse4077; GFX90A-NEXT: v_accvgpr_write_b32 a0, v04078; GFX90A-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload4079; GFX90A-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload4080; GFX90A-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload4081; GFX90A-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload4082; GFX90A-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload4083; GFX90A-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload4084; GFX90A-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload4085; GFX90A-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload4086; GFX90A-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload4087; GFX90A-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload4088; GFX90A-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload4089; GFX90A-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload4090; GFX90A-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload4091; GFX90A-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload4092; GFX90A-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload4093; GFX90A-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload4094; GFX90A-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload4095; GFX90A-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload4096; GFX90A-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload4097; GFX90A-NEXT: v_accvgpr_read_b32 v19, a31 ; Reload Reuse4098; GFX90A-NEXT: v_accvgpr_read_b32 v20, a30 ; Reload Reuse4099; GFX90A-NEXT: v_accvgpr_read_b32 v21, a29 ; Reload Reuse4100; GFX90A-NEXT: v_accvgpr_read_b32 v22, a28 ; Reload Reuse4101; GFX90A-NEXT: v_accvgpr_read_b32 v23, a27 ; Reload Reuse4102; GFX90A-NEXT: v_accvgpr_read_b32 v24, a26 ; Reload Reuse4103; GFX90A-NEXT: v_accvgpr_read_b32 v25, a25 ; Reload Reuse4104; GFX90A-NEXT: v_accvgpr_read_b32 v26, a24 ; Reload Reuse4105; GFX90A-NEXT: v_accvgpr_read_b32 v27, a23 ; Reload Reuse4106; GFX90A-NEXT: v_accvgpr_read_b32 v28, a22 ; Reload Reuse4107; GFX90A-NEXT: v_accvgpr_read_b32 v29, a21 ; Reload Reuse4108; GFX90A-NEXT: v_accvgpr_read_b32 v30, a20 ; Reload Reuse4109; GFX90A-NEXT: ;;#ASMSTART4110; GFX90A-NEXT: ; use a04111; GFX90A-NEXT: ;;#ASMEND4112; GFX90A-NEXT: s_waitcnt vmcnt(0)4113; GFX90A-NEXT: v_accvgpr_read_b32 v31, a19 ; Reload Reuse4114; GFX90A-NEXT: ;;#ASMSTART4115; GFX90A-NEXT: ; use v[0:31]4116; GFX90A-NEXT: ;;#ASMEND4117; GFX90A-NEXT: v_accvgpr_read_b32 v63, a18 ; Reload Reuse4118; GFX90A-NEXT: v_accvgpr_read_b32 v62, a17 ; Reload Reuse4119; GFX90A-NEXT: v_accvgpr_read_b32 v61, a16 ; Reload Reuse4120; GFX90A-NEXT: v_accvgpr_read_b32 v60, a15 ; Reload Reuse4121; GFX90A-NEXT: v_accvgpr_read_b32 v59, a14 ; Reload Reuse4122; GFX90A-NEXT: v_accvgpr_read_b32 v58, a13 ; Reload Reuse4123; GFX90A-NEXT: v_accvgpr_read_b32 v57, a12 ; Reload Reuse4124; GFX90A-NEXT: v_accvgpr_read_b32 v56, a11 ; Reload Reuse4125; GFX90A-NEXT: v_accvgpr_read_b32 v47, a10 ; Reload Reuse4126; GFX90A-NEXT: v_accvgpr_read_b32 v46, a9 ; Reload Reuse4127; GFX90A-NEXT: v_accvgpr_read_b32 v45, a8 ; Reload Reuse4128; GFX90A-NEXT: v_accvgpr_read_b32 v44, a7 ; Reload Reuse4129; GFX90A-NEXT: v_accvgpr_read_b32 v43, a6 ; Reload Reuse4130; GFX90A-NEXT: v_accvgpr_read_b32 v42, a5 ; Reload Reuse4131; GFX90A-NEXT: v_accvgpr_read_b32 v41, a4 ; Reload Reuse4132; GFX90A-NEXT: v_accvgpr_read_b32 v40, a3 ; Reload Reuse4133; GFX90A-NEXT: s_setpc_b64 s[30:31]4134;4135; GFX950-LABEL: flat_atomic_xor_i32_ret_av_av_no_agprs:4136; GFX950: ; %bb.0:4137; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4138; GFX950-NEXT: v_accvgpr_write_b32 a0, v04139; GFX950-NEXT: v_accvgpr_write_b32 a3, v40 ; Reload Reuse4140; GFX950-NEXT: v_accvgpr_write_b32 a4, v41 ; Reload Reuse4141; GFX950-NEXT: v_accvgpr_write_b32 a5, v42 ; Reload Reuse4142; GFX950-NEXT: v_accvgpr_write_b32 a6, v43 ; Reload Reuse4143; GFX950-NEXT: v_accvgpr_write_b32 a7, v44 ; Reload Reuse4144; GFX950-NEXT: v_accvgpr_write_b32 a8, v45 ; Reload Reuse4145; GFX950-NEXT: v_accvgpr_write_b32 a9, v46 ; Reload Reuse4146; GFX950-NEXT: v_accvgpr_write_b32 a10, v47 ; Reload Reuse4147; GFX950-NEXT: v_accvgpr_write_b32 a11, v56 ; Reload Reuse4148; GFX950-NEXT: v_accvgpr_write_b32 a12, v57 ; Reload Reuse4149; GFX950-NEXT: v_accvgpr_write_b32 a13, v58 ; Reload Reuse4150; GFX950-NEXT: v_accvgpr_write_b32 a14, v59 ; Reload Reuse4151; GFX950-NEXT: v_accvgpr_write_b32 a15, v60 ; Reload Reuse4152; GFX950-NEXT: v_accvgpr_write_b32 a16, v61 ; Reload Reuse4153; GFX950-NEXT: v_accvgpr_write_b32 a17, v62 ; Reload Reuse4154; GFX950-NEXT: v_accvgpr_write_b32 a18, v63 ; Reload Reuse4155; GFX950-NEXT: v_accvgpr_write_b32 a1, v14156; GFX950-NEXT: ;;#ASMSTART4157; GFX950-NEXT: ; def v[0:31]4158; GFX950-NEXT: ;;#ASMEND4159; GFX950-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; 16-byte Folded Spill4160; GFX950-NEXT: s_nop 04161; GFX950-NEXT: scratch_store_dwordx4 off, v[4:7], s32 offset:16 ; 16-byte Folded Spill4162; GFX950-NEXT: scratch_store_dwordx4 off, v[8:11], s32 offset:32 ; 16-byte Folded Spill4163; GFX950-NEXT: scratch_store_dwordx4 off, v[12:15], s32 offset:48 ; 16-byte Folded Spill4164; GFX950-NEXT: ;;#ASMSTART4165; GFX950-NEXT: ; def a24166; GFX950-NEXT: ;;#ASMEND4167; GFX950-NEXT: v_accvgpr_write_b32 a19, v31 ; Reload Reuse4168; GFX950-NEXT: v_accvgpr_write_b32 a20, v30 ; Reload Reuse4169; GFX950-NEXT: v_accvgpr_write_b32 a21, v29 ; Reload Reuse4170; GFX950-NEXT: v_accvgpr_write_b32 a22, v28 ; Reload Reuse4171; GFX950-NEXT: v_accvgpr_read_b32 v0, a04172; GFX950-NEXT: scratch_store_dwordx3 off, v[16:18], s32 offset:64 ; 12-byte Folded Spill4173; GFX950-NEXT: v_accvgpr_read_b32 v1, a14174; GFX950-NEXT: v_accvgpr_read_b32 v2, a24175; GFX950-NEXT: buffer_wbl2 sc14176; GFX950-NEXT: s_waitcnt vmcnt(0)4177; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 sc04178; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4179; GFX950-NEXT: buffer_inv sc14180; GFX950-NEXT: v_accvgpr_write_b32 a31, v19 ; Reload Reuse4181; GFX950-NEXT: v_accvgpr_write_b32 a27, v23 ; Reload Reuse4182; GFX950-NEXT: v_accvgpr_write_b32 a28, v22 ; Reload Reuse4183; GFX950-NEXT: v_accvgpr_write_b32 a29, v21 ; Reload Reuse4184; GFX950-NEXT: v_accvgpr_write_b32 a30, v20 ; Reload Reuse4185; GFX950-NEXT: v_accvgpr_write_b32 a23, v27 ; Reload Reuse4186; GFX950-NEXT: v_accvgpr_write_b32 a24, v26 ; Reload Reuse4187; GFX950-NEXT: v_accvgpr_write_b32 a25, v25 ; Reload Reuse4188; GFX950-NEXT: v_accvgpr_write_b32 a26, v24 ; Reload Reuse4189; GFX950-NEXT: v_accvgpr_write_b32 a0, v04190; GFX950-NEXT: scratch_load_dwordx4 v[0:3], off, s32 ; 16-byte Folded Reload4191; GFX950-NEXT: scratch_load_dwordx4 v[4:7], off, s32 offset:16 ; 16-byte Folded Reload4192; GFX950-NEXT: scratch_load_dwordx4 v[8:11], off, s32 offset:32 ; 16-byte Folded Reload4193; GFX950-NEXT: scratch_load_dwordx4 v[12:15], off, s32 offset:48 ; 16-byte Folded Reload4194; GFX950-NEXT: v_accvgpr_read_b32 v19, a31 ; Reload Reuse4195; GFX950-NEXT: scratch_load_dwordx3 v[16:18], off, s32 offset:64 ; 12-byte Folded Reload4196; GFX950-NEXT: v_accvgpr_read_b32 v23, a27 ; Reload Reuse4197; GFX950-NEXT: v_accvgpr_read_b32 v22, a28 ; Reload Reuse4198; GFX950-NEXT: v_accvgpr_read_b32 v21, a29 ; Reload Reuse4199; GFX950-NEXT: v_accvgpr_read_b32 v20, a30 ; Reload Reuse4200; GFX950-NEXT: v_accvgpr_read_b32 v27, a23 ; Reload Reuse4201; GFX950-NEXT: v_accvgpr_read_b32 v26, a24 ; Reload Reuse4202; GFX950-NEXT: v_accvgpr_read_b32 v25, a25 ; Reload Reuse4203; GFX950-NEXT: v_accvgpr_read_b32 v24, a26 ; Reload Reuse4204; GFX950-NEXT: ;;#ASMSTART4205; GFX950-NEXT: ; use a04206; GFX950-NEXT: ;;#ASMEND4207; GFX950-NEXT: s_waitcnt vmcnt(0)4208; GFX950-NEXT: v_accvgpr_read_b32 v31, a19 ; Reload Reuse4209; GFX950-NEXT: v_accvgpr_read_b32 v30, a20 ; Reload Reuse4210; GFX950-NEXT: v_accvgpr_read_b32 v29, a21 ; Reload Reuse4211; GFX950-NEXT: v_accvgpr_read_b32 v28, a22 ; Reload Reuse4212; GFX950-NEXT: ;;#ASMSTART4213; GFX950-NEXT: ; use v[0:31]4214; GFX950-NEXT: ;;#ASMEND4215; GFX950-NEXT: v_accvgpr_read_b32 v63, a18 ; Reload Reuse4216; GFX950-NEXT: v_accvgpr_read_b32 v62, a17 ; Reload Reuse4217; GFX950-NEXT: v_accvgpr_read_b32 v61, a16 ; Reload Reuse4218; GFX950-NEXT: v_accvgpr_read_b32 v60, a15 ; Reload Reuse4219; GFX950-NEXT: v_accvgpr_read_b32 v59, a14 ; Reload Reuse4220; GFX950-NEXT: v_accvgpr_read_b32 v58, a13 ; Reload Reuse4221; GFX950-NEXT: v_accvgpr_read_b32 v57, a12 ; Reload Reuse4222; GFX950-NEXT: v_accvgpr_read_b32 v56, a11 ; Reload Reuse4223; GFX950-NEXT: v_accvgpr_read_b32 v47, a10 ; Reload Reuse4224; GFX950-NEXT: v_accvgpr_read_b32 v46, a9 ; Reload Reuse4225; GFX950-NEXT: v_accvgpr_read_b32 v45, a8 ; Reload Reuse4226; GFX950-NEXT: v_accvgpr_read_b32 v44, a7 ; Reload Reuse4227; GFX950-NEXT: v_accvgpr_read_b32 v43, a6 ; Reload Reuse4228; GFX950-NEXT: v_accvgpr_read_b32 v42, a5 ; Reload Reuse4229; GFX950-NEXT: v_accvgpr_read_b32 v41, a4 ; Reload Reuse4230; GFX950-NEXT: v_accvgpr_read_b32 v40, a3 ; Reload Reuse4231; GFX950-NEXT: s_setpc_b64 s[30:31]4232 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 104233 %data = call i32 asm "; def $0", "=^VA"()4234 %vgpr.def = call { <32 x i32>, <32 x i32> } asm sideeffect "; def $0", "=${v[0:31]},=${v[32:63]}"()4235 %vgpr.0 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 04236 %vgpr.1 = extractvalue { <32 x i32>, <32 x i32> } %vgpr.def, 14237 %result = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04238 call void asm sideeffect "; use $0", "{v[0:31]},{v[32:63]}"(<32 x i32> %vgpr.0, <32 x i32> %vgpr.1)4239 call void asm "; use $0", "^VA"(i32 %result)4240 ret void4241}4242 4243define void @flat_atomic_xor_i32_noret_a(ptr %ptr) #0 {4244; GFX90A-LABEL: flat_atomic_xor_i32_noret_a:4245; GFX90A: ; %bb.0:4246; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4247; GFX90A-NEXT: ;;#ASMSTART4248; GFX90A-NEXT: ; def a04249; GFX90A-NEXT: ;;#ASMEND4250; GFX90A-NEXT: flat_atomic_xor v[0:1], a04251; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4252; GFX90A-NEXT: buffer_wbinvl1_vol4253; GFX90A-NEXT: s_setpc_b64 s[30:31]4254;4255; GFX950-LABEL: flat_atomic_xor_i32_noret_a:4256; GFX950: ; %bb.0:4257; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4258; GFX950-NEXT: ;;#ASMSTART4259; GFX950-NEXT: ; def a04260; GFX950-NEXT: ;;#ASMEND4261; GFX950-NEXT: buffer_wbl2 sc14262; GFX950-NEXT: flat_atomic_xor v[0:1], a04263; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4264; GFX950-NEXT: buffer_inv sc14265; GFX950-NEXT: s_setpc_b64 s[30:31]4266 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 104267 %data = call i32 asm "; def $0", "=a"()4268 %unused = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04269 ret void4270}4271 4272define void @flat_atomic_xor_i32_noret_av(ptr %ptr) #0 {4273; GFX90A-LABEL: flat_atomic_xor_i32_noret_av:4274; GFX90A: ; %bb.0:4275; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4276; GFX90A-NEXT: ;;#ASMSTART4277; GFX90A-NEXT: ; def v24278; GFX90A-NEXT: ;;#ASMEND4279; GFX90A-NEXT: flat_atomic_xor v[0:1], v24280; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4281; GFX90A-NEXT: buffer_wbinvl1_vol4282; GFX90A-NEXT: s_setpc_b64 s[30:31]4283;4284; GFX950-LABEL: flat_atomic_xor_i32_noret_av:4285; GFX950: ; %bb.0:4286; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4287; GFX950-NEXT: ;;#ASMSTART4288; GFX950-NEXT: ; def v24289; GFX950-NEXT: ;;#ASMEND4290; GFX950-NEXT: buffer_wbl2 sc14291; GFX950-NEXT: flat_atomic_xor v[0:1], v24292; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4293; GFX950-NEXT: buffer_inv sc14294; GFX950-NEXT: s_setpc_b64 s[30:31]4295 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 104296 %data = call i32 asm "; def $0", "=^VA"()4297 %unused = atomicrmw xor ptr %ptr, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04298 ret void4299}4300 4301;---------------------------------------------------------------------4302; xor i64 cases with instruction4303;---------------------------------------------------------------------4304 4305; Input and result use AGPR4306define void @flat_atomic_xor_i64_ret_a_a(ptr %ptr) #0 {4307; GFX90A-LABEL: flat_atomic_xor_i64_ret_a_a:4308; GFX90A: ; %bb.0:4309; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4310; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4311; GFX90A-NEXT: ;;#ASMSTART4312; GFX90A-NEXT: ; def a[0:1]4313; GFX90A-NEXT: ;;#ASMEND4314; GFX90A-NEXT: v_accvgpr_read_b32 v3, a14315; GFX90A-NEXT: v_accvgpr_read_b32 v2, a04316; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14317; GFX90A-NEXT: ; implicit-def: $agpr0_agpr14318; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4319; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4320; GFX90A-NEXT: s_cbranch_execz .LBB53_24321; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4322; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc4323; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4324; GFX90A-NEXT: buffer_wbinvl1_vol4325; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34326; GFX90A-NEXT: v_accvgpr_write_b32 a0, v04327; GFX90A-NEXT: v_accvgpr_write_b32 a1, v14328; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14329; GFX90A-NEXT: .LBB53_2: ; %Flow4330; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4331; GFX90A-NEXT: s_cbranch_execz .LBB53_44332; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4333; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4334; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4335; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen4336; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:44337; GFX90A-NEXT: s_waitcnt vmcnt(1)4338; GFX90A-NEXT: v_accvgpr_write_b32 a0, v14339; GFX90A-NEXT: s_waitcnt vmcnt(0)4340; GFX90A-NEXT: v_accvgpr_write_b32 a1, v44341; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v24342; GFX90A-NEXT: v_xor_b32_e32 v3, v4, v34343; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen4344; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44345; GFX90A-NEXT: .LBB53_4: ; %atomicrmw.phi4346; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4347; GFX90A-NEXT: ;;#ASMSTART4348; GFX90A-NEXT: ; use a[0:1]4349; GFX90A-NEXT: ;;#ASMEND4350; GFX90A-NEXT: s_waitcnt vmcnt(0)4351; GFX90A-NEXT: s_setpc_b64 s[30:31]4352;4353; GFX950-LABEL: flat_atomic_xor_i64_ret_a_a:4354; GFX950: ; %bb.0:4355; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4356; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4357; GFX950-NEXT: ;;#ASMSTART4358; GFX950-NEXT: ; def a[0:1]4359; GFX950-NEXT: ;;#ASMEND4360; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14361; GFX950-NEXT: v_accvgpr_read_b32 v3, a14362; GFX950-NEXT: v_accvgpr_read_b32 v2, a04363; GFX950-NEXT: ; implicit-def: $agpr0_agpr14364; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4365; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4366; GFX950-NEXT: s_cbranch_execz .LBB53_24367; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4368; GFX950-NEXT: buffer_wbl2 sc14369; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc04370; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4371; GFX950-NEXT: buffer_inv sc14372; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34373; GFX950-NEXT: v_accvgpr_write_b32 a0, v04374; GFX950-NEXT: v_accvgpr_write_b32 a1, v14375; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14376; GFX950-NEXT: .LBB53_2: ; %Flow4377; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4378; GFX950-NEXT: s_cbranch_execz .LBB53_44379; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4380; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4381; GFX950-NEXT: s_nop 14382; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc4383; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off4384; GFX950-NEXT: s_waitcnt vmcnt(0)4385; GFX950-NEXT: v_accvgpr_write_b32 a0, v04386; GFX950-NEXT: v_xor_b32_e32 v3, v1, v34387; GFX950-NEXT: v_xor_b32_e32 v2, v0, v24388; GFX950-NEXT: v_accvgpr_write_b32 a1, v14389; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off4390; GFX950-NEXT: .LBB53_4: ; %atomicrmw.phi4391; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4392; GFX950-NEXT: ;;#ASMSTART4393; GFX950-NEXT: ; use a[0:1]4394; GFX950-NEXT: ;;#ASMEND4395; GFX950-NEXT: s_waitcnt vmcnt(0)4396; GFX950-NEXT: s_setpc_b64 s[30:31]4397 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104398 %data = call i64 asm "; def $0", "=a"()4399 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04400 call void asm "; use $0", "a"(i64 %result)4401 ret void4402}4403 4404; Input is AGPR, result used as VGPR.4405define void @flat_atomic_xor_i64_ret_a_v(ptr %ptr) #0 {4406; GFX90A-LABEL: flat_atomic_xor_i64_ret_a_v:4407; GFX90A: ; %bb.0:4408; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4409; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4410; GFX90A-NEXT: ;;#ASMSTART4411; GFX90A-NEXT: ; def a[0:1]4412; GFX90A-NEXT: ;;#ASMEND4413; GFX90A-NEXT: v_accvgpr_read_b32 v5, a14414; GFX90A-NEXT: v_accvgpr_read_b32 v4, a04415; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14416; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34417; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4418; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4419; GFX90A-NEXT: s_cbranch_execz .LBB54_24420; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4421; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] glc4422; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4423; GFX90A-NEXT: buffer_wbinvl1_vol4424; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14425; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr54426; GFX90A-NEXT: .LBB54_2: ; %Flow4427; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4428; GFX90A-NEXT: s_cbranch_execz .LBB54_44429; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4430; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4431; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4432; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:44433; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen4434; GFX90A-NEXT: s_waitcnt vmcnt(1)4435; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v54436; GFX90A-NEXT: s_waitcnt vmcnt(0)4437; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v44438; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen4439; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44440; GFX90A-NEXT: .LBB54_4: ; %atomicrmw.phi4441; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4442; GFX90A-NEXT: ;;#ASMSTART4443; GFX90A-NEXT: ; use v[2:3]4444; GFX90A-NEXT: ;;#ASMEND4445; GFX90A-NEXT: s_waitcnt vmcnt(0)4446; GFX90A-NEXT: s_setpc_b64 s[30:31]4447;4448; GFX950-LABEL: flat_atomic_xor_i64_ret_a_v:4449; GFX950: ; %bb.0:4450; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4451; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4452; GFX950-NEXT: ;;#ASMSTART4453; GFX950-NEXT: ; def a[0:1]4454; GFX950-NEXT: ;;#ASMEND4455; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14456; GFX950-NEXT: v_accvgpr_read_b32 v5, a14457; GFX950-NEXT: v_accvgpr_read_b32 v4, a04458; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34459; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4460; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4461; GFX950-NEXT: s_cbranch_execz .LBB54_24462; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4463; GFX950-NEXT: buffer_wbl2 sc14464; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] sc04465; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4466; GFX950-NEXT: buffer_inv sc14467; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14468; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr54469; GFX950-NEXT: .LBB54_2: ; %Flow4470; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4471; GFX950-NEXT: s_cbranch_execz .LBB54_44472; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4473; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4474; GFX950-NEXT: s_nop 14475; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc4476; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off4477; GFX950-NEXT: s_waitcnt vmcnt(0)4478; GFX950-NEXT: v_xor_b32_e32 v1, v3, v54479; GFX950-NEXT: v_xor_b32_e32 v0, v2, v44480; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off4481; GFX950-NEXT: .LBB54_4: ; %atomicrmw.phi4482; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4483; GFX950-NEXT: ;;#ASMSTART4484; GFX950-NEXT: ; use v[2:3]4485; GFX950-NEXT: ;;#ASMEND4486; GFX950-NEXT: s_waitcnt vmcnt(0)4487; GFX950-NEXT: s_setpc_b64 s[30:31]4488 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104489 %data = call i64 asm "; def $0", "=a"()4490 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04491 call void asm "; use $0", "v"(i64 %result)4492 ret void4493}4494 4495; Input is VGPR, result used as AGPR4496define void @flat_atomic_xor_i64_ret_v_a(ptr %ptr) #0 {4497; GFX90A-LABEL: flat_atomic_xor_i64_ret_v_a:4498; GFX90A: ; %bb.0:4499; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4500; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4501; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14502; GFX90A-NEXT: ;;#ASMSTART4503; GFX90A-NEXT: ; def v[2:3]4504; GFX90A-NEXT: ;;#ASMEND4505; GFX90A-NEXT: ; implicit-def: $agpr0_agpr14506; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4507; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4508; GFX90A-NEXT: s_cbranch_execz .LBB55_24509; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4510; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc4511; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4512; GFX90A-NEXT: buffer_wbinvl1_vol4513; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34514; GFX90A-NEXT: v_accvgpr_write_b32 a0, v04515; GFX90A-NEXT: v_accvgpr_write_b32 a1, v14516; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14517; GFX90A-NEXT: .LBB55_2: ; %Flow4518; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4519; GFX90A-NEXT: s_cbranch_execz .LBB55_44520; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4521; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4522; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4523; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen4524; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:44525; GFX90A-NEXT: s_waitcnt vmcnt(1)4526; GFX90A-NEXT: v_accvgpr_write_b32 a0, v14527; GFX90A-NEXT: s_waitcnt vmcnt(0)4528; GFX90A-NEXT: v_accvgpr_write_b32 a1, v44529; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v24530; GFX90A-NEXT: v_xor_b32_e32 v3, v4, v34531; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen4532; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44533; GFX90A-NEXT: .LBB55_4: ; %atomicrmw.phi4534; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4535; GFX90A-NEXT: ;;#ASMSTART4536; GFX90A-NEXT: ; use a[0:1]4537; GFX90A-NEXT: ;;#ASMEND4538; GFX90A-NEXT: s_waitcnt vmcnt(0)4539; GFX90A-NEXT: s_setpc_b64 s[30:31]4540;4541; GFX950-LABEL: flat_atomic_xor_i64_ret_v_a:4542; GFX950: ; %bb.0:4543; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4544; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4545; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14546; GFX950-NEXT: ;;#ASMSTART4547; GFX950-NEXT: ; def v[2:3]4548; GFX950-NEXT: ;;#ASMEND4549; GFX950-NEXT: ; implicit-def: $agpr0_agpr14550; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4551; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4552; GFX950-NEXT: s_cbranch_execz .LBB55_24553; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4554; GFX950-NEXT: buffer_wbl2 sc14555; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc04556; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4557; GFX950-NEXT: buffer_inv sc14558; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34559; GFX950-NEXT: v_accvgpr_write_b32 a0, v04560; GFX950-NEXT: v_accvgpr_write_b32 a1, v14561; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14562; GFX950-NEXT: .LBB55_2: ; %Flow4563; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4564; GFX950-NEXT: s_cbranch_execz .LBB55_44565; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4566; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4567; GFX950-NEXT: s_nop 14568; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc4569; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off4570; GFX950-NEXT: s_waitcnt vmcnt(0)4571; GFX950-NEXT: v_accvgpr_write_b32 a0, v04572; GFX950-NEXT: v_xor_b32_e32 v3, v1, v34573; GFX950-NEXT: v_xor_b32_e32 v2, v0, v24574; GFX950-NEXT: v_accvgpr_write_b32 a1, v14575; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off4576; GFX950-NEXT: .LBB55_4: ; %atomicrmw.phi4577; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4578; GFX950-NEXT: ;;#ASMSTART4579; GFX950-NEXT: ; use a[0:1]4580; GFX950-NEXT: ;;#ASMEND4581; GFX950-NEXT: s_waitcnt vmcnt(0)4582; GFX950-NEXT: s_setpc_b64 s[30:31]4583 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104584 %data = call i64 asm "; def $0", "=v"()4585 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04586 call void asm "; use $0", "a"(i64 %result)4587 ret void4588}4589 4590; Input is AV, result also used as AV4591define void @flat_atomic_xor_i64_ret_av_av(ptr %ptr) #0 {4592; GFX90A-LABEL: flat_atomic_xor_i64_ret_av_av:4593; GFX90A: ; %bb.0:4594; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4595; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4596; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14597; GFX90A-NEXT: ;;#ASMSTART4598; GFX90A-NEXT: ; def v[4:5]4599; GFX90A-NEXT: ;;#ASMEND4600; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34601; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4602; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4603; GFX90A-NEXT: s_cbranch_execz .LBB56_24604; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4605; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] glc4606; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4607; GFX90A-NEXT: buffer_wbinvl1_vol4608; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14609; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr54610; GFX90A-NEXT: .LBB56_2: ; %Flow4611; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4612; GFX90A-NEXT: s_cbranch_execz .LBB56_44613; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4614; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4615; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4616; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:44617; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen4618; GFX90A-NEXT: s_waitcnt vmcnt(1)4619; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v54620; GFX90A-NEXT: s_waitcnt vmcnt(0)4621; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v44622; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen4623; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44624; GFX90A-NEXT: .LBB56_4: ; %atomicrmw.phi4625; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4626; GFX90A-NEXT: ;;#ASMSTART4627; GFX90A-NEXT: ; use v[2:3]4628; GFX90A-NEXT: ;;#ASMEND4629; GFX90A-NEXT: s_waitcnt vmcnt(0)4630; GFX90A-NEXT: s_setpc_b64 s[30:31]4631;4632; GFX950-LABEL: flat_atomic_xor_i64_ret_av_av:4633; GFX950: ; %bb.0:4634; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4635; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4636; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14637; GFX950-NEXT: ;;#ASMSTART4638; GFX950-NEXT: ; def v[4:5]4639; GFX950-NEXT: ;;#ASMEND4640; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34641; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4642; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4643; GFX950-NEXT: s_cbranch_execz .LBB56_24644; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4645; GFX950-NEXT: buffer_wbl2 sc14646; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] sc04647; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4648; GFX950-NEXT: buffer_inv sc14649; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14650; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr54651; GFX950-NEXT: .LBB56_2: ; %Flow4652; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4653; GFX950-NEXT: s_cbranch_execz .LBB56_44654; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4655; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4656; GFX950-NEXT: s_nop 14657; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc4658; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off4659; GFX950-NEXT: s_waitcnt vmcnt(0)4660; GFX950-NEXT: v_xor_b32_e32 v1, v3, v54661; GFX950-NEXT: v_xor_b32_e32 v0, v2, v44662; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off4663; GFX950-NEXT: .LBB56_4: ; %atomicrmw.phi4664; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4665; GFX950-NEXT: ;;#ASMSTART4666; GFX950-NEXT: ; use v[2:3]4667; GFX950-NEXT: ;;#ASMEND4668; GFX950-NEXT: s_waitcnt vmcnt(0)4669; GFX950-NEXT: s_setpc_b64 s[30:31]4670 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104671 %data = call i64 asm "; def $0", "=^VA"()4672 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04673 call void asm "; use $0", "^VA"(i64 %result)4674 ret void4675}4676 4677; Input is AV, used as v4678define void @flat_atomic_xor_i64_ret_av_v(ptr %ptr) #0 {4679; GFX90A-LABEL: flat_atomic_xor_i64_ret_av_v:4680; GFX90A: ; %bb.0:4681; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4682; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4683; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14684; GFX90A-NEXT: ;;#ASMSTART4685; GFX90A-NEXT: ; def v[4:5]4686; GFX90A-NEXT: ;;#ASMEND4687; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34688; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4689; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4690; GFX90A-NEXT: s_cbranch_execz .LBB57_24691; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4692; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] glc4693; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4694; GFX90A-NEXT: buffer_wbinvl1_vol4695; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14696; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr54697; GFX90A-NEXT: .LBB57_2: ; %Flow4698; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4699; GFX90A-NEXT: s_cbranch_execz .LBB57_44700; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4701; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4702; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4703; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:44704; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen4705; GFX90A-NEXT: s_waitcnt vmcnt(1)4706; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v54707; GFX90A-NEXT: s_waitcnt vmcnt(0)4708; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v44709; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen4710; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44711; GFX90A-NEXT: .LBB57_4: ; %atomicrmw.phi4712; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4713; GFX90A-NEXT: ;;#ASMSTART4714; GFX90A-NEXT: ; use v[2:3]4715; GFX90A-NEXT: ;;#ASMEND4716; GFX90A-NEXT: s_waitcnt vmcnt(0)4717; GFX90A-NEXT: s_setpc_b64 s[30:31]4718;4719; GFX950-LABEL: flat_atomic_xor_i64_ret_av_v:4720; GFX950: ; %bb.0:4721; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4722; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4723; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14724; GFX950-NEXT: ;;#ASMSTART4725; GFX950-NEXT: ; def v[4:5]4726; GFX950-NEXT: ;;#ASMEND4727; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34728; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4729; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4730; GFX950-NEXT: s_cbranch_execz .LBB57_24731; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4732; GFX950-NEXT: buffer_wbl2 sc14733; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] sc04734; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4735; GFX950-NEXT: buffer_inv sc14736; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14737; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr54738; GFX950-NEXT: .LBB57_2: ; %Flow4739; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4740; GFX950-NEXT: s_cbranch_execz .LBB57_44741; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4742; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4743; GFX950-NEXT: s_nop 14744; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc4745; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off4746; GFX950-NEXT: s_waitcnt vmcnt(0)4747; GFX950-NEXT: v_xor_b32_e32 v1, v3, v54748; GFX950-NEXT: v_xor_b32_e32 v0, v2, v44749; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off4750; GFX950-NEXT: .LBB57_4: ; %atomicrmw.phi4751; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4752; GFX950-NEXT: ;;#ASMSTART4753; GFX950-NEXT: ; use v[2:3]4754; GFX950-NEXT: ;;#ASMEND4755; GFX950-NEXT: s_waitcnt vmcnt(0)4756; GFX950-NEXT: s_setpc_b64 s[30:31]4757 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104758 %data = call i64 asm "; def $0", "=^VA"()4759 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04760 call void asm "; use $0", "v"(i64 %result)4761 ret void4762}4763 4764; Input is AV, used as a4765define void @flat_atomic_xor_i64_ret_av_a(ptr %ptr) #0 {4766; GFX90A-LABEL: flat_atomic_xor_i64_ret_av_a:4767; GFX90A: ; %bb.0:4768; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4769; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4770; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14771; GFX90A-NEXT: ;;#ASMSTART4772; GFX90A-NEXT: ; def v[2:3]4773; GFX90A-NEXT: ;;#ASMEND4774; GFX90A-NEXT: ; implicit-def: $agpr0_agpr14775; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4776; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4777; GFX90A-NEXT: s_cbranch_execz .LBB58_24778; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4779; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc4780; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4781; GFX90A-NEXT: buffer_wbinvl1_vol4782; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34783; GFX90A-NEXT: v_accvgpr_write_b32 a0, v04784; GFX90A-NEXT: v_accvgpr_write_b32 a1, v14785; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14786; GFX90A-NEXT: .LBB58_2: ; %Flow4787; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4788; GFX90A-NEXT: s_cbranch_execz .LBB58_44789; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4790; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4791; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4792; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen4793; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:44794; GFX90A-NEXT: s_waitcnt vmcnt(1)4795; GFX90A-NEXT: v_accvgpr_write_b32 a0, v14796; GFX90A-NEXT: s_waitcnt vmcnt(0)4797; GFX90A-NEXT: v_accvgpr_write_b32 a1, v44798; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v24799; GFX90A-NEXT: v_xor_b32_e32 v3, v4, v34800; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen4801; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:44802; GFX90A-NEXT: .LBB58_4: ; %atomicrmw.phi4803; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4804; GFX90A-NEXT: ;;#ASMSTART4805; GFX90A-NEXT: ; use a[0:1]4806; GFX90A-NEXT: ;;#ASMEND4807; GFX90A-NEXT: s_waitcnt vmcnt(0)4808; GFX90A-NEXT: s_setpc_b64 s[30:31]4809;4810; GFX950-LABEL: flat_atomic_xor_i64_ret_av_a:4811; GFX950: ; %bb.0:4812; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4813; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4814; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14815; GFX950-NEXT: ;;#ASMSTART4816; GFX950-NEXT: ; def v[2:3]4817; GFX950-NEXT: ;;#ASMEND4818; GFX950-NEXT: ; implicit-def: $agpr0_agpr14819; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4820; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4821; GFX950-NEXT: s_cbranch_execz .LBB58_24822; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4823; GFX950-NEXT: buffer_wbl2 sc14824; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc04825; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4826; GFX950-NEXT: buffer_inv sc14827; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34828; GFX950-NEXT: v_accvgpr_write_b32 a0, v04829; GFX950-NEXT: v_accvgpr_write_b32 a1, v14830; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14831; GFX950-NEXT: .LBB58_2: ; %Flow4832; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4833; GFX950-NEXT: s_cbranch_execz .LBB58_44834; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4835; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4836; GFX950-NEXT: s_nop 14837; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc4838; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off4839; GFX950-NEXT: s_waitcnt vmcnt(0)4840; GFX950-NEXT: v_accvgpr_write_b32 a0, v04841; GFX950-NEXT: v_xor_b32_e32 v3, v1, v34842; GFX950-NEXT: v_xor_b32_e32 v2, v0, v24843; GFX950-NEXT: v_accvgpr_write_b32 a1, v14844; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off4845; GFX950-NEXT: .LBB58_4: ; %atomicrmw.phi4846; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4847; GFX950-NEXT: ;;#ASMSTART4848; GFX950-NEXT: ; use a[0:1]4849; GFX950-NEXT: ;;#ASMEND4850; GFX950-NEXT: s_waitcnt vmcnt(0)4851; GFX950-NEXT: s_setpc_b64 s[30:31]4852 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104853 %data = call i64 asm "; def $0", "=^VA"()4854 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04855 call void asm "; use $0", "a"(i64 %result)4856 ret void4857}4858 4859; Input is a, result used as AV4860define void @flat_atomic_xor_i64_ret_a_av(ptr %ptr) #0 {4861; GFX90A-LABEL: flat_atomic_xor_i64_ret_a_av:4862; GFX90A: ; %bb.0:4863; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4864; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4865; GFX90A-NEXT: ;;#ASMSTART4866; GFX90A-NEXT: ; def a[0:1]4867; GFX90A-NEXT: ;;#ASMEND4868; GFX90A-NEXT: v_accvgpr_read_b32 v5, a14869; GFX90A-NEXT: v_accvgpr_read_b32 v4, a04870; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14871; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34872; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4873; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4874; GFX90A-NEXT: s_cbranch_execz .LBB59_24875; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4876; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] glc4877; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4878; GFX90A-NEXT: buffer_wbinvl1_vol4879; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14880; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr54881; GFX90A-NEXT: .LBB59_2: ; %Flow4882; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4883; GFX90A-NEXT: s_cbranch_execz .LBB59_44884; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4885; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4886; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4887; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:44888; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen4889; GFX90A-NEXT: s_waitcnt vmcnt(1)4890; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v54891; GFX90A-NEXT: s_waitcnt vmcnt(0)4892; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v44893; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen4894; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44895; GFX90A-NEXT: .LBB59_4: ; %atomicrmw.phi4896; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4897; GFX90A-NEXT: ;;#ASMSTART4898; GFX90A-NEXT: ; use v[2:3]4899; GFX90A-NEXT: ;;#ASMEND4900; GFX90A-NEXT: s_waitcnt vmcnt(0)4901; GFX90A-NEXT: s_setpc_b64 s[30:31]4902;4903; GFX950-LABEL: flat_atomic_xor_i64_ret_a_av:4904; GFX950: ; %bb.0:4905; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4906; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4907; GFX950-NEXT: ;;#ASMSTART4908; GFX950-NEXT: ; def a[0:1]4909; GFX950-NEXT: ;;#ASMEND4910; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14911; GFX950-NEXT: v_accvgpr_read_b32 v5, a14912; GFX950-NEXT: v_accvgpr_read_b32 v4, a04913; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr34914; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc4915; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4916; GFX950-NEXT: s_cbranch_execz .LBB59_24917; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global4918; GFX950-NEXT: buffer_wbl2 sc14919; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] sc04920; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4921; GFX950-NEXT: buffer_inv sc14922; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr14923; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr54924; GFX950-NEXT: .LBB59_2: ; %Flow4925; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4926; GFX950-NEXT: s_cbranch_execz .LBB59_44927; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private4928; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4929; GFX950-NEXT: s_nop 14930; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc4931; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off4932; GFX950-NEXT: s_waitcnt vmcnt(0)4933; GFX950-NEXT: v_xor_b32_e32 v1, v3, v54934; GFX950-NEXT: v_xor_b32_e32 v0, v2, v44935; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off4936; GFX950-NEXT: .LBB59_4: ; %atomicrmw.phi4937; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]4938; GFX950-NEXT: ;;#ASMSTART4939; GFX950-NEXT: ; use v[2:3]4940; GFX950-NEXT: ;;#ASMEND4941; GFX950-NEXT: s_waitcnt vmcnt(0)4942; GFX950-NEXT: s_setpc_b64 s[30:31]4943 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 104944 %data = call i64 asm "; def $0", "=a"()4945 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04946 call void asm "; use $0", "^VA"(i64 %result)4947 ret void4948}4949 4950; Input is v, result used as AV4951define void @flat_atomic_xor_i64_ret_v_av(ptr %ptr) #0 {4952; GFX90A-LABEL: flat_atomic_xor_i64_ret_v_av:4953; GFX90A: ; %bb.0:4954; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4955; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base4956; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v14957; GFX90A-NEXT: ;;#ASMSTART4958; GFX90A-NEXT: ; def v[4:5]4959; GFX90A-NEXT: ;;#ASMEND4960; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr34961; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc4962; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4963; GFX90A-NEXT: s_cbranch_execz .LBB60_24964; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global4965; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] glc4966; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4967; GFX90A-NEXT: buffer_wbinvl1_vol4968; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr14969; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr54970; GFX90A-NEXT: .LBB60_2: ; %Flow4971; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]4972; GFX90A-NEXT: s_cbranch_execz .LBB60_44973; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private4974; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4975; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc4976; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:44977; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen4978; GFX90A-NEXT: s_waitcnt vmcnt(1)4979; GFX90A-NEXT: v_xor_b32_e32 v1, v3, v54980; GFX90A-NEXT: s_waitcnt vmcnt(0)4981; GFX90A-NEXT: v_xor_b32_e32 v4, v2, v44982; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen4983; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:44984; GFX90A-NEXT: .LBB60_4: ; %atomicrmw.phi4985; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4986; GFX90A-NEXT: ;;#ASMSTART4987; GFX90A-NEXT: ; use v[2:3]4988; GFX90A-NEXT: ;;#ASMEND4989; GFX90A-NEXT: s_waitcnt vmcnt(0)4990; GFX90A-NEXT: s_setpc_b64 s[30:31]4991;4992; GFX950-LABEL: flat_atomic_xor_i64_ret_v_av:4993; GFX950: ; %bb.0:4994; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4995; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base4996; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14997; GFX950-NEXT: ;;#ASMSTART4998; GFX950-NEXT: ; def v[4:5]4999; GFX950-NEXT: ;;#ASMEND5000; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr35001; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc5002; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5003; GFX950-NEXT: s_cbranch_execz .LBB60_25004; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global5005; GFX950-NEXT: buffer_wbl2 sc15006; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1], v[4:5] sc05007; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5008; GFX950-NEXT: buffer_inv sc15009; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr15010; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr55011; GFX950-NEXT: .LBB60_2: ; %Flow5012; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5013; GFX950-NEXT: s_cbranch_execz .LBB60_45014; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private5015; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5016; GFX950-NEXT: s_nop 15017; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc5018; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off5019; GFX950-NEXT: s_waitcnt vmcnt(0)5020; GFX950-NEXT: v_xor_b32_e32 v1, v3, v55021; GFX950-NEXT: v_xor_b32_e32 v0, v2, v45022; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off5023; GFX950-NEXT: .LBB60_4: ; %atomicrmw.phi5024; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5025; GFX950-NEXT: ;;#ASMSTART5026; GFX950-NEXT: ; use v[2:3]5027; GFX950-NEXT: ;;#ASMEND5028; GFX950-NEXT: s_waitcnt vmcnt(0)5029; GFX950-NEXT: s_setpc_b64 s[30:31]5030 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 105031 %data = call i64 asm "; def $0", "=v"()5032 %result = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05033 call void asm "; use $0", "^VA"(i64 %result)5034 ret void5035}5036 5037define void @flat_atomic_xor_i64_noret_a(ptr %ptr) #0 {5038; GFX90A-LABEL: flat_atomic_xor_i64_noret_a:5039; GFX90A: ; %bb.0:5040; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5041; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base5042; GFX90A-NEXT: ;;#ASMSTART5043; GFX90A-NEXT: ; def a[0:1]5044; GFX90A-NEXT: ;;#ASMEND5045; GFX90A-NEXT: v_accvgpr_read_b32 v3, a15046; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05047; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v15048; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc5049; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]5050; GFX90A-NEXT: s_cbranch_execnz .LBB61_35051; GFX90A-NEXT: ; %bb.1: ; %Flow5052; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]5053; GFX90A-NEXT: s_cbranch_execnz .LBB61_45054; GFX90A-NEXT: .LBB61_2: ; %atomicrmw.phi5055; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5056; GFX90A-NEXT: s_setpc_b64 s[30:31]5057; GFX90A-NEXT: .LBB61_3: ; %atomicrmw.global5058; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], a[0:1]5059; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5060; GFX90A-NEXT: buffer_wbinvl1_vol5061; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr15062; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr35063; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]5064; GFX90A-NEXT: s_cbranch_execz .LBB61_25065; GFX90A-NEXT: .LBB61_4: ; %atomicrmw.private5066; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5067; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc5068; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:45069; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen5070; GFX90A-NEXT: s_waitcnt vmcnt(1)5071; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v35072; GFX90A-NEXT: s_waitcnt vmcnt(0)5073; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v25074; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen5075; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:45076; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5077; GFX90A-NEXT: s_waitcnt vmcnt(0)5078; GFX90A-NEXT: s_setpc_b64 s[30:31]5079;5080; GFX950-LABEL: flat_atomic_xor_i64_noret_a:5081; GFX950: ; %bb.0:5082; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5083; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base5084; GFX950-NEXT: ;;#ASMSTART5085; GFX950-NEXT: ; def a[0:1]5086; GFX950-NEXT: ;;#ASMEND5087; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15088; GFX950-NEXT: v_accvgpr_read_b32 v3, a15089; GFX950-NEXT: v_accvgpr_read_b32 v2, a05090; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc5091; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5092; GFX950-NEXT: s_cbranch_execnz .LBB61_35093; GFX950-NEXT: ; %bb.1: ; %Flow5094; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5095; GFX950-NEXT: s_cbranch_execnz .LBB61_45096; GFX950-NEXT: .LBB61_2: ; %atomicrmw.phi5097; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5098; GFX950-NEXT: s_setpc_b64 s[30:31]5099; GFX950-NEXT: .LBB61_3: ; %atomicrmw.global5100; GFX950-NEXT: buffer_wbl2 sc15101; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], a[0:1]5102; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5103; GFX950-NEXT: buffer_inv sc15104; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr15105; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr35106; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5107; GFX950-NEXT: s_cbranch_execz .LBB61_25108; GFX950-NEXT: .LBB61_4: ; %atomicrmw.private5109; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5110; GFX950-NEXT: s_nop 15111; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc5112; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off5113; GFX950-NEXT: s_waitcnt vmcnt(0)5114; GFX950-NEXT: v_xor_b32_e32 v1, v1, v35115; GFX950-NEXT: v_xor_b32_e32 v0, v0, v25116; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off5117; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5118; GFX950-NEXT: s_waitcnt vmcnt(0)5119; GFX950-NEXT: s_setpc_b64 s[30:31]5120 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 105121 %data = call i64 asm "; def $0", "=a"()5122 %unused = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05123 ret void5124}5125 5126define void @flat_atomic_xor_i64_noret_av(ptr %ptr) #0 {5127; GFX90A-LABEL: flat_atomic_xor_i64_noret_av:5128; GFX90A: ; %bb.0:5129; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5130; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base5131; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v15132; GFX90A-NEXT: ;;#ASMSTART5133; GFX90A-NEXT: ; def v[2:3]5134; GFX90A-NEXT: ;;#ASMEND5135; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc5136; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]5137; GFX90A-NEXT: s_cbranch_execnz .LBB62_35138; GFX90A-NEXT: ; %bb.1: ; %Flow5139; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]5140; GFX90A-NEXT: s_cbranch_execnz .LBB62_45141; GFX90A-NEXT: .LBB62_2: ; %atomicrmw.phi5142; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5143; GFX90A-NEXT: s_setpc_b64 s[30:31]5144; GFX90A-NEXT: .LBB62_3: ; %atomicrmw.global5145; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5146; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5147; GFX90A-NEXT: buffer_wbinvl1_vol5148; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr15149; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr35150; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]5151; GFX90A-NEXT: s_cbranch_execz .LBB62_25152; GFX90A-NEXT: .LBB62_4: ; %atomicrmw.private5153; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5154; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc5155; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen offset:45156; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen5157; GFX90A-NEXT: s_waitcnt vmcnt(1)5158; GFX90A-NEXT: v_xor_b32_e32 v1, v1, v35159; GFX90A-NEXT: s_waitcnt vmcnt(0)5160; GFX90A-NEXT: v_xor_b32_e32 v2, v4, v25161; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen5162; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:45163; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5164; GFX90A-NEXT: s_waitcnt vmcnt(0)5165; GFX90A-NEXT: s_setpc_b64 s[30:31]5166;5167; GFX950-LABEL: flat_atomic_xor_i64_noret_av:5168; GFX950: ; %bb.0:5169; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5170; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base5171; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15172; GFX950-NEXT: ;;#ASMSTART5173; GFX950-NEXT: ; def v[2:3]5174; GFX950-NEXT: ;;#ASMEND5175; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc5176; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5177; GFX950-NEXT: s_cbranch_execnz .LBB62_35178; GFX950-NEXT: ; %bb.1: ; %Flow5179; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5180; GFX950-NEXT: s_cbranch_execnz .LBB62_45181; GFX950-NEXT: .LBB62_2: ; %atomicrmw.phi5182; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5183; GFX950-NEXT: s_setpc_b64 s[30:31]5184; GFX950-NEXT: .LBB62_3: ; %atomicrmw.global5185; GFX950-NEXT: buffer_wbl2 sc15186; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5187; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5188; GFX950-NEXT: buffer_inv sc15189; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr15190; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr35191; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5192; GFX950-NEXT: s_cbranch_execz .LBB62_25193; GFX950-NEXT: .LBB62_4: ; %atomicrmw.private5194; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5195; GFX950-NEXT: s_nop 15196; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc5197; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off5198; GFX950-NEXT: s_waitcnt vmcnt(0)5199; GFX950-NEXT: v_xor_b32_e32 v1, v1, v35200; GFX950-NEXT: v_xor_b32_e32 v0, v0, v25201; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off5202; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5203; GFX950-NEXT: s_waitcnt vmcnt(0)5204; GFX950-NEXT: s_setpc_b64 s[30:31]5205 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 105206 %data = call i64 asm "; def $0", "=^VA"()5207 %unused = atomicrmw xor ptr %ptr, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05208 ret void5209}5210 5211;---------------------------------------------------------------------5212; other atomics i32, with aa+av cases5213;---------------------------------------------------------------------5214 5215define void @flat_atomic_add_i32_ret_a_a(ptr %ptr) #0 {5216; GFX90A-LABEL: flat_atomic_add_i32_ret_a_a:5217; GFX90A: ; %bb.0:5218; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5219; GFX90A-NEXT: ;;#ASMSTART5220; GFX90A-NEXT: ; def a05221; GFX90A-NEXT: ;;#ASMEND5222; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05223; GFX90A-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 glc5224; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5225; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05226; GFX90A-NEXT: ;;#ASMSTART5227; GFX90A-NEXT: ; use a05228; GFX90A-NEXT: ;;#ASMEND5229; GFX90A-NEXT: s_setpc_b64 s[30:31]5230;5231; GFX950-LABEL: flat_atomic_add_i32_ret_a_a:5232; GFX950: ; %bb.0:5233; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5234; GFX950-NEXT: ;;#ASMSTART5235; GFX950-NEXT: ; def a05236; GFX950-NEXT: ;;#ASMEND5237; GFX950-NEXT: s_nop 05238; GFX950-NEXT: v_accvgpr_read_b32 v2, a05239; GFX950-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 sc05240; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5241; GFX950-NEXT: v_accvgpr_write_b32 a0, v05242; GFX950-NEXT: ;;#ASMSTART5243; GFX950-NEXT: ; use a05244; GFX950-NEXT: ;;#ASMEND5245; GFX950-NEXT: s_setpc_b64 s[30:31]5246 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105247 %data = call i32 asm "; def $0", "=a"()5248 %result = atomicrmw add ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05249 call void asm "; use $0", "a"(i32 %result)5250 ret void5251}5252 5253define void @flat_atomic_add_i32_ret_av_av(ptr %ptr) #0 {5254; GFX90A-LABEL: flat_atomic_add_i32_ret_av_av:5255; GFX90A: ; %bb.0:5256; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5257; GFX90A-NEXT: ;;#ASMSTART5258; GFX90A-NEXT: ; def v25259; GFX90A-NEXT: ;;#ASMEND5260; GFX90A-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 glc5261; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5262; GFX90A-NEXT: ;;#ASMSTART5263; GFX90A-NEXT: ; use v05264; GFX90A-NEXT: ;;#ASMEND5265; GFX90A-NEXT: s_setpc_b64 s[30:31]5266;5267; GFX950-LABEL: flat_atomic_add_i32_ret_av_av:5268; GFX950: ; %bb.0:5269; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5270; GFX950-NEXT: ;;#ASMSTART5271; GFX950-NEXT: ; def v25272; GFX950-NEXT: ;;#ASMEND5273; GFX950-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 sc05274; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5275; GFX950-NEXT: ;;#ASMSTART5276; GFX950-NEXT: ; use v05277; GFX950-NEXT: ;;#ASMEND5278; GFX950-NEXT: s_setpc_b64 s[30:31]5279 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105280 %data = call i32 asm "; def $0", "=^VA"()5281 %result = atomicrmw add ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05282 call void asm "; use $0", "^VA"(i32 %result)5283 ret void5284}5285 5286define void @flat_atomic_sub_i32_ret_a_a(ptr %ptr) #0 {5287; GFX90A-LABEL: flat_atomic_sub_i32_ret_a_a:5288; GFX90A: ; %bb.0:5289; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5290; GFX90A-NEXT: ;;#ASMSTART5291; GFX90A-NEXT: ; def a05292; GFX90A-NEXT: ;;#ASMEND5293; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05294; GFX90A-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 glc5295; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5296; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05297; GFX90A-NEXT: ;;#ASMSTART5298; GFX90A-NEXT: ; use a05299; GFX90A-NEXT: ;;#ASMEND5300; GFX90A-NEXT: s_setpc_b64 s[30:31]5301;5302; GFX950-LABEL: flat_atomic_sub_i32_ret_a_a:5303; GFX950: ; %bb.0:5304; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5305; GFX950-NEXT: ;;#ASMSTART5306; GFX950-NEXT: ; def a05307; GFX950-NEXT: ;;#ASMEND5308; GFX950-NEXT: s_nop 05309; GFX950-NEXT: v_accvgpr_read_b32 v2, a05310; GFX950-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 sc05311; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5312; GFX950-NEXT: v_accvgpr_write_b32 a0, v05313; GFX950-NEXT: ;;#ASMSTART5314; GFX950-NEXT: ; use a05315; GFX950-NEXT: ;;#ASMEND5316; GFX950-NEXT: s_setpc_b64 s[30:31]5317 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105318 %data = call i32 asm "; def $0", "=a"()5319 %result = atomicrmw sub ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05320 call void asm "; use $0", "a"(i32 %result)5321 ret void5322}5323 5324define void @flat_atomic_sub_i32_ret_av_av(ptr %ptr) #0 {5325; GFX90A-LABEL: flat_atomic_sub_i32_ret_av_av:5326; GFX90A: ; %bb.0:5327; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5328; GFX90A-NEXT: ;;#ASMSTART5329; GFX90A-NEXT: ; def v25330; GFX90A-NEXT: ;;#ASMEND5331; GFX90A-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 glc5332; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5333; GFX90A-NEXT: ;;#ASMSTART5334; GFX90A-NEXT: ; use v05335; GFX90A-NEXT: ;;#ASMEND5336; GFX90A-NEXT: s_setpc_b64 s[30:31]5337;5338; GFX950-LABEL: flat_atomic_sub_i32_ret_av_av:5339; GFX950: ; %bb.0:5340; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5341; GFX950-NEXT: ;;#ASMSTART5342; GFX950-NEXT: ; def v25343; GFX950-NEXT: ;;#ASMEND5344; GFX950-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 sc05345; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5346; GFX950-NEXT: ;;#ASMSTART5347; GFX950-NEXT: ; use v05348; GFX950-NEXT: ;;#ASMEND5349; GFX950-NEXT: s_setpc_b64 s[30:31]5350 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105351 %data = call i32 asm "; def $0", "=^VA"()5352 %result = atomicrmw sub ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05353 call void asm "; use $0", "^VA"(i32 %result)5354 ret void5355}5356 5357define void @flat_atomic_and_i32_ret_a_a(ptr %ptr) #0 {5358; GFX90A-LABEL: flat_atomic_and_i32_ret_a_a:5359; GFX90A: ; %bb.0:5360; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5361; GFX90A-NEXT: ;;#ASMSTART5362; GFX90A-NEXT: ; def a05363; GFX90A-NEXT: ;;#ASMEND5364; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05365; GFX90A-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 glc5366; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5367; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05368; GFX90A-NEXT: ;;#ASMSTART5369; GFX90A-NEXT: ; use a05370; GFX90A-NEXT: ;;#ASMEND5371; GFX90A-NEXT: s_setpc_b64 s[30:31]5372;5373; GFX950-LABEL: flat_atomic_and_i32_ret_a_a:5374; GFX950: ; %bb.0:5375; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5376; GFX950-NEXT: ;;#ASMSTART5377; GFX950-NEXT: ; def a05378; GFX950-NEXT: ;;#ASMEND5379; GFX950-NEXT: s_nop 05380; GFX950-NEXT: v_accvgpr_read_b32 v2, a05381; GFX950-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 sc05382; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5383; GFX950-NEXT: v_accvgpr_write_b32 a0, v05384; GFX950-NEXT: ;;#ASMSTART5385; GFX950-NEXT: ; use a05386; GFX950-NEXT: ;;#ASMEND5387; GFX950-NEXT: s_setpc_b64 s[30:31]5388 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105389 %data = call i32 asm "; def $0", "=a"()5390 %result = atomicrmw and ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05391 call void asm "; use $0", "a"(i32 %result)5392 ret void5393}5394 5395define void @flat_atomic_and_i32_ret_av_av(ptr %ptr) #0 {5396; GFX90A-LABEL: flat_atomic_and_i32_ret_av_av:5397; GFX90A: ; %bb.0:5398; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5399; GFX90A-NEXT: ;;#ASMSTART5400; GFX90A-NEXT: ; def v25401; GFX90A-NEXT: ;;#ASMEND5402; GFX90A-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 glc5403; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5404; GFX90A-NEXT: ;;#ASMSTART5405; GFX90A-NEXT: ; use v05406; GFX90A-NEXT: ;;#ASMEND5407; GFX90A-NEXT: s_setpc_b64 s[30:31]5408;5409; GFX950-LABEL: flat_atomic_and_i32_ret_av_av:5410; GFX950: ; %bb.0:5411; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5412; GFX950-NEXT: ;;#ASMSTART5413; GFX950-NEXT: ; def v25414; GFX950-NEXT: ;;#ASMEND5415; GFX950-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 sc05416; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5417; GFX950-NEXT: ;;#ASMSTART5418; GFX950-NEXT: ; use v05419; GFX950-NEXT: ;;#ASMEND5420; GFX950-NEXT: s_setpc_b64 s[30:31]5421 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105422 %data = call i32 asm "; def $0", "=^VA"()5423 %result = atomicrmw and ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05424 call void asm "; use $0", "^VA"(i32 %result)5425 ret void5426}5427 5428define void @flat_atomic_nand_i32_ret_a_a(ptr %ptr) #0 {5429; GFX90A-LABEL: flat_atomic_nand_i32_ret_a_a:5430; GFX90A: ; %bb.0:5431; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5432; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:405433; GFX90A-NEXT: ;;#ASMSTART5434; GFX90A-NEXT: ; def a05435; GFX90A-NEXT: ;;#ASMEND5436; GFX90A-NEXT: v_accvgpr_read_b32 v4, a05437; GFX90A-NEXT: s_mov_b64 s[4:5], 05438; GFX90A-NEXT: .LBB69_1: ; %atomicrmw.start5439; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15440; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5441; GFX90A-NEXT: v_and_b32_e32 v2, v3, v45442; GFX90A-NEXT: v_not_b32_e32 v2, v25443; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc5444; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5445; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v35446; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5447; GFX90A-NEXT: v_mov_b32_e32 v3, v25448; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5449; GFX90A-NEXT: s_cbranch_execnz .LBB69_15450; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5451; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5452; GFX90A-NEXT: v_accvgpr_write_b32 a0, v25453; GFX90A-NEXT: ;;#ASMSTART5454; GFX90A-NEXT: ; use a05455; GFX90A-NEXT: ;;#ASMEND5456; GFX90A-NEXT: s_setpc_b64 s[30:31]5457;5458; GFX950-LABEL: flat_atomic_nand_i32_ret_a_a:5459; GFX950: ; %bb.0:5460; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5461; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:405462; GFX950-NEXT: ;;#ASMSTART5463; GFX950-NEXT: ; def a05464; GFX950-NEXT: ;;#ASMEND5465; GFX950-NEXT: s_mov_b64 s[0:1], 05466; GFX950-NEXT: v_accvgpr_read_b32 v4, a05467; GFX950-NEXT: .LBB69_1: ; %atomicrmw.start5468; GFX950-NEXT: ; =>This Inner Loop Header: Depth=15469; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5470; GFX950-NEXT: v_bitop3_b32 v2, v3, v4, v3 bitop3:0x3f5471; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc05472; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5473; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v35474; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]5475; GFX950-NEXT: v_mov_b32_e32 v3, v25476; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]5477; GFX950-NEXT: s_cbranch_execnz .LBB69_15478; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end5479; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5480; GFX950-NEXT: v_accvgpr_write_b32 a0, v25481; GFX950-NEXT: ;;#ASMSTART5482; GFX950-NEXT: ; use a05483; GFX950-NEXT: ;;#ASMEND5484; GFX950-NEXT: s_setpc_b64 s[30:31]5485 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105486 %data = call i32 asm "; def $0", "=a"()5487 %result = atomicrmw nand ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05488 call void asm "; use $0", "a"(i32 %result)5489 ret void5490}5491 5492define void @flat_atomic_nand_i32_ret_av_av(ptr %ptr) #0 {5493; GFX90A-LABEL: flat_atomic_nand_i32_ret_av_av:5494; GFX90A: ; %bb.0:5495; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5496; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:405497; GFX90A-NEXT: s_mov_b64 s[4:5], 05498; GFX90A-NEXT: ;;#ASMSTART5499; GFX90A-NEXT: ; def v45500; GFX90A-NEXT: ;;#ASMEND5501; GFX90A-NEXT: .LBB70_1: ; %atomicrmw.start5502; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15503; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5504; GFX90A-NEXT: v_and_b32_e32 v2, v3, v45505; GFX90A-NEXT: v_not_b32_e32 v2, v25506; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc5507; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5508; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v35509; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5510; GFX90A-NEXT: v_mov_b32_e32 v3, v25511; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5512; GFX90A-NEXT: s_cbranch_execnz .LBB70_15513; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5514; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5515; GFX90A-NEXT: ;;#ASMSTART5516; GFX90A-NEXT: ; use v25517; GFX90A-NEXT: ;;#ASMEND5518; GFX90A-NEXT: s_setpc_b64 s[30:31]5519;5520; GFX950-LABEL: flat_atomic_nand_i32_ret_av_av:5521; GFX950: ; %bb.0:5522; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5523; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:405524; GFX950-NEXT: s_mov_b64 s[0:1], 05525; GFX950-NEXT: ;;#ASMSTART5526; GFX950-NEXT: ; def v45527; GFX950-NEXT: ;;#ASMEND5528; GFX950-NEXT: .LBB70_1: ; %atomicrmw.start5529; GFX950-NEXT: ; =>This Inner Loop Header: Depth=15530; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5531; GFX950-NEXT: v_bitop3_b32 v2, v3, v4, v3 bitop3:0x3f5532; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc05533; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5534; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v35535; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]5536; GFX950-NEXT: v_mov_b32_e32 v3, v25537; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]5538; GFX950-NEXT: s_cbranch_execnz .LBB70_15539; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end5540; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]5541; GFX950-NEXT: ;;#ASMSTART5542; GFX950-NEXT: ; use v25543; GFX950-NEXT: ;;#ASMEND5544; GFX950-NEXT: s_setpc_b64 s[30:31]5545 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105546 %data = call i32 asm "; def $0", "=^VA"()5547 %result = atomicrmw nand ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05548 call void asm "; use $0", "^VA"(i32 %result)5549 ret void5550}5551 5552define void @flat_atomic_or_i32_ret_a_a(ptr %ptr) #0 {5553; GFX90A-LABEL: flat_atomic_or_i32_ret_a_a:5554; GFX90A: ; %bb.0:5555; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5556; GFX90A-NEXT: ;;#ASMSTART5557; GFX90A-NEXT: ; def a05558; GFX90A-NEXT: ;;#ASMEND5559; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05560; GFX90A-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 glc5561; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5562; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05563; GFX90A-NEXT: ;;#ASMSTART5564; GFX90A-NEXT: ; use a05565; GFX90A-NEXT: ;;#ASMEND5566; GFX90A-NEXT: s_setpc_b64 s[30:31]5567;5568; GFX950-LABEL: flat_atomic_or_i32_ret_a_a:5569; GFX950: ; %bb.0:5570; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5571; GFX950-NEXT: ;;#ASMSTART5572; GFX950-NEXT: ; def a05573; GFX950-NEXT: ;;#ASMEND5574; GFX950-NEXT: s_nop 05575; GFX950-NEXT: v_accvgpr_read_b32 v2, a05576; GFX950-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 sc05577; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5578; GFX950-NEXT: v_accvgpr_write_b32 a0, v05579; GFX950-NEXT: ;;#ASMSTART5580; GFX950-NEXT: ; use a05581; GFX950-NEXT: ;;#ASMEND5582; GFX950-NEXT: s_setpc_b64 s[30:31]5583 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105584 %data = call i32 asm "; def $0", "=a"()5585 %result = atomicrmw or ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05586 call void asm "; use $0", "a"(i32 %result)5587 ret void5588}5589 5590define void @flat_atomic_or_i32_ret_av_av(ptr %ptr) #0 {5591; GFX90A-LABEL: flat_atomic_or_i32_ret_av_av:5592; GFX90A: ; %bb.0:5593; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5594; GFX90A-NEXT: ;;#ASMSTART5595; GFX90A-NEXT: ; def v25596; GFX90A-NEXT: ;;#ASMEND5597; GFX90A-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 glc5598; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5599; GFX90A-NEXT: ;;#ASMSTART5600; GFX90A-NEXT: ; use v05601; GFX90A-NEXT: ;;#ASMEND5602; GFX90A-NEXT: s_setpc_b64 s[30:31]5603;5604; GFX950-LABEL: flat_atomic_or_i32_ret_av_av:5605; GFX950: ; %bb.0:5606; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5607; GFX950-NEXT: ;;#ASMSTART5608; GFX950-NEXT: ; def v25609; GFX950-NEXT: ;;#ASMEND5610; GFX950-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 sc05611; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5612; GFX950-NEXT: ;;#ASMSTART5613; GFX950-NEXT: ; use v05614; GFX950-NEXT: ;;#ASMEND5615; GFX950-NEXT: s_setpc_b64 s[30:31]5616 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105617 %data = call i32 asm "; def $0", "=^VA"()5618 %result = atomicrmw or ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05619 call void asm "; use $0", "^VA"(i32 %result)5620 ret void5621}5622 5623define void @flat_atomic_max_i32_ret_a_a(ptr %ptr) #0 {5624; GFX90A-LABEL: flat_atomic_max_i32_ret_a_a:5625; GFX90A: ; %bb.0:5626; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5627; GFX90A-NEXT: ;;#ASMSTART5628; GFX90A-NEXT: ; def a05629; GFX90A-NEXT: ;;#ASMEND5630; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05631; GFX90A-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 glc5632; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5633; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05634; GFX90A-NEXT: ;;#ASMSTART5635; GFX90A-NEXT: ; use a05636; GFX90A-NEXT: ;;#ASMEND5637; GFX90A-NEXT: s_setpc_b64 s[30:31]5638;5639; GFX950-LABEL: flat_atomic_max_i32_ret_a_a:5640; GFX950: ; %bb.0:5641; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5642; GFX950-NEXT: ;;#ASMSTART5643; GFX950-NEXT: ; def a05644; GFX950-NEXT: ;;#ASMEND5645; GFX950-NEXT: s_nop 05646; GFX950-NEXT: v_accvgpr_read_b32 v2, a05647; GFX950-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 sc05648; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5649; GFX950-NEXT: v_accvgpr_write_b32 a0, v05650; GFX950-NEXT: ;;#ASMSTART5651; GFX950-NEXT: ; use a05652; GFX950-NEXT: ;;#ASMEND5653; GFX950-NEXT: s_setpc_b64 s[30:31]5654 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105655 %data = call i32 asm "; def $0", "=a"()5656 %result = atomicrmw max ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05657 call void asm "; use $0", "a"(i32 %result)5658 ret void5659}5660 5661define void @flat_atomic_max_i32_ret_av_av(ptr %ptr) #0 {5662; GFX90A-LABEL: flat_atomic_max_i32_ret_av_av:5663; GFX90A: ; %bb.0:5664; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5665; GFX90A-NEXT: ;;#ASMSTART5666; GFX90A-NEXT: ; def v25667; GFX90A-NEXT: ;;#ASMEND5668; GFX90A-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 glc5669; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5670; GFX90A-NEXT: ;;#ASMSTART5671; GFX90A-NEXT: ; use v05672; GFX90A-NEXT: ;;#ASMEND5673; GFX90A-NEXT: s_setpc_b64 s[30:31]5674;5675; GFX950-LABEL: flat_atomic_max_i32_ret_av_av:5676; GFX950: ; %bb.0:5677; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5678; GFX950-NEXT: ;;#ASMSTART5679; GFX950-NEXT: ; def v25680; GFX950-NEXT: ;;#ASMEND5681; GFX950-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 sc05682; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5683; GFX950-NEXT: ;;#ASMSTART5684; GFX950-NEXT: ; use v05685; GFX950-NEXT: ;;#ASMEND5686; GFX950-NEXT: s_setpc_b64 s[30:31]5687 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105688 %data = call i32 asm "; def $0", "=^VA"()5689 %result = atomicrmw max ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05690 call void asm "; use $0", "^VA"(i32 %result)5691 ret void5692}5693 5694define void @flat_atomic_min_i32_ret_a_a(ptr %ptr) #0 {5695; GFX90A-LABEL: flat_atomic_min_i32_ret_a_a:5696; GFX90A: ; %bb.0:5697; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5698; GFX90A-NEXT: ;;#ASMSTART5699; GFX90A-NEXT: ; def a05700; GFX90A-NEXT: ;;#ASMEND5701; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05702; GFX90A-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 glc5703; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5704; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05705; GFX90A-NEXT: ;;#ASMSTART5706; GFX90A-NEXT: ; use a05707; GFX90A-NEXT: ;;#ASMEND5708; GFX90A-NEXT: s_setpc_b64 s[30:31]5709;5710; GFX950-LABEL: flat_atomic_min_i32_ret_a_a:5711; GFX950: ; %bb.0:5712; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5713; GFX950-NEXT: ;;#ASMSTART5714; GFX950-NEXT: ; def a05715; GFX950-NEXT: ;;#ASMEND5716; GFX950-NEXT: s_nop 05717; GFX950-NEXT: v_accvgpr_read_b32 v2, a05718; GFX950-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 sc05719; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5720; GFX950-NEXT: v_accvgpr_write_b32 a0, v05721; GFX950-NEXT: ;;#ASMSTART5722; GFX950-NEXT: ; use a05723; GFX950-NEXT: ;;#ASMEND5724; GFX950-NEXT: s_setpc_b64 s[30:31]5725 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105726 %data = call i32 asm "; def $0", "=a"()5727 %result = atomicrmw min ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05728 call void asm "; use $0", "a"(i32 %result)5729 ret void5730}5731 5732define void @flat_atomic_min_i32_ret_av_av(ptr %ptr) #0 {5733; GFX90A-LABEL: flat_atomic_min_i32_ret_av_av:5734; GFX90A: ; %bb.0:5735; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5736; GFX90A-NEXT: ;;#ASMSTART5737; GFX90A-NEXT: ; def v25738; GFX90A-NEXT: ;;#ASMEND5739; GFX90A-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 glc5740; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5741; GFX90A-NEXT: ;;#ASMSTART5742; GFX90A-NEXT: ; use v05743; GFX90A-NEXT: ;;#ASMEND5744; GFX90A-NEXT: s_setpc_b64 s[30:31]5745;5746; GFX950-LABEL: flat_atomic_min_i32_ret_av_av:5747; GFX950: ; %bb.0:5748; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5749; GFX950-NEXT: ;;#ASMSTART5750; GFX950-NEXT: ; def v25751; GFX950-NEXT: ;;#ASMEND5752; GFX950-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 sc05753; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5754; GFX950-NEXT: ;;#ASMSTART5755; GFX950-NEXT: ; use v05756; GFX950-NEXT: ;;#ASMEND5757; GFX950-NEXT: s_setpc_b64 s[30:31]5758 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105759 %data = call i32 asm "; def $0", "=^VA"()5760 %result = atomicrmw min ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05761 call void asm "; use $0", "^VA"(i32 %result)5762 ret void5763}5764 5765define void @flat_atomic_umax_i32_ret_a_a(ptr %ptr) #0 {5766; GFX90A-LABEL: flat_atomic_umax_i32_ret_a_a:5767; GFX90A: ; %bb.0:5768; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5769; GFX90A-NEXT: ;;#ASMSTART5770; GFX90A-NEXT: ; def a05771; GFX90A-NEXT: ;;#ASMEND5772; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05773; GFX90A-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 glc5774; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5775; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05776; GFX90A-NEXT: ;;#ASMSTART5777; GFX90A-NEXT: ; use a05778; GFX90A-NEXT: ;;#ASMEND5779; GFX90A-NEXT: s_setpc_b64 s[30:31]5780;5781; GFX950-LABEL: flat_atomic_umax_i32_ret_a_a:5782; GFX950: ; %bb.0:5783; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5784; GFX950-NEXT: ;;#ASMSTART5785; GFX950-NEXT: ; def a05786; GFX950-NEXT: ;;#ASMEND5787; GFX950-NEXT: s_nop 05788; GFX950-NEXT: v_accvgpr_read_b32 v2, a05789; GFX950-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 sc05790; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5791; GFX950-NEXT: v_accvgpr_write_b32 a0, v05792; GFX950-NEXT: ;;#ASMSTART5793; GFX950-NEXT: ; use a05794; GFX950-NEXT: ;;#ASMEND5795; GFX950-NEXT: s_setpc_b64 s[30:31]5796 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105797 %data = call i32 asm "; def $0", "=a"()5798 %result = atomicrmw umax ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05799 call void asm "; use $0", "a"(i32 %result)5800 ret void5801}5802 5803define void @flat_atomic_umax_i32_ret_av_av(ptr %ptr) #0 {5804; GFX90A-LABEL: flat_atomic_umax_i32_ret_av_av:5805; GFX90A: ; %bb.0:5806; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5807; GFX90A-NEXT: ;;#ASMSTART5808; GFX90A-NEXT: ; def v25809; GFX90A-NEXT: ;;#ASMEND5810; GFX90A-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 glc5811; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5812; GFX90A-NEXT: ;;#ASMSTART5813; GFX90A-NEXT: ; use v05814; GFX90A-NEXT: ;;#ASMEND5815; GFX90A-NEXT: s_setpc_b64 s[30:31]5816;5817; GFX950-LABEL: flat_atomic_umax_i32_ret_av_av:5818; GFX950: ; %bb.0:5819; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5820; GFX950-NEXT: ;;#ASMSTART5821; GFX950-NEXT: ; def v25822; GFX950-NEXT: ;;#ASMEND5823; GFX950-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 sc05824; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5825; GFX950-NEXT: ;;#ASMSTART5826; GFX950-NEXT: ; use v05827; GFX950-NEXT: ;;#ASMEND5828; GFX950-NEXT: s_setpc_b64 s[30:31]5829 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105830 %data = call i32 asm "; def $0", "=^VA"()5831 %result = atomicrmw umax ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05832 call void asm "; use $0", "^VA"(i32 %result)5833 ret void5834}5835 5836define void @flat_atomic_umin_i32_ret_a_a(ptr %ptr) #0 {5837; GFX90A-LABEL: flat_atomic_umin_i32_ret_a_a:5838; GFX90A: ; %bb.0:5839; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5840; GFX90A-NEXT: ;;#ASMSTART5841; GFX90A-NEXT: ; def a05842; GFX90A-NEXT: ;;#ASMEND5843; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05844; GFX90A-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 glc5845; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5846; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05847; GFX90A-NEXT: ;;#ASMSTART5848; GFX90A-NEXT: ; use a05849; GFX90A-NEXT: ;;#ASMEND5850; GFX90A-NEXT: s_setpc_b64 s[30:31]5851;5852; GFX950-LABEL: flat_atomic_umin_i32_ret_a_a:5853; GFX950: ; %bb.0:5854; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5855; GFX950-NEXT: ;;#ASMSTART5856; GFX950-NEXT: ; def a05857; GFX950-NEXT: ;;#ASMEND5858; GFX950-NEXT: s_nop 05859; GFX950-NEXT: v_accvgpr_read_b32 v2, a05860; GFX950-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 sc05861; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5862; GFX950-NEXT: v_accvgpr_write_b32 a0, v05863; GFX950-NEXT: ;;#ASMSTART5864; GFX950-NEXT: ; use a05865; GFX950-NEXT: ;;#ASMEND5866; GFX950-NEXT: s_setpc_b64 s[30:31]5867 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105868 %data = call i32 asm "; def $0", "=a"()5869 %result = atomicrmw umin ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05870 call void asm "; use $0", "a"(i32 %result)5871 ret void5872}5873 5874define void @flat_atomic_umin_i32_ret_av_av(ptr %ptr) #0 {5875; GFX90A-LABEL: flat_atomic_umin_i32_ret_av_av:5876; GFX90A: ; %bb.0:5877; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5878; GFX90A-NEXT: ;;#ASMSTART5879; GFX90A-NEXT: ; def v25880; GFX90A-NEXT: ;;#ASMEND5881; GFX90A-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 glc5882; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5883; GFX90A-NEXT: ;;#ASMSTART5884; GFX90A-NEXT: ; use v05885; GFX90A-NEXT: ;;#ASMEND5886; GFX90A-NEXT: s_setpc_b64 s[30:31]5887;5888; GFX950-LABEL: flat_atomic_umin_i32_ret_av_av:5889; GFX950: ; %bb.0:5890; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5891; GFX950-NEXT: ;;#ASMSTART5892; GFX950-NEXT: ; def v25893; GFX950-NEXT: ;;#ASMEND5894; GFX950-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 sc05895; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5896; GFX950-NEXT: ;;#ASMSTART5897; GFX950-NEXT: ; use v05898; GFX950-NEXT: ;;#ASMEND5899; GFX950-NEXT: s_setpc_b64 s[30:31]5900 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105901 %data = call i32 asm "; def $0", "=^VA"()5902 %result = atomicrmw umin ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05903 call void asm "; use $0", "^VA"(i32 %result)5904 ret void5905}5906 5907define void @flat_atomic_uinc_wrap_i32_ret_a_a(ptr %ptr) #0 {5908; GFX90A-LABEL: flat_atomic_uinc_wrap_i32_ret_a_a:5909; GFX90A: ; %bb.0:5910; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5911; GFX90A-NEXT: ;;#ASMSTART5912; GFX90A-NEXT: ; def a05913; GFX90A-NEXT: ;;#ASMEND5914; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05915; GFX90A-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 glc5916; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5917; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05918; GFX90A-NEXT: ;;#ASMSTART5919; GFX90A-NEXT: ; use a05920; GFX90A-NEXT: ;;#ASMEND5921; GFX90A-NEXT: s_setpc_b64 s[30:31]5922;5923; GFX950-LABEL: flat_atomic_uinc_wrap_i32_ret_a_a:5924; GFX950: ; %bb.0:5925; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5926; GFX950-NEXT: ;;#ASMSTART5927; GFX950-NEXT: ; def a05928; GFX950-NEXT: ;;#ASMEND5929; GFX950-NEXT: s_nop 05930; GFX950-NEXT: v_accvgpr_read_b32 v2, a05931; GFX950-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 sc05932; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5933; GFX950-NEXT: v_accvgpr_write_b32 a0, v05934; GFX950-NEXT: ;;#ASMSTART5935; GFX950-NEXT: ; use a05936; GFX950-NEXT: ;;#ASMEND5937; GFX950-NEXT: s_setpc_b64 s[30:31]5938 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105939 %data = call i32 asm "; def $0", "=a"()5940 %result = atomicrmw uinc_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05941 call void asm "; use $0", "a"(i32 %result)5942 ret void5943}5944 5945define void @flat_atomic_uinc_wrap_i32_ret_av_av(ptr %ptr) #0 {5946; GFX90A-LABEL: flat_atomic_uinc_wrap_i32_ret_av_av:5947; GFX90A: ; %bb.0:5948; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5949; GFX90A-NEXT: ;;#ASMSTART5950; GFX90A-NEXT: ; def v25951; GFX90A-NEXT: ;;#ASMEND5952; GFX90A-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 glc5953; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5954; GFX90A-NEXT: ;;#ASMSTART5955; GFX90A-NEXT: ; use v05956; GFX90A-NEXT: ;;#ASMEND5957; GFX90A-NEXT: s_setpc_b64 s[30:31]5958;5959; GFX950-LABEL: flat_atomic_uinc_wrap_i32_ret_av_av:5960; GFX950: ; %bb.0:5961; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5962; GFX950-NEXT: ;;#ASMSTART5963; GFX950-NEXT: ; def v25964; GFX950-NEXT: ;;#ASMEND5965; GFX950-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 sc05966; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5967; GFX950-NEXT: ;;#ASMSTART5968; GFX950-NEXT: ; use v05969; GFX950-NEXT: ;;#ASMEND5970; GFX950-NEXT: s_setpc_b64 s[30:31]5971 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 105972 %data = call i32 asm "; def $0", "=^VA"()5973 %result = atomicrmw uinc_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !05974 call void asm "; use $0", "^VA"(i32 %result)5975 ret void5976}5977 5978define void @flat_atomic_udec_wrap_i32_ret_a_a(ptr %ptr) #0 {5979; GFX90A-LABEL: flat_atomic_udec_wrap_i32_ret_a_a:5980; GFX90A: ; %bb.0:5981; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5982; GFX90A-NEXT: ;;#ASMSTART5983; GFX90A-NEXT: ; def a05984; GFX90A-NEXT: ;;#ASMEND5985; GFX90A-NEXT: v_accvgpr_read_b32 v2, a05986; GFX90A-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 glc5987; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5988; GFX90A-NEXT: v_accvgpr_write_b32 a0, v05989; GFX90A-NEXT: ;;#ASMSTART5990; GFX90A-NEXT: ; use a05991; GFX90A-NEXT: ;;#ASMEND5992; GFX90A-NEXT: s_setpc_b64 s[30:31]5993;5994; GFX950-LABEL: flat_atomic_udec_wrap_i32_ret_a_a:5995; GFX950: ; %bb.0:5996; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5997; GFX950-NEXT: ;;#ASMSTART5998; GFX950-NEXT: ; def a05999; GFX950-NEXT: ;;#ASMEND6000; GFX950-NEXT: s_nop 06001; GFX950-NEXT: v_accvgpr_read_b32 v2, a06002; GFX950-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 sc06003; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6004; GFX950-NEXT: v_accvgpr_write_b32 a0, v06005; GFX950-NEXT: ;;#ASMSTART6006; GFX950-NEXT: ; use a06007; GFX950-NEXT: ;;#ASMEND6008; GFX950-NEXT: s_setpc_b64 s[30:31]6009 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106010 %data = call i32 asm "; def $0", "=a"()6011 %result = atomicrmw udec_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06012 call void asm "; use $0", "a"(i32 %result)6013 ret void6014}6015 6016define void @flat_atomic_udec_wrap_i32_ret_av_av(ptr %ptr) #0 {6017; GFX90A-LABEL: flat_atomic_udec_wrap_i32_ret_av_av:6018; GFX90A: ; %bb.0:6019; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6020; GFX90A-NEXT: ;;#ASMSTART6021; GFX90A-NEXT: ; def v26022; GFX90A-NEXT: ;;#ASMEND6023; GFX90A-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 glc6024; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6025; GFX90A-NEXT: ;;#ASMSTART6026; GFX90A-NEXT: ; use v06027; GFX90A-NEXT: ;;#ASMEND6028; GFX90A-NEXT: s_setpc_b64 s[30:31]6029;6030; GFX950-LABEL: flat_atomic_udec_wrap_i32_ret_av_av:6031; GFX950: ; %bb.0:6032; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6033; GFX950-NEXT: ;;#ASMSTART6034; GFX950-NEXT: ; def v26035; GFX950-NEXT: ;;#ASMEND6036; GFX950-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 sc06037; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6038; GFX950-NEXT: ;;#ASMSTART6039; GFX950-NEXT: ; use v06040; GFX950-NEXT: ;;#ASMEND6041; GFX950-NEXT: s_setpc_b64 s[30:31]6042 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106043 %data = call i32 asm "; def $0", "=^VA"()6044 %result = atomicrmw udec_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06045 call void asm "; use $0", "^VA"(i32 %result)6046 ret void6047}6048 6049define void @flat_atomic_usub_cond_i32_ret_a_a(ptr %ptr) #0 {6050; GFX90A-LABEL: flat_atomic_usub_cond_i32_ret_a_a:6051; GFX90A: ; %bb.0:6052; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6053; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:406054; GFX90A-NEXT: ;;#ASMSTART6055; GFX90A-NEXT: ; def a06056; GFX90A-NEXT: ;;#ASMEND6057; GFX90A-NEXT: v_accvgpr_read_b32 v4, a06058; GFX90A-NEXT: s_mov_b64 s[4:5], 06059; GFX90A-NEXT: .LBB85_1: ; %atomicrmw.start6060; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16061; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6062; GFX90A-NEXT: v_sub_u32_e32 v2, v3, v46063; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v3, v46064; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc6065; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc6066; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6067; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36068; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6069; GFX90A-NEXT: v_mov_b32_e32 v3, v26070; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6071; GFX90A-NEXT: s_cbranch_execnz .LBB85_16072; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6073; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6074; GFX90A-NEXT: v_accvgpr_write_b32 a0, v26075; GFX90A-NEXT: ;;#ASMSTART6076; GFX90A-NEXT: ; use a06077; GFX90A-NEXT: ;;#ASMEND6078; GFX90A-NEXT: s_setpc_b64 s[30:31]6079;6080; GFX950-LABEL: flat_atomic_usub_cond_i32_ret_a_a:6081; GFX950: ; %bb.0:6082; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6083; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:406084; GFX950-NEXT: ;;#ASMSTART6085; GFX950-NEXT: ; def a06086; GFX950-NEXT: ;;#ASMEND6087; GFX950-NEXT: s_mov_b64 s[0:1], 06088; GFX950-NEXT: v_accvgpr_read_b32 v4, a06089; GFX950-NEXT: .LBB85_1: ; %atomicrmw.start6090; GFX950-NEXT: ; =>This Inner Loop Header: Depth=16091; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6092; GFX950-NEXT: v_sub_u32_e32 v2, v3, v46093; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v3, v46094; GFX950-NEXT: s_nop 16095; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc6096; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc06097; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6098; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36099; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6100; GFX950-NEXT: v_mov_b32_e32 v3, v26101; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]6102; GFX950-NEXT: s_cbranch_execnz .LBB85_16103; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end6104; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6105; GFX950-NEXT: v_accvgpr_write_b32 a0, v26106; GFX950-NEXT: ;;#ASMSTART6107; GFX950-NEXT: ; use a06108; GFX950-NEXT: ;;#ASMEND6109; GFX950-NEXT: s_setpc_b64 s[30:31]6110 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106111 %data = call i32 asm "; def $0", "=a"()6112 %result = atomicrmw usub_cond ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06113 call void asm "; use $0", "a"(i32 %result)6114 ret void6115}6116 6117define void @flat_atomic_usub_cond_i32_ret_av_av(ptr %ptr) #0 {6118; GFX90A-LABEL: flat_atomic_usub_cond_i32_ret_av_av:6119; GFX90A: ; %bb.0:6120; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6121; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:406122; GFX90A-NEXT: s_mov_b64 s[4:5], 06123; GFX90A-NEXT: ;;#ASMSTART6124; GFX90A-NEXT: ; def v46125; GFX90A-NEXT: ;;#ASMEND6126; GFX90A-NEXT: .LBB86_1: ; %atomicrmw.start6127; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16128; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6129; GFX90A-NEXT: v_sub_u32_e32 v2, v3, v46130; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v3, v46131; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc6132; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc6133; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6134; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36135; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6136; GFX90A-NEXT: v_mov_b32_e32 v3, v26137; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6138; GFX90A-NEXT: s_cbranch_execnz .LBB86_16139; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6140; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6141; GFX90A-NEXT: ;;#ASMSTART6142; GFX90A-NEXT: ; use v26143; GFX90A-NEXT: ;;#ASMEND6144; GFX90A-NEXT: s_setpc_b64 s[30:31]6145;6146; GFX950-LABEL: flat_atomic_usub_cond_i32_ret_av_av:6147; GFX950: ; %bb.0:6148; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6149; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:406150; GFX950-NEXT: s_mov_b64 s[0:1], 06151; GFX950-NEXT: ;;#ASMSTART6152; GFX950-NEXT: ; def v46153; GFX950-NEXT: ;;#ASMEND6154; GFX950-NEXT: .LBB86_1: ; %atomicrmw.start6155; GFX950-NEXT: ; =>This Inner Loop Header: Depth=16156; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6157; GFX950-NEXT: v_sub_u32_e32 v2, v3, v46158; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v3, v46159; GFX950-NEXT: s_nop 16160; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc6161; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc06162; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6163; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36164; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6165; GFX950-NEXT: v_mov_b32_e32 v3, v26166; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]6167; GFX950-NEXT: s_cbranch_execnz .LBB86_16168; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end6169; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6170; GFX950-NEXT: ;;#ASMSTART6171; GFX950-NEXT: ; use v26172; GFX950-NEXT: ;;#ASMEND6173; GFX950-NEXT: s_setpc_b64 s[30:31]6174 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106175 %data = call i32 asm "; def $0", "=^VA"()6176 %result = atomicrmw usub_cond ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06177 call void asm "; use $0", "^VA"(i32 %result)6178 ret void6179}6180 6181define void @flat_atomic_usub_sat_i32_ret_a_a(ptr %ptr) #0 {6182; GFX90A-LABEL: flat_atomic_usub_sat_i32_ret_a_a:6183; GFX90A: ; %bb.0:6184; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6185; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:406186; GFX90A-NEXT: ;;#ASMSTART6187; GFX90A-NEXT: ; def a06188; GFX90A-NEXT: ;;#ASMEND6189; GFX90A-NEXT: v_accvgpr_read_b32 v4, a06190; GFX90A-NEXT: s_mov_b64 s[4:5], 06191; GFX90A-NEXT: .LBB87_1: ; %atomicrmw.start6192; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16193; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6194; GFX90A-NEXT: v_sub_u32_e64 v2, v3, v4 clamp6195; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc6196; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6197; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36198; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6199; GFX90A-NEXT: v_mov_b32_e32 v3, v26200; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6201; GFX90A-NEXT: s_cbranch_execnz .LBB87_16202; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6203; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6204; GFX90A-NEXT: v_accvgpr_write_b32 a0, v26205; GFX90A-NEXT: ;;#ASMSTART6206; GFX90A-NEXT: ; use a06207; GFX90A-NEXT: ;;#ASMEND6208; GFX90A-NEXT: s_setpc_b64 s[30:31]6209;6210; GFX950-LABEL: flat_atomic_usub_sat_i32_ret_a_a:6211; GFX950: ; %bb.0:6212; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6213; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:406214; GFX950-NEXT: ;;#ASMSTART6215; GFX950-NEXT: ; def a06216; GFX950-NEXT: ;;#ASMEND6217; GFX950-NEXT: s_mov_b64 s[0:1], 06218; GFX950-NEXT: v_accvgpr_read_b32 v4, a06219; GFX950-NEXT: .LBB87_1: ; %atomicrmw.start6220; GFX950-NEXT: ; =>This Inner Loop Header: Depth=16221; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6222; GFX950-NEXT: v_sub_u32_e64 v2, v3, v4 clamp6223; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc06224; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6225; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36226; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6227; GFX950-NEXT: v_mov_b32_e32 v3, v26228; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]6229; GFX950-NEXT: s_cbranch_execnz .LBB87_16230; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end6231; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6232; GFX950-NEXT: v_accvgpr_write_b32 a0, v26233; GFX950-NEXT: ;;#ASMSTART6234; GFX950-NEXT: ; use a06235; GFX950-NEXT: ;;#ASMEND6236; GFX950-NEXT: s_setpc_b64 s[30:31]6237 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106238 %data = call i32 asm "; def $0", "=a"()6239 %result = atomicrmw usub_sat ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06240 call void asm "; use $0", "a"(i32 %result)6241 ret void6242}6243 6244define void @flat_atomic_usub_sat_i32_ret_av_av(ptr %ptr) #0 {6245; GFX90A-LABEL: flat_atomic_usub_sat_i32_ret_av_av:6246; GFX90A: ; %bb.0:6247; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6248; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:406249; GFX90A-NEXT: s_mov_b64 s[4:5], 06250; GFX90A-NEXT: ;;#ASMSTART6251; GFX90A-NEXT: ; def v46252; GFX90A-NEXT: ;;#ASMEND6253; GFX90A-NEXT: .LBB88_1: ; %atomicrmw.start6254; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16255; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6256; GFX90A-NEXT: v_sub_u32_e64 v2, v3, v4 clamp6257; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc6258; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6259; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36260; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6261; GFX90A-NEXT: v_mov_b32_e32 v3, v26262; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6263; GFX90A-NEXT: s_cbranch_execnz .LBB88_16264; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6265; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6266; GFX90A-NEXT: ;;#ASMSTART6267; GFX90A-NEXT: ; use v26268; GFX90A-NEXT: ;;#ASMEND6269; GFX90A-NEXT: s_setpc_b64 s[30:31]6270;6271; GFX950-LABEL: flat_atomic_usub_sat_i32_ret_av_av:6272; GFX950: ; %bb.0:6273; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6274; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:406275; GFX950-NEXT: s_mov_b64 s[0:1], 06276; GFX950-NEXT: ;;#ASMSTART6277; GFX950-NEXT: ; def v46278; GFX950-NEXT: ;;#ASMEND6279; GFX950-NEXT: .LBB88_1: ; %atomicrmw.start6280; GFX950-NEXT: ; =>This Inner Loop Header: Depth=16281; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6282; GFX950-NEXT: v_sub_u32_e64 v2, v3, v4 clamp6283; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc06284; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6285; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36286; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6287; GFX950-NEXT: v_mov_b32_e32 v3, v26288; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]6289; GFX950-NEXT: s_cbranch_execnz .LBB88_16290; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end6291; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6292; GFX950-NEXT: ;;#ASMSTART6293; GFX950-NEXT: ; use v26294; GFX950-NEXT: ;;#ASMEND6295; GFX950-NEXT: s_setpc_b64 s[30:31]6296 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 106297 %data = call i32 asm "; def $0", "=^VA"()6298 %result = atomicrmw usub_sat ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06299 call void asm "; use $0", "^VA"(i32 %result)6300 ret void6301}6302 6303;---------------------------------------------------------------------6304; other atomics i64, with aa+av cases6305;---------------------------------------------------------------------6306 6307define void @flat_atomic_add_i64_ret_a_a(ptr %ptr) #0 {6308; GFX90A-LABEL: flat_atomic_add_i64_ret_a_a:6309; GFX90A: ; %bb.0:6310; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6311; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v06312; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6313; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc6314; GFX90A-NEXT: ;;#ASMSTART6315; GFX90A-NEXT: ; def a[0:1]6316; GFX90A-NEXT: ;;#ASMEND6317; GFX90A-NEXT: v_accvgpr_read_b32 v3, a16318; GFX90A-NEXT: v_accvgpr_read_b32 v2, a06319; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v16320; GFX90A-NEXT: ; implicit-def: $agpr0_agpr16321; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6322; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6323; GFX90A-NEXT: s_cbranch_execz .LBB89_26324; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6325; GFX90A-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc6326; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6327; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36328; GFX90A-NEXT: v_accvgpr_write_b32 a0, v06329; GFX90A-NEXT: v_accvgpr_write_b32 a1, v16330; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16331; GFX90A-NEXT: .LBB89_2: ; %Flow6332; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6333; GFX90A-NEXT: s_cbranch_execz .LBB89_46334; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6335; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6336; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc6337; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen6338; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:46339; GFX90A-NEXT: s_waitcnt vmcnt(1)6340; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v1, v26341; GFX90A-NEXT: v_accvgpr_write_b32 a0, v16342; GFX90A-NEXT: s_waitcnt vmcnt(0)6343; GFX90A-NEXT: v_accvgpr_write_b32 a1, v46344; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v4, v3, vcc6345; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen6346; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:46347; GFX90A-NEXT: .LBB89_4: ; %atomicrmw.phi6348; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6349; GFX90A-NEXT: ;;#ASMSTART6350; GFX90A-NEXT: ; use a[0:1]6351; GFX90A-NEXT: ;;#ASMEND6352; GFX90A-NEXT: s_waitcnt vmcnt(0)6353; GFX90A-NEXT: s_setpc_b64 s[30:31]6354;6355; GFX950-LABEL: flat_atomic_add_i64_ret_a_a:6356; GFX950: ; %bb.0:6357; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6358; GFX950-NEXT: s_mov_b64 s[2:3], 0x506359; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6360; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]6361; GFX950-NEXT: ;;#ASMSTART6362; GFX950-NEXT: ; def a[0:1]6363; GFX950-NEXT: ;;#ASMEND6364; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16365; GFX950-NEXT: v_accvgpr_read_b32 v3, a16366; GFX950-NEXT: v_accvgpr_read_b32 v2, a06367; GFX950-NEXT: ; implicit-def: $agpr0_agpr16368; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6369; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6370; GFX950-NEXT: s_cbranch_execz .LBB89_26371; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6372; GFX950-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] sc06373; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6374; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36375; GFX950-NEXT: v_accvgpr_write_b32 a0, v06376; GFX950-NEXT: v_accvgpr_write_b32 a1, v16377; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16378; GFX950-NEXT: .LBB89_2: ; %Flow6379; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6380; GFX950-NEXT: s_cbranch_execz .LBB89_46381; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6382; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6383; GFX950-NEXT: s_nop 16384; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc6385; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6386; GFX950-NEXT: s_waitcnt vmcnt(0)6387; GFX950-NEXT: v_accvgpr_write_b32 a0, v06388; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3]6389; GFX950-NEXT: v_accvgpr_write_b32 a1, v16390; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6391; GFX950-NEXT: .LBB89_4: ; %atomicrmw.phi6392; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6393; GFX950-NEXT: ;;#ASMSTART6394; GFX950-NEXT: ; use a[0:1]6395; GFX950-NEXT: ;;#ASMEND6396; GFX950-NEXT: s_waitcnt vmcnt(0)6397; GFX950-NEXT: s_setpc_b64 s[30:31]6398 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106399 %data = call i64 asm "; def $0", "=a"()6400 %result = atomicrmw add ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06401 call void asm "; use $0", "a"(i64 %result)6402 ret void6403}6404 6405define void @flat_atomic_add_i64_ret_av_av(ptr %ptr) #0 {6406; GFX90A-LABEL: flat_atomic_add_i64_ret_av_av:6407; GFX90A: ; %bb.0:6408; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6409; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v06410; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6411; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc6412; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v36413; GFX90A-NEXT: ;;#ASMSTART6414; GFX90A-NEXT: ; def v[4:5]6415; GFX90A-NEXT: ;;#ASMEND6416; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16417; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6418; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6419; GFX90A-NEXT: s_cbranch_execz .LBB90_26420; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6421; GFX90A-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[4:5] glc6422; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6423; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36424; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr56425; GFX90A-NEXT: .LBB90_2: ; %Flow6426; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6427; GFX90A-NEXT: s_cbranch_execz .LBB90_46428; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6429; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6430; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc6431; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen6432; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:46433; GFX90A-NEXT: s_waitcnt vmcnt(0)6434; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, v0, v46435; GFX90A-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc6436; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen6437; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:46438; GFX90A-NEXT: .LBB90_4: ; %atomicrmw.phi6439; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6440; GFX90A-NEXT: s_waitcnt vmcnt(0)6441; GFX90A-NEXT: ;;#ASMSTART6442; GFX90A-NEXT: ; use v[0:1]6443; GFX90A-NEXT: ;;#ASMEND6444; GFX90A-NEXT: s_setpc_b64 s[30:31]6445;6446; GFX950-LABEL: flat_atomic_add_i64_ret_av_av:6447; GFX950: ; %bb.0:6448; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6449; GFX950-NEXT: s_mov_b64 s[2:3], 0x506450; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6451; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]6452; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v36453; GFX950-NEXT: ;;#ASMSTART6454; GFX950-NEXT: ; def v[4:5]6455; GFX950-NEXT: ;;#ASMEND6456; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16457; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6458; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6459; GFX950-NEXT: s_cbranch_execz .LBB90_26460; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6461; GFX950-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[4:5] sc06462; GFX950-NEXT: s_waitcnt lgkmcnt(0)6463; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36464; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr56465; GFX950-NEXT: .LBB90_2: ; %Flow6466; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6467; GFX950-NEXT: s_cbranch_execz .LBB90_46468; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6469; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6470; GFX950-NEXT: s_nop 16471; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc6472; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off6473; GFX950-NEXT: s_waitcnt vmcnt(0)6474; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[4:5]6475; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off6476; GFX950-NEXT: .LBB90_4: ; %atomicrmw.phi6477; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6478; GFX950-NEXT: s_waitcnt vmcnt(0)6479; GFX950-NEXT: ;;#ASMSTART6480; GFX950-NEXT: ; use v[0:1]6481; GFX950-NEXT: ;;#ASMEND6482; GFX950-NEXT: s_setpc_b64 s[30:31]6483 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106484 %data = call i64 asm "; def $0", "=^VA"()6485 %result = atomicrmw add ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06486 call void asm "; use $0", "^VA"(i64 %result)6487 ret void6488}6489 6490define void @flat_atomic_sub_i64_ret_a_a(ptr %ptr) #0 {6491; GFX90A-LABEL: flat_atomic_sub_i64_ret_a_a:6492; GFX90A: ; %bb.0:6493; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6494; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v06495; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6496; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc6497; GFX90A-NEXT: ;;#ASMSTART6498; GFX90A-NEXT: ; def a[0:1]6499; GFX90A-NEXT: ;;#ASMEND6500; GFX90A-NEXT: v_accvgpr_read_b32 v3, a16501; GFX90A-NEXT: v_accvgpr_read_b32 v2, a06502; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v16503; GFX90A-NEXT: ; implicit-def: $agpr0_agpr16504; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6505; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6506; GFX90A-NEXT: s_cbranch_execz .LBB91_26507; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6508; GFX90A-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc6509; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6510; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36511; GFX90A-NEXT: v_accvgpr_write_b32 a0, v06512; GFX90A-NEXT: v_accvgpr_write_b32 a1, v16513; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16514; GFX90A-NEXT: .LBB91_2: ; %Flow6515; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6516; GFX90A-NEXT: s_cbranch_execz .LBB91_46517; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6518; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6519; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc6520; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen6521; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:46522; GFX90A-NEXT: s_waitcnt vmcnt(1)6523; GFX90A-NEXT: v_sub_co_u32_e32 v2, vcc, v1, v26524; GFX90A-NEXT: v_accvgpr_write_b32 a0, v16525; GFX90A-NEXT: s_waitcnt vmcnt(0)6526; GFX90A-NEXT: v_accvgpr_write_b32 a1, v46527; GFX90A-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc6528; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen6529; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:46530; GFX90A-NEXT: .LBB91_4: ; %atomicrmw.phi6531; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6532; GFX90A-NEXT: ;;#ASMSTART6533; GFX90A-NEXT: ; use a[0:1]6534; GFX90A-NEXT: ;;#ASMEND6535; GFX90A-NEXT: s_waitcnt vmcnt(0)6536; GFX90A-NEXT: s_setpc_b64 s[30:31]6537;6538; GFX950-LABEL: flat_atomic_sub_i64_ret_a_a:6539; GFX950: ; %bb.0:6540; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6541; GFX950-NEXT: s_mov_b64 s[2:3], 0x506542; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6543; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]6544; GFX950-NEXT: ;;#ASMSTART6545; GFX950-NEXT: ; def a[0:1]6546; GFX950-NEXT: ;;#ASMEND6547; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16548; GFX950-NEXT: v_accvgpr_read_b32 v3, a16549; GFX950-NEXT: v_accvgpr_read_b32 v2, a06550; GFX950-NEXT: ; implicit-def: $agpr0_agpr16551; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6552; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6553; GFX950-NEXT: s_cbranch_execz .LBB91_26554; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6555; GFX950-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] sc06556; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6557; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36558; GFX950-NEXT: v_accvgpr_write_b32 a0, v06559; GFX950-NEXT: v_accvgpr_write_b32 a1, v16560; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16561; GFX950-NEXT: .LBB91_2: ; %Flow6562; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6563; GFX950-NEXT: s_cbranch_execz .LBB91_46564; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6565; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6566; GFX950-NEXT: s_nop 16567; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc6568; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6569; GFX950-NEXT: s_waitcnt vmcnt(0)6570; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v26571; GFX950-NEXT: v_accvgpr_write_b32 a0, v06572; GFX950-NEXT: s_nop 06573; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc6574; GFX950-NEXT: v_accvgpr_write_b32 a1, v16575; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6576; GFX950-NEXT: .LBB91_4: ; %atomicrmw.phi6577; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6578; GFX950-NEXT: ;;#ASMSTART6579; GFX950-NEXT: ; use a[0:1]6580; GFX950-NEXT: ;;#ASMEND6581; GFX950-NEXT: s_waitcnt vmcnt(0)6582; GFX950-NEXT: s_setpc_b64 s[30:31]6583 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106584 %data = call i64 asm "; def $0", "=a"()6585 %result = atomicrmw sub ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06586 call void asm "; use $0", "a"(i64 %result)6587 ret void6588}6589 6590define void @flat_atomic_sub_i64_ret_av_av(ptr %ptr) #0 {6591; GFX90A-LABEL: flat_atomic_sub_i64_ret_av_av:6592; GFX90A: ; %bb.0:6593; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6594; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v06595; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6596; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc6597; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v36598; GFX90A-NEXT: ;;#ASMSTART6599; GFX90A-NEXT: ; def v[4:5]6600; GFX90A-NEXT: ;;#ASMEND6601; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16602; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6603; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6604; GFX90A-NEXT: s_cbranch_execz .LBB92_26605; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6606; GFX90A-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[4:5] glc6607; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6608; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36609; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr56610; GFX90A-NEXT: .LBB92_2: ; %Flow6611; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6612; GFX90A-NEXT: s_cbranch_execz .LBB92_46613; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6614; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6615; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc6616; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen6617; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:46618; GFX90A-NEXT: s_waitcnt vmcnt(0)6619; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v46620; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v5, vcc6621; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen6622; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:46623; GFX90A-NEXT: .LBB92_4: ; %atomicrmw.phi6624; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6625; GFX90A-NEXT: s_waitcnt vmcnt(0)6626; GFX90A-NEXT: ;;#ASMSTART6627; GFX90A-NEXT: ; use v[0:1]6628; GFX90A-NEXT: ;;#ASMEND6629; GFX90A-NEXT: s_setpc_b64 s[30:31]6630;6631; GFX950-LABEL: flat_atomic_sub_i64_ret_av_av:6632; GFX950: ; %bb.0:6633; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6634; GFX950-NEXT: s_mov_b64 s[2:3], 0x506635; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6636; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]6637; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56638; GFX950-NEXT: ;;#ASMSTART6639; GFX950-NEXT: ; def v[2:3]6640; GFX950-NEXT: ;;#ASMEND6641; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16642; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6643; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6644; GFX950-NEXT: s_cbranch_execz .LBB92_26645; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6646; GFX950-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] sc06647; GFX950-NEXT: s_waitcnt lgkmcnt(0)6648; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr56649; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36650; GFX950-NEXT: .LBB92_2: ; %Flow6651; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6652; GFX950-NEXT: s_cbranch_execz .LBB92_46653; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6654; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6655; GFX950-NEXT: s_nop 16656; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6657; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6658; GFX950-NEXT: s_waitcnt vmcnt(0)6659; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v26660; GFX950-NEXT: s_nop 16661; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc6662; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6663; GFX950-NEXT: .LBB92_4: ; %atomicrmw.phi6664; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6665; GFX950-NEXT: s_waitcnt vmcnt(0)6666; GFX950-NEXT: ;;#ASMSTART6667; GFX950-NEXT: ; use v[0:1]6668; GFX950-NEXT: ;;#ASMEND6669; GFX950-NEXT: s_setpc_b64 s[30:31]6670 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106671 %data = call i64 asm "; def $0", "=^VA"()6672 %result = atomicrmw sub ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06673 call void asm "; use $0", "^VA"(i64 %result)6674 ret void6675}6676 6677define void @flat_atomic_and_i64_ret_a_a(ptr %ptr) #0 {6678; GFX90A-LABEL: flat_atomic_and_i64_ret_a_a:6679; GFX90A: ; %bb.0:6680; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6681; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v06682; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6683; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc6684; GFX90A-NEXT: ;;#ASMSTART6685; GFX90A-NEXT: ; def a[0:1]6686; GFX90A-NEXT: ;;#ASMEND6687; GFX90A-NEXT: v_accvgpr_read_b32 v3, a16688; GFX90A-NEXT: v_accvgpr_read_b32 v2, a06689; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v16690; GFX90A-NEXT: ; implicit-def: $agpr0_agpr16691; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6692; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6693; GFX90A-NEXT: s_cbranch_execz .LBB93_26694; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6695; GFX90A-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc6696; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6697; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36698; GFX90A-NEXT: v_accvgpr_write_b32 a0, v06699; GFX90A-NEXT: v_accvgpr_write_b32 a1, v16700; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16701; GFX90A-NEXT: .LBB93_2: ; %Flow6702; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6703; GFX90A-NEXT: s_cbranch_execz .LBB93_46704; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6705; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6706; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc6707; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen6708; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:46709; GFX90A-NEXT: s_waitcnt vmcnt(1)6710; GFX90A-NEXT: v_accvgpr_write_b32 a0, v16711; GFX90A-NEXT: s_waitcnt vmcnt(0)6712; GFX90A-NEXT: v_accvgpr_write_b32 a1, v46713; GFX90A-NEXT: v_and_b32_e32 v1, v1, v26714; GFX90A-NEXT: v_and_b32_e32 v3, v4, v36715; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen6716; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:46717; GFX90A-NEXT: .LBB93_4: ; %atomicrmw.phi6718; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6719; GFX90A-NEXT: ;;#ASMSTART6720; GFX90A-NEXT: ; use a[0:1]6721; GFX90A-NEXT: ;;#ASMEND6722; GFX90A-NEXT: s_waitcnt vmcnt(0)6723; GFX90A-NEXT: s_setpc_b64 s[30:31]6724;6725; GFX950-LABEL: flat_atomic_and_i64_ret_a_a:6726; GFX950: ; %bb.0:6727; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6728; GFX950-NEXT: s_mov_b64 s[2:3], 0x506729; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6730; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]6731; GFX950-NEXT: ;;#ASMSTART6732; GFX950-NEXT: ; def a[0:1]6733; GFX950-NEXT: ;;#ASMEND6734; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16735; GFX950-NEXT: v_accvgpr_read_b32 v3, a16736; GFX950-NEXT: v_accvgpr_read_b32 v2, a06737; GFX950-NEXT: ; implicit-def: $agpr0_agpr16738; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6739; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6740; GFX950-NEXT: s_cbranch_execz .LBB93_26741; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6742; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] sc06743; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6744; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36745; GFX950-NEXT: v_accvgpr_write_b32 a0, v06746; GFX950-NEXT: v_accvgpr_write_b32 a1, v16747; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16748; GFX950-NEXT: .LBB93_2: ; %Flow6749; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6750; GFX950-NEXT: s_cbranch_execz .LBB93_46751; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6752; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6753; GFX950-NEXT: s_nop 16754; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc6755; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6756; GFX950-NEXT: s_waitcnt vmcnt(0)6757; GFX950-NEXT: v_accvgpr_write_b32 a0, v06758; GFX950-NEXT: v_and_b32_e32 v3, v1, v36759; GFX950-NEXT: v_and_b32_e32 v2, v0, v26760; GFX950-NEXT: v_accvgpr_write_b32 a1, v16761; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6762; GFX950-NEXT: .LBB93_4: ; %atomicrmw.phi6763; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6764; GFX950-NEXT: ;;#ASMSTART6765; GFX950-NEXT: ; use a[0:1]6766; GFX950-NEXT: ;;#ASMEND6767; GFX950-NEXT: s_waitcnt vmcnt(0)6768; GFX950-NEXT: s_setpc_b64 s[30:31]6769 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106770 %data = call i64 asm "; def $0", "=a"()6771 %result = atomicrmw and ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06772 call void asm "; use $0", "a"(i64 %result)6773 ret void6774}6775 6776define void @flat_atomic_and_i64_ret_av_av(ptr %ptr) #0 {6777; GFX90A-LABEL: flat_atomic_and_i64_ret_av_av:6778; GFX90A: ; %bb.0:6779; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6780; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v06781; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6782; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc6783; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v56784; GFX90A-NEXT: ;;#ASMSTART6785; GFX90A-NEXT: ; def v[2:3]6786; GFX90A-NEXT: ;;#ASMEND6787; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr16788; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6789; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6790; GFX90A-NEXT: s_cbranch_execz .LBB94_26791; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6792; GFX90A-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc6793; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6794; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr56795; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr36796; GFX90A-NEXT: .LBB94_2: ; %Flow6797; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6798; GFX90A-NEXT: s_cbranch_execz .LBB94_46799; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private6800; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6801; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6802; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:46803; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen6804; GFX90A-NEXT: s_waitcnt vmcnt(0)6805; GFX90A-NEXT: v_and_b32_e32 v3, v1, v36806; GFX90A-NEXT: v_and_b32_e32 v2, v0, v26807; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen6808; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:46809; GFX90A-NEXT: .LBB94_4: ; %atomicrmw.phi6810; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6811; GFX90A-NEXT: s_waitcnt vmcnt(0)6812; GFX90A-NEXT: ;;#ASMSTART6813; GFX90A-NEXT: ; use v[0:1]6814; GFX90A-NEXT: ;;#ASMEND6815; GFX90A-NEXT: s_setpc_b64 s[30:31]6816;6817; GFX950-LABEL: flat_atomic_and_i64_ret_av_av:6818; GFX950: ; %bb.0:6819; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6820; GFX950-NEXT: s_mov_b64 s[2:3], 0x506821; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6822; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]6823; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56824; GFX950-NEXT: ;;#ASMSTART6825; GFX950-NEXT: ; def v[2:3]6826; GFX950-NEXT: ;;#ASMEND6827; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr16828; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6829; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6830; GFX950-NEXT: s_cbranch_execz .LBB94_26831; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6832; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc06833; GFX950-NEXT: s_waitcnt lgkmcnt(0)6834; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr56835; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr36836; GFX950-NEXT: .LBB94_2: ; %Flow6837; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6838; GFX950-NEXT: s_cbranch_execz .LBB94_46839; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private6840; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6841; GFX950-NEXT: s_nop 16842; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6843; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6844; GFX950-NEXT: s_waitcnt vmcnt(0)6845; GFX950-NEXT: v_and_b32_e32 v3, v1, v36846; GFX950-NEXT: v_and_b32_e32 v2, v0, v26847; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6848; GFX950-NEXT: .LBB94_4: ; %atomicrmw.phi6849; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6850; GFX950-NEXT: s_waitcnt vmcnt(0)6851; GFX950-NEXT: ;;#ASMSTART6852; GFX950-NEXT: ; use v[0:1]6853; GFX950-NEXT: ;;#ASMEND6854; GFX950-NEXT: s_setpc_b64 s[30:31]6855 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106856 %data = call i64 asm "; def $0", "=^VA"()6857 %result = atomicrmw and ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06858 call void asm "; use $0", "^VA"(i64 %result)6859 ret void6860}6861 6862define void @flat_atomic_nand_i64_ret_a_a(ptr %ptr) #0 {6863; GFX90A-LABEL: flat_atomic_nand_i64_ret_a_a:6864; GFX90A: ; %bb.0:6865; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6866; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v06867; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base6868; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc6869; GFX90A-NEXT: ;;#ASMSTART6870; GFX90A-NEXT: ; def a[0:1]6871; GFX90A-NEXT: ;;#ASMEND6872; GFX90A-NEXT: v_accvgpr_read_b32 v7, a16873; GFX90A-NEXT: v_accvgpr_read_b32 v6, a06874; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v56875; GFX90A-NEXT: ; implicit-def: $agpr0_agpr16876; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc6877; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]6878; GFX90A-NEXT: s_cbranch_execz .LBB95_46879; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global6880; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6881; GFX90A-NEXT: s_mov_b64 s[6:7], 06882; GFX90A-NEXT: .LBB95_2: ; %atomicrmw.start6883; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16884; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6885; GFX90A-NEXT: v_and_b32_e32 v0, v3, v76886; GFX90A-NEXT: v_and_b32_e32 v8, v2, v66887; GFX90A-NEXT: v_not_b32_e32 v1, v06888; GFX90A-NEXT: v_not_b32_e32 v0, v86889; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6890; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6891; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6892; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6893; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]6894; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]6895; GFX90A-NEXT: s_cbranch_execnz .LBB95_26896; GFX90A-NEXT: ; %bb.3: ; %Flow6897; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]6898; GFX90A-NEXT: v_accvgpr_write_b32 a0, v06899; GFX90A-NEXT: v_accvgpr_write_b32 a1, v16900; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr56901; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr76902; GFX90A-NEXT: .LBB95_4: ; %Flow36903; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]6904; GFX90A-NEXT: s_cbranch_execz .LBB95_66905; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private6906; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6907; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v4, vcc6908; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen6909; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:46910; GFX90A-NEXT: s_waitcnt vmcnt(1)6911; GFX90A-NEXT: v_accvgpr_write_b32 a0, v16912; GFX90A-NEXT: v_and_b32_e32 v1, v1, v66913; GFX90A-NEXT: s_waitcnt vmcnt(0)6914; GFX90A-NEXT: v_accvgpr_write_b32 a1, v26915; GFX90A-NEXT: v_and_b32_e32 v2, v2, v76916; GFX90A-NEXT: v_not_b32_e32 v1, v16917; GFX90A-NEXT: v_not_b32_e32 v2, v26918; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen6919; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:46920; GFX90A-NEXT: .LBB95_6: ; %atomicrmw.phi6921; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6922; GFX90A-NEXT: ;;#ASMSTART6923; GFX90A-NEXT: ; use a[0:1]6924; GFX90A-NEXT: ;;#ASMEND6925; GFX90A-NEXT: s_waitcnt vmcnt(0)6926; GFX90A-NEXT: s_setpc_b64 s[30:31]6927;6928; GFX950-LABEL: flat_atomic_nand_i64_ret_a_a:6929; GFX950: ; %bb.0:6930; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6931; GFX950-NEXT: s_mov_b64 s[2:3], 0x506932; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base6933; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]6934; GFX950-NEXT: ;;#ASMSTART6935; GFX950-NEXT: ; def a[0:1]6936; GFX950-NEXT: ;;#ASMEND6937; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56938; GFX950-NEXT: v_accvgpr_read_b32 v7, a16939; GFX950-NEXT: v_accvgpr_read_b32 v6, a06940; GFX950-NEXT: ; implicit-def: $agpr0_agpr16941; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc6942; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6943; GFX950-NEXT: s_cbranch_execz .LBB95_46944; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global6945; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6946; GFX950-NEXT: s_mov_b64 s[2:3], 06947; GFX950-NEXT: .LBB95_2: ; %atomicrmw.start6948; GFX950-NEXT: ; =>This Inner Loop Header: Depth=16949; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6950; GFX950-NEXT: v_and_b32_e32 v0, v3, v76951; GFX950-NEXT: v_and_b32_e32 v8, v2, v66952; GFX950-NEXT: v_not_b32_e32 v1, v06953; GFX950-NEXT: v_not_b32_e32 v0, v86954; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] sc06955; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6956; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6957; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]6958; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]6959; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]6960; GFX950-NEXT: s_cbranch_execnz .LBB95_26961; GFX950-NEXT: ; %bb.3: ; %Flow6962; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]6963; GFX950-NEXT: v_accvgpr_write_b32 a0, v06964; GFX950-NEXT: v_accvgpr_write_b32 a1, v16965; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr56966; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr76967; GFX950-NEXT: .LBB95_4: ; %Flow36968; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6969; GFX950-NEXT: s_cbranch_execz .LBB95_66970; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private6971; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6972; GFX950-NEXT: s_nop 16973; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6974; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off6975; GFX950-NEXT: s_waitcnt vmcnt(0)6976; GFX950-NEXT: v_and_b32_e32 v2, v1, v76977; GFX950-NEXT: v_and_b32_e32 v5, v0, v66978; GFX950-NEXT: v_accvgpr_write_b32 a0, v06979; GFX950-NEXT: v_not_b32_e32 v3, v26980; GFX950-NEXT: v_not_b32_e32 v2, v56981; GFX950-NEXT: v_accvgpr_write_b32 a1, v16982; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off6983; GFX950-NEXT: .LBB95_6: ; %atomicrmw.phi6984; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]6985; GFX950-NEXT: ;;#ASMSTART6986; GFX950-NEXT: ; use a[0:1]6987; GFX950-NEXT: ;;#ASMEND6988; GFX950-NEXT: s_waitcnt vmcnt(0)6989; GFX950-NEXT: s_setpc_b64 s[30:31]6990 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 106991 %data = call i64 asm "; def $0", "=a"()6992 %result = atomicrmw nand ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !06993 call void asm "; use $0", "a"(i64 %result)6994 ret void6995}6996 6997define void @flat_atomic_nand_i64_ret_av_av(ptr %ptr) #0 {6998; GFX90A-LABEL: flat_atomic_nand_i64_ret_av_av:6999; GFX90A: ; %bb.0:7000; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7001; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v07002; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7003; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc7004; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v77005; GFX90A-NEXT: ;;#ASMSTART7006; GFX90A-NEXT: ; def v[4:5]7007; GFX90A-NEXT: ;;#ASMEND7008; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17009; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7010; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7011; GFX90A-NEXT: s_cbranch_execz .LBB96_47012; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7013; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]7014; GFX90A-NEXT: s_mov_b64 s[6:7], 07015; GFX90A-NEXT: .LBB96_2: ; %atomicrmw.start7016; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=17017; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7018; GFX90A-NEXT: v_and_b32_e32 v0, v3, v57019; GFX90A-NEXT: v_and_b32_e32 v8, v2, v47020; GFX90A-NEXT: v_not_b32_e32 v1, v07021; GFX90A-NEXT: v_not_b32_e32 v0, v87022; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc7023; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7024; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7025; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7026; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]7027; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]7028; GFX90A-NEXT: s_cbranch_execnz .LBB96_27029; GFX90A-NEXT: ; %bb.3: ; %Flow7030; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]7031; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr77032; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57033; GFX90A-NEXT: .LBB96_4: ; %Flow37034; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7035; GFX90A-NEXT: s_cbranch_execz .LBB96_67036; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private7037; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]7038; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc7039; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:47040; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen7041; GFX90A-NEXT: s_waitcnt vmcnt(1)7042; GFX90A-NEXT: v_and_b32_e32 v3, v1, v57043; GFX90A-NEXT: s_waitcnt vmcnt(0)7044; GFX90A-NEXT: v_and_b32_e32 v4, v0, v47045; GFX90A-NEXT: v_not_b32_e32 v4, v47046; GFX90A-NEXT: v_not_b32_e32 v3, v37047; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen7048; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:47049; GFX90A-NEXT: .LBB96_6: ; %atomicrmw.phi7050; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7051; GFX90A-NEXT: ;;#ASMSTART7052; GFX90A-NEXT: ; use v[0:1]7053; GFX90A-NEXT: ;;#ASMEND7054; GFX90A-NEXT: s_waitcnt vmcnt(0)7055; GFX90A-NEXT: s_setpc_b64 s[30:31]7056;7057; GFX950-LABEL: flat_atomic_nand_i64_ret_av_av:7058; GFX950: ; %bb.0:7059; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7060; GFX950-NEXT: s_mov_b64 s[2:3], 0x507061; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7062; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]7063; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v77064; GFX950-NEXT: ;;#ASMSTART7065; GFX950-NEXT: ; def v[4:5]7066; GFX950-NEXT: ;;#ASMEND7067; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17068; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7069; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7070; GFX950-NEXT: s_cbranch_execz .LBB96_47071; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7072; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]7073; GFX950-NEXT: s_mov_b64 s[2:3], 07074; GFX950-NEXT: .LBB96_2: ; %atomicrmw.start7075; GFX950-NEXT: ; =>This Inner Loop Header: Depth=17076; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7077; GFX950-NEXT: v_and_b32_e32 v0, v3, v57078; GFX950-NEXT: v_and_b32_e32 v8, v2, v47079; GFX950-NEXT: v_not_b32_e32 v1, v07080; GFX950-NEXT: v_not_b32_e32 v0, v87081; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc07082; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7083; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7084; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]7085; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]7086; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]7087; GFX950-NEXT: s_cbranch_execnz .LBB96_27088; GFX950-NEXT: ; %bb.3: ; %Flow7089; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]7090; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr77091; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr57092; GFX950-NEXT: .LBB96_4: ; %Flow37093; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7094; GFX950-NEXT: s_cbranch_execz .LBB96_67095; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private7096; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]7097; GFX950-NEXT: s_nop 17098; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc7099; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off7100; GFX950-NEXT: s_waitcnt vmcnt(0)7101; GFX950-NEXT: v_and_b32_e32 v2, v1, v57102; GFX950-NEXT: v_and_b32_e32 v4, v0, v47103; GFX950-NEXT: v_not_b32_e32 v3, v27104; GFX950-NEXT: v_not_b32_e32 v2, v47105; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off7106; GFX950-NEXT: .LBB96_6: ; %atomicrmw.phi7107; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7108; GFX950-NEXT: ;;#ASMSTART7109; GFX950-NEXT: ; use v[0:1]7110; GFX950-NEXT: ;;#ASMEND7111; GFX950-NEXT: s_waitcnt vmcnt(0)7112; GFX950-NEXT: s_setpc_b64 s[30:31]7113 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107114 %data = call i64 asm "; def $0", "=^VA"()7115 %result = atomicrmw nand ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07116 call void asm "; use $0", "^VA"(i64 %result)7117 ret void7118}7119 7120define void @flat_atomic_or_i64_ret_a_a(ptr %ptr) #0 {7121; GFX90A-LABEL: flat_atomic_or_i64_ret_a_a:7122; GFX90A: ; %bb.0:7123; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7124; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v07125; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7126; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc7127; GFX90A-NEXT: ;;#ASMSTART7128; GFX90A-NEXT: ; def a[0:1]7129; GFX90A-NEXT: ;;#ASMEND7130; GFX90A-NEXT: v_accvgpr_read_b32 v3, a17131; GFX90A-NEXT: v_accvgpr_read_b32 v2, a07132; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v17133; GFX90A-NEXT: ; implicit-def: $agpr0_agpr17134; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7135; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7136; GFX90A-NEXT: s_cbranch_execz .LBB97_27137; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7138; GFX90A-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc7139; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7140; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37141; GFX90A-NEXT: v_accvgpr_write_b32 a0, v07142; GFX90A-NEXT: v_accvgpr_write_b32 a1, v17143; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17144; GFX90A-NEXT: .LBB97_2: ; %Flow7145; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7146; GFX90A-NEXT: s_cbranch_execz .LBB97_47147; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7148; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7149; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc7150; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen7151; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:47152; GFX90A-NEXT: s_waitcnt vmcnt(1)7153; GFX90A-NEXT: v_accvgpr_write_b32 a0, v17154; GFX90A-NEXT: s_waitcnt vmcnt(0)7155; GFX90A-NEXT: v_accvgpr_write_b32 a1, v47156; GFX90A-NEXT: v_or_b32_e32 v1, v1, v27157; GFX90A-NEXT: v_or_b32_e32 v3, v4, v37158; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen7159; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen offset:47160; GFX90A-NEXT: .LBB97_4: ; %atomicrmw.phi7161; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7162; GFX90A-NEXT: ;;#ASMSTART7163; GFX90A-NEXT: ; use a[0:1]7164; GFX90A-NEXT: ;;#ASMEND7165; GFX90A-NEXT: s_waitcnt vmcnt(0)7166; GFX90A-NEXT: s_setpc_b64 s[30:31]7167;7168; GFX950-LABEL: flat_atomic_or_i64_ret_a_a:7169; GFX950: ; %bb.0:7170; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7171; GFX950-NEXT: s_mov_b64 s[2:3], 0x507172; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7173; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]7174; GFX950-NEXT: ;;#ASMSTART7175; GFX950-NEXT: ; def a[0:1]7176; GFX950-NEXT: ;;#ASMEND7177; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17178; GFX950-NEXT: v_accvgpr_read_b32 v3, a17179; GFX950-NEXT: v_accvgpr_read_b32 v2, a07180; GFX950-NEXT: ; implicit-def: $agpr0_agpr17181; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7182; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7183; GFX950-NEXT: s_cbranch_execz .LBB97_27184; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7185; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] sc07186; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7187; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37188; GFX950-NEXT: v_accvgpr_write_b32 a0, v07189; GFX950-NEXT: v_accvgpr_write_b32 a1, v17190; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17191; GFX950-NEXT: .LBB97_2: ; %Flow7192; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7193; GFX950-NEXT: s_cbranch_execz .LBB97_47194; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7195; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7196; GFX950-NEXT: s_nop 17197; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc7198; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off7199; GFX950-NEXT: s_waitcnt vmcnt(0)7200; GFX950-NEXT: v_accvgpr_write_b32 a0, v07201; GFX950-NEXT: v_or_b32_e32 v3, v1, v37202; GFX950-NEXT: v_or_b32_e32 v2, v0, v27203; GFX950-NEXT: v_accvgpr_write_b32 a1, v17204; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off7205; GFX950-NEXT: .LBB97_4: ; %atomicrmw.phi7206; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7207; GFX950-NEXT: ;;#ASMSTART7208; GFX950-NEXT: ; use a[0:1]7209; GFX950-NEXT: ;;#ASMEND7210; GFX950-NEXT: s_waitcnt vmcnt(0)7211; GFX950-NEXT: s_setpc_b64 s[30:31]7212 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107213 %data = call i64 asm "; def $0", "=a"()7214 %result = atomicrmw or ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07215 call void asm "; use $0", "a"(i64 %result)7216 ret void7217}7218 7219define void @flat_atomic_or_i64_ret_av_av(ptr %ptr) #0 {7220; GFX90A-LABEL: flat_atomic_or_i64_ret_av_av:7221; GFX90A: ; %bb.0:7222; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7223; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v07224; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7225; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc7226; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v57227; GFX90A-NEXT: ;;#ASMSTART7228; GFX90A-NEXT: ; def v[2:3]7229; GFX90A-NEXT: ;;#ASMEND7230; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17231; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7232; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7233; GFX90A-NEXT: s_cbranch_execz .LBB98_27234; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7235; GFX90A-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc7236; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7237; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57238; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37239; GFX90A-NEXT: .LBB98_2: ; %Flow7240; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7241; GFX90A-NEXT: s_cbranch_execz .LBB98_47242; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7243; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7244; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7245; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:47246; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen7247; GFX90A-NEXT: s_waitcnt vmcnt(0)7248; GFX90A-NEXT: v_or_b32_e32 v3, v1, v37249; GFX90A-NEXT: v_or_b32_e32 v2, v0, v27250; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen7251; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:47252; GFX90A-NEXT: .LBB98_4: ; %atomicrmw.phi7253; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7254; GFX90A-NEXT: s_waitcnt vmcnt(0)7255; GFX90A-NEXT: ;;#ASMSTART7256; GFX90A-NEXT: ; use v[0:1]7257; GFX90A-NEXT: ;;#ASMEND7258; GFX90A-NEXT: s_setpc_b64 s[30:31]7259;7260; GFX950-LABEL: flat_atomic_or_i64_ret_av_av:7261; GFX950: ; %bb.0:7262; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7263; GFX950-NEXT: s_mov_b64 s[2:3], 0x507264; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7265; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]7266; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57267; GFX950-NEXT: ;;#ASMSTART7268; GFX950-NEXT: ; def v[2:3]7269; GFX950-NEXT: ;;#ASMEND7270; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17271; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7272; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7273; GFX950-NEXT: s_cbranch_execz .LBB98_27274; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7275; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc07276; GFX950-NEXT: s_waitcnt lgkmcnt(0)7277; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr57278; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37279; GFX950-NEXT: .LBB98_2: ; %Flow7280; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7281; GFX950-NEXT: s_cbranch_execz .LBB98_47282; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7283; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7284; GFX950-NEXT: s_nop 17285; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7286; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off7287; GFX950-NEXT: s_waitcnt vmcnt(0)7288; GFX950-NEXT: v_or_b32_e32 v3, v1, v37289; GFX950-NEXT: v_or_b32_e32 v2, v0, v27290; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off7291; GFX950-NEXT: .LBB98_4: ; %atomicrmw.phi7292; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7293; GFX950-NEXT: s_waitcnt vmcnt(0)7294; GFX950-NEXT: ;;#ASMSTART7295; GFX950-NEXT: ; use v[0:1]7296; GFX950-NEXT: ;;#ASMEND7297; GFX950-NEXT: s_setpc_b64 s[30:31]7298 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107299 %data = call i64 asm "; def $0", "=^VA"()7300 %result = atomicrmw or ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07301 call void asm "; use $0", "^VA"(i64 %result)7302 ret void7303}7304 7305define void @flat_atomic_max_i64_ret_a_a(ptr %ptr) #0 {7306; GFX90A-LABEL: flat_atomic_max_i64_ret_a_a:7307; GFX90A: ; %bb.0:7308; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7309; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v07310; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7311; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc7312; GFX90A-NEXT: ;;#ASMSTART7313; GFX90A-NEXT: ; def a[0:1]7314; GFX90A-NEXT: ;;#ASMEND7315; GFX90A-NEXT: v_accvgpr_read_b32 v0, a07316; GFX90A-NEXT: v_accvgpr_read_b32 v1, a17317; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v37318; GFX90A-NEXT: ; implicit-def: $agpr0_agpr17319; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7320; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7321; GFX90A-NEXT: s_cbranch_execz .LBB99_27322; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7323; GFX90A-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc7324; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7325; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37326; GFX90A-NEXT: v_accvgpr_write_b32 a0, v07327; GFX90A-NEXT: v_accvgpr_write_b32 a1, v17328; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17329; GFX90A-NEXT: .LBB99_2: ; %Flow7330; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7331; GFX90A-NEXT: s_cbranch_execz .LBB99_47332; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7333; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7334; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7335; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen7336; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:47337; GFX90A-NEXT: s_waitcnt vmcnt(1)7338; GFX90A-NEXT: v_accvgpr_write_b32 a0, v27339; GFX90A-NEXT: s_waitcnt vmcnt(0)7340; GFX90A-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[0:1]7341; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7342; GFX90A-NEXT: v_accvgpr_write_b32 a1, v37343; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7344; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:47345; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen7346; GFX90A-NEXT: .LBB99_4: ; %atomicrmw.phi7347; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7348; GFX90A-NEXT: ;;#ASMSTART7349; GFX90A-NEXT: ; use a[0:1]7350; GFX90A-NEXT: ;;#ASMEND7351; GFX90A-NEXT: s_waitcnt vmcnt(0)7352; GFX90A-NEXT: s_setpc_b64 s[30:31]7353;7354; GFX950-LABEL: flat_atomic_max_i64_ret_a_a:7355; GFX950: ; %bb.0:7356; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7357; GFX950-NEXT: s_mov_b64 s[2:3], 0x507358; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7359; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]7360; GFX950-NEXT: ;;#ASMSTART7361; GFX950-NEXT: ; def a[0:1]7362; GFX950-NEXT: ;;#ASMEND7363; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37364; GFX950-NEXT: v_accvgpr_read_b32 v0, a07365; GFX950-NEXT: v_accvgpr_read_b32 v1, a17366; GFX950-NEXT: ; implicit-def: $agpr0_agpr17367; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7368; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7369; GFX950-NEXT: s_cbranch_execz .LBB99_27370; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7371; GFX950-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] sc07372; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7373; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37374; GFX950-NEXT: v_accvgpr_write_b32 a0, v07375; GFX950-NEXT: v_accvgpr_write_b32 a1, v17376; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17377; GFX950-NEXT: .LBB99_2: ; %Flow7378; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7379; GFX950-NEXT: s_cbranch_execz .LBB99_47380; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7381; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7382; GFX950-NEXT: s_nop 17383; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7384; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off7385; GFX950-NEXT: s_waitcnt vmcnt(0)7386; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[0:1]7387; GFX950-NEXT: v_accvgpr_write_b32 a0, v27388; GFX950-NEXT: s_nop 07389; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7390; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7391; GFX950-NEXT: v_accvgpr_write_b32 a1, v37392; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off7393; GFX950-NEXT: .LBB99_4: ; %atomicrmw.phi7394; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7395; GFX950-NEXT: ;;#ASMSTART7396; GFX950-NEXT: ; use a[0:1]7397; GFX950-NEXT: ;;#ASMEND7398; GFX950-NEXT: s_waitcnt vmcnt(0)7399; GFX950-NEXT: s_setpc_b64 s[30:31]7400 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107401 %data = call i64 asm "; def $0", "=a"()7402 %result = atomicrmw max ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07403 call void asm "; use $0", "a"(i64 %result)7404 ret void7405}7406 7407define void @flat_atomic_max_i64_ret_av_av(ptr %ptr) #0 {7408; GFX90A-LABEL: flat_atomic_max_i64_ret_av_av:7409; GFX90A: ; %bb.0:7410; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7411; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v07412; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7413; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc7414; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v57415; GFX90A-NEXT: ;;#ASMSTART7416; GFX90A-NEXT: ; def v[2:3]7417; GFX90A-NEXT: ;;#ASMEND7418; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17419; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7420; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7421; GFX90A-NEXT: s_cbranch_execz .LBB100_27422; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7423; GFX90A-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc7424; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7425; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57426; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37427; GFX90A-NEXT: .LBB100_2: ; %Flow7428; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7429; GFX90A-NEXT: s_cbranch_execz .LBB100_47430; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7431; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7432; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7433; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen7434; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:47435; GFX90A-NEXT: s_waitcnt vmcnt(0)7436; GFX90A-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]7437; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7438; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7439; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen7440; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:47441; GFX90A-NEXT: .LBB100_4: ; %atomicrmw.phi7442; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7443; GFX90A-NEXT: s_waitcnt vmcnt(0)7444; GFX90A-NEXT: ;;#ASMSTART7445; GFX90A-NEXT: ; use v[0:1]7446; GFX90A-NEXT: ;;#ASMEND7447; GFX90A-NEXT: s_setpc_b64 s[30:31]7448;7449; GFX950-LABEL: flat_atomic_max_i64_ret_av_av:7450; GFX950: ; %bb.0:7451; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7452; GFX950-NEXT: s_mov_b64 s[2:3], 0x507453; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7454; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]7455; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57456; GFX950-NEXT: ;;#ASMSTART7457; GFX950-NEXT: ; def v[2:3]7458; GFX950-NEXT: ;;#ASMEND7459; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17460; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7461; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7462; GFX950-NEXT: s_cbranch_execz .LBB100_27463; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7464; GFX950-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc07465; GFX950-NEXT: s_waitcnt lgkmcnt(0)7466; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr57467; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37468; GFX950-NEXT: .LBB100_2: ; %Flow7469; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7470; GFX950-NEXT: s_cbranch_execz .LBB100_47471; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7472; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7473; GFX950-NEXT: s_nop 17474; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7475; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off7476; GFX950-NEXT: s_waitcnt vmcnt(0)7477; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]7478; GFX950-NEXT: s_nop 17479; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7480; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7481; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off7482; GFX950-NEXT: .LBB100_4: ; %atomicrmw.phi7483; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7484; GFX950-NEXT: s_waitcnt vmcnt(0)7485; GFX950-NEXT: ;;#ASMSTART7486; GFX950-NEXT: ; use v[0:1]7487; GFX950-NEXT: ;;#ASMEND7488; GFX950-NEXT: s_setpc_b64 s[30:31]7489 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107490 %data = call i64 asm "; def $0", "=^VA"()7491 %result = atomicrmw max ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07492 call void asm "; use $0", "^VA"(i64 %result)7493 ret void7494}7495 7496define void @flat_atomic_min_i64_ret_a_a(ptr %ptr) #0 {7497; GFX90A-LABEL: flat_atomic_min_i64_ret_a_a:7498; GFX90A: ; %bb.0:7499; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7500; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v07501; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7502; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc7503; GFX90A-NEXT: ;;#ASMSTART7504; GFX90A-NEXT: ; def a[0:1]7505; GFX90A-NEXT: ;;#ASMEND7506; GFX90A-NEXT: v_accvgpr_read_b32 v0, a07507; GFX90A-NEXT: v_accvgpr_read_b32 v1, a17508; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v37509; GFX90A-NEXT: ; implicit-def: $agpr0_agpr17510; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7511; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7512; GFX90A-NEXT: s_cbranch_execz .LBB101_27513; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7514; GFX90A-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc7515; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7516; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37517; GFX90A-NEXT: v_accvgpr_write_b32 a0, v07518; GFX90A-NEXT: v_accvgpr_write_b32 a1, v17519; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17520; GFX90A-NEXT: .LBB101_2: ; %Flow7521; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7522; GFX90A-NEXT: s_cbranch_execz .LBB101_47523; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7524; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7525; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7526; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen7527; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:47528; GFX90A-NEXT: s_waitcnt vmcnt(1)7529; GFX90A-NEXT: v_accvgpr_write_b32 a0, v27530; GFX90A-NEXT: s_waitcnt vmcnt(0)7531; GFX90A-NEXT: v_cmp_le_i64_e32 vcc, v[2:3], v[0:1]7532; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7533; GFX90A-NEXT: v_accvgpr_write_b32 a1, v37534; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7535; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:47536; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen7537; GFX90A-NEXT: .LBB101_4: ; %atomicrmw.phi7538; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7539; GFX90A-NEXT: ;;#ASMSTART7540; GFX90A-NEXT: ; use a[0:1]7541; GFX90A-NEXT: ;;#ASMEND7542; GFX90A-NEXT: s_waitcnt vmcnt(0)7543; GFX90A-NEXT: s_setpc_b64 s[30:31]7544;7545; GFX950-LABEL: flat_atomic_min_i64_ret_a_a:7546; GFX950: ; %bb.0:7547; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7548; GFX950-NEXT: s_mov_b64 s[2:3], 0x507549; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7550; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]7551; GFX950-NEXT: ;;#ASMSTART7552; GFX950-NEXT: ; def a[0:1]7553; GFX950-NEXT: ;;#ASMEND7554; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37555; GFX950-NEXT: v_accvgpr_read_b32 v0, a07556; GFX950-NEXT: v_accvgpr_read_b32 v1, a17557; GFX950-NEXT: ; implicit-def: $agpr0_agpr17558; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7559; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7560; GFX950-NEXT: s_cbranch_execz .LBB101_27561; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7562; GFX950-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] sc07563; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7564; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37565; GFX950-NEXT: v_accvgpr_write_b32 a0, v07566; GFX950-NEXT: v_accvgpr_write_b32 a1, v17567; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17568; GFX950-NEXT: .LBB101_2: ; %Flow7569; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7570; GFX950-NEXT: s_cbranch_execz .LBB101_47571; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7572; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7573; GFX950-NEXT: s_nop 17574; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7575; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off7576; GFX950-NEXT: s_waitcnt vmcnt(0)7577; GFX950-NEXT: v_cmp_le_i64_e32 vcc, v[2:3], v[0:1]7578; GFX950-NEXT: v_accvgpr_write_b32 a0, v27579; GFX950-NEXT: s_nop 07580; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7581; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7582; GFX950-NEXT: v_accvgpr_write_b32 a1, v37583; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off7584; GFX950-NEXT: .LBB101_4: ; %atomicrmw.phi7585; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7586; GFX950-NEXT: ;;#ASMSTART7587; GFX950-NEXT: ; use a[0:1]7588; GFX950-NEXT: ;;#ASMEND7589; GFX950-NEXT: s_waitcnt vmcnt(0)7590; GFX950-NEXT: s_setpc_b64 s[30:31]7591 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107592 %data = call i64 asm "; def $0", "=a"()7593 %result = atomicrmw min ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07594 call void asm "; use $0", "a"(i64 %result)7595 ret void7596}7597 7598define void @flat_atomic_min_i64_ret_av_av(ptr %ptr) #0 {7599; GFX90A-LABEL: flat_atomic_min_i64_ret_av_av:7600; GFX90A: ; %bb.0:7601; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7602; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v07603; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7604; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc7605; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v57606; GFX90A-NEXT: ;;#ASMSTART7607; GFX90A-NEXT: ; def v[2:3]7608; GFX90A-NEXT: ;;#ASMEND7609; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17610; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7611; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7612; GFX90A-NEXT: s_cbranch_execz .LBB102_27613; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7614; GFX90A-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc7615; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7616; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57617; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37618; GFX90A-NEXT: .LBB102_2: ; %Flow7619; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7620; GFX90A-NEXT: s_cbranch_execz .LBB102_47621; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7622; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7623; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7624; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen7625; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:47626; GFX90A-NEXT: s_waitcnt vmcnt(0)7627; GFX90A-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]7628; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7629; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7630; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen7631; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:47632; GFX90A-NEXT: .LBB102_4: ; %atomicrmw.phi7633; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7634; GFX90A-NEXT: s_waitcnt vmcnt(0)7635; GFX90A-NEXT: ;;#ASMSTART7636; GFX90A-NEXT: ; use v[0:1]7637; GFX90A-NEXT: ;;#ASMEND7638; GFX90A-NEXT: s_setpc_b64 s[30:31]7639;7640; GFX950-LABEL: flat_atomic_min_i64_ret_av_av:7641; GFX950: ; %bb.0:7642; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7643; GFX950-NEXT: s_mov_b64 s[2:3], 0x507644; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7645; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]7646; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57647; GFX950-NEXT: ;;#ASMSTART7648; GFX950-NEXT: ; def v[2:3]7649; GFX950-NEXT: ;;#ASMEND7650; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17651; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7652; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7653; GFX950-NEXT: s_cbranch_execz .LBB102_27654; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7655; GFX950-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] sc07656; GFX950-NEXT: s_waitcnt lgkmcnt(0)7657; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr57658; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37659; GFX950-NEXT: .LBB102_2: ; %Flow7660; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7661; GFX950-NEXT: s_cbranch_execz .LBB102_47662; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7663; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7664; GFX950-NEXT: s_nop 17665; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7666; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off7667; GFX950-NEXT: s_waitcnt vmcnt(0)7668; GFX950-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]7669; GFX950-NEXT: s_nop 17670; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7671; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7672; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off7673; GFX950-NEXT: .LBB102_4: ; %atomicrmw.phi7674; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7675; GFX950-NEXT: s_waitcnt vmcnt(0)7676; GFX950-NEXT: ;;#ASMSTART7677; GFX950-NEXT: ; use v[0:1]7678; GFX950-NEXT: ;;#ASMEND7679; GFX950-NEXT: s_setpc_b64 s[30:31]7680 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107681 %data = call i64 asm "; def $0", "=^VA"()7682 %result = atomicrmw min ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07683 call void asm "; use $0", "^VA"(i64 %result)7684 ret void7685}7686 7687define void @flat_atomic_umax_i64_ret_a_a(ptr %ptr) #0 {7688; GFX90A-LABEL: flat_atomic_umax_i64_ret_a_a:7689; GFX90A: ; %bb.0:7690; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7691; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v07692; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7693; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc7694; GFX90A-NEXT: ;;#ASMSTART7695; GFX90A-NEXT: ; def a[0:1]7696; GFX90A-NEXT: ;;#ASMEND7697; GFX90A-NEXT: v_accvgpr_read_b32 v0, a07698; GFX90A-NEXT: v_accvgpr_read_b32 v1, a17699; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v37700; GFX90A-NEXT: ; implicit-def: $agpr0_agpr17701; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7702; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7703; GFX90A-NEXT: s_cbranch_execz .LBB103_27704; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7705; GFX90A-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc7706; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7707; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37708; GFX90A-NEXT: v_accvgpr_write_b32 a0, v07709; GFX90A-NEXT: v_accvgpr_write_b32 a1, v17710; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17711; GFX90A-NEXT: .LBB103_2: ; %Flow7712; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7713; GFX90A-NEXT: s_cbranch_execz .LBB103_47714; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7715; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7716; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7717; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen7718; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:47719; GFX90A-NEXT: s_waitcnt vmcnt(1)7720; GFX90A-NEXT: v_accvgpr_write_b32 a0, v27721; GFX90A-NEXT: s_waitcnt vmcnt(0)7722; GFX90A-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]7723; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7724; GFX90A-NEXT: v_accvgpr_write_b32 a1, v37725; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7726; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:47727; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen7728; GFX90A-NEXT: .LBB103_4: ; %atomicrmw.phi7729; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7730; GFX90A-NEXT: ;;#ASMSTART7731; GFX90A-NEXT: ; use a[0:1]7732; GFX90A-NEXT: ;;#ASMEND7733; GFX90A-NEXT: s_waitcnt vmcnt(0)7734; GFX90A-NEXT: s_setpc_b64 s[30:31]7735;7736; GFX950-LABEL: flat_atomic_umax_i64_ret_a_a:7737; GFX950: ; %bb.0:7738; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7739; GFX950-NEXT: s_mov_b64 s[2:3], 0x507740; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7741; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]7742; GFX950-NEXT: ;;#ASMSTART7743; GFX950-NEXT: ; def a[0:1]7744; GFX950-NEXT: ;;#ASMEND7745; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37746; GFX950-NEXT: v_accvgpr_read_b32 v0, a07747; GFX950-NEXT: v_accvgpr_read_b32 v1, a17748; GFX950-NEXT: ; implicit-def: $agpr0_agpr17749; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7750; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7751; GFX950-NEXT: s_cbranch_execz .LBB103_27752; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7753; GFX950-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] sc07754; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7755; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37756; GFX950-NEXT: v_accvgpr_write_b32 a0, v07757; GFX950-NEXT: v_accvgpr_write_b32 a1, v17758; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17759; GFX950-NEXT: .LBB103_2: ; %Flow7760; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7761; GFX950-NEXT: s_cbranch_execz .LBB103_47762; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7763; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7764; GFX950-NEXT: s_nop 17765; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7766; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off7767; GFX950-NEXT: s_waitcnt vmcnt(0)7768; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]7769; GFX950-NEXT: v_accvgpr_write_b32 a0, v27770; GFX950-NEXT: s_nop 07771; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7772; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7773; GFX950-NEXT: v_accvgpr_write_b32 a1, v37774; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off7775; GFX950-NEXT: .LBB103_4: ; %atomicrmw.phi7776; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7777; GFX950-NEXT: ;;#ASMSTART7778; GFX950-NEXT: ; use a[0:1]7779; GFX950-NEXT: ;;#ASMEND7780; GFX950-NEXT: s_waitcnt vmcnt(0)7781; GFX950-NEXT: s_setpc_b64 s[30:31]7782 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107783 %data = call i64 asm "; def $0", "=a"()7784 %result = atomicrmw umax ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07785 call void asm "; use $0", "a"(i64 %result)7786 ret void7787}7788 7789define void @flat_atomic_umax_i64_ret_av_av(ptr %ptr) #0 {7790; GFX90A-LABEL: flat_atomic_umax_i64_ret_av_av:7791; GFX90A: ; %bb.0:7792; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7793; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v07794; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7795; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc7796; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v57797; GFX90A-NEXT: ;;#ASMSTART7798; GFX90A-NEXT: ; def v[2:3]7799; GFX90A-NEXT: ;;#ASMEND7800; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17801; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7802; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7803; GFX90A-NEXT: s_cbranch_execz .LBB104_27804; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7805; GFX90A-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc7806; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7807; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57808; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37809; GFX90A-NEXT: .LBB104_2: ; %Flow7810; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7811; GFX90A-NEXT: s_cbranch_execz .LBB104_47812; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7813; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7814; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7815; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen7816; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:47817; GFX90A-NEXT: s_waitcnt vmcnt(0)7818; GFX90A-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]7819; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7820; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7821; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen7822; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:47823; GFX90A-NEXT: .LBB104_4: ; %atomicrmw.phi7824; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7825; GFX90A-NEXT: s_waitcnt vmcnt(0)7826; GFX90A-NEXT: ;;#ASMSTART7827; GFX90A-NEXT: ; use v[0:1]7828; GFX90A-NEXT: ;;#ASMEND7829; GFX90A-NEXT: s_setpc_b64 s[30:31]7830;7831; GFX950-LABEL: flat_atomic_umax_i64_ret_av_av:7832; GFX950: ; %bb.0:7833; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7834; GFX950-NEXT: s_mov_b64 s[2:3], 0x507835; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7836; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]7837; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57838; GFX950-NEXT: ;;#ASMSTART7839; GFX950-NEXT: ; def v[2:3]7840; GFX950-NEXT: ;;#ASMEND7841; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17842; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7843; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7844; GFX950-NEXT: s_cbranch_execz .LBB104_27845; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7846; GFX950-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] sc07847; GFX950-NEXT: s_waitcnt lgkmcnt(0)7848; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr57849; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37850; GFX950-NEXT: .LBB104_2: ; %Flow7851; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7852; GFX950-NEXT: s_cbranch_execz .LBB104_47853; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7854; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7855; GFX950-NEXT: s_nop 17856; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7857; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off7858; GFX950-NEXT: s_waitcnt vmcnt(0)7859; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]7860; GFX950-NEXT: s_nop 17861; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7862; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7863; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off7864; GFX950-NEXT: .LBB104_4: ; %atomicrmw.phi7865; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7866; GFX950-NEXT: s_waitcnt vmcnt(0)7867; GFX950-NEXT: ;;#ASMSTART7868; GFX950-NEXT: ; use v[0:1]7869; GFX950-NEXT: ;;#ASMEND7870; GFX950-NEXT: s_setpc_b64 s[30:31]7871 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107872 %data = call i64 asm "; def $0", "=^VA"()7873 %result = atomicrmw umax ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07874 call void asm "; use $0", "^VA"(i64 %result)7875 ret void7876}7877 7878define void @flat_atomic_umin_i64_ret_a_a(ptr %ptr) #0 {7879; GFX90A-LABEL: flat_atomic_umin_i64_ret_a_a:7880; GFX90A: ; %bb.0:7881; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7882; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v07883; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7884; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc7885; GFX90A-NEXT: ;;#ASMSTART7886; GFX90A-NEXT: ; def a[0:1]7887; GFX90A-NEXT: ;;#ASMEND7888; GFX90A-NEXT: v_accvgpr_read_b32 v0, a07889; GFX90A-NEXT: v_accvgpr_read_b32 v1, a17890; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v37891; GFX90A-NEXT: ; implicit-def: $agpr0_agpr17892; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7893; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7894; GFX90A-NEXT: s_cbranch_execz .LBB105_27895; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7896; GFX90A-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc7897; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7898; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr37899; GFX90A-NEXT: v_accvgpr_write_b32 a0, v07900; GFX90A-NEXT: v_accvgpr_write_b32 a1, v17901; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17902; GFX90A-NEXT: .LBB105_2: ; %Flow7903; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]7904; GFX90A-NEXT: s_cbranch_execz .LBB105_47905; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private7906; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7907; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7908; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen7909; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:47910; GFX90A-NEXT: s_waitcnt vmcnt(1)7911; GFX90A-NEXT: v_accvgpr_write_b32 a0, v27912; GFX90A-NEXT: s_waitcnt vmcnt(0)7913; GFX90A-NEXT: v_cmp_le_u64_e32 vcc, v[2:3], v[0:1]7914; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7915; GFX90A-NEXT: v_accvgpr_write_b32 a1, v37916; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7917; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:47918; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen7919; GFX90A-NEXT: .LBB105_4: ; %atomicrmw.phi7920; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7921; GFX90A-NEXT: ;;#ASMSTART7922; GFX90A-NEXT: ; use a[0:1]7923; GFX90A-NEXT: ;;#ASMEND7924; GFX90A-NEXT: s_waitcnt vmcnt(0)7925; GFX90A-NEXT: s_setpc_b64 s[30:31]7926;7927; GFX950-LABEL: flat_atomic_umin_i64_ret_a_a:7928; GFX950: ; %bb.0:7929; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7930; GFX950-NEXT: s_mov_b64 s[2:3], 0x507931; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base7932; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]7933; GFX950-NEXT: ;;#ASMSTART7934; GFX950-NEXT: ; def a[0:1]7935; GFX950-NEXT: ;;#ASMEND7936; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37937; GFX950-NEXT: v_accvgpr_read_b32 v0, a07938; GFX950-NEXT: v_accvgpr_read_b32 v1, a17939; GFX950-NEXT: ; implicit-def: $agpr0_agpr17940; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc7941; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7942; GFX950-NEXT: s_cbranch_execz .LBB105_27943; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global7944; GFX950-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] sc07945; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7946; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr37947; GFX950-NEXT: v_accvgpr_write_b32 a0, v07948; GFX950-NEXT: v_accvgpr_write_b32 a1, v17949; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr17950; GFX950-NEXT: .LBB105_2: ; %Flow7951; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7952; GFX950-NEXT: s_cbranch_execz .LBB105_47953; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private7954; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7955; GFX950-NEXT: s_nop 17956; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc7957; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v4, off7958; GFX950-NEXT: s_waitcnt vmcnt(0)7959; GFX950-NEXT: v_cmp_le_u64_e32 vcc, v[2:3], v[0:1]7960; GFX950-NEXT: v_accvgpr_write_b32 a0, v27961; GFX950-NEXT: s_nop 07962; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc7963; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc7964; GFX950-NEXT: v_accvgpr_write_b32 a1, v37965; GFX950-NEXT: scratch_store_dwordx2 v4, v[0:1], off7966; GFX950-NEXT: .LBB105_4: ; %atomicrmw.phi7967; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]7968; GFX950-NEXT: ;;#ASMSTART7969; GFX950-NEXT: ; use a[0:1]7970; GFX950-NEXT: ;;#ASMEND7971; GFX950-NEXT: s_waitcnt vmcnt(0)7972; GFX950-NEXT: s_setpc_b64 s[30:31]7973 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 107974 %data = call i64 asm "; def $0", "=a"()7975 %result = atomicrmw umin ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !07976 call void asm "; use $0", "a"(i64 %result)7977 ret void7978}7979 7980define void @flat_atomic_umin_i64_ret_av_av(ptr %ptr) #0 {7981; GFX90A-LABEL: flat_atomic_umin_i64_ret_av_av:7982; GFX90A: ; %bb.0:7983; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7984; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v07985; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base7986; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc7987; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v57988; GFX90A-NEXT: ;;#ASMSTART7989; GFX90A-NEXT: ; def v[2:3]7990; GFX90A-NEXT: ;;#ASMEND7991; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr17992; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc7993; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7994; GFX90A-NEXT: s_cbranch_execz .LBB106_27995; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global7996; GFX90A-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc7997; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7998; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr57999; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr38000; GFX90A-NEXT: .LBB106_2: ; %Flow8001; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8002; GFX90A-NEXT: s_cbranch_execz .LBB106_48003; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private8004; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8005; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8006; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen8007; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:48008; GFX90A-NEXT: s_waitcnt vmcnt(0)8009; GFX90A-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]8010; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc8011; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc8012; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen8013; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:48014; GFX90A-NEXT: .LBB106_4: ; %atomicrmw.phi8015; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8016; GFX90A-NEXT: s_waitcnt vmcnt(0)8017; GFX90A-NEXT: ;;#ASMSTART8018; GFX90A-NEXT: ; use v[0:1]8019; GFX90A-NEXT: ;;#ASMEND8020; GFX90A-NEXT: s_setpc_b64 s[30:31]8021;8022; GFX950-LABEL: flat_atomic_umin_i64_ret_av_av:8023; GFX950: ; %bb.0:8024; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8025; GFX950-NEXT: s_mov_b64 s[2:3], 0x508026; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8027; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]8028; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58029; GFX950-NEXT: ;;#ASMSTART8030; GFX950-NEXT: ; def v[2:3]8031; GFX950-NEXT: ;;#ASMEND8032; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18033; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8034; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8035; GFX950-NEXT: s_cbranch_execz .LBB106_28036; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8037; GFX950-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] sc08038; GFX950-NEXT: s_waitcnt lgkmcnt(0)8039; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58040; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr38041; GFX950-NEXT: .LBB106_2: ; %Flow8042; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8043; GFX950-NEXT: s_cbranch_execz .LBB106_48044; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private8045; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8046; GFX950-NEXT: s_nop 18047; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8048; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off8049; GFX950-NEXT: s_waitcnt vmcnt(0)8050; GFX950-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]8051; GFX950-NEXT: s_nop 18052; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc8053; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc8054; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off8055; GFX950-NEXT: .LBB106_4: ; %atomicrmw.phi8056; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8057; GFX950-NEXT: s_waitcnt vmcnt(0)8058; GFX950-NEXT: ;;#ASMSTART8059; GFX950-NEXT: ; use v[0:1]8060; GFX950-NEXT: ;;#ASMEND8061; GFX950-NEXT: s_setpc_b64 s[30:31]8062 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108063 %data = call i64 asm "; def $0", "=^VA"()8064 %result = atomicrmw umin ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08065 call void asm "; use $0", "^VA"(i64 %result)8066 ret void8067}8068 8069define void @flat_atomic_uinc_wrap_i64_ret_a_a(ptr %ptr) #0 {8070; GFX90A-LABEL: flat_atomic_uinc_wrap_i64_ret_a_a:8071; GFX90A: ; %bb.0:8072; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8073; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v08074; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8075; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc8076; GFX90A-NEXT: ;;#ASMSTART8077; GFX90A-NEXT: ; def a[0:1]8078; GFX90A-NEXT: ;;#ASMEND8079; GFX90A-NEXT: v_accvgpr_read_b32 v0, a08080; GFX90A-NEXT: v_accvgpr_read_b32 v1, a18081; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v38082; GFX90A-NEXT: ; implicit-def: $agpr0_agpr18083; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8084; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8085; GFX90A-NEXT: s_cbranch_execz .LBB107_28086; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8087; GFX90A-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc8088; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8089; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr38090; GFX90A-NEXT: v_accvgpr_write_b32 a0, v08091; GFX90A-NEXT: v_accvgpr_write_b32 a1, v18092; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18093; GFX90A-NEXT: .LBB107_2: ; %Flow8094; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8095; GFX90A-NEXT: s_cbranch_execz .LBB107_48096; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private8097; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]8098; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc8099; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen8100; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:48101; GFX90A-NEXT: s_waitcnt vmcnt(1)8102; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, 1, v28103; GFX90A-NEXT: s_waitcnt vmcnt(0)8104; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v3, vcc8105; GFX90A-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]8106; GFX90A-NEXT: v_accvgpr_write_b32 a0, v28107; GFX90A-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc8108; GFX90A-NEXT: v_accvgpr_write_b32 a1, v38109; GFX90A-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc8110; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen8111; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:48112; GFX90A-NEXT: .LBB107_4: ; %atomicrmw.phi8113; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8114; GFX90A-NEXT: ;;#ASMSTART8115; GFX90A-NEXT: ; use a[0:1]8116; GFX90A-NEXT: ;;#ASMEND8117; GFX90A-NEXT: s_waitcnt vmcnt(0)8118; GFX90A-NEXT: s_setpc_b64 s[30:31]8119;8120; GFX950-LABEL: flat_atomic_uinc_wrap_i64_ret_a_a:8121; GFX950: ; %bb.0:8122; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8123; GFX950-NEXT: s_mov_b64 s[2:3], 0x508124; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8125; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]8126; GFX950-NEXT: ;;#ASMSTART8127; GFX950-NEXT: ; def a[0:1]8128; GFX950-NEXT: ;;#ASMEND8129; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18130; GFX950-NEXT: v_accvgpr_read_b32 v3, a18131; GFX950-NEXT: v_accvgpr_read_b32 v2, a08132; GFX950-NEXT: ; implicit-def: $agpr0_agpr18133; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8134; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8135; GFX950-NEXT: s_cbranch_execz .LBB107_28136; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8137; GFX950-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] sc08138; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8139; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr38140; GFX950-NEXT: v_accvgpr_write_b32 a0, v08141; GFX950-NEXT: v_accvgpr_write_b32 a1, v18142; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18143; GFX950-NEXT: .LBB107_2: ; %Flow8144; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8145; GFX950-NEXT: s_cbranch_execz .LBB107_48146; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private8147; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8148; GFX950-NEXT: s_nop 18149; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc8150; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8151; GFX950-NEXT: s_waitcnt vmcnt(0)8152; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 18153; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]8154; GFX950-NEXT: v_accvgpr_write_b32 a0, v08155; GFX950-NEXT: v_accvgpr_write_b32 a1, v18156; GFX950-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc8157; GFX950-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc8158; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8159; GFX950-NEXT: .LBB107_4: ; %atomicrmw.phi8160; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8161; GFX950-NEXT: ;;#ASMSTART8162; GFX950-NEXT: ; use a[0:1]8163; GFX950-NEXT: ;;#ASMEND8164; GFX950-NEXT: s_waitcnt vmcnt(0)8165; GFX950-NEXT: s_setpc_b64 s[30:31]8166 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108167 %data = call i64 asm "; def $0", "=a"()8168 %result = atomicrmw uinc_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08169 call void asm "; use $0", "a"(i64 %result)8170 ret void8171}8172 8173define void @flat_atomic_uinc_wrap_i64_ret_av_av(ptr %ptr) #0 {8174; GFX90A-LABEL: flat_atomic_uinc_wrap_i64_ret_av_av:8175; GFX90A: ; %bb.0:8176; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8177; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v08178; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8179; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc8180; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v58181; GFX90A-NEXT: ;;#ASMSTART8182; GFX90A-NEXT: ; def v[2:3]8183; GFX90A-NEXT: ;;#ASMEND8184; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18185; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8186; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8187; GFX90A-NEXT: s_cbranch_execz .LBB108_28188; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8189; GFX90A-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc8190; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8191; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58192; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr38193; GFX90A-NEXT: .LBB108_2: ; %Flow8194; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8195; GFX90A-NEXT: s_cbranch_execz .LBB108_48196; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private8197; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8198; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8199; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen8200; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:48201; GFX90A-NEXT: s_waitcnt vmcnt(0)8202; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, 1, v08203; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc8204; GFX90A-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]8205; GFX90A-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc8206; GFX90A-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc8207; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen8208; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:48209; GFX90A-NEXT: .LBB108_4: ; %atomicrmw.phi8210; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8211; GFX90A-NEXT: s_waitcnt vmcnt(0)8212; GFX90A-NEXT: ;;#ASMSTART8213; GFX90A-NEXT: ; use v[0:1]8214; GFX90A-NEXT: ;;#ASMEND8215; GFX90A-NEXT: s_setpc_b64 s[30:31]8216;8217; GFX950-LABEL: flat_atomic_uinc_wrap_i64_ret_av_av:8218; GFX950: ; %bb.0:8219; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8220; GFX950-NEXT: s_mov_b64 s[2:3], 0x508221; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8222; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]8223; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58224; GFX950-NEXT: ;;#ASMSTART8225; GFX950-NEXT: ; def v[2:3]8226; GFX950-NEXT: ;;#ASMEND8227; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18228; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8229; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8230; GFX950-NEXT: s_cbranch_execz .LBB108_28231; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8232; GFX950-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] sc08233; GFX950-NEXT: s_waitcnt lgkmcnt(0)8234; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58235; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr38236; GFX950-NEXT: .LBB108_2: ; %Flow8237; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8238; GFX950-NEXT: s_cbranch_execz .LBB108_48239; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private8240; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8241; GFX950-NEXT: s_nop 18242; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc8243; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8244; GFX950-NEXT: s_waitcnt vmcnt(0)8245; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 18246; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]8247; GFX950-NEXT: s_nop 18248; GFX950-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc8249; GFX950-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc8250; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8251; GFX950-NEXT: .LBB108_4: ; %atomicrmw.phi8252; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8253; GFX950-NEXT: s_waitcnt vmcnt(0)8254; GFX950-NEXT: ;;#ASMSTART8255; GFX950-NEXT: ; use v[0:1]8256; GFX950-NEXT: ;;#ASMEND8257; GFX950-NEXT: s_setpc_b64 s[30:31]8258 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108259 %data = call i64 asm "; def $0", "=^VA"()8260 %result = atomicrmw uinc_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08261 call void asm "; use $0", "^VA"(i64 %result)8262 ret void8263}8264 8265define void @flat_atomic_udec_wrap_i64_ret_a_a(ptr %ptr) #0 {8266; GFX90A-LABEL: flat_atomic_udec_wrap_i64_ret_a_a:8267; GFX90A: ; %bb.0:8268; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8269; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v08270; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8271; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc8272; GFX90A-NEXT: ;;#ASMSTART8273; GFX90A-NEXT: ; def a[0:1]8274; GFX90A-NEXT: ;;#ASMEND8275; GFX90A-NEXT: v_accvgpr_read_b32 v0, a08276; GFX90A-NEXT: v_accvgpr_read_b32 v1, a18277; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v38278; GFX90A-NEXT: ; implicit-def: $agpr0_agpr18279; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8280; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8281; GFX90A-NEXT: s_cbranch_execz .LBB109_28282; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8283; GFX90A-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc8284; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8285; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr38286; GFX90A-NEXT: v_accvgpr_write_b32 a0, v08287; GFX90A-NEXT: v_accvgpr_write_b32 a1, v18288; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18289; GFX90A-NEXT: .LBB109_2: ; %Flow8290; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[4:5]8291; GFX90A-NEXT: s_cbranch_execz .LBB109_48292; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private8293; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]8294; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc8295; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen8296; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:48297; GFX90A-NEXT: s_waitcnt vmcnt(1)8298; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, -1, v28299; GFX90A-NEXT: s_waitcnt vmcnt(0)8300; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v3, vcc8301; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]8302; GFX90A-NEXT: v_cmp_gt_u64_e64 s[4:5], v[2:3], v[0:1]8303; GFX90A-NEXT: s_or_b64 vcc, vcc, s[4:5]8304; GFX90A-NEXT: v_accvgpr_write_b32 a0, v28305; GFX90A-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc8306; GFX90A-NEXT: v_accvgpr_write_b32 a1, v38307; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc8308; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:48309; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen8310; GFX90A-NEXT: .LBB109_4: ; %atomicrmw.phi8311; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8312; GFX90A-NEXT: ;;#ASMSTART8313; GFX90A-NEXT: ; use a[0:1]8314; GFX90A-NEXT: ;;#ASMEND8315; GFX90A-NEXT: s_waitcnt vmcnt(0)8316; GFX90A-NEXT: s_setpc_b64 s[30:31]8317;8318; GFX950-LABEL: flat_atomic_udec_wrap_i64_ret_a_a:8319; GFX950: ; %bb.0:8320; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8321; GFX950-NEXT: s_mov_b64 s[2:3], 0x508322; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8323; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]8324; GFX950-NEXT: ;;#ASMSTART8325; GFX950-NEXT: ; def a[0:1]8326; GFX950-NEXT: ;;#ASMEND8327; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v38328; GFX950-NEXT: v_accvgpr_read_b32 v0, a08329; GFX950-NEXT: v_accvgpr_read_b32 v1, a18330; GFX950-NEXT: ; implicit-def: $agpr0_agpr18331; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8332; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8333; GFX950-NEXT: s_cbranch_execz .LBB109_28334; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8335; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] sc08336; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8337; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr38338; GFX950-NEXT: v_accvgpr_write_b32 a0, v08339; GFX950-NEXT: v_accvgpr_write_b32 a1, v18340; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18341; GFX950-NEXT: .LBB109_2: ; %Flow8342; GFX950-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]8343; GFX950-NEXT: s_cbranch_execz .LBB109_48344; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private8345; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]8346; GFX950-NEXT: s_nop 18347; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc8348; GFX950-NEXT: scratch_load_dwordx2 v[2:3], v6, off8349; GFX950-NEXT: s_waitcnt vmcnt(0)8350; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]8351; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[0:1]8352; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, -18353; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]8354; GFX950-NEXT: v_accvgpr_write_b32 a0, v28355; GFX950-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc8356; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc8357; GFX950-NEXT: v_accvgpr_write_b32 a1, v38358; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off8359; GFX950-NEXT: .LBB109_4: ; %atomicrmw.phi8360; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8361; GFX950-NEXT: ;;#ASMSTART8362; GFX950-NEXT: ; use a[0:1]8363; GFX950-NEXT: ;;#ASMEND8364; GFX950-NEXT: s_waitcnt vmcnt(0)8365; GFX950-NEXT: s_setpc_b64 s[30:31]8366 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108367 %data = call i64 asm "; def $0", "=a"()8368 %result = atomicrmw udec_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08369 call void asm "; use $0", "a"(i64 %result)8370 ret void8371}8372 8373define void @flat_atomic_udec_wrap_i64_ret_av_av(ptr %ptr) #0 {8374; GFX90A-LABEL: flat_atomic_udec_wrap_i64_ret_av_av:8375; GFX90A: ; %bb.0:8376; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8377; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v08378; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8379; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc8380; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v58381; GFX90A-NEXT: ;;#ASMSTART8382; GFX90A-NEXT: ; def v[2:3]8383; GFX90A-NEXT: ;;#ASMEND8384; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18385; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8386; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8387; GFX90A-NEXT: s_cbranch_execz .LBB110_28388; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8389; GFX90A-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc8390; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8391; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58392; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr38393; GFX90A-NEXT: .LBB110_2: ; %Flow8394; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[4:5]8395; GFX90A-NEXT: s_cbranch_execz .LBB110_48396; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private8397; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8398; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8399; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen8400; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:48401; GFX90A-NEXT: s_waitcnt vmcnt(0)8402; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, -1, v08403; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v1, vcc8404; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]8405; GFX90A-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]8406; GFX90A-NEXT: s_or_b64 vcc, vcc, s[4:5]8407; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc8408; GFX90A-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc8409; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen8410; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:48411; GFX90A-NEXT: .LBB110_4: ; %atomicrmw.phi8412; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8413; GFX90A-NEXT: s_waitcnt vmcnt(0)8414; GFX90A-NEXT: ;;#ASMSTART8415; GFX90A-NEXT: ; use v[0:1]8416; GFX90A-NEXT: ;;#ASMEND8417; GFX90A-NEXT: s_setpc_b64 s[30:31]8418;8419; GFX950-LABEL: flat_atomic_udec_wrap_i64_ret_av_av:8420; GFX950: ; %bb.0:8421; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8422; GFX950-NEXT: s_mov_b64 s[2:3], 0x508423; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8424; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]8425; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58426; GFX950-NEXT: ;;#ASMSTART8427; GFX950-NEXT: ; def v[2:3]8428; GFX950-NEXT: ;;#ASMEND8429; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18430; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8431; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8432; GFX950-NEXT: s_cbranch_execz .LBB110_28433; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8434; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] sc08435; GFX950-NEXT: s_waitcnt lgkmcnt(0)8436; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58437; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr38438; GFX950-NEXT: .LBB110_2: ; %Flow8439; GFX950-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]8440; GFX950-NEXT: s_cbranch_execz .LBB110_48441; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private8442; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8443; GFX950-NEXT: s_nop 18444; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc8445; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8446; GFX950-NEXT: s_waitcnt vmcnt(0)8447; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]8448; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]8449; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -18450; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]8451; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc8452; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc8453; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8454; GFX950-NEXT: .LBB110_4: ; %atomicrmw.phi8455; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8456; GFX950-NEXT: s_waitcnt vmcnt(0)8457; GFX950-NEXT: ;;#ASMSTART8458; GFX950-NEXT: ; use v[0:1]8459; GFX950-NEXT: ;;#ASMEND8460; GFX950-NEXT: s_setpc_b64 s[30:31]8461 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108462 %data = call i64 asm "; def $0", "=^VA"()8463 %result = atomicrmw udec_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08464 call void asm "; use $0", "^VA"(i64 %result)8465 ret void8466}8467 8468define void @flat_atomic_usub_cond_i64_ret_a_a(ptr %ptr) #0 {8469; GFX90A-LABEL: flat_atomic_usub_cond_i64_ret_a_a:8470; GFX90A: ; %bb.0:8471; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8472; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v08473; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8474; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc8475; GFX90A-NEXT: ;;#ASMSTART8476; GFX90A-NEXT: ; def a[0:1]8477; GFX90A-NEXT: ;;#ASMEND8478; GFX90A-NEXT: v_accvgpr_read_b32 v5, a18479; GFX90A-NEXT: v_accvgpr_read_b32 v4, a08480; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v78481; GFX90A-NEXT: ; implicit-def: $agpr0_agpr18482; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8483; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8484; GFX90A-NEXT: s_cbranch_execz .LBB111_48485; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8486; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]8487; GFX90A-NEXT: s_mov_b64 s[6:7], 08488; GFX90A-NEXT: .LBB111_2: ; %atomicrmw.start8489; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18490; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8491; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v48492; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc8493; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]8494; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8495; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8496; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc8497; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8498; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8499; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8500; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]8501; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]8502; GFX90A-NEXT: s_cbranch_execnz .LBB111_28503; GFX90A-NEXT: ; %bb.3: ; %Flow8504; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8505; GFX90A-NEXT: v_accvgpr_write_b32 a0, v08506; GFX90A-NEXT: v_accvgpr_write_b32 a1, v18507; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr78508; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58509; GFX90A-NEXT: .LBB111_4: ; %Flow38510; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8511; GFX90A-NEXT: s_cbranch_execz .LBB111_68512; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private8513; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]8514; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc8515; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen8516; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:48517; GFX90A-NEXT: s_waitcnt vmcnt(1)8518; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v48519; GFX90A-NEXT: s_waitcnt vmcnt(0)8520; GFX90A-NEXT: v_subb_co_u32_e32 v6, vcc, v1, v5, vcc8521; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]8522; GFX90A-NEXT: v_accvgpr_write_b32 a0, v08523; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc8524; GFX90A-NEXT: v_accvgpr_write_b32 a1, v18525; GFX90A-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc8526; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen8527; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:48528; GFX90A-NEXT: .LBB111_6: ; %atomicrmw.phi8529; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8530; GFX90A-NEXT: ;;#ASMSTART8531; GFX90A-NEXT: ; use a[0:1]8532; GFX90A-NEXT: ;;#ASMEND8533; GFX90A-NEXT: s_waitcnt vmcnt(0)8534; GFX90A-NEXT: s_setpc_b64 s[30:31]8535;8536; GFX950-LABEL: flat_atomic_usub_cond_i64_ret_a_a:8537; GFX950: ; %bb.0:8538; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8539; GFX950-NEXT: s_mov_b64 s[2:3], 0x508540; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8541; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]8542; GFX950-NEXT: ;;#ASMSTART8543; GFX950-NEXT: ; def a[0:1]8544; GFX950-NEXT: ;;#ASMEND8545; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v78546; GFX950-NEXT: v_accvgpr_read_b32 v5, a18547; GFX950-NEXT: v_accvgpr_read_b32 v4, a08548; GFX950-NEXT: ; implicit-def: $agpr0_agpr18549; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8550; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8551; GFX950-NEXT: s_cbranch_execz .LBB111_48552; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8553; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]8554; GFX950-NEXT: s_mov_b64 s[2:3], 08555; GFX950-NEXT: .LBB111_2: ; %atomicrmw.start8556; GFX950-NEXT: ; =>This Inner Loop Header: Depth=18557; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8558; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v48559; GFX950-NEXT: s_nop 18560; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc8561; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]8562; GFX950-NEXT: s_nop 18563; GFX950-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8564; GFX950-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8565; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc08566; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8567; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8568; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8569; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]8570; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]8571; GFX950-NEXT: s_cbranch_execnz .LBB111_28572; GFX950-NEXT: ; %bb.3: ; %Flow8573; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8574; GFX950-NEXT: v_accvgpr_write_b32 a0, v08575; GFX950-NEXT: v_accvgpr_write_b32 a1, v18576; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr78577; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58578; GFX950-NEXT: .LBB111_4: ; %Flow38579; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8580; GFX950-NEXT: s_cbranch_execz .LBB111_68581; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private8582; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]8583; GFX950-NEXT: s_nop 18584; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc8585; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8586; GFX950-NEXT: s_waitcnt vmcnt(0)8587; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v48588; GFX950-NEXT: s_nop 18589; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc8590; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]8591; GFX950-NEXT: v_accvgpr_write_b32 a0, v08592; GFX950-NEXT: v_accvgpr_write_b32 a1, v18593; GFX950-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc8594; GFX950-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc8595; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8596; GFX950-NEXT: .LBB111_6: ; %atomicrmw.phi8597; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8598; GFX950-NEXT: ;;#ASMSTART8599; GFX950-NEXT: ; use a[0:1]8600; GFX950-NEXT: ;;#ASMEND8601; GFX950-NEXT: s_waitcnt vmcnt(0)8602; GFX950-NEXT: s_setpc_b64 s[30:31]8603 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108604 %data = call i64 asm "; def $0", "=a"()8605 %result = atomicrmw usub_cond ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08606 call void asm "; use $0", "a"(i64 %result)8607 ret void8608}8609 8610define void @flat_atomic_usub_cond_i64_ret_av_av(ptr %ptr) #0 {8611; GFX90A-LABEL: flat_atomic_usub_cond_i64_ret_av_av:8612; GFX90A: ; %bb.0:8613; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8614; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v08615; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8616; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc8617; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v78618; GFX90A-NEXT: ;;#ASMSTART8619; GFX90A-NEXT: ; def v[4:5]8620; GFX90A-NEXT: ;;#ASMEND8621; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18622; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8623; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8624; GFX90A-NEXT: s_cbranch_execz .LBB112_48625; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8626; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]8627; GFX90A-NEXT: s_mov_b64 s[6:7], 08628; GFX90A-NEXT: .LBB112_2: ; %atomicrmw.start8629; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18630; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8631; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v48632; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc8633; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]8634; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8635; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8636; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc8637; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8638; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8639; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8640; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]8641; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]8642; GFX90A-NEXT: s_cbranch_execnz .LBB112_28643; GFX90A-NEXT: ; %bb.3: ; %Flow8644; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8645; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr78646; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58647; GFX90A-NEXT: .LBB112_4: ; %Flow38648; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8649; GFX90A-NEXT: s_cbranch_execz .LBB112_68650; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private8651; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]8652; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc8653; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen8654; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:48655; GFX90A-NEXT: s_waitcnt vmcnt(1)8656; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v48657; GFX90A-NEXT: s_waitcnt vmcnt(0)8658; GFX90A-NEXT: v_subb_co_u32_e32 v6, vcc, v1, v5, vcc8659; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]8660; GFX90A-NEXT: v_cndmask_b32_e32 v3, v0, v3, vcc8661; GFX90A-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc8662; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen8663; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:48664; GFX90A-NEXT: .LBB112_6: ; %atomicrmw.phi8665; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8666; GFX90A-NEXT: ;;#ASMSTART8667; GFX90A-NEXT: ; use v[0:1]8668; GFX90A-NEXT: ;;#ASMEND8669; GFX90A-NEXT: s_waitcnt vmcnt(0)8670; GFX90A-NEXT: s_setpc_b64 s[30:31]8671;8672; GFX950-LABEL: flat_atomic_usub_cond_i64_ret_av_av:8673; GFX950: ; %bb.0:8674; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8675; GFX950-NEXT: s_mov_b64 s[2:3], 0x508676; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8677; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]8678; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v78679; GFX950-NEXT: ;;#ASMSTART8680; GFX950-NEXT: ; def v[4:5]8681; GFX950-NEXT: ;;#ASMEND8682; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18683; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8684; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8685; GFX950-NEXT: s_cbranch_execz .LBB112_48686; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8687; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]8688; GFX950-NEXT: s_mov_b64 s[2:3], 08689; GFX950-NEXT: .LBB112_2: ; %atomicrmw.start8690; GFX950-NEXT: ; =>This Inner Loop Header: Depth=18691; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8692; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v48693; GFX950-NEXT: s_nop 18694; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc8695; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]8696; GFX950-NEXT: s_nop 18697; GFX950-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8698; GFX950-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8699; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc08700; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8701; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8702; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8703; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]8704; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]8705; GFX950-NEXT: s_cbranch_execnz .LBB112_28706; GFX950-NEXT: ; %bb.3: ; %Flow8707; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8708; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr78709; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58710; GFX950-NEXT: .LBB112_4: ; %Flow38711; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8712; GFX950-NEXT: s_cbranch_execz .LBB112_68713; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private8714; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]8715; GFX950-NEXT: s_nop 18716; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc8717; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8718; GFX950-NEXT: s_waitcnt vmcnt(0)8719; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v48720; GFX950-NEXT: s_nop 18721; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc8722; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]8723; GFX950-NEXT: s_nop 18724; GFX950-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc8725; GFX950-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc8726; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8727; GFX950-NEXT: .LBB112_6: ; %atomicrmw.phi8728; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8729; GFX950-NEXT: ;;#ASMSTART8730; GFX950-NEXT: ; use v[0:1]8731; GFX950-NEXT: ;;#ASMEND8732; GFX950-NEXT: s_waitcnt vmcnt(0)8733; GFX950-NEXT: s_setpc_b64 s[30:31]8734 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108735 %data = call i64 asm "; def $0", "=^VA"()8736 %result = atomicrmw usub_cond ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08737 call void asm "; use $0", "^VA"(i64 %result)8738 ret void8739}8740 8741define void @flat_atomic_usub_sat_i64_ret_a_a(ptr %ptr) #0 {8742; GFX90A-LABEL: flat_atomic_usub_sat_i64_ret_a_a:8743; GFX90A: ; %bb.0:8744; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8745; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v08746; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8747; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc8748; GFX90A-NEXT: ;;#ASMSTART8749; GFX90A-NEXT: ; def a[0:1]8750; GFX90A-NEXT: ;;#ASMEND8751; GFX90A-NEXT: v_accvgpr_read_b32 v7, a18752; GFX90A-NEXT: v_accvgpr_read_b32 v6, a08753; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v58754; GFX90A-NEXT: ; implicit-def: $agpr0_agpr18755; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8756; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8757; GFX90A-NEXT: s_cbranch_execz .LBB113_48758; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8759; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]8760; GFX90A-NEXT: s_mov_b64 s[6:7], 08761; GFX90A-NEXT: .LBB113_2: ; %atomicrmw.start8762; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18763; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8764; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v68765; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v7, vcc8766; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8767; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc8768; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8769; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8770; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8771; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8772; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]8773; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]8774; GFX90A-NEXT: s_cbranch_execnz .LBB113_28775; GFX90A-NEXT: ; %bb.3: ; %Flow8776; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8777; GFX90A-NEXT: v_accvgpr_write_b32 a0, v08778; GFX90A-NEXT: v_accvgpr_write_b32 a1, v18779; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58780; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr78781; GFX90A-NEXT: .LBB113_4: ; %Flow38782; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8783; GFX90A-NEXT: s_cbranch_execz .LBB113_68784; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private8785; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8786; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v4, vcc8787; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen8788; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:48789; GFX90A-NEXT: s_waitcnt vmcnt(1)8790; GFX90A-NEXT: v_accvgpr_write_b32 a0, v18791; GFX90A-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v68792; GFX90A-NEXT: s_waitcnt vmcnt(0)8793; GFX90A-NEXT: v_accvgpr_write_b32 a1, v28794; GFX90A-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v7, vcc8795; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc8796; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc8797; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen8798; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:48799; GFX90A-NEXT: .LBB113_6: ; %atomicrmw.phi8800; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8801; GFX90A-NEXT: ;;#ASMSTART8802; GFX90A-NEXT: ; use a[0:1]8803; GFX90A-NEXT: ;;#ASMEND8804; GFX90A-NEXT: s_waitcnt vmcnt(0)8805; GFX90A-NEXT: s_setpc_b64 s[30:31]8806;8807; GFX950-LABEL: flat_atomic_usub_sat_i64_ret_a_a:8808; GFX950: ; %bb.0:8809; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8810; GFX950-NEXT: s_mov_b64 s[2:3], 0x508811; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8812; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]8813; GFX950-NEXT: ;;#ASMSTART8814; GFX950-NEXT: ; def a[0:1]8815; GFX950-NEXT: ;;#ASMEND8816; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58817; GFX950-NEXT: v_accvgpr_read_b32 v7, a18818; GFX950-NEXT: v_accvgpr_read_b32 v6, a08819; GFX950-NEXT: ; implicit-def: $agpr0_agpr18820; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8821; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8822; GFX950-NEXT: s_cbranch_execz .LBB113_48823; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8824; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[4:5]8825; GFX950-NEXT: s_mov_b64 s[2:3], 08826; GFX950-NEXT: .LBB113_2: ; %atomicrmw.start8827; GFX950-NEXT: ; =>This Inner Loop Header: Depth=18828; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8829; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v68830; GFX950-NEXT: s_nop 18831; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v7, vcc8832; GFX950-NEXT: s_nop 18833; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8834; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc8835; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] sc08836; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8837; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8838; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8839; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]8840; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]8841; GFX950-NEXT: s_cbranch_execnz .LBB113_28842; GFX950-NEXT: ; %bb.3: ; %Flow8843; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8844; GFX950-NEXT: v_accvgpr_write_b32 a0, v08845; GFX950-NEXT: v_accvgpr_write_b32 a1, v18846; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58847; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr78848; GFX950-NEXT: .LBB113_4: ; %Flow38849; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8850; GFX950-NEXT: s_cbranch_execz .LBB113_68851; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private8852; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8853; GFX950-NEXT: s_nop 18854; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8855; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off8856; GFX950-NEXT: s_waitcnt vmcnt(0)8857; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v68858; GFX950-NEXT: s_nop 18859; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v7, vcc8860; GFX950-NEXT: v_accvgpr_write_b32 a0, v08861; GFX950-NEXT: s_nop 08862; GFX950-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc8863; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc8864; GFX950-NEXT: v_accvgpr_write_b32 a1, v18865; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off8866; GFX950-NEXT: .LBB113_6: ; %atomicrmw.phi8867; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8868; GFX950-NEXT: ;;#ASMSTART8869; GFX950-NEXT: ; use a[0:1]8870; GFX950-NEXT: ;;#ASMEND8871; GFX950-NEXT: s_waitcnt vmcnt(0)8872; GFX950-NEXT: s_setpc_b64 s[30:31]8873 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 108874 %data = call i64 asm "; def $0", "=a"()8875 %result = atomicrmw usub_sat ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !08876 call void asm "; use $0", "a"(i64 %result)8877 ret void8878}8879 8880define void @flat_atomic_usub_sat_i64_ret_av_av(ptr %ptr) #0 {8881; GFX90A-LABEL: flat_atomic_usub_sat_i64_ret_av_av:8882; GFX90A: ; %bb.0:8883; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8884; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v08885; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base8886; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc8887; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v58888; GFX90A-NEXT: ;;#ASMSTART8889; GFX90A-NEXT: ; def v[6:7]8890; GFX90A-NEXT: ;;#ASMEND8891; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr18892; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc8893; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]8894; GFX90A-NEXT: s_cbranch_execz .LBB114_48895; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global8896; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]8897; GFX90A-NEXT: s_mov_b64 s[6:7], 08898; GFX90A-NEXT: .LBB114_2: ; %atomicrmw.start8899; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18900; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8901; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v68902; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v7, vcc8903; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8904; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc8905; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8906; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8907; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8908; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8909; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]8910; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]8911; GFX90A-NEXT: s_cbranch_execnz .LBB114_28912; GFX90A-NEXT: ; %bb.3: ; %Flow8913; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8914; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr58915; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr78916; GFX90A-NEXT: .LBB114_4: ; %Flow38917; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]8918; GFX90A-NEXT: s_cbranch_execz .LBB114_68919; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private8920; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8921; GFX90A-NEXT: v_cndmask_b32_e32 v2, -1, v4, vcc8922; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen8923; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:48924; GFX90A-NEXT: s_waitcnt vmcnt(1)8925; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v68926; GFX90A-NEXT: s_waitcnt vmcnt(0)8927; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v7, vcc8928; GFX90A-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc8929; GFX90A-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc8930; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen8931; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:48932; GFX90A-NEXT: .LBB114_6: ; %atomicrmw.phi8933; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8934; GFX90A-NEXT: ;;#ASMSTART8935; GFX90A-NEXT: ; use v[0:1]8936; GFX90A-NEXT: ;;#ASMEND8937; GFX90A-NEXT: s_waitcnt vmcnt(0)8938; GFX90A-NEXT: s_setpc_b64 s[30:31]8939;8940; GFX950-LABEL: flat_atomic_usub_sat_i64_ret_av_av:8941; GFX950: ; %bb.0:8942; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8943; GFX950-NEXT: s_mov_b64 s[2:3], 0x508944; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base8945; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]8946; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v78947; GFX950-NEXT: ;;#ASMSTART8948; GFX950-NEXT: ; def v[4:5]8949; GFX950-NEXT: ;;#ASMEND8950; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr18951; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc8952; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8953; GFX950-NEXT: s_cbranch_execz .LBB114_48954; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global8955; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]8956; GFX950-NEXT: s_mov_b64 s[2:3], 08957; GFX950-NEXT: .LBB114_2: ; %atomicrmw.start8958; GFX950-NEXT: ; =>This Inner Loop Header: Depth=18959; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8960; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v48961; GFX950-NEXT: s_nop 18962; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc8963; GFX950-NEXT: s_nop 18964; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8965; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc8966; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc08967; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8968; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8969; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8970; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]8971; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]8972; GFX950-NEXT: s_cbranch_execnz .LBB114_28973; GFX950-NEXT: ; %bb.3: ; %Flow8974; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]8975; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr78976; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr58977; GFX950-NEXT: .LBB114_4: ; %Flow38978; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8979; GFX950-NEXT: s_cbranch_execz .LBB114_68980; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private8981; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]8982; GFX950-NEXT: s_nop 18983; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc8984; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off8985; GFX950-NEXT: s_waitcnt vmcnt(0)8986; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v48987; GFX950-NEXT: s_nop 18988; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc8989; GFX950-NEXT: s_nop 18990; GFX950-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc8991; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc8992; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off8993; GFX950-NEXT: .LBB114_6: ; %atomicrmw.phi8994; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]8995; GFX950-NEXT: ;;#ASMSTART8996; GFX950-NEXT: ; use v[0:1]8997; GFX950-NEXT: ;;#ASMEND8998; GFX950-NEXT: s_waitcnt vmcnt(0)8999; GFX950-NEXT: s_setpc_b64 s[30:31]9000 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 109001 %data = call i64 asm "; def $0", "=^VA"()9002 %result = atomicrmw usub_sat ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !09003 call void asm "; use $0", "^VA"(i64 %result)9004 ret void9005}9006 9007;---------------------------------------------------------------------9008; other atomics f32, with aa+av cases9009;---------------------------------------------------------------------9010 9011define void @flat_atomic_fadd_f32_ret_a_a(ptr %ptr) #0 {9012; GFX90A-LABEL: flat_atomic_fadd_f32_ret_a_a:9013; GFX90A: ; %bb.0:9014; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9015; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 40, v09016; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base9017; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc9018; GFX90A-NEXT: ;;#ASMSTART9019; GFX90A-NEXT: ; def a09020; GFX90A-NEXT: ;;#ASMEND9021; GFX90A-NEXT: v_accvgpr_read_b32 v2, a09022; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v19023; GFX90A-NEXT: ; implicit-def: $agpr09024; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc9025; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]9026; GFX90A-NEXT: s_cbranch_execz .LBB115_69027; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private9028; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base9029; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v19030; GFX90A-NEXT: ; implicit-def: $vgpr39031; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc9032; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]9033; GFX90A-NEXT: s_cbranch_execz .LBB115_39034; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global9035; GFX90A-NEXT: global_atomic_add_f32 v3, v[0:1], v2, off glc9036; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19037; GFX90A-NEXT: ; implicit-def: $vgpr29038; GFX90A-NEXT: .LBB115_3: ; %Flow9039; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]9040; GFX90A-NEXT: s_cbranch_execz .LBB115_59041; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private9042; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9043; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9044; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen9045; GFX90A-NEXT: s_waitcnt vmcnt(0)9046; GFX90A-NEXT: v_add_f32_e32 v1, v3, v29047; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen9048; GFX90A-NEXT: .LBB115_5: ; %Flow19049; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9050; GFX90A-NEXT: s_waitcnt vmcnt(0)9051; GFX90A-NEXT: v_accvgpr_write_b32 a0, v39052; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19053; GFX90A-NEXT: ; implicit-def: $vgpr29054; GFX90A-NEXT: .LBB115_6: ; %Flow29055; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]9056; GFX90A-NEXT: s_cbranch_execz .LBB115_89057; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared9058; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9059; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9060; GFX90A-NEXT: ds_add_rtn_f32 v0, v0, v29061; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9062; GFX90A-NEXT: v_accvgpr_write_b32 a0, v09063; GFX90A-NEXT: .LBB115_8: ; %atomicrmw.phi9064; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9065; GFX90A-NEXT: ;;#ASMSTART9066; GFX90A-NEXT: ; use a09067; GFX90A-NEXT: ;;#ASMEND9068; GFX90A-NEXT: s_setpc_b64 s[30:31]9069;9070; GFX950-LABEL: flat_atomic_fadd_f32_ret_a_a:9071; GFX950: ; %bb.0:9072; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9073; GFX950-NEXT: ;;#ASMSTART9074; GFX950-NEXT: ; def a09075; GFX950-NEXT: ;;#ASMEND9076; GFX950-NEXT: s_nop 09077; GFX950-NEXT: v_accvgpr_read_b32 v2, a09078; GFX950-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:40 sc09079; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9080; GFX950-NEXT: v_accvgpr_write_b32 a0, v09081; GFX950-NEXT: ;;#ASMSTART9082; GFX950-NEXT: ; use a09083; GFX950-NEXT: ;;#ASMEND9084; GFX950-NEXT: s_setpc_b64 s[30:31]9085 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109086 %data = call float asm "; def $0", "=a"()9087 %result = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09088 call void asm "; use $0", "a"(float %result)9089 ret void9090}9091 9092define void @flat_atomic_fadd_f32_ret_av_av(ptr %ptr) #0 {9093; GFX90A-LABEL: flat_atomic_fadd_f32_ret_av_av:9094; GFX90A: ; %bb.0:9095; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9096; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 40, v09097; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base9098; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc9099; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v19100; GFX90A-NEXT: ;;#ASMSTART9101; GFX90A-NEXT: ; def v39102; GFX90A-NEXT: ;;#ASMEND9103; GFX90A-NEXT: ; implicit-def: $vgpr29104; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc9105; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]9106; GFX90A-NEXT: s_cbranch_execz .LBB116_69107; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private9108; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base9109; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v19110; GFX90A-NEXT: ; implicit-def: $vgpr29111; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc9112; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]9113; GFX90A-NEXT: s_cbranch_execz .LBB116_39114; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global9115; GFX90A-NEXT: global_atomic_add_f32 v2, v[0:1], v3, off glc9116; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19117; GFX90A-NEXT: ; implicit-def: $vgpr39118; GFX90A-NEXT: .LBB116_3: ; %Flow9119; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]9120; GFX90A-NEXT: s_cbranch_execz .LBB116_59121; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private9122; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9123; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9124; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen9125; GFX90A-NEXT: s_waitcnt vmcnt(0)9126; GFX90A-NEXT: v_add_f32_e32 v1, v2, v39127; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen9128; GFX90A-NEXT: .LBB116_5: ; %Flow19129; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9130; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19131; GFX90A-NEXT: ; implicit-def: $vgpr39132; GFX90A-NEXT: .LBB116_6: ; %Flow29133; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]9134; GFX90A-NEXT: s_cbranch_execz .LBB116_89135; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared9136; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9137; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9138; GFX90A-NEXT: s_waitcnt vmcnt(0)9139; GFX90A-NEXT: ds_add_rtn_f32 v2, v0, v39140; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9141; GFX90A-NEXT: .LBB116_8: ; %atomicrmw.phi9142; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9143; GFX90A-NEXT: s_waitcnt vmcnt(0)9144; GFX90A-NEXT: ;;#ASMSTART9145; GFX90A-NEXT: ; use v29146; GFX90A-NEXT: ;;#ASMEND9147; GFX90A-NEXT: s_setpc_b64 s[30:31]9148;9149; GFX950-LABEL: flat_atomic_fadd_f32_ret_av_av:9150; GFX950: ; %bb.0:9151; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9152; GFX950-NEXT: ;;#ASMSTART9153; GFX950-NEXT: ; def v29154; GFX950-NEXT: ;;#ASMEND9155; GFX950-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:40 sc09156; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9157; GFX950-NEXT: ;;#ASMSTART9158; GFX950-NEXT: ; use v09159; GFX950-NEXT: ;;#ASMEND9160; GFX950-NEXT: s_setpc_b64 s[30:31]9161 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109162 %data = call float asm "; def $0", "=^VA"()9163 %result = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09164 call void asm "; use $0", "^VA"(float %result)9165 ret void9166}9167 9168define void @flat_atomic_fsub_f32_ret_a_a(ptr %ptr) #0 {9169; GFX90A-LABEL: flat_atomic_fsub_f32_ret_a_a:9170; GFX90A: ; %bb.0:9171; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9172; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409173; GFX90A-NEXT: ;;#ASMSTART9174; GFX90A-NEXT: ; def a09175; GFX90A-NEXT: ;;#ASMEND9176; GFX90A-NEXT: v_accvgpr_read_b32 v4, a09177; GFX90A-NEXT: s_mov_b64 s[4:5], 09178; GFX90A-NEXT: .LBB117_1: ; %atomicrmw.start9179; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19180; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9181; GFX90A-NEXT: v_sub_f32_e32 v2, v3, v49182; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9183; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9184; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39185; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9186; GFX90A-NEXT: v_mov_b32_e32 v3, v29187; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9188; GFX90A-NEXT: s_cbranch_execnz .LBB117_19189; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9190; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9191; GFX90A-NEXT: v_accvgpr_write_b32 a0, v29192; GFX90A-NEXT: ;;#ASMSTART9193; GFX90A-NEXT: ; use a09194; GFX90A-NEXT: ;;#ASMEND9195; GFX90A-NEXT: s_setpc_b64 s[30:31]9196;9197; GFX950-LABEL: flat_atomic_fsub_f32_ret_a_a:9198; GFX950: ; %bb.0:9199; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9200; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409201; GFX950-NEXT: ;;#ASMSTART9202; GFX950-NEXT: ; def a09203; GFX950-NEXT: ;;#ASMEND9204; GFX950-NEXT: s_mov_b64 s[0:1], 09205; GFX950-NEXT: v_accvgpr_read_b32 v4, a09206; GFX950-NEXT: .LBB117_1: ; %atomicrmw.start9207; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19208; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9209; GFX950-NEXT: v_sub_f32_e32 v2, v3, v49210; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09211; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9212; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39213; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9214; GFX950-NEXT: v_mov_b32_e32 v3, v29215; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9216; GFX950-NEXT: s_cbranch_execnz .LBB117_19217; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9218; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9219; GFX950-NEXT: v_accvgpr_write_b32 a0, v29220; GFX950-NEXT: ;;#ASMSTART9221; GFX950-NEXT: ; use a09222; GFX950-NEXT: ;;#ASMEND9223; GFX950-NEXT: s_setpc_b64 s[30:31]9224 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109225 %data = call float asm "; def $0", "=a"()9226 %result = atomicrmw fsub ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09227 call void asm "; use $0", "a"(float %result)9228 ret void9229}9230 9231define void @flat_atomic_fsub_f32_ret_av_av(ptr %ptr) #0 {9232; GFX90A-LABEL: flat_atomic_fsub_f32_ret_av_av:9233; GFX90A: ; %bb.0:9234; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9235; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409236; GFX90A-NEXT: s_mov_b64 s[4:5], 09237; GFX90A-NEXT: ;;#ASMSTART9238; GFX90A-NEXT: ; def v49239; GFX90A-NEXT: ;;#ASMEND9240; GFX90A-NEXT: .LBB118_1: ; %atomicrmw.start9241; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19242; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9243; GFX90A-NEXT: v_sub_f32_e32 v2, v3, v49244; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9245; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9246; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39247; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9248; GFX90A-NEXT: v_mov_b32_e32 v3, v29249; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9250; GFX90A-NEXT: s_cbranch_execnz .LBB118_19251; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9252; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9253; GFX90A-NEXT: ;;#ASMSTART9254; GFX90A-NEXT: ; use v29255; GFX90A-NEXT: ;;#ASMEND9256; GFX90A-NEXT: s_setpc_b64 s[30:31]9257;9258; GFX950-LABEL: flat_atomic_fsub_f32_ret_av_av:9259; GFX950: ; %bb.0:9260; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9261; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409262; GFX950-NEXT: s_mov_b64 s[0:1], 09263; GFX950-NEXT: ;;#ASMSTART9264; GFX950-NEXT: ; def v49265; GFX950-NEXT: ;;#ASMEND9266; GFX950-NEXT: .LBB118_1: ; %atomicrmw.start9267; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19268; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9269; GFX950-NEXT: v_sub_f32_e32 v2, v3, v49270; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09271; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9272; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39273; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9274; GFX950-NEXT: v_mov_b32_e32 v3, v29275; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9276; GFX950-NEXT: s_cbranch_execnz .LBB118_19277; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9278; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9279; GFX950-NEXT: ;;#ASMSTART9280; GFX950-NEXT: ; use v29281; GFX950-NEXT: ;;#ASMEND9282; GFX950-NEXT: s_setpc_b64 s[30:31]9283 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109284 %data = call float asm "; def $0", "=^VA"()9285 %result = atomicrmw fsub ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09286 call void asm "; use $0", "^VA"(float %result)9287 ret void9288}9289 9290define void @flat_atomic_fmax_f32_ret_a_a(ptr %ptr) #0 {9291; GFX90A-LABEL: flat_atomic_fmax_f32_ret_a_a:9292; GFX90A: ; %bb.0:9293; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9294; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409295; GFX90A-NEXT: ;;#ASMSTART9296; GFX90A-NEXT: ; def a09297; GFX90A-NEXT: ;;#ASMEND9298; GFX90A-NEXT: v_accvgpr_read_b32 v2, a09299; GFX90A-NEXT: s_mov_b64 s[4:5], 09300; GFX90A-NEXT: v_max_f32_e32 v4, v2, v29301; GFX90A-NEXT: .LBB119_1: ; %atomicrmw.start9302; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19303; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9304; GFX90A-NEXT: v_max_f32_e32 v2, v3, v39305; GFX90A-NEXT: v_max_f32_e32 v2, v2, v49306; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9307; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9308; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39309; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9310; GFX90A-NEXT: v_mov_b32_e32 v3, v29311; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9312; GFX90A-NEXT: s_cbranch_execnz .LBB119_19313; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9314; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9315; GFX90A-NEXT: v_accvgpr_write_b32 a0, v29316; GFX90A-NEXT: ;;#ASMSTART9317; GFX90A-NEXT: ; use a09318; GFX90A-NEXT: ;;#ASMEND9319; GFX90A-NEXT: s_setpc_b64 s[30:31]9320;9321; GFX950-LABEL: flat_atomic_fmax_f32_ret_a_a:9322; GFX950: ; %bb.0:9323; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9324; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409325; GFX950-NEXT: ;;#ASMSTART9326; GFX950-NEXT: ; def a09327; GFX950-NEXT: ;;#ASMEND9328; GFX950-NEXT: s_mov_b64 s[0:1], 09329; GFX950-NEXT: v_accvgpr_read_b32 v2, a09330; GFX950-NEXT: v_max_f32_e32 v4, v2, v29331; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start9332; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19333; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9334; GFX950-NEXT: v_max_f32_e32 v2, v3, v39335; GFX950-NEXT: v_max_f32_e32 v2, v2, v49336; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09337; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9338; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39339; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9340; GFX950-NEXT: v_mov_b32_e32 v3, v29341; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9342; GFX950-NEXT: s_cbranch_execnz .LBB119_19343; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9344; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9345; GFX950-NEXT: v_accvgpr_write_b32 a0, v29346; GFX950-NEXT: ;;#ASMSTART9347; GFX950-NEXT: ; use a09348; GFX950-NEXT: ;;#ASMEND9349; GFX950-NEXT: s_setpc_b64 s[30:31]9350 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109351 %data = call float asm "; def $0", "=a"()9352 %result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09353 call void asm "; use $0", "a"(float %result)9354 ret void9355}9356 9357define void @flat_atomic_fmax_f32_ret_av_av(ptr %ptr) #0 {9358; GFX90A-LABEL: flat_atomic_fmax_f32_ret_av_av:9359; GFX90A: ; %bb.0:9360; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9361; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409362; GFX90A-NEXT: ;;#ASMSTART9363; GFX90A-NEXT: ; def v29364; GFX90A-NEXT: ;;#ASMEND9365; GFX90A-NEXT: s_mov_b64 s[4:5], 09366; GFX90A-NEXT: v_max_f32_e32 v4, v2, v29367; GFX90A-NEXT: .LBB120_1: ; %atomicrmw.start9368; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19369; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9370; GFX90A-NEXT: v_max_f32_e32 v2, v3, v39371; GFX90A-NEXT: v_max_f32_e32 v2, v2, v49372; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9373; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9374; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39375; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9376; GFX90A-NEXT: v_mov_b32_e32 v3, v29377; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9378; GFX90A-NEXT: s_cbranch_execnz .LBB120_19379; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9380; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9381; GFX90A-NEXT: ;;#ASMSTART9382; GFX90A-NEXT: ; use v29383; GFX90A-NEXT: ;;#ASMEND9384; GFX90A-NEXT: s_setpc_b64 s[30:31]9385;9386; GFX950-LABEL: flat_atomic_fmax_f32_ret_av_av:9387; GFX950: ; %bb.0:9388; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9389; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409390; GFX950-NEXT: ;;#ASMSTART9391; GFX950-NEXT: ; def v29392; GFX950-NEXT: ;;#ASMEND9393; GFX950-NEXT: s_mov_b64 s[0:1], 09394; GFX950-NEXT: v_max_f32_e32 v4, v2, v29395; GFX950-NEXT: .LBB120_1: ; %atomicrmw.start9396; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19397; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9398; GFX950-NEXT: v_max_f32_e32 v2, v3, v39399; GFX950-NEXT: v_max_f32_e32 v2, v2, v49400; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09401; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9402; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39403; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9404; GFX950-NEXT: v_mov_b32_e32 v3, v29405; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9406; GFX950-NEXT: s_cbranch_execnz .LBB120_19407; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9408; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9409; GFX950-NEXT: ;;#ASMSTART9410; GFX950-NEXT: ; use v29411; GFX950-NEXT: ;;#ASMEND9412; GFX950-NEXT: s_setpc_b64 s[30:31]9413 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109414 %data = call float asm "; def $0", "=^VA"()9415 %result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09416 call void asm "; use $0", "^VA"(float %result)9417 ret void9418}9419 9420define void @flat_atomic_fmin_f32_ret_a_a(ptr %ptr) #0 {9421; GFX90A-LABEL: flat_atomic_fmin_f32_ret_a_a:9422; GFX90A: ; %bb.0:9423; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9424; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409425; GFX90A-NEXT: ;;#ASMSTART9426; GFX90A-NEXT: ; def a09427; GFX90A-NEXT: ;;#ASMEND9428; GFX90A-NEXT: v_accvgpr_read_b32 v2, a09429; GFX90A-NEXT: s_mov_b64 s[4:5], 09430; GFX90A-NEXT: v_max_f32_e32 v4, v2, v29431; GFX90A-NEXT: .LBB121_1: ; %atomicrmw.start9432; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19433; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9434; GFX90A-NEXT: v_max_f32_e32 v2, v3, v39435; GFX90A-NEXT: v_min_f32_e32 v2, v2, v49436; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9437; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9438; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39439; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9440; GFX90A-NEXT: v_mov_b32_e32 v3, v29441; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9442; GFX90A-NEXT: s_cbranch_execnz .LBB121_19443; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9444; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9445; GFX90A-NEXT: v_accvgpr_write_b32 a0, v29446; GFX90A-NEXT: ;;#ASMSTART9447; GFX90A-NEXT: ; use a09448; GFX90A-NEXT: ;;#ASMEND9449; GFX90A-NEXT: s_setpc_b64 s[30:31]9450;9451; GFX950-LABEL: flat_atomic_fmin_f32_ret_a_a:9452; GFX950: ; %bb.0:9453; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9454; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409455; GFX950-NEXT: ;;#ASMSTART9456; GFX950-NEXT: ; def a09457; GFX950-NEXT: ;;#ASMEND9458; GFX950-NEXT: s_mov_b64 s[0:1], 09459; GFX950-NEXT: v_accvgpr_read_b32 v2, a09460; GFX950-NEXT: v_max_f32_e32 v4, v2, v29461; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start9462; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19463; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9464; GFX950-NEXT: v_max_f32_e32 v2, v3, v39465; GFX950-NEXT: v_min_f32_e32 v2, v2, v49466; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09467; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9468; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39469; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9470; GFX950-NEXT: v_mov_b32_e32 v3, v29471; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9472; GFX950-NEXT: s_cbranch_execnz .LBB121_19473; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9474; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9475; GFX950-NEXT: v_accvgpr_write_b32 a0, v29476; GFX950-NEXT: ;;#ASMSTART9477; GFX950-NEXT: ; use a09478; GFX950-NEXT: ;;#ASMEND9479; GFX950-NEXT: s_setpc_b64 s[30:31]9480 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109481 %data = call float asm "; def $0", "=a"()9482 %result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09483 call void asm "; use $0", "a"(float %result)9484 ret void9485}9486 9487define void @flat_atomic_fmin_f32_ret_av_av(ptr %ptr) #0 {9488; GFX90A-LABEL: flat_atomic_fmin_f32_ret_av_av:9489; GFX90A: ; %bb.0:9490; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9491; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409492; GFX90A-NEXT: ;;#ASMSTART9493; GFX90A-NEXT: ; def v29494; GFX90A-NEXT: ;;#ASMEND9495; GFX90A-NEXT: s_mov_b64 s[4:5], 09496; GFX90A-NEXT: v_max_f32_e32 v4, v2, v29497; GFX90A-NEXT: .LBB122_1: ; %atomicrmw.start9498; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19499; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9500; GFX90A-NEXT: v_max_f32_e32 v2, v3, v39501; GFX90A-NEXT: v_min_f32_e32 v2, v2, v49502; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9503; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9504; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39505; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9506; GFX90A-NEXT: v_mov_b32_e32 v3, v29507; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9508; GFX90A-NEXT: s_cbranch_execnz .LBB122_19509; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9510; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9511; GFX90A-NEXT: ;;#ASMSTART9512; GFX90A-NEXT: ; use v29513; GFX90A-NEXT: ;;#ASMEND9514; GFX90A-NEXT: s_setpc_b64 s[30:31]9515;9516; GFX950-LABEL: flat_atomic_fmin_f32_ret_av_av:9517; GFX950: ; %bb.0:9518; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9519; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409520; GFX950-NEXT: ;;#ASMSTART9521; GFX950-NEXT: ; def v29522; GFX950-NEXT: ;;#ASMEND9523; GFX950-NEXT: s_mov_b64 s[0:1], 09524; GFX950-NEXT: v_max_f32_e32 v4, v2, v29525; GFX950-NEXT: .LBB122_1: ; %atomicrmw.start9526; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19527; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9528; GFX950-NEXT: v_max_f32_e32 v2, v3, v39529; GFX950-NEXT: v_min_f32_e32 v2, v2, v49530; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09531; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9532; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39533; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9534; GFX950-NEXT: v_mov_b32_e32 v3, v29535; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9536; GFX950-NEXT: s_cbranch_execnz .LBB122_19537; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9538; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9539; GFX950-NEXT: ;;#ASMSTART9540; GFX950-NEXT: ; use v29541; GFX950-NEXT: ;;#ASMEND9542; GFX950-NEXT: s_setpc_b64 s[30:31]9543 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109544 %data = call float asm "; def $0", "=^VA"()9545 %result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09546 call void asm "; use $0", "^VA"(float %result)9547 ret void9548}9549 9550define void @flat_atomic_fmaximum_f32_ret_a_a(ptr %ptr) #0 {9551; GFX90A-LABEL: flat_atomic_fmaximum_f32_ret_a_a:9552; GFX90A: ; %bb.0:9553; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9554; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409555; GFX90A-NEXT: ;;#ASMSTART9556; GFX90A-NEXT: ; def a09557; GFX90A-NEXT: ;;#ASMEND9558; GFX90A-NEXT: v_accvgpr_read_b32 v4, a09559; GFX90A-NEXT: s_mov_b64 s[4:5], 09560; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc000009561; GFX90A-NEXT: .LBB123_1: ; %atomicrmw.start9562; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19563; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9564; GFX90A-NEXT: v_max_f32_e32 v2, v3, v49565; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v3, v49566; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc9567; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9568; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9569; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39570; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9571; GFX90A-NEXT: v_mov_b32_e32 v3, v29572; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9573; GFX90A-NEXT: s_cbranch_execnz .LBB123_19574; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9575; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9576; GFX90A-NEXT: v_accvgpr_write_b32 a0, v29577; GFX90A-NEXT: ;;#ASMSTART9578; GFX90A-NEXT: ; use a09579; GFX90A-NEXT: ;;#ASMEND9580; GFX90A-NEXT: s_setpc_b64 s[30:31]9581;9582; GFX950-LABEL: flat_atomic_fmaximum_f32_ret_a_a:9583; GFX950: ; %bb.0:9584; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9585; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409586; GFX950-NEXT: ;;#ASMSTART9587; GFX950-NEXT: ; def a09588; GFX950-NEXT: ;;#ASMEND9589; GFX950-NEXT: s_mov_b64 s[0:1], 09590; GFX950-NEXT: v_accvgpr_read_b32 v4, a09591; GFX950-NEXT: .LBB123_1: ; %atomicrmw.start9592; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19593; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9594; GFX950-NEXT: v_maximum3_f32 v2, v3, v4, v49595; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09596; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9597; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39598; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9599; GFX950-NEXT: v_mov_b32_e32 v3, v29600; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9601; GFX950-NEXT: s_cbranch_execnz .LBB123_19602; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9603; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9604; GFX950-NEXT: v_accvgpr_write_b32 a0, v29605; GFX950-NEXT: ;;#ASMSTART9606; GFX950-NEXT: ; use a09607; GFX950-NEXT: ;;#ASMEND9608; GFX950-NEXT: s_setpc_b64 s[30:31]9609 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109610 %data = call float asm "; def $0", "=a"()9611 %result = atomicrmw fmaximum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09612 call void asm "; use $0", "a"(float %result)9613 ret void9614}9615 9616define void @flat_atomic_fmaximum_f32_ret_av_av(ptr %ptr) #0 {9617; GFX90A-LABEL: flat_atomic_fmaximum_f32_ret_av_av:9618; GFX90A: ; %bb.0:9619; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9620; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409621; GFX90A-NEXT: s_mov_b64 s[4:5], 09622; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc000009623; GFX90A-NEXT: ;;#ASMSTART9624; GFX90A-NEXT: ; def v49625; GFX90A-NEXT: ;;#ASMEND9626; GFX90A-NEXT: .LBB124_1: ; %atomicrmw.start9627; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19628; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9629; GFX90A-NEXT: v_max_f32_e32 v2, v3, v49630; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v3, v49631; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc9632; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9633; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9634; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39635; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9636; GFX90A-NEXT: v_mov_b32_e32 v3, v29637; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9638; GFX90A-NEXT: s_cbranch_execnz .LBB124_19639; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9640; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9641; GFX90A-NEXT: ;;#ASMSTART9642; GFX90A-NEXT: ; use v29643; GFX90A-NEXT: ;;#ASMEND9644; GFX90A-NEXT: s_setpc_b64 s[30:31]9645;9646; GFX950-LABEL: flat_atomic_fmaximum_f32_ret_av_av:9647; GFX950: ; %bb.0:9648; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9649; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409650; GFX950-NEXT: s_mov_b64 s[0:1], 09651; GFX950-NEXT: ;;#ASMSTART9652; GFX950-NEXT: ; def v49653; GFX950-NEXT: ;;#ASMEND9654; GFX950-NEXT: .LBB124_1: ; %atomicrmw.start9655; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19656; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9657; GFX950-NEXT: v_maximum3_f32 v2, v3, v4, v49658; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09659; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9660; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39661; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9662; GFX950-NEXT: v_mov_b32_e32 v3, v29663; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9664; GFX950-NEXT: s_cbranch_execnz .LBB124_19665; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9666; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9667; GFX950-NEXT: ;;#ASMSTART9668; GFX950-NEXT: ; use v29669; GFX950-NEXT: ;;#ASMEND9670; GFX950-NEXT: s_setpc_b64 s[30:31]9671 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109672 %data = call float asm "; def $0", "=^VA"()9673 %result = atomicrmw fmaximum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09674 call void asm "; use $0", "^VA"(float %result)9675 ret void9676}9677 9678define void @flat_atomic_fminimum_f32_ret_a_a(ptr %ptr) #0 {9679; GFX90A-LABEL: flat_atomic_fminimum_f32_ret_a_a:9680; GFX90A: ; %bb.0:9681; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9682; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409683; GFX90A-NEXT: ;;#ASMSTART9684; GFX90A-NEXT: ; def a09685; GFX90A-NEXT: ;;#ASMEND9686; GFX90A-NEXT: v_accvgpr_read_b32 v4, a09687; GFX90A-NEXT: s_mov_b64 s[4:5], 09688; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc000009689; GFX90A-NEXT: .LBB125_1: ; %atomicrmw.start9690; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19691; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9692; GFX90A-NEXT: v_min_f32_e32 v2, v3, v49693; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v3, v49694; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc9695; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9696; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9697; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39698; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9699; GFX90A-NEXT: v_mov_b32_e32 v3, v29700; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9701; GFX90A-NEXT: s_cbranch_execnz .LBB125_19702; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9703; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9704; GFX90A-NEXT: v_accvgpr_write_b32 a0, v29705; GFX90A-NEXT: ;;#ASMSTART9706; GFX90A-NEXT: ; use a09707; GFX90A-NEXT: ;;#ASMEND9708; GFX90A-NEXT: s_setpc_b64 s[30:31]9709;9710; GFX950-LABEL: flat_atomic_fminimum_f32_ret_a_a:9711; GFX950: ; %bb.0:9712; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9713; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409714; GFX950-NEXT: ;;#ASMSTART9715; GFX950-NEXT: ; def a09716; GFX950-NEXT: ;;#ASMEND9717; GFX950-NEXT: s_mov_b64 s[0:1], 09718; GFX950-NEXT: v_accvgpr_read_b32 v4, a09719; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start9720; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19721; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9722; GFX950-NEXT: v_minimum3_f32 v2, v3, v4, v49723; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09724; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9725; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39726; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9727; GFX950-NEXT: v_mov_b32_e32 v3, v29728; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9729; GFX950-NEXT: s_cbranch_execnz .LBB125_19730; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9731; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9732; GFX950-NEXT: v_accvgpr_write_b32 a0, v29733; GFX950-NEXT: ;;#ASMSTART9734; GFX950-NEXT: ; use a09735; GFX950-NEXT: ;;#ASMEND9736; GFX950-NEXT: s_setpc_b64 s[30:31]9737 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109738 %data = call float asm "; def $0", "=a"()9739 %result = atomicrmw fminimum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09740 call void asm "; use $0", "a"(float %result)9741 ret void9742}9743 9744define void @flat_atomic_fminimum_f32_ret_av_av(ptr %ptr) #0 {9745; GFX90A-LABEL: flat_atomic_fminimum_f32_ret_av_av:9746; GFX90A: ; %bb.0:9747; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9748; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:409749; GFX90A-NEXT: s_mov_b64 s[4:5], 09750; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc000009751; GFX90A-NEXT: ;;#ASMSTART9752; GFX90A-NEXT: ; def v49753; GFX90A-NEXT: ;;#ASMEND9754; GFX90A-NEXT: .LBB126_1: ; %atomicrmw.start9755; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19756; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9757; GFX90A-NEXT: v_min_f32_e32 v2, v3, v49758; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v3, v49759; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc9760; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc9761; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9762; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39763; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9764; GFX90A-NEXT: v_mov_b32_e32 v3, v29765; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]9766; GFX90A-NEXT: s_cbranch_execnz .LBB126_19767; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9768; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9769; GFX90A-NEXT: ;;#ASMSTART9770; GFX90A-NEXT: ; use v29771; GFX90A-NEXT: ;;#ASMEND9772; GFX90A-NEXT: s_setpc_b64 s[30:31]9773;9774; GFX950-LABEL: flat_atomic_fminimum_f32_ret_av_av:9775; GFX950: ; %bb.0:9776; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9777; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:409778; GFX950-NEXT: s_mov_b64 s[0:1], 09779; GFX950-NEXT: ;;#ASMSTART9780; GFX950-NEXT: ; def v49781; GFX950-NEXT: ;;#ASMEND9782; GFX950-NEXT: .LBB126_1: ; %atomicrmw.start9783; GFX950-NEXT: ; =>This Inner Loop Header: Depth=19784; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9785; GFX950-NEXT: v_minimum3_f32 v2, v3, v4, v49786; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc09787; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9788; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v39789; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9790; GFX950-NEXT: v_mov_b32_e32 v3, v29791; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]9792; GFX950-NEXT: s_cbranch_execnz .LBB126_19793; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end9794; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9795; GFX950-NEXT: ;;#ASMSTART9796; GFX950-NEXT: ; use v29797; GFX950-NEXT: ;;#ASMEND9798; GFX950-NEXT: s_setpc_b64 s[30:31]9799 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 109800 %data = call float asm "; def $0", "=^VA"()9801 %result = atomicrmw fminimum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !09802 call void asm "; use $0", "^VA"(float %result)9803 ret void9804}9805 9806;---------------------------------------------------------------------9807; other atomics f64, with aa+av cases9808;---------------------------------------------------------------------9809 9810define void @flat_atomic_fadd_f64_ret_a_a(ptr %ptr) #0 {9811; GFX90A-LABEL: flat_atomic_fadd_f64_ret_a_a:9812; GFX90A: ; %bb.0:9813; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9814; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v09815; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base9816; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc9817; GFX90A-NEXT: ;;#ASMSTART9818; GFX90A-NEXT: ; def a[0:1]9819; GFX90A-NEXT: ;;#ASMEND9820; GFX90A-NEXT: v_accvgpr_read_b32 v3, a19821; GFX90A-NEXT: v_accvgpr_read_b32 v2, a09822; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v19823; GFX90A-NEXT: ; implicit-def: $agpr0_agpr19824; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc9825; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]9826; GFX90A-NEXT: s_cbranch_execz .LBB127_69827; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private9828; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base9829; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v19830; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr59831; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc9832; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]9833; GFX90A-NEXT: s_cbranch_execz .LBB127_39834; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global9835; GFX90A-NEXT: global_atomic_add_f64 v[4:5], v[0:1], v[2:3], off glc9836; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19837; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr39838; GFX90A-NEXT: .LBB127_3: ; %Flow9839; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]9840; GFX90A-NEXT: s_cbranch_execz .LBB127_59841; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private9842; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9843; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc9844; GFX90A-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen9845; GFX90A-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:49846; GFX90A-NEXT: s_waitcnt vmcnt(0)9847; GFX90A-NEXT: v_add_f64 v[0:1], v[4:5], v[2:3]9848; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen9849; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:49850; GFX90A-NEXT: .LBB127_5: ; %Flow19851; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9852; GFX90A-NEXT: s_waitcnt vmcnt(0)9853; GFX90A-NEXT: v_accvgpr_write_b32 a0, v49854; GFX90A-NEXT: v_accvgpr_write_b32 a1, v59855; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19856; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr39857; GFX90A-NEXT: .LBB127_6: ; %Flow29858; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]9859; GFX90A-NEXT: s_cbranch_execz .LBB127_89860; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared9861; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9862; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9863; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]9864; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9865; GFX90A-NEXT: v_accvgpr_write_b32 a0, v09866; GFX90A-NEXT: v_accvgpr_write_b32 a1, v19867; GFX90A-NEXT: .LBB127_8: ; %atomicrmw.phi9868; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9869; GFX90A-NEXT: ;;#ASMSTART9870; GFX90A-NEXT: ; use a[0:1]9871; GFX90A-NEXT: ;;#ASMEND9872; GFX90A-NEXT: s_setpc_b64 s[30:31]9873;9874; GFX950-LABEL: flat_atomic_fadd_f64_ret_a_a:9875; GFX950: ; %bb.0:9876; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9877; GFX950-NEXT: s_mov_b64 s[2:3], 0x509878; GFX950-NEXT: s_mov_b64 s[0:1], src_shared_base9879; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]9880; GFX950-NEXT: ;;#ASMSTART9881; GFX950-NEXT: ; def a[0:1]9882; GFX950-NEXT: ;;#ASMEND9883; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19884; GFX950-NEXT: v_accvgpr_read_b32 v3, a19885; GFX950-NEXT: v_accvgpr_read_b32 v2, a09886; GFX950-NEXT: ; implicit-def: $agpr0_agpr19887; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc9888; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9889; GFX950-NEXT: s_cbranch_execz .LBB127_69890; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private9891; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base9892; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s3, v19893; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr59894; GFX950-NEXT: s_and_saveexec_b64 s[2:3], vcc9895; GFX950-NEXT: s_xor_b64 s[2:3], exec, s[2:3]9896; GFX950-NEXT: s_cbranch_execz .LBB127_39897; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global9898; GFX950-NEXT: global_atomic_add_f64 v[4:5], v[0:1], v[2:3], off sc09899; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr19900; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr39901; GFX950-NEXT: .LBB127_3: ; %Flow9902; GFX950-NEXT: s_andn2_saveexec_b64 s[2:3], s[2:3]9903; GFX950-NEXT: s_cbranch_execz .LBB127_59904; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private9905; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9906; GFX950-NEXT: s_nop 19907; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc9908; GFX950-NEXT: scratch_load_dwordx2 v[4:5], v6, off9909; GFX950-NEXT: s_waitcnt vmcnt(0)9910; GFX950-NEXT: v_add_f64 v[0:1], v[4:5], v[2:3]9911; GFX950-NEXT: scratch_store_dwordx2 v6, v[0:1], off9912; GFX950-NEXT: .LBB127_5: ; %Flow19913; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]9914; GFX950-NEXT: s_waitcnt vmcnt(0)9915; GFX950-NEXT: v_accvgpr_write_b32 a0, v49916; GFX950-NEXT: v_accvgpr_write_b32 a1, v59917; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr19918; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr39919; GFX950-NEXT: .LBB127_6: ; %Flow29920; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9921; GFX950-NEXT: s_cbranch_execz .LBB127_89922; GFX950-NEXT: ; %bb.7: ; %atomicrmw.shared9923; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9924; GFX950-NEXT: s_nop 19925; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc9926; GFX950-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]9927; GFX950-NEXT: s_waitcnt lgkmcnt(0)9928; GFX950-NEXT: v_accvgpr_write_b32 a0, v09929; GFX950-NEXT: v_accvgpr_write_b32 a1, v19930; GFX950-NEXT: .LBB127_8: ; %atomicrmw.phi9931; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]9932; GFX950-NEXT: ;;#ASMSTART9933; GFX950-NEXT: ; use a[0:1]9934; GFX950-NEXT: ;;#ASMEND9935; GFX950-NEXT: s_setpc_b64 s[30:31]9936 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 109937 %data = call double asm "; def $0", "=a"()9938 %result = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !09939 call void asm "; use $0", "a"(double %result)9940 ret void9941}9942 9943define void @flat_atomic_fadd_f64_ret_av_av(ptr %ptr) #0 {9944; GFX90A-LABEL: flat_atomic_fadd_f64_ret_av_av:9945; GFX90A: ; %bb.0:9946; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9947; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v09948; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base9949; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc9950; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v39951; GFX90A-NEXT: ;;#ASMSTART9952; GFX90A-NEXT: ; def v[4:5]9953; GFX90A-NEXT: ;;#ASMEND9954; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19955; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc9956; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]9957; GFX90A-NEXT: s_cbranch_execz .LBB128_69958; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private9959; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base9960; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v39961; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr19962; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc9963; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]9964; GFX90A-NEXT: s_cbranch_execz .LBB128_39965; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global9966; GFX90A-NEXT: global_atomic_add_f64 v[0:1], v[2:3], v[4:5], off glc9967; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr39968; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr59969; GFX90A-NEXT: .LBB128_3: ; %Flow9970; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]9971; GFX90A-NEXT: s_cbranch_execz .LBB128_59972; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private9973; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9974; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc9975; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen9976; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:49977; GFX90A-NEXT: s_waitcnt vmcnt(0)9978; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], v[4:5]9979; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen9980; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:49981; GFX90A-NEXT: .LBB128_5: ; %Flow19982; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9983; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr39984; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr59985; GFX90A-NEXT: .LBB128_6: ; %Flow29986; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]9987; GFX90A-NEXT: s_cbranch_execz .LBB128_89988; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared9989; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9990; GFX90A-NEXT: s_waitcnt vmcnt(0)9991; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc9992; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[4:5]9993; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9994; GFX90A-NEXT: .LBB128_8: ; %atomicrmw.phi9995; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]9996; GFX90A-NEXT: s_waitcnt vmcnt(0)9997; GFX90A-NEXT: ;;#ASMSTART9998; GFX90A-NEXT: ; use v[0:1]9999; GFX90A-NEXT: ;;#ASMEND10000; GFX90A-NEXT: s_setpc_b64 s[30:31]10001;10002; GFX950-LABEL: flat_atomic_fadd_f64_ret_av_av:10003; GFX950: ; %bb.0:10004; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10005; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010006; GFX950-NEXT: s_mov_b64 s[0:1], src_shared_base10007; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]10008; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v310009; GFX950-NEXT: ;;#ASMSTART10010; GFX950-NEXT: ; def v[4:5]10011; GFX950-NEXT: ;;#ASMEND10012; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110013; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10014; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10015; GFX950-NEXT: s_cbranch_execz .LBB128_610016; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private10017; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base10018; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s3, v310019; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110020; GFX950-NEXT: s_and_saveexec_b64 s[2:3], vcc10021; GFX950-NEXT: s_xor_b64 s[2:3], exec, s[2:3]10022; GFX950-NEXT: s_cbranch_execz .LBB128_310023; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global10024; GFX950-NEXT: global_atomic_add_f64 v[0:1], v[2:3], v[4:5], off sc010025; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310026; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510027; GFX950-NEXT: .LBB128_3: ; %Flow10028; GFX950-NEXT: s_andn2_saveexec_b64 s[2:3], s[2:3]10029; GFX950-NEXT: s_cbranch_execz .LBB128_510030; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private10031; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10032; GFX950-NEXT: s_nop 110033; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc10034; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10035; GFX950-NEXT: s_waitcnt vmcnt(0)10036; GFX950-NEXT: v_add_f64 v[2:3], v[0:1], v[4:5]10037; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10038; GFX950-NEXT: .LBB128_5: ; %Flow110039; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]10040; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310041; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510042; GFX950-NEXT: .LBB128_6: ; %Flow210043; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10044; GFX950-NEXT: s_cbranch_execz .LBB128_810045; GFX950-NEXT: ; %bb.7: ; %atomicrmw.shared10046; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10047; GFX950-NEXT: s_waitcnt vmcnt(0)10048; GFX950-NEXT: s_nop 010049; GFX950-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc10050; GFX950-NEXT: ds_add_rtn_f64 v[0:1], v0, v[4:5]10051; GFX950-NEXT: s_waitcnt lgkmcnt(0)10052; GFX950-NEXT: .LBB128_8: ; %atomicrmw.phi10053; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10054; GFX950-NEXT: s_waitcnt vmcnt(0)10055; GFX950-NEXT: ;;#ASMSTART10056; GFX950-NEXT: ; use v[0:1]10057; GFX950-NEXT: ;;#ASMEND10058; GFX950-NEXT: s_setpc_b64 s[30:31]10059 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010060 %data = call double asm "; def $0", "=^VA"()10061 %result = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010062 call void asm "; use $0", "^VA"(double %result)10063 ret void10064}10065 10066define void @flat_atomic_fsub_f64_ret_a_a(ptr %ptr) #0 {10067; GFX90A-LABEL: flat_atomic_fsub_f64_ret_a_a:10068; GFX90A: ; %bb.0:10069; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10070; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v010071; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10072; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc10073; GFX90A-NEXT: ;;#ASMSTART10074; GFX90A-NEXT: ; def a[0:1]10075; GFX90A-NEXT: ;;#ASMEND10076; GFX90A-NEXT: v_accvgpr_read_b32 v7, a110077; GFX90A-NEXT: v_accvgpr_read_b32 v6, a010078; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v510079; GFX90A-NEXT: ; implicit-def: $agpr0_agpr110080; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10081; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10082; GFX90A-NEXT: s_cbranch_execz .LBB129_410083; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10084; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]10085; GFX90A-NEXT: s_mov_b64 s[6:7], 010086; GFX90A-NEXT: .LBB129_2: ; %atomicrmw.start10087; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110088; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10089; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[6:7]10090; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10091; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10092; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10093; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10094; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]10095; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10096; GFX90A-NEXT: s_cbranch_execnz .LBB129_210097; GFX90A-NEXT: ; %bb.3: ; %Flow10098; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10099; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010100; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110101; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510102; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr710103; GFX90A-NEXT: .LBB129_4: ; %Flow310104; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10105; GFX90A-NEXT: s_cbranch_execz .LBB129_610106; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private10107; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10108; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc10109; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen10110; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:410111; GFX90A-NEXT: s_waitcnt vmcnt(0)10112; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010113; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], -v[6:7]10114; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110115; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen10116; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:410117; GFX90A-NEXT: .LBB129_6: ; %atomicrmw.phi10118; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10119; GFX90A-NEXT: ;;#ASMSTART10120; GFX90A-NEXT: ; use a[0:1]10121; GFX90A-NEXT: ;;#ASMEND10122; GFX90A-NEXT: s_waitcnt vmcnt(0)10123; GFX90A-NEXT: s_setpc_b64 s[30:31]10124;10125; GFX950-LABEL: flat_atomic_fsub_f64_ret_a_a:10126; GFX950: ; %bb.0:10127; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10128; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010129; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10130; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]10131; GFX950-NEXT: ;;#ASMSTART10132; GFX950-NEXT: ; def a[0:1]10133; GFX950-NEXT: ;;#ASMEND10134; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510135; GFX950-NEXT: v_accvgpr_read_b32 v7, a110136; GFX950-NEXT: v_accvgpr_read_b32 v6, a010137; GFX950-NEXT: ; implicit-def: $agpr0_agpr110138; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10139; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10140; GFX950-NEXT: s_cbranch_execz .LBB129_410141; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10142; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[4:5]10143; GFX950-NEXT: s_mov_b64 s[2:3], 010144; GFX950-NEXT: .LBB129_2: ; %atomicrmw.start10145; GFX950-NEXT: ; =>This Inner Loop Header: Depth=110146; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10147; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[6:7]10148; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] sc010149; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10150; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10151; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10152; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]10153; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]10154; GFX950-NEXT: s_cbranch_execnz .LBB129_210155; GFX950-NEXT: ; %bb.3: ; %Flow10156; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]10157; GFX950-NEXT: v_accvgpr_write_b32 a0, v010158; GFX950-NEXT: v_accvgpr_write_b32 a1, v110159; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510160; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr710161; GFX950-NEXT: .LBB129_4: ; %Flow310162; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10163; GFX950-NEXT: s_cbranch_execz .LBB129_610164; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private10165; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10166; GFX950-NEXT: s_nop 110167; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc10168; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off10169; GFX950-NEXT: s_waitcnt vmcnt(0)10170; GFX950-NEXT: v_accvgpr_write_b32 a0, v010171; GFX950-NEXT: v_add_f64 v[2:3], v[0:1], -v[6:7]10172; GFX950-NEXT: v_accvgpr_write_b32 a1, v110173; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off10174; GFX950-NEXT: .LBB129_6: ; %atomicrmw.phi10175; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10176; GFX950-NEXT: ;;#ASMSTART10177; GFX950-NEXT: ; use a[0:1]10178; GFX950-NEXT: ;;#ASMEND10179; GFX950-NEXT: s_waitcnt vmcnt(0)10180; GFX950-NEXT: s_setpc_b64 s[30:31]10181 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010182 %data = call double asm "; def $0", "=a"()10183 %result = atomicrmw fsub ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010184 call void asm "; use $0", "a"(double %result)10185 ret void10186}10187 10188define void @flat_atomic_fsub_f64_ret_av_av(ptr %ptr) #0 {10189; GFX90A-LABEL: flat_atomic_fsub_f64_ret_av_av:10190; GFX90A: ; %bb.0:10191; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10192; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x50, v010193; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10194; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc10195; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v510196; GFX90A-NEXT: ;;#ASMSTART10197; GFX90A-NEXT: ; def v[6:7]10198; GFX90A-NEXT: ;;#ASMEND10199; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110200; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10201; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10202; GFX90A-NEXT: s_cbranch_execz .LBB130_410203; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10204; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]10205; GFX90A-NEXT: s_mov_b64 s[6:7], 010206; GFX90A-NEXT: .LBB130_2: ; %atomicrmw.start10207; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110208; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10209; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[6:7]10210; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10211; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10212; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10213; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10214; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]10215; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10216; GFX90A-NEXT: s_cbranch_execnz .LBB130_210217; GFX90A-NEXT: ; %bb.3: ; %Flow10218; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10219; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510220; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr710221; GFX90A-NEXT: .LBB130_4: ; %Flow310222; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10223; GFX90A-NEXT: s_cbranch_execz .LBB130_610224; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private10225; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10226; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc10227; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen10228; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:410229; GFX90A-NEXT: s_waitcnt vmcnt(0)10230; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], -v[6:7]10231; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen10232; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:410233; GFX90A-NEXT: .LBB130_6: ; %atomicrmw.phi10234; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10235; GFX90A-NEXT: ;;#ASMSTART10236; GFX90A-NEXT: ; use v[0:1]10237; GFX90A-NEXT: ;;#ASMEND10238; GFX90A-NEXT: s_waitcnt vmcnt(0)10239; GFX90A-NEXT: s_setpc_b64 s[30:31]10240;10241; GFX950-LABEL: flat_atomic_fsub_f64_ret_av_av:10242; GFX950: ; %bb.0:10243; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10244; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010245; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10246; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]10247; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510248; GFX950-NEXT: ;;#ASMSTART10249; GFX950-NEXT: ; def v[6:7]10250; GFX950-NEXT: ;;#ASMEND10251; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110252; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10253; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10254; GFX950-NEXT: s_cbranch_execz .LBB130_410255; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10256; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[4:5]10257; GFX950-NEXT: s_mov_b64 s[2:3], 010258; GFX950-NEXT: .LBB130_2: ; %atomicrmw.start10259; GFX950-NEXT: ; =>This Inner Loop Header: Depth=110260; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10261; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[6:7]10262; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] sc010263; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10264; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10265; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10266; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]10267; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]10268; GFX950-NEXT: s_cbranch_execnz .LBB130_210269; GFX950-NEXT: ; %bb.3: ; %Flow10270; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]10271; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510272; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr710273; GFX950-NEXT: .LBB130_4: ; %Flow310274; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10275; GFX950-NEXT: s_cbranch_execz .LBB130_610276; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private10277; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10278; GFX950-NEXT: s_nop 110279; GFX950-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc10280; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v4, off10281; GFX950-NEXT: s_waitcnt vmcnt(0)10282; GFX950-NEXT: v_add_f64 v[2:3], v[0:1], -v[6:7]10283; GFX950-NEXT: scratch_store_dwordx2 v4, v[2:3], off10284; GFX950-NEXT: .LBB130_6: ; %atomicrmw.phi10285; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10286; GFX950-NEXT: ;;#ASMSTART10287; GFX950-NEXT: ; use v[0:1]10288; GFX950-NEXT: ;;#ASMEND10289; GFX950-NEXT: s_waitcnt vmcnt(0)10290; GFX950-NEXT: s_setpc_b64 s[30:31]10291 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010292 %data = call double asm "; def $0", "=^VA"()10293 %result = atomicrmw fsub ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010294 call void asm "; use $0", "^VA"(double %result)10295 ret void10296}10297 10298define void @flat_atomic_fmax_f64_ret_a_a(ptr %ptr) #0 {10299; GFX90A-LABEL: flat_atomic_fmax_f64_ret_a_a:10300; GFX90A: ; %bb.0:10301; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10302; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v010303; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10304; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc10305; GFX90A-NEXT: ;;#ASMSTART10306; GFX90A-NEXT: ; def a[0:1]10307; GFX90A-NEXT: ;;#ASMEND10308; GFX90A-NEXT: v_accvgpr_read_b32 v3, a110309; GFX90A-NEXT: v_accvgpr_read_b32 v2, a010310; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v110311; GFX90A-NEXT: ; implicit-def: $agpr0_agpr110312; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10313; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10314; GFX90A-NEXT: s_cbranch_execz .LBB131_210315; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10316; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc10317; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10318; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr310319; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010320; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110321; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110322; GFX90A-NEXT: .LBB131_2: ; %Flow10323; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10324; GFX90A-NEXT: s_cbranch_execz .LBB131_410325; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private10326; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10327; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10328; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10329; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410330; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10331; GFX90A-NEXT: s_waitcnt vmcnt(0)10332; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10333; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010334; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]10335; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110336; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10337; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410338; GFX90A-NEXT: .LBB131_4: ; %atomicrmw.phi10339; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10340; GFX90A-NEXT: ;;#ASMSTART10341; GFX90A-NEXT: ; use a[0:1]10342; GFX90A-NEXT: ;;#ASMEND10343; GFX90A-NEXT: s_waitcnt vmcnt(0)10344; GFX90A-NEXT: s_setpc_b64 s[30:31]10345;10346; GFX950-LABEL: flat_atomic_fmax_f64_ret_a_a:10347; GFX950: ; %bb.0:10348; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10349; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010350; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10351; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]10352; GFX950-NEXT: ;;#ASMSTART10353; GFX950-NEXT: ; def a[0:1]10354; GFX950-NEXT: ;;#ASMEND10355; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110356; GFX950-NEXT: v_accvgpr_read_b32 v3, a110357; GFX950-NEXT: v_accvgpr_read_b32 v2, a010358; GFX950-NEXT: ; implicit-def: $agpr0_agpr110359; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10360; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10361; GFX950-NEXT: s_cbranch_execz .LBB131_210362; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10363; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc010364; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10365; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310366; GFX950-NEXT: v_accvgpr_write_b32 a0, v010367; GFX950-NEXT: v_accvgpr_write_b32 a1, v110368; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110369; GFX950-NEXT: .LBB131_2: ; %Flow10370; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10371; GFX950-NEXT: s_cbranch_execz .LBB131_410372; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private10373; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10374; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10375; GFX950-NEXT: s_nop 010376; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10377; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10378; GFX950-NEXT: s_waitcnt vmcnt(0)10379; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10380; GFX950-NEXT: v_accvgpr_write_b32 a0, v010381; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]10382; GFX950-NEXT: v_accvgpr_write_b32 a1, v110383; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10384; GFX950-NEXT: .LBB131_4: ; %atomicrmw.phi10385; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10386; GFX950-NEXT: ;;#ASMSTART10387; GFX950-NEXT: ; use a[0:1]10388; GFX950-NEXT: ;;#ASMEND10389; GFX950-NEXT: s_waitcnt vmcnt(0)10390; GFX950-NEXT: s_setpc_b64 s[30:31]10391 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010392 %data = call double asm "; def $0", "=a"()10393 %result = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010394 call void asm "; use $0", "a"(double %result)10395 ret void10396}10397 10398define void @flat_atomic_fmax_f64_ret_av_av(ptr %ptr) #0 {10399; GFX90A-LABEL: flat_atomic_fmax_f64_ret_av_av:10400; GFX90A: ; %bb.0:10401; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10402; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v010403; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10404; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc10405; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v310406; GFX90A-NEXT: ;;#ASMSTART10407; GFX90A-NEXT: ; def v[4:5]10408; GFX90A-NEXT: ;;#ASMEND10409; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110410; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10411; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10412; GFX90A-NEXT: s_cbranch_execz .LBB132_210413; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10414; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[2:3], v[4:5] glc10415; GFX90A-NEXT: s_waitcnt lgkmcnt(0)10416; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr310417; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510418; GFX90A-NEXT: .LBB132_2: ; %Flow10419; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10420; GFX90A-NEXT: s_cbranch_execz .LBB132_410421; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private10422; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10423; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc10424; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10425; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410426; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]10427; GFX90A-NEXT: s_waitcnt vmcnt(0)10428; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10429; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]10430; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10431; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410432; GFX90A-NEXT: .LBB132_4: ; %atomicrmw.phi10433; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10434; GFX90A-NEXT: s_waitcnt vmcnt(0)10435; GFX90A-NEXT: ;;#ASMSTART10436; GFX90A-NEXT: ; use v[0:1]10437; GFX90A-NEXT: ;;#ASMEND10438; GFX90A-NEXT: s_setpc_b64 s[30:31]10439;10440; GFX950-LABEL: flat_atomic_fmax_f64_ret_av_av:10441; GFX950: ; %bb.0:10442; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10443; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010444; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10445; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]10446; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510447; GFX950-NEXT: ;;#ASMSTART10448; GFX950-NEXT: ; def v[2:3]10449; GFX950-NEXT: ;;#ASMEND10450; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110451; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10452; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10453; GFX950-NEXT: s_cbranch_execz .LBB132_210454; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10455; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[4:5], v[2:3] sc010456; GFX950-NEXT: s_waitcnt lgkmcnt(0)10457; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510458; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310459; GFX950-NEXT: .LBB132_2: ; %Flow10460; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10461; GFX950-NEXT: s_cbranch_execz .LBB132_410462; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private10463; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10464; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10465; GFX950-NEXT: s_nop 010466; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc10467; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10468; GFX950-NEXT: s_waitcnt vmcnt(0)10469; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10470; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]10471; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10472; GFX950-NEXT: .LBB132_4: ; %atomicrmw.phi10473; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10474; GFX950-NEXT: s_waitcnt vmcnt(0)10475; GFX950-NEXT: ;;#ASMSTART10476; GFX950-NEXT: ; use v[0:1]10477; GFX950-NEXT: ;;#ASMEND10478; GFX950-NEXT: s_setpc_b64 s[30:31]10479 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010480 %data = call double asm "; def $0", "=^VA"()10481 %result = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010482 call void asm "; use $0", "^VA"(double %result)10483 ret void10484}10485 10486define void @flat_atomic_fmin_f64_ret_a_a(ptr %ptr) #0 {10487; GFX90A-LABEL: flat_atomic_fmin_f64_ret_a_a:10488; GFX90A: ; %bb.0:10489; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10490; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x50, v010491; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10492; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc10493; GFX90A-NEXT: ;;#ASMSTART10494; GFX90A-NEXT: ; def a[0:1]10495; GFX90A-NEXT: ;;#ASMEND10496; GFX90A-NEXT: v_accvgpr_read_b32 v3, a110497; GFX90A-NEXT: v_accvgpr_read_b32 v2, a010498; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v110499; GFX90A-NEXT: ; implicit-def: $agpr0_agpr110500; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10501; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10502; GFX90A-NEXT: s_cbranch_execz .LBB133_210503; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10504; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc10505; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10506; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr310507; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010508; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110509; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110510; GFX90A-NEXT: .LBB133_2: ; %Flow10511; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10512; GFX90A-NEXT: s_cbranch_execz .LBB133_410513; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private10514; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10515; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10516; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10517; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410518; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10519; GFX90A-NEXT: s_waitcnt vmcnt(0)10520; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10521; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010522; GFX90A-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]10523; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110524; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10525; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410526; GFX90A-NEXT: .LBB133_4: ; %atomicrmw.phi10527; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10528; GFX90A-NEXT: ;;#ASMSTART10529; GFX90A-NEXT: ; use a[0:1]10530; GFX90A-NEXT: ;;#ASMEND10531; GFX90A-NEXT: s_waitcnt vmcnt(0)10532; GFX90A-NEXT: s_setpc_b64 s[30:31]10533;10534; GFX950-LABEL: flat_atomic_fmin_f64_ret_a_a:10535; GFX950: ; %bb.0:10536; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10537; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010538; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10539; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]10540; GFX950-NEXT: ;;#ASMSTART10541; GFX950-NEXT: ; def a[0:1]10542; GFX950-NEXT: ;;#ASMEND10543; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110544; GFX950-NEXT: v_accvgpr_read_b32 v3, a110545; GFX950-NEXT: v_accvgpr_read_b32 v2, a010546; GFX950-NEXT: ; implicit-def: $agpr0_agpr110547; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10548; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10549; GFX950-NEXT: s_cbranch_execz .LBB133_210550; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10551; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc010552; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10553; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310554; GFX950-NEXT: v_accvgpr_write_b32 a0, v010555; GFX950-NEXT: v_accvgpr_write_b32 a1, v110556; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110557; GFX950-NEXT: .LBB133_2: ; %Flow10558; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10559; GFX950-NEXT: s_cbranch_execz .LBB133_410560; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private10561; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10562; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10563; GFX950-NEXT: s_nop 010564; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10565; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10566; GFX950-NEXT: s_waitcnt vmcnt(0)10567; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10568; GFX950-NEXT: v_accvgpr_write_b32 a0, v010569; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]10570; GFX950-NEXT: v_accvgpr_write_b32 a1, v110571; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10572; GFX950-NEXT: .LBB133_4: ; %atomicrmw.phi10573; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10574; GFX950-NEXT: ;;#ASMSTART10575; GFX950-NEXT: ; use a[0:1]10576; GFX950-NEXT: ;;#ASMEND10577; GFX950-NEXT: s_waitcnt vmcnt(0)10578; GFX950-NEXT: s_setpc_b64 s[30:31]10579 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010580 %data = call double asm "; def $0", "=a"()10581 %result = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010582 call void asm "; use $0", "a"(double %result)10583 ret void10584}10585 10586define void @flat_atomic_fmin_f64_ret_av_av(ptr %ptr) #0 {10587; GFX90A-LABEL: flat_atomic_fmin_f64_ret_av_av:10588; GFX90A: ; %bb.0:10589; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10590; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x50, v010591; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10592; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc10593; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v310594; GFX90A-NEXT: ;;#ASMSTART10595; GFX90A-NEXT: ; def v[4:5]10596; GFX90A-NEXT: ;;#ASMEND10597; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110598; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10599; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10600; GFX90A-NEXT: s_cbranch_execz .LBB134_210601; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10602; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[2:3], v[4:5] glc10603; GFX90A-NEXT: s_waitcnt lgkmcnt(0)10604; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr310605; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510606; GFX90A-NEXT: .LBB134_2: ; %Flow10607; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10608; GFX90A-NEXT: s_cbranch_execz .LBB134_410609; GFX90A-NEXT: ; %bb.3: ; %atomicrmw.private10610; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10611; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc10612; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10613; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410614; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]10615; GFX90A-NEXT: s_waitcnt vmcnt(0)10616; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10617; GFX90A-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]10618; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10619; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410620; GFX90A-NEXT: .LBB134_4: ; %atomicrmw.phi10621; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10622; GFX90A-NEXT: s_waitcnt vmcnt(0)10623; GFX90A-NEXT: ;;#ASMSTART10624; GFX90A-NEXT: ; use v[0:1]10625; GFX90A-NEXT: ;;#ASMEND10626; GFX90A-NEXT: s_setpc_b64 s[30:31]10627;10628; GFX950-LABEL: flat_atomic_fmin_f64_ret_av_av:10629; GFX950: ; %bb.0:10630; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10631; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010632; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10633; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]10634; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510635; GFX950-NEXT: ;;#ASMSTART10636; GFX950-NEXT: ; def v[2:3]10637; GFX950-NEXT: ;;#ASMEND10638; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110639; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10640; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10641; GFX950-NEXT: s_cbranch_execz .LBB134_210642; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10643; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[4:5], v[2:3] sc010644; GFX950-NEXT: s_waitcnt lgkmcnt(0)10645; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510646; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr310647; GFX950-NEXT: .LBB134_2: ; %Flow10648; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10649; GFX950-NEXT: s_cbranch_execz .LBB134_410650; GFX950-NEXT: ; %bb.3: ; %atomicrmw.private10651; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10652; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]10653; GFX950-NEXT: s_nop 010654; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc10655; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10656; GFX950-NEXT: s_waitcnt vmcnt(0)10657; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]10658; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]10659; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10660; GFX950-NEXT: .LBB134_4: ; %atomicrmw.phi10661; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10662; GFX950-NEXT: s_waitcnt vmcnt(0)10663; GFX950-NEXT: ;;#ASMSTART10664; GFX950-NEXT: ; use v[0:1]10665; GFX950-NEXT: ;;#ASMEND10666; GFX950-NEXT: s_setpc_b64 s[30:31]10667 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010668 %data = call double asm "; def $0", "=^VA"()10669 %result = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010670 call void asm "; use $0", "^VA"(double %result)10671 ret void10672}10673 10674define void @flat_atomic_fmaximum_f64_ret_a_a(ptr %ptr) #0 {10675; GFX90A-LABEL: flat_atomic_fmaximum_f64_ret_a_a:10676; GFX90A: ; %bb.0:10677; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10678; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v010679; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10680; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc10681; GFX90A-NEXT: ;;#ASMSTART10682; GFX90A-NEXT: ; def a[0:1]10683; GFX90A-NEXT: ;;#ASMEND10684; GFX90A-NEXT: v_accvgpr_read_b32 v5, a110685; GFX90A-NEXT: v_accvgpr_read_b32 v4, a010686; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v710687; GFX90A-NEXT: ; implicit-def: $agpr0_agpr110688; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10689; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10690; GFX90A-NEXT: s_cbranch_execz .LBB135_410691; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10692; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]10693; GFX90A-NEXT: s_mov_b64 s[6:7], 010694; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000010695; GFX90A-NEXT: .LBB135_2: ; %atomicrmw.start10696; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110697; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10698; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]10699; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]10700; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc10701; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10702; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc10703; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10704; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10705; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10706; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]10707; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10708; GFX90A-NEXT: s_cbranch_execnz .LBB135_210709; GFX90A-NEXT: ; %bb.3: ; %Flow10710; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10711; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010712; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110713; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr710714; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510715; GFX90A-NEXT: .LBB135_4: ; %Flow210716; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10717; GFX90A-NEXT: s_cbranch_execz .LBB135_610718; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private10719; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]10720; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc10721; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10722; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410723; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000010724; GFX90A-NEXT: s_waitcnt vmcnt(0)10725; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]10726; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]10727; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010728; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10729; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110730; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc10731; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10732; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410733; GFX90A-NEXT: .LBB135_6: ; %atomicrmw.phi10734; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10735; GFX90A-NEXT: ;;#ASMSTART10736; GFX90A-NEXT: ; use a[0:1]10737; GFX90A-NEXT: ;;#ASMEND10738; GFX90A-NEXT: s_waitcnt vmcnt(0)10739; GFX90A-NEXT: s_setpc_b64 s[30:31]10740;10741; GFX950-LABEL: flat_atomic_fmaximum_f64_ret_a_a:10742; GFX950: ; %bb.0:10743; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10744; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010745; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10746; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]10747; GFX950-NEXT: ;;#ASMSTART10748; GFX950-NEXT: ; def a[0:1]10749; GFX950-NEXT: ;;#ASMEND10750; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v710751; GFX950-NEXT: v_accvgpr_read_b32 v5, a110752; GFX950-NEXT: v_accvgpr_read_b32 v4, a010753; GFX950-NEXT: ; implicit-def: $agpr0_agpr110754; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10755; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10756; GFX950-NEXT: s_cbranch_execz .LBB135_410757; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10758; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]10759; GFX950-NEXT: s_mov_b64 s[2:3], 010760; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000010761; GFX950-NEXT: .LBB135_2: ; %atomicrmw.start10762; GFX950-NEXT: ; =>This Inner Loop Header: Depth=110763; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10764; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]10765; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]10766; GFX950-NEXT: s_nop 110767; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc10768; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10769; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc010770; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10771; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10772; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10773; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]10774; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]10775; GFX950-NEXT: s_cbranch_execnz .LBB135_210776; GFX950-NEXT: ; %bb.3: ; %Flow10777; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]10778; GFX950-NEXT: v_accvgpr_write_b32 a0, v010779; GFX950-NEXT: v_accvgpr_write_b32 a1, v110780; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr710781; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510782; GFX950-NEXT: .LBB135_4: ; %Flow210783; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10784; GFX950-NEXT: s_cbranch_execz .LBB135_610785; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private10786; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]10787; GFX950-NEXT: v_mov_b32_e32 v7, 0x7ff8000010788; GFX950-NEXT: s_nop 010789; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc10790; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10791; GFX950-NEXT: s_waitcnt vmcnt(0)10792; GFX950-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]10793; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]10794; GFX950-NEXT: v_accvgpr_write_b32 a0, v010795; GFX950-NEXT: v_accvgpr_write_b32 a1, v110796; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc10797; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10798; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10799; GFX950-NEXT: .LBB135_6: ; %atomicrmw.phi10800; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10801; GFX950-NEXT: ;;#ASMSTART10802; GFX950-NEXT: ; use a[0:1]10803; GFX950-NEXT: ;;#ASMEND10804; GFX950-NEXT: s_waitcnt vmcnt(0)10805; GFX950-NEXT: s_setpc_b64 s[30:31]10806 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010807 %data = call double asm "; def $0", "=a"()10808 %result = atomicrmw fmaximum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010809 call void asm "; use $0", "a"(double %result)10810 ret void10811}10812 10813define void @flat_atomic_fmaximum_f64_ret_av_av(ptr %ptr) #0 {10814; GFX90A-LABEL: flat_atomic_fmaximum_f64_ret_av_av:10815; GFX90A: ; %bb.0:10816; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10817; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v010818; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10819; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc10820; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v710821; GFX90A-NEXT: ;;#ASMSTART10822; GFX90A-NEXT: ; def v[4:5]10823; GFX90A-NEXT: ;;#ASMEND10824; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr110825; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10826; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10827; GFX90A-NEXT: s_cbranch_execz .LBB136_410828; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10829; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]10830; GFX90A-NEXT: s_mov_b64 s[6:7], 010831; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000010832; GFX90A-NEXT: .LBB136_2: ; %atomicrmw.start10833; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110834; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10835; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]10836; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]10837; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc10838; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10839; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc10840; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10841; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10842; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10843; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]10844; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10845; GFX90A-NEXT: s_cbranch_execnz .LBB136_210846; GFX90A-NEXT: ; %bb.3: ; %Flow10847; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10848; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr710849; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510850; GFX90A-NEXT: .LBB136_4: ; %Flow210851; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10852; GFX90A-NEXT: s_cbranch_execz .LBB136_610853; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private10854; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]10855; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc10856; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10857; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410858; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000010859; GFX90A-NEXT: s_waitcnt vmcnt(0)10860; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]10861; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]10862; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10863; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc10864; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10865; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:410866; GFX90A-NEXT: .LBB136_6: ; %atomicrmw.phi10867; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]10868; GFX90A-NEXT: ;;#ASMSTART10869; GFX90A-NEXT: ; use v[0:1]10870; GFX90A-NEXT: ;;#ASMEND10871; GFX90A-NEXT: s_waitcnt vmcnt(0)10872; GFX90A-NEXT: s_setpc_b64 s[30:31]10873;10874; GFX950-LABEL: flat_atomic_fmaximum_f64_ret_av_av:10875; GFX950: ; %bb.0:10876; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10877; GFX950-NEXT: s_mov_b64 s[2:3], 0x5010878; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base10879; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]10880; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v710881; GFX950-NEXT: ;;#ASMSTART10882; GFX950-NEXT: ; def v[4:5]10883; GFX950-NEXT: ;;#ASMEND10884; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr110885; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc10886; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10887; GFX950-NEXT: s_cbranch_execz .LBB136_410888; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global10889; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]10890; GFX950-NEXT: s_mov_b64 s[2:3], 010891; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000010892; GFX950-NEXT: .LBB136_2: ; %atomicrmw.start10893; GFX950-NEXT: ; =>This Inner Loop Header: Depth=110894; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10895; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]10896; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]10897; GFX950-NEXT: s_nop 110898; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc10899; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10900; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc010901; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10902; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10903; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10904; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]10905; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]10906; GFX950-NEXT: s_cbranch_execnz .LBB136_210907; GFX950-NEXT: ; %bb.3: ; %Flow10908; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]10909; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr710910; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr510911; GFX950-NEXT: .LBB136_4: ; %Flow210912; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10913; GFX950-NEXT: s_cbranch_execz .LBB136_610914; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private10915; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]10916; GFX950-NEXT: v_mov_b32_e32 v7, 0x7ff8000010917; GFX950-NEXT: s_nop 010918; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc10919; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off10920; GFX950-NEXT: s_waitcnt vmcnt(0)10921; GFX950-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]10922; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]10923; GFX950-NEXT: s_nop 110924; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc10925; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10926; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off10927; GFX950-NEXT: .LBB136_6: ; %atomicrmw.phi10928; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]10929; GFX950-NEXT: ;;#ASMSTART10930; GFX950-NEXT: ; use v[0:1]10931; GFX950-NEXT: ;;#ASMEND10932; GFX950-NEXT: s_waitcnt vmcnt(0)10933; GFX950-NEXT: s_setpc_b64 s[30:31]10934 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1010935 %data = call double asm "; def $0", "=^VA"()10936 %result = atomicrmw fmaximum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !010937 call void asm "; use $0", "^VA"(double %result)10938 ret void10939}10940 10941define void @flat_atomic_fminimum_f64_ret_a_a(ptr %ptr) #0 {10942; GFX90A-LABEL: flat_atomic_fminimum_f64_ret_a_a:10943; GFX90A: ; %bb.0:10944; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10945; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v010946; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base10947; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc10948; GFX90A-NEXT: ;;#ASMSTART10949; GFX90A-NEXT: ; def a[0:1]10950; GFX90A-NEXT: ;;#ASMEND10951; GFX90A-NEXT: v_accvgpr_read_b32 v5, a110952; GFX90A-NEXT: v_accvgpr_read_b32 v4, a010953; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v710954; GFX90A-NEXT: ; implicit-def: $agpr0_agpr110955; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc10956; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]10957; GFX90A-NEXT: s_cbranch_execz .LBB137_410958; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global10959; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]10960; GFX90A-NEXT: s_mov_b64 s[6:7], 010961; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000010962; GFX90A-NEXT: .LBB137_2: ; %atomicrmw.start10963; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110964; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10965; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]10966; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]10967; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc10968; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10969; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc10970; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10971; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10972; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10973; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]10974; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10975; GFX90A-NEXT: s_cbranch_execnz .LBB137_210976; GFX90A-NEXT: ; %bb.3: ; %Flow10977; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10978; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010979; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110980; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr710981; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr510982; GFX90A-NEXT: .LBB137_4: ; %Flow210983; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]10984; GFX90A-NEXT: s_cbranch_execz .LBB137_610985; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private10986; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]10987; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc10988; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen10989; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:410990; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000010991; GFX90A-NEXT: s_waitcnt vmcnt(0)10992; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]10993; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]10994; GFX90A-NEXT: v_accvgpr_write_b32 a0, v010995; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10996; GFX90A-NEXT: v_accvgpr_write_b32 a1, v110997; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc10998; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen10999; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:411000; GFX90A-NEXT: .LBB137_6: ; %atomicrmw.phi11001; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11002; GFX90A-NEXT: ;;#ASMSTART11003; GFX90A-NEXT: ; use a[0:1]11004; GFX90A-NEXT: ;;#ASMEND11005; GFX90A-NEXT: s_waitcnt vmcnt(0)11006; GFX90A-NEXT: s_setpc_b64 s[30:31]11007;11008; GFX950-LABEL: flat_atomic_fminimum_f64_ret_a_a:11009; GFX950: ; %bb.0:11010; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11011; GFX950-NEXT: s_mov_b64 s[2:3], 0x5011012; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base11013; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]11014; GFX950-NEXT: ;;#ASMSTART11015; GFX950-NEXT: ; def a[0:1]11016; GFX950-NEXT: ;;#ASMEND11017; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v711018; GFX950-NEXT: v_accvgpr_read_b32 v5, a111019; GFX950-NEXT: v_accvgpr_read_b32 v4, a011020; GFX950-NEXT: ; implicit-def: $agpr0_agpr111021; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc11022; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11023; GFX950-NEXT: s_cbranch_execz .LBB137_411024; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global11025; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]11026; GFX950-NEXT: s_mov_b64 s[2:3], 011027; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000011028; GFX950-NEXT: .LBB137_2: ; %atomicrmw.start11029; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111030; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11031; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]11032; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]11033; GFX950-NEXT: s_nop 111034; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc11035; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc11036; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc011037; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11038; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11039; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11040; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]11041; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]11042; GFX950-NEXT: s_cbranch_execnz .LBB137_211043; GFX950-NEXT: ; %bb.3: ; %Flow11044; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]11045; GFX950-NEXT: v_accvgpr_write_b32 a0, v011046; GFX950-NEXT: v_accvgpr_write_b32 a1, v111047; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr711048; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr511049; GFX950-NEXT: .LBB137_4: ; %Flow211050; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11051; GFX950-NEXT: s_cbranch_execz .LBB137_611052; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private11053; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]11054; GFX950-NEXT: v_mov_b32_e32 v7, 0x7ff8000011055; GFX950-NEXT: s_nop 011056; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc11057; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off11058; GFX950-NEXT: s_waitcnt vmcnt(0)11059; GFX950-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]11060; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]11061; GFX950-NEXT: v_accvgpr_write_b32 a0, v011062; GFX950-NEXT: v_accvgpr_write_b32 a1, v111063; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc11064; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc11065; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off11066; GFX950-NEXT: .LBB137_6: ; %atomicrmw.phi11067; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11068; GFX950-NEXT: ;;#ASMSTART11069; GFX950-NEXT: ; use a[0:1]11070; GFX950-NEXT: ;;#ASMEND11071; GFX950-NEXT: s_waitcnt vmcnt(0)11072; GFX950-NEXT: s_setpc_b64 s[30:31]11073 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1011074 %data = call double asm "; def $0", "=a"()11075 %result = atomicrmw fminimum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011076 call void asm "; use $0", "a"(double %result)11077 ret void11078}11079 11080define void @flat_atomic_fminimum_f64_ret_av_av(ptr %ptr) #0 {11081; GFX90A-LABEL: flat_atomic_fminimum_f64_ret_av_av:11082; GFX90A: ; %bb.0:11083; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11084; GFX90A-NEXT: v_add_co_u32_e32 v6, vcc, 0x50, v011085; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base11086; GFX90A-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc11087; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v711088; GFX90A-NEXT: ;;#ASMSTART11089; GFX90A-NEXT: ; def v[4:5]11090; GFX90A-NEXT: ;;#ASMEND11091; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr111092; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc11093; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]11094; GFX90A-NEXT: s_cbranch_execz .LBB138_411095; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global11096; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]11097; GFX90A-NEXT: s_mov_b64 s[6:7], 011098; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000011099; GFX90A-NEXT: .LBB138_2: ; %atomicrmw.start11100; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111101; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11102; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]11103; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]11104; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc11105; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc11106; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc11107; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11108; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11109; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11110; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]11111; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11112; GFX90A-NEXT: s_cbranch_execnz .LBB138_211113; GFX90A-NEXT: ; %bb.3: ; %Flow11114; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11115; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr711116; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr511117; GFX90A-NEXT: .LBB138_4: ; %Flow211118; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]11119; GFX90A-NEXT: s_cbranch_execz .LBB138_611120; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private11121; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]11122; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc11123; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen11124; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:411125; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000011126; GFX90A-NEXT: s_waitcnt vmcnt(0)11127; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]11128; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]11129; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc11130; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc11131; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen11132; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:411133; GFX90A-NEXT: .LBB138_6: ; %atomicrmw.phi11134; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11135; GFX90A-NEXT: ;;#ASMSTART11136; GFX90A-NEXT: ; use v[0:1]11137; GFX90A-NEXT: ;;#ASMEND11138; GFX90A-NEXT: s_waitcnt vmcnt(0)11139; GFX90A-NEXT: s_setpc_b64 s[30:31]11140;11141; GFX950-LABEL: flat_atomic_fminimum_f64_ret_av_av:11142; GFX950: ; %bb.0:11143; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11144; GFX950-NEXT: s_mov_b64 s[2:3], 0x5011145; GFX950-NEXT: s_mov_b64 s[0:1], src_private_base11146; GFX950-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[2:3]11147; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, s1, v711148; GFX950-NEXT: ;;#ASMSTART11149; GFX950-NEXT: ; def v[4:5]11150; GFX950-NEXT: ;;#ASMEND11151; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr111152; GFX950-NEXT: s_and_saveexec_b64 s[0:1], vcc11153; GFX950-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11154; GFX950-NEXT: s_cbranch_execz .LBB138_411155; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global11156; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]11157; GFX950-NEXT: s_mov_b64 s[2:3], 011158; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000011159; GFX950-NEXT: .LBB138_2: ; %atomicrmw.start11160; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111161; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11162; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]11163; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]11164; GFX950-NEXT: s_nop 111165; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc11166; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc11167; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc011168; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11169; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11170; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11171; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]11172; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]11173; GFX950-NEXT: s_cbranch_execnz .LBB138_211174; GFX950-NEXT: ; %bb.3: ; %Flow11175; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]11176; GFX950-NEXT: ; implicit-def: $vgpr6_vgpr711177; GFX950-NEXT: ; implicit-def: $vgpr4_vgpr511178; GFX950-NEXT: .LBB138_4: ; %Flow211179; GFX950-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11180; GFX950-NEXT: s_cbranch_execz .LBB138_611181; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private11182; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]11183; GFX950-NEXT: v_mov_b32_e32 v7, 0x7ff8000011184; GFX950-NEXT: s_nop 011185; GFX950-NEXT: v_cndmask_b32_e32 v6, -1, v6, vcc11186; GFX950-NEXT: scratch_load_dwordx2 v[0:1], v6, off11187; GFX950-NEXT: s_waitcnt vmcnt(0)11188; GFX950-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]11189; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]11190; GFX950-NEXT: s_nop 111191; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc11192; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc11193; GFX950-NEXT: scratch_store_dwordx2 v6, v[2:3], off11194; GFX950-NEXT: .LBB138_6: ; %atomicrmw.phi11195; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11196; GFX950-NEXT: ;;#ASMSTART11197; GFX950-NEXT: ; use v[0:1]11198; GFX950-NEXT: ;;#ASMEND11199; GFX950-NEXT: s_waitcnt vmcnt(0)11200; GFX950-NEXT: s_setpc_b64 s[30:31]11201 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1011202 %data = call double asm "; def $0", "=^VA"()11203 %result = atomicrmw fminimum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011204 call void asm "; use $0", "^VA"(double %result)11205 ret void11206}11207 11208;---------------------------------------------------------------------11209; other atomics v2f16, with aa+av cases11210;---------------------------------------------------------------------11211 11212define void @flat_atomic_fadd_v2f16_ret_a_a(ptr %ptr) #0 {11213; GFX90A-LABEL: flat_atomic_fadd_v2f16_ret_a_a:11214; GFX90A: ; %bb.0:11215; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11216; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011217; GFX90A-NEXT: ;;#ASMSTART11218; GFX90A-NEXT: ; def a011219; GFX90A-NEXT: ;;#ASMEND11220; GFX90A-NEXT: v_accvgpr_read_b32 v4, a011221; GFX90A-NEXT: s_mov_b64 s[4:5], 011222; GFX90A-NEXT: .LBB139_1: ; %atomicrmw.start11223; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111224; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11225; GFX90A-NEXT: v_pk_add_f16 v2, v3, v411226; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11227; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11228; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311229; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11230; GFX90A-NEXT: v_mov_b32_e32 v3, v211231; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11232; GFX90A-NEXT: s_cbranch_execnz .LBB139_111233; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11234; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11235; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211236; GFX90A-NEXT: ;;#ASMSTART11237; GFX90A-NEXT: ; use a011238; GFX90A-NEXT: ;;#ASMEND11239; GFX90A-NEXT: s_setpc_b64 s[30:31]11240;11241; GFX950-LABEL: flat_atomic_fadd_v2f16_ret_a_a:11242; GFX950: ; %bb.0:11243; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11244; GFX950-NEXT: ;;#ASMSTART11245; GFX950-NEXT: ; def a011246; GFX950-NEXT: ;;#ASMEND11247; GFX950-NEXT: s_nop 011248; GFX950-NEXT: v_accvgpr_read_b32 v2, a011249; GFX950-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:40 sc011250; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11251; GFX950-NEXT: v_accvgpr_write_b32 a0, v011252; GFX950-NEXT: ;;#ASMSTART11253; GFX950-NEXT: ; use a011254; GFX950-NEXT: ;;#ASMEND11255; GFX950-NEXT: s_setpc_b64 s[30:31]11256 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011257 %data = call <2 x half> asm "; def $0", "=a"()11258 %result = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011259 call void asm "; use $0", "a"(<2 x half> %result)11260 ret void11261}11262 11263define void @flat_atomic_fadd_v2f16_ret_av_av(ptr %ptr) #0 {11264; GFX90A-LABEL: flat_atomic_fadd_v2f16_ret_av_av:11265; GFX90A: ; %bb.0:11266; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11267; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011268; GFX90A-NEXT: s_mov_b64 s[4:5], 011269; GFX90A-NEXT: ;;#ASMSTART11270; GFX90A-NEXT: ; def v411271; GFX90A-NEXT: ;;#ASMEND11272; GFX90A-NEXT: .LBB140_1: ; %atomicrmw.start11273; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111274; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11275; GFX90A-NEXT: v_pk_add_f16 v2, v3, v411276; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11277; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11278; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311279; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11280; GFX90A-NEXT: v_mov_b32_e32 v3, v211281; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11282; GFX90A-NEXT: s_cbranch_execnz .LBB140_111283; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11284; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11285; GFX90A-NEXT: ;;#ASMSTART11286; GFX90A-NEXT: ; use v211287; GFX90A-NEXT: ;;#ASMEND11288; GFX90A-NEXT: s_setpc_b64 s[30:31]11289;11290; GFX950-LABEL: flat_atomic_fadd_v2f16_ret_av_av:11291; GFX950: ; %bb.0:11292; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11293; GFX950-NEXT: ;;#ASMSTART11294; GFX950-NEXT: ; def v211295; GFX950-NEXT: ;;#ASMEND11296; GFX950-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:40 sc011297; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11298; GFX950-NEXT: ;;#ASMSTART11299; GFX950-NEXT: ; use v011300; GFX950-NEXT: ;;#ASMEND11301; GFX950-NEXT: s_setpc_b64 s[30:31]11302 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011303 %data = call <2 x half> asm "; def $0", "=^VA"()11304 %result = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011305 call void asm "; use $0", "^VA"(<2 x half> %result)11306 ret void11307}11308 11309define void @flat_atomic_fsub_v2f16_ret_a_a(ptr %ptr) #0 {11310; GFX90A-LABEL: flat_atomic_fsub_v2f16_ret_a_a:11311; GFX90A: ; %bb.0:11312; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11313; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011314; GFX90A-NEXT: ;;#ASMSTART11315; GFX90A-NEXT: ; def a011316; GFX90A-NEXT: ;;#ASMEND11317; GFX90A-NEXT: v_accvgpr_read_b32 v4, a011318; GFX90A-NEXT: s_mov_b64 s[4:5], 011319; GFX90A-NEXT: .LBB141_1: ; %atomicrmw.start11320; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111321; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11322; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]11323; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11324; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11325; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311326; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11327; GFX90A-NEXT: v_mov_b32_e32 v3, v211328; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11329; GFX90A-NEXT: s_cbranch_execnz .LBB141_111330; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11331; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11332; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211333; GFX90A-NEXT: ;;#ASMSTART11334; GFX90A-NEXT: ; use a011335; GFX90A-NEXT: ;;#ASMEND11336; GFX90A-NEXT: s_setpc_b64 s[30:31]11337;11338; GFX950-LABEL: flat_atomic_fsub_v2f16_ret_a_a:11339; GFX950: ; %bb.0:11340; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11341; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011342; GFX950-NEXT: ;;#ASMSTART11343; GFX950-NEXT: ; def a011344; GFX950-NEXT: ;;#ASMEND11345; GFX950-NEXT: s_mov_b64 s[0:1], 011346; GFX950-NEXT: v_accvgpr_read_b32 v4, a011347; GFX950-NEXT: .LBB141_1: ; %atomicrmw.start11348; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111349; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11350; GFX950-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]11351; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011352; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11353; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311354; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11355; GFX950-NEXT: v_mov_b32_e32 v3, v211356; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11357; GFX950-NEXT: s_cbranch_execnz .LBB141_111358; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11359; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11360; GFX950-NEXT: v_accvgpr_write_b32 a0, v211361; GFX950-NEXT: ;;#ASMSTART11362; GFX950-NEXT: ; use a011363; GFX950-NEXT: ;;#ASMEND11364; GFX950-NEXT: s_setpc_b64 s[30:31]11365 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011366 %data = call <2 x half> asm "; def $0", "=a"()11367 %result = atomicrmw fsub ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011368 call void asm "; use $0", "a"(<2 x half> %result)11369 ret void11370}11371 11372define void @flat_atomic_fsub_v2f16_ret_av_av(ptr %ptr) #0 {11373; GFX90A-LABEL: flat_atomic_fsub_v2f16_ret_av_av:11374; GFX90A: ; %bb.0:11375; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11376; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011377; GFX90A-NEXT: s_mov_b64 s[4:5], 011378; GFX90A-NEXT: ;;#ASMSTART11379; GFX90A-NEXT: ; def v411380; GFX90A-NEXT: ;;#ASMEND11381; GFX90A-NEXT: .LBB142_1: ; %atomicrmw.start11382; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111383; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11384; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]11385; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11386; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11387; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311388; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11389; GFX90A-NEXT: v_mov_b32_e32 v3, v211390; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11391; GFX90A-NEXT: s_cbranch_execnz .LBB142_111392; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11393; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11394; GFX90A-NEXT: ;;#ASMSTART11395; GFX90A-NEXT: ; use v211396; GFX90A-NEXT: ;;#ASMEND11397; GFX90A-NEXT: s_setpc_b64 s[30:31]11398;11399; GFX950-LABEL: flat_atomic_fsub_v2f16_ret_av_av:11400; GFX950: ; %bb.0:11401; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11402; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011403; GFX950-NEXT: s_mov_b64 s[0:1], 011404; GFX950-NEXT: ;;#ASMSTART11405; GFX950-NEXT: ; def v411406; GFX950-NEXT: ;;#ASMEND11407; GFX950-NEXT: .LBB142_1: ; %atomicrmw.start11408; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111409; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11410; GFX950-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]11411; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011412; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11413; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311414; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11415; GFX950-NEXT: v_mov_b32_e32 v3, v211416; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11417; GFX950-NEXT: s_cbranch_execnz .LBB142_111418; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11419; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11420; GFX950-NEXT: ;;#ASMSTART11421; GFX950-NEXT: ; use v211422; GFX950-NEXT: ;;#ASMEND11423; GFX950-NEXT: s_setpc_b64 s[30:31]11424 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011425 %data = call <2 x half> asm "; def $0", "=^VA"()11426 %result = atomicrmw fsub ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011427 call void asm "; use $0", "^VA"(<2 x half> %result)11428 ret void11429}11430 11431define void @flat_atomic_fmax_v2f16_ret_a_a(ptr %ptr) #0 {11432; GFX90A-LABEL: flat_atomic_fmax_v2f16_ret_a_a:11433; GFX90A: ; %bb.0:11434; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11435; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011436; GFX90A-NEXT: ;;#ASMSTART11437; GFX90A-NEXT: ; def a011438; GFX90A-NEXT: ;;#ASMEND11439; GFX90A-NEXT: v_accvgpr_read_b32 v2, a011440; GFX90A-NEXT: s_mov_b64 s[4:5], 011441; GFX90A-NEXT: v_pk_max_f16 v4, v2, v211442; GFX90A-NEXT: .LBB143_1: ; %atomicrmw.start11443; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111444; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11445; GFX90A-NEXT: v_pk_max_f16 v2, v3, v311446; GFX90A-NEXT: v_pk_max_f16 v2, v2, v411447; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11448; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11449; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311450; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11451; GFX90A-NEXT: v_mov_b32_e32 v3, v211452; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11453; GFX90A-NEXT: s_cbranch_execnz .LBB143_111454; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11455; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11456; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211457; GFX90A-NEXT: ;;#ASMSTART11458; GFX90A-NEXT: ; use a011459; GFX90A-NEXT: ;;#ASMEND11460; GFX90A-NEXT: s_setpc_b64 s[30:31]11461;11462; GFX950-LABEL: flat_atomic_fmax_v2f16_ret_a_a:11463; GFX950: ; %bb.0:11464; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11465; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011466; GFX950-NEXT: ;;#ASMSTART11467; GFX950-NEXT: ; def a011468; GFX950-NEXT: ;;#ASMEND11469; GFX950-NEXT: s_mov_b64 s[0:1], 011470; GFX950-NEXT: v_accvgpr_read_b32 v2, a011471; GFX950-NEXT: v_pk_max_f16 v4, v2, v211472; GFX950-NEXT: .LBB143_1: ; %atomicrmw.start11473; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111474; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11475; GFX950-NEXT: v_pk_max_f16 v2, v3, v311476; GFX950-NEXT: s_nop 011477; GFX950-NEXT: v_pk_max_f16 v2, v2, v411478; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011479; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11480; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311481; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11482; GFX950-NEXT: v_mov_b32_e32 v3, v211483; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11484; GFX950-NEXT: s_cbranch_execnz .LBB143_111485; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11486; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11487; GFX950-NEXT: v_accvgpr_write_b32 a0, v211488; GFX950-NEXT: ;;#ASMSTART11489; GFX950-NEXT: ; use a011490; GFX950-NEXT: ;;#ASMEND11491; GFX950-NEXT: s_setpc_b64 s[30:31]11492 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011493 %data = call <2 x half> asm "; def $0", "=a"()11494 %result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011495 call void asm "; use $0", "a"(<2 x half> %result)11496 ret void11497}11498 11499define void @flat_atomic_fmax_v2f16_ret_av_av(ptr %ptr) #0 {11500; GFX90A-LABEL: flat_atomic_fmax_v2f16_ret_av_av:11501; GFX90A: ; %bb.0:11502; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11503; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011504; GFX90A-NEXT: ;;#ASMSTART11505; GFX90A-NEXT: ; def v211506; GFX90A-NEXT: ;;#ASMEND11507; GFX90A-NEXT: s_mov_b64 s[4:5], 011508; GFX90A-NEXT: v_pk_max_f16 v4, v2, v211509; GFX90A-NEXT: .LBB144_1: ; %atomicrmw.start11510; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111511; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11512; GFX90A-NEXT: v_pk_max_f16 v2, v3, v311513; GFX90A-NEXT: v_pk_max_f16 v2, v2, v411514; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11515; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11516; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311517; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11518; GFX90A-NEXT: v_mov_b32_e32 v3, v211519; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11520; GFX90A-NEXT: s_cbranch_execnz .LBB144_111521; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11522; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11523; GFX90A-NEXT: ;;#ASMSTART11524; GFX90A-NEXT: ; use v211525; GFX90A-NEXT: ;;#ASMEND11526; GFX90A-NEXT: s_setpc_b64 s[30:31]11527;11528; GFX950-LABEL: flat_atomic_fmax_v2f16_ret_av_av:11529; GFX950: ; %bb.0:11530; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11531; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011532; GFX950-NEXT: ;;#ASMSTART11533; GFX950-NEXT: ; def v211534; GFX950-NEXT: ;;#ASMEND11535; GFX950-NEXT: s_mov_b64 s[0:1], 011536; GFX950-NEXT: v_pk_max_f16 v4, v2, v211537; GFX950-NEXT: .LBB144_1: ; %atomicrmw.start11538; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111539; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11540; GFX950-NEXT: v_pk_max_f16 v2, v3, v311541; GFX950-NEXT: s_nop 011542; GFX950-NEXT: v_pk_max_f16 v2, v2, v411543; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011544; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11545; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311546; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11547; GFX950-NEXT: v_mov_b32_e32 v3, v211548; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11549; GFX950-NEXT: s_cbranch_execnz .LBB144_111550; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11551; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11552; GFX950-NEXT: ;;#ASMSTART11553; GFX950-NEXT: ; use v211554; GFX950-NEXT: ;;#ASMEND11555; GFX950-NEXT: s_setpc_b64 s[30:31]11556 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011557 %data = call <2 x half> asm "; def $0", "=^VA"()11558 %result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011559 call void asm "; use $0", "^VA"(<2 x half> %result)11560 ret void11561}11562 11563define void @flat_atomic_fmin_v2f16_ret_a_a(ptr %ptr) #0 {11564; GFX90A-LABEL: flat_atomic_fmin_v2f16_ret_a_a:11565; GFX90A: ; %bb.0:11566; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11567; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011568; GFX90A-NEXT: ;;#ASMSTART11569; GFX90A-NEXT: ; def a011570; GFX90A-NEXT: ;;#ASMEND11571; GFX90A-NEXT: v_accvgpr_read_b32 v2, a011572; GFX90A-NEXT: s_mov_b64 s[4:5], 011573; GFX90A-NEXT: v_pk_max_f16 v4, v2, v211574; GFX90A-NEXT: .LBB145_1: ; %atomicrmw.start11575; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111576; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11577; GFX90A-NEXT: v_pk_max_f16 v2, v3, v311578; GFX90A-NEXT: v_pk_min_f16 v2, v2, v411579; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11580; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11581; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311582; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11583; GFX90A-NEXT: v_mov_b32_e32 v3, v211584; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11585; GFX90A-NEXT: s_cbranch_execnz .LBB145_111586; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11587; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11588; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211589; GFX90A-NEXT: ;;#ASMSTART11590; GFX90A-NEXT: ; use a011591; GFX90A-NEXT: ;;#ASMEND11592; GFX90A-NEXT: s_setpc_b64 s[30:31]11593;11594; GFX950-LABEL: flat_atomic_fmin_v2f16_ret_a_a:11595; GFX950: ; %bb.0:11596; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11597; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011598; GFX950-NEXT: ;;#ASMSTART11599; GFX950-NEXT: ; def a011600; GFX950-NEXT: ;;#ASMEND11601; GFX950-NEXT: s_mov_b64 s[0:1], 011602; GFX950-NEXT: v_accvgpr_read_b32 v2, a011603; GFX950-NEXT: v_pk_max_f16 v4, v2, v211604; GFX950-NEXT: .LBB145_1: ; %atomicrmw.start11605; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111606; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11607; GFX950-NEXT: v_pk_max_f16 v2, v3, v311608; GFX950-NEXT: s_nop 011609; GFX950-NEXT: v_pk_min_f16 v2, v2, v411610; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011611; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11612; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311613; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11614; GFX950-NEXT: v_mov_b32_e32 v3, v211615; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11616; GFX950-NEXT: s_cbranch_execnz .LBB145_111617; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11618; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11619; GFX950-NEXT: v_accvgpr_write_b32 a0, v211620; GFX950-NEXT: ;;#ASMSTART11621; GFX950-NEXT: ; use a011622; GFX950-NEXT: ;;#ASMEND11623; GFX950-NEXT: s_setpc_b64 s[30:31]11624 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011625 %data = call <2 x half> asm "; def $0", "=a"()11626 %result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011627 call void asm "; use $0", "a"(<2 x half> %result)11628 ret void11629}11630 11631define void @flat_atomic_fmin_v2f16_ret_av_av(ptr %ptr) #0 {11632; GFX90A-LABEL: flat_atomic_fmin_v2f16_ret_av_av:11633; GFX90A: ; %bb.0:11634; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11635; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011636; GFX90A-NEXT: ;;#ASMSTART11637; GFX90A-NEXT: ; def v211638; GFX90A-NEXT: ;;#ASMEND11639; GFX90A-NEXT: s_mov_b64 s[4:5], 011640; GFX90A-NEXT: v_pk_max_f16 v4, v2, v211641; GFX90A-NEXT: .LBB146_1: ; %atomicrmw.start11642; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111643; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11644; GFX90A-NEXT: v_pk_max_f16 v2, v3, v311645; GFX90A-NEXT: v_pk_min_f16 v2, v2, v411646; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11647; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11648; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311649; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11650; GFX90A-NEXT: v_mov_b32_e32 v3, v211651; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]11652; GFX90A-NEXT: s_cbranch_execnz .LBB146_111653; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11654; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]11655; GFX90A-NEXT: ;;#ASMSTART11656; GFX90A-NEXT: ; use v211657; GFX90A-NEXT: ;;#ASMEND11658; GFX90A-NEXT: s_setpc_b64 s[30:31]11659;11660; GFX950-LABEL: flat_atomic_fmin_v2f16_ret_av_av:11661; GFX950: ; %bb.0:11662; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11663; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011664; GFX950-NEXT: ;;#ASMSTART11665; GFX950-NEXT: ; def v211666; GFX950-NEXT: ;;#ASMEND11667; GFX950-NEXT: s_mov_b64 s[0:1], 011668; GFX950-NEXT: v_pk_max_f16 v4, v2, v211669; GFX950-NEXT: .LBB146_1: ; %atomicrmw.start11670; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111671; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11672; GFX950-NEXT: v_pk_max_f16 v2, v3, v311673; GFX950-NEXT: s_nop 011674; GFX950-NEXT: v_pk_min_f16 v2, v2, v411675; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011676; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11677; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311678; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11679; GFX950-NEXT: v_mov_b32_e32 v3, v211680; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11681; GFX950-NEXT: s_cbranch_execnz .LBB146_111682; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11683; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11684; GFX950-NEXT: ;;#ASMSTART11685; GFX950-NEXT: ; use v211686; GFX950-NEXT: ;;#ASMEND11687; GFX950-NEXT: s_setpc_b64 s[30:31]11688 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011689 %data = call <2 x half> asm "; def $0", "=^VA"()11690 %result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011691 call void asm "; use $0", "^VA"(<2 x half> %result)11692 ret void11693}11694 11695define void @flat_atomic_fmaximum_v2f16_ret_a_a(ptr %ptr) #0 {11696; GFX90A-LABEL: flat_atomic_fmaximum_v2f16_ret_a_a:11697; GFX90A: ; %bb.0:11698; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11699; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011700; GFX90A-NEXT: ;;#ASMSTART11701; GFX90A-NEXT: ; def a011702; GFX90A-NEXT: ;;#ASMEND11703; GFX90A-NEXT: v_accvgpr_read_b32 v4, a011704; GFX90A-NEXT: s_mov_b64 s[6:7], 011705; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0011706; GFX90A-NEXT: s_mov_b32 s8, 0x504010011707; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start11708; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111709; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11710; GFX90A-NEXT: v_pk_max_f16 v2, v3, v411711; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_111712; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v411713; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]11714; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_111715; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s811716; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11717; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11718; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311719; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11720; GFX90A-NEXT: v_mov_b32_e32 v3, v211721; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11722; GFX90A-NEXT: s_cbranch_execnz .LBB147_111723; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11724; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11725; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211726; GFX90A-NEXT: ;;#ASMSTART11727; GFX90A-NEXT: ; use a011728; GFX90A-NEXT: ;;#ASMEND11729; GFX90A-NEXT: s_setpc_b64 s[30:31]11730;11731; GFX950-LABEL: flat_atomic_fmaximum_v2f16_ret_a_a:11732; GFX950: ; %bb.0:11733; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11734; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011735; GFX950-NEXT: ;;#ASMSTART11736; GFX950-NEXT: ; def a011737; GFX950-NEXT: ;;#ASMEND11738; GFX950-NEXT: s_mov_b64 s[0:1], 011739; GFX950-NEXT: v_accvgpr_read_b32 v4, a011740; GFX950-NEXT: .LBB147_1: ; %atomicrmw.start11741; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111742; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11743; GFX950-NEXT: v_pk_maximum3_f16 v2, v3, v4, v411744; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011745; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11746; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311747; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11748; GFX950-NEXT: v_mov_b32_e32 v3, v211749; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11750; GFX950-NEXT: s_cbranch_execnz .LBB147_111751; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11752; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11753; GFX950-NEXT: v_accvgpr_write_b32 a0, v211754; GFX950-NEXT: ;;#ASMSTART11755; GFX950-NEXT: ; use a011756; GFX950-NEXT: ;;#ASMEND11757; GFX950-NEXT: s_setpc_b64 s[30:31]11758 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011759 %data = call <2 x half> asm "; def $0", "=a"()11760 %result = atomicrmw fmaximum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011761 call void asm "; use $0", "a"(<2 x half> %result)11762 ret void11763}11764 11765define void @flat_atomic_fmaximum_v2f16_ret_av_av(ptr %ptr) #0 {11766; GFX90A-LABEL: flat_atomic_fmaximum_v2f16_ret_av_av:11767; GFX90A: ; %bb.0:11768; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11769; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011770; GFX90A-NEXT: s_mov_b64 s[6:7], 011771; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0011772; GFX90A-NEXT: s_mov_b32 s8, 0x504010011773; GFX90A-NEXT: ;;#ASMSTART11774; GFX90A-NEXT: ; def v411775; GFX90A-NEXT: ;;#ASMEND11776; GFX90A-NEXT: .LBB148_1: ; %atomicrmw.start11777; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111778; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11779; GFX90A-NEXT: v_pk_max_f16 v2, v3, v411780; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_111781; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v411782; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]11783; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_111784; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s811785; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11786; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11787; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311788; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11789; GFX90A-NEXT: v_mov_b32_e32 v3, v211790; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11791; GFX90A-NEXT: s_cbranch_execnz .LBB148_111792; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11793; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11794; GFX90A-NEXT: ;;#ASMSTART11795; GFX90A-NEXT: ; use v211796; GFX90A-NEXT: ;;#ASMEND11797; GFX90A-NEXT: s_setpc_b64 s[30:31]11798;11799; GFX950-LABEL: flat_atomic_fmaximum_v2f16_ret_av_av:11800; GFX950: ; %bb.0:11801; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11802; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011803; GFX950-NEXT: s_mov_b64 s[0:1], 011804; GFX950-NEXT: ;;#ASMSTART11805; GFX950-NEXT: ; def v411806; GFX950-NEXT: ;;#ASMEND11807; GFX950-NEXT: .LBB148_1: ; %atomicrmw.start11808; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111809; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11810; GFX950-NEXT: v_pk_maximum3_f16 v2, v3, v4, v411811; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011812; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11813; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311814; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11815; GFX950-NEXT: v_mov_b32_e32 v3, v211816; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11817; GFX950-NEXT: s_cbranch_execnz .LBB148_111818; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11819; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11820; GFX950-NEXT: ;;#ASMSTART11821; GFX950-NEXT: ; use v211822; GFX950-NEXT: ;;#ASMEND11823; GFX950-NEXT: s_setpc_b64 s[30:31]11824 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011825 %data = call <2 x half> asm "; def $0", "=^VA"()11826 %result = atomicrmw fmaximum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011827 call void asm "; use $0", "^VA"(<2 x half> %result)11828 ret void11829}11830 11831define void @flat_atomic_fminimum_v2f16_ret_a_a(ptr %ptr) #0 {11832; GFX90A-LABEL: flat_atomic_fminimum_v2f16_ret_a_a:11833; GFX90A: ; %bb.0:11834; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11835; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011836; GFX90A-NEXT: ;;#ASMSTART11837; GFX90A-NEXT: ; def a011838; GFX90A-NEXT: ;;#ASMEND11839; GFX90A-NEXT: v_accvgpr_read_b32 v4, a011840; GFX90A-NEXT: s_mov_b64 s[6:7], 011841; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0011842; GFX90A-NEXT: s_mov_b32 s8, 0x504010011843; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start11844; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111845; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11846; GFX90A-NEXT: v_pk_min_f16 v2, v3, v411847; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_111848; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v411849; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]11850; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_111851; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s811852; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11853; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11854; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311855; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11856; GFX90A-NEXT: v_mov_b32_e32 v3, v211857; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11858; GFX90A-NEXT: s_cbranch_execnz .LBB149_111859; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11860; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11861; GFX90A-NEXT: v_accvgpr_write_b32 a0, v211862; GFX90A-NEXT: ;;#ASMSTART11863; GFX90A-NEXT: ; use a011864; GFX90A-NEXT: ;;#ASMEND11865; GFX90A-NEXT: s_setpc_b64 s[30:31]11866;11867; GFX950-LABEL: flat_atomic_fminimum_v2f16_ret_a_a:11868; GFX950: ; %bb.0:11869; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11870; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011871; GFX950-NEXT: ;;#ASMSTART11872; GFX950-NEXT: ; def a011873; GFX950-NEXT: ;;#ASMEND11874; GFX950-NEXT: s_mov_b64 s[0:1], 011875; GFX950-NEXT: v_accvgpr_read_b32 v4, a011876; GFX950-NEXT: .LBB149_1: ; %atomicrmw.start11877; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111878; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11879; GFX950-NEXT: v_pk_minimum3_f16 v2, v3, v4, v411880; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011881; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11882; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311883; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11884; GFX950-NEXT: v_mov_b32_e32 v3, v211885; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11886; GFX950-NEXT: s_cbranch_execnz .LBB149_111887; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11888; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11889; GFX950-NEXT: v_accvgpr_write_b32 a0, v211890; GFX950-NEXT: ;;#ASMSTART11891; GFX950-NEXT: ; use a011892; GFX950-NEXT: ;;#ASMEND11893; GFX950-NEXT: s_setpc_b64 s[30:31]11894 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011895 %data = call <2 x half> asm "; def $0", "=a"()11896 %result = atomicrmw fminimum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011897 call void asm "; use $0", "a"(<2 x half> %result)11898 ret void11899}11900 11901define void @flat_atomic_fminimum_v2f16_ret_av_av(ptr %ptr) #0 {11902; GFX90A-LABEL: flat_atomic_fminimum_v2f16_ret_av_av:11903; GFX90A: ; %bb.0:11904; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11905; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011906; GFX90A-NEXT: s_mov_b64 s[6:7], 011907; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0011908; GFX90A-NEXT: s_mov_b32 s8, 0x504010011909; GFX90A-NEXT: ;;#ASMSTART11910; GFX90A-NEXT: ; def v411911; GFX90A-NEXT: ;;#ASMEND11912; GFX90A-NEXT: .LBB150_1: ; %atomicrmw.start11913; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111914; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11915; GFX90A-NEXT: v_pk_min_f16 v2, v3, v411916; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_111917; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v411918; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]11919; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_111920; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s811921; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc11922; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11923; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311924; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11925; GFX90A-NEXT: v_mov_b32_e32 v3, v211926; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11927; GFX90A-NEXT: s_cbranch_execnz .LBB150_111928; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11929; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11930; GFX90A-NEXT: ;;#ASMSTART11931; GFX90A-NEXT: ; use v211932; GFX90A-NEXT: ;;#ASMEND11933; GFX90A-NEXT: s_setpc_b64 s[30:31]11934;11935; GFX950-LABEL: flat_atomic_fminimum_v2f16_ret_av_av:11936; GFX950: ; %bb.0:11937; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11938; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4011939; GFX950-NEXT: s_mov_b64 s[0:1], 011940; GFX950-NEXT: ;;#ASMSTART11941; GFX950-NEXT: ; def v411942; GFX950-NEXT: ;;#ASMEND11943; GFX950-NEXT: .LBB150_1: ; %atomicrmw.start11944; GFX950-NEXT: ; =>This Inner Loop Header: Depth=111945; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11946; GFX950-NEXT: v_pk_minimum3_f16 v2, v3, v4, v411947; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc011948; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11949; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v311950; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]11951; GFX950-NEXT: v_mov_b32_e32 v3, v211952; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]11953; GFX950-NEXT: s_cbranch_execnz .LBB150_111954; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end11955; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]11956; GFX950-NEXT: ;;#ASMSTART11957; GFX950-NEXT: ; use v211958; GFX950-NEXT: ;;#ASMEND11959; GFX950-NEXT: s_setpc_b64 s[30:31]11960 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1011961 %data = call <2 x half> asm "; def $0", "=^VA"()11962 %result = atomicrmw fminimum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !011963 call void asm "; use $0", "^VA"(<2 x half> %result)11964 ret void11965}11966 11967;---------------------------------------------------------------------11968; other atomics v2bf16, with aa+av cases11969;---------------------------------------------------------------------11970 11971define void @flat_atomic_fadd_v2bf16_ret_a_a(ptr %ptr) #0 {11972; GFX90A-LABEL: flat_atomic_fadd_v2bf16_ret_a_a:11973; GFX90A: ; %bb.0:11974; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11975; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4011976; GFX90A-NEXT: ;;#ASMSTART11977; GFX90A-NEXT: ; def a011978; GFX90A-NEXT: ;;#ASMEND11979; GFX90A-NEXT: v_accvgpr_read_b32 v2, a011980; GFX90A-NEXT: s_mov_b64 s[6:7], 011981; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v211982; GFX90A-NEXT: s_movk_i32 s8, 0x7fff11983; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v211984; GFX90A-NEXT: s_mov_b32 s9, 0x706030211985; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start11986; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111987; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11988; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v311989; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v311990; GFX90A-NEXT: v_add_f32_e32 v2, v2, v411991; GFX90A-NEXT: v_add_f32_e32 v6, v6, v511992; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 111993; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 111994; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v211995; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v611996; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s811997; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s811998; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v611999; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212000; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12001; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12002; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912003; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12004; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12005; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312006; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12007; GFX90A-NEXT: v_mov_b32_e32 v3, v212008; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12009; GFX90A-NEXT: s_cbranch_execnz .LBB151_112010; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12011; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12012; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212013; GFX90A-NEXT: ;;#ASMSTART12014; GFX90A-NEXT: ; use a012015; GFX90A-NEXT: ;;#ASMEND12016; GFX90A-NEXT: s_setpc_b64 s[30:31]12017;12018; GFX950-LABEL: flat_atomic_fadd_v2bf16_ret_a_a:12019; GFX950: ; %bb.0:12020; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12021; GFX950-NEXT: ;;#ASMSTART12022; GFX950-NEXT: ; def a012023; GFX950-NEXT: ;;#ASMEND12024; GFX950-NEXT: s_nop 012025; GFX950-NEXT: v_accvgpr_read_b32 v2, a012026; GFX950-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:40 sc012027; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12028; GFX950-NEXT: v_accvgpr_write_b32 a0, v012029; GFX950-NEXT: ;;#ASMSTART12030; GFX950-NEXT: ; use a012031; GFX950-NEXT: ;;#ASMEND12032; GFX950-NEXT: s_setpc_b64 s[30:31]12033 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012034 %data = call <2 x bfloat> asm "; def $0", "=a"()12035 %result = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012036 call void asm "; use $0", "a"(<2 x bfloat> %result)12037 ret void12038}12039 12040define void @flat_atomic_fadd_v2bf16_ret_av_av(ptr %ptr) #0 {12041; GFX90A-LABEL: flat_atomic_fadd_v2bf16_ret_av_av:12042; GFX90A: ; %bb.0:12043; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12044; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012045; GFX90A-NEXT: ;;#ASMSTART12046; GFX90A-NEXT: ; def v212047; GFX90A-NEXT: ;;#ASMEND12048; GFX90A-NEXT: s_mov_b64 s[6:7], 012049; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212050; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12051; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212052; GFX90A-NEXT: s_mov_b32 s9, 0x706030212053; GFX90A-NEXT: .LBB152_1: ; %atomicrmw.start12054; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112055; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12056; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312057; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312058; GFX90A-NEXT: v_add_f32_e32 v2, v2, v412059; GFX90A-NEXT: v_add_f32_e32 v6, v6, v512060; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112061; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112062; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212063; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612064; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812065; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812066; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612067; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212068; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12069; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12070; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912071; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12072; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12073; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312074; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12075; GFX90A-NEXT: v_mov_b32_e32 v3, v212076; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12077; GFX90A-NEXT: s_cbranch_execnz .LBB152_112078; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12079; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12080; GFX90A-NEXT: ;;#ASMSTART12081; GFX90A-NEXT: ; use v212082; GFX90A-NEXT: ;;#ASMEND12083; GFX90A-NEXT: s_setpc_b64 s[30:31]12084;12085; GFX950-LABEL: flat_atomic_fadd_v2bf16_ret_av_av:12086; GFX950: ; %bb.0:12087; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12088; GFX950-NEXT: ;;#ASMSTART12089; GFX950-NEXT: ; def v212090; GFX950-NEXT: ;;#ASMEND12091; GFX950-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:40 sc012092; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12093; GFX950-NEXT: ;;#ASMSTART12094; GFX950-NEXT: ; use v012095; GFX950-NEXT: ;;#ASMEND12096; GFX950-NEXT: s_setpc_b64 s[30:31]12097 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012098 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12099 %result = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012100 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12101 ret void12102}12103 12104define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {12105; GFX90A-LABEL: flat_atomic_fsub_v2bf16_ret_a_a:12106; GFX90A: ; %bb.0:12107; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12108; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012109; GFX90A-NEXT: ;;#ASMSTART12110; GFX90A-NEXT: ; def a012111; GFX90A-NEXT: ;;#ASMEND12112; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012113; GFX90A-NEXT: s_mov_b64 s[6:7], 012114; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212115; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12116; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212117; GFX90A-NEXT: s_mov_b32 s9, 0x706030212118; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start12119; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112120; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12121; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312122; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312123; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v412124; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v512125; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112126; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112127; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212128; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612129; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812130; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812131; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612132; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212133; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12134; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12135; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912136; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12137; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12138; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312139; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12140; GFX90A-NEXT: v_mov_b32_e32 v3, v212141; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12142; GFX90A-NEXT: s_cbranch_execnz .LBB153_112143; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12144; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12145; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212146; GFX90A-NEXT: ;;#ASMSTART12147; GFX90A-NEXT: ; use a012148; GFX90A-NEXT: ;;#ASMEND12149; GFX90A-NEXT: s_setpc_b64 s[30:31]12150;12151; GFX950-LABEL: flat_atomic_fsub_v2bf16_ret_a_a:12152; GFX950: ; %bb.0:12153; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12154; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012155; GFX950-NEXT: ;;#ASMSTART12156; GFX950-NEXT: ; def a012157; GFX950-NEXT: ;;#ASMEND12158; GFX950-NEXT: s_mov_b64 s[0:1], 012159; GFX950-NEXT: v_accvgpr_read_b32 v2, a012160; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212161; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212162; GFX950-NEXT: .LBB153_1: ; %atomicrmw.start12163; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112164; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12165; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312166; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312167; GFX950-NEXT: v_sub_f32_e32 v2, v2, v412168; GFX950-NEXT: v_sub_f32_e32 v6, v6, v512169; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212170; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012171; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12172; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312173; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12174; GFX950-NEXT: v_mov_b32_e32 v3, v212175; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12176; GFX950-NEXT: s_cbranch_execnz .LBB153_112177; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12178; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12179; GFX950-NEXT: v_accvgpr_write_b32 a0, v212180; GFX950-NEXT: ;;#ASMSTART12181; GFX950-NEXT: ; use a012182; GFX950-NEXT: ;;#ASMEND12183; GFX950-NEXT: s_setpc_b64 s[30:31]12184 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012185 %data = call <2 x bfloat> asm "; def $0", "=a"()12186 %result = atomicrmw fsub ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012187 call void asm "; use $0", "a"(<2 x bfloat> %result)12188 ret void12189}12190 12191define void @flat_atomic_fsub_v2bf16_ret_av_av(ptr %ptr) #0 {12192; GFX90A-LABEL: flat_atomic_fsub_v2bf16_ret_av_av:12193; GFX90A: ; %bb.0:12194; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12195; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012196; GFX90A-NEXT: ;;#ASMSTART12197; GFX90A-NEXT: ; def v212198; GFX90A-NEXT: ;;#ASMEND12199; GFX90A-NEXT: s_mov_b64 s[6:7], 012200; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212201; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12202; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212203; GFX90A-NEXT: s_mov_b32 s9, 0x706030212204; GFX90A-NEXT: .LBB154_1: ; %atomicrmw.start12205; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112206; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12207; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312208; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312209; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v412210; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v512211; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112212; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112213; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212214; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612215; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812216; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812217; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612218; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212219; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12220; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12221; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912222; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12223; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12224; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312225; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12226; GFX90A-NEXT: v_mov_b32_e32 v3, v212227; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12228; GFX90A-NEXT: s_cbranch_execnz .LBB154_112229; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12230; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12231; GFX90A-NEXT: ;;#ASMSTART12232; GFX90A-NEXT: ; use v212233; GFX90A-NEXT: ;;#ASMEND12234; GFX90A-NEXT: s_setpc_b64 s[30:31]12235;12236; GFX950-LABEL: flat_atomic_fsub_v2bf16_ret_av_av:12237; GFX950: ; %bb.0:12238; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12239; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012240; GFX950-NEXT: ;;#ASMSTART12241; GFX950-NEXT: ; def v212242; GFX950-NEXT: ;;#ASMEND12243; GFX950-NEXT: s_mov_b64 s[0:1], 012244; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212245; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212246; GFX950-NEXT: .LBB154_1: ; %atomicrmw.start12247; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112248; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12249; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312250; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312251; GFX950-NEXT: v_sub_f32_e32 v2, v2, v412252; GFX950-NEXT: v_sub_f32_e32 v6, v6, v512253; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212254; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012255; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12256; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312257; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12258; GFX950-NEXT: v_mov_b32_e32 v3, v212259; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12260; GFX950-NEXT: s_cbranch_execnz .LBB154_112261; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12262; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12263; GFX950-NEXT: ;;#ASMSTART12264; GFX950-NEXT: ; use v212265; GFX950-NEXT: ;;#ASMEND12266; GFX950-NEXT: s_setpc_b64 s[30:31]12267 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012268 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12269 %result = atomicrmw fsub ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012270 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12271 ret void12272}12273 12274define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {12275; GFX90A-LABEL: flat_atomic_fmax_v2bf16_ret_a_a:12276; GFX90A: ; %bb.0:12277; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12278; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012279; GFX90A-NEXT: ;;#ASMSTART12280; GFX90A-NEXT: ; def a012281; GFX90A-NEXT: ;;#ASMEND12282; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012283; GFX90A-NEXT: s_mov_b64 s[6:7], 012284; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212285; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12286; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212287; GFX90A-NEXT: s_mov_b32 s9, 0x706030212288; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start12289; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112290; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12291; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312292; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312293; GFX90A-NEXT: v_max_f32_e32 v2, v2, v412294; GFX90A-NEXT: v_max_f32_e32 v6, v6, v512295; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112296; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112297; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212298; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612299; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812300; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812301; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612302; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212303; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12304; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12305; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912306; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12307; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12308; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312309; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12310; GFX90A-NEXT: v_mov_b32_e32 v3, v212311; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12312; GFX90A-NEXT: s_cbranch_execnz .LBB155_112313; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12314; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12315; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212316; GFX90A-NEXT: ;;#ASMSTART12317; GFX90A-NEXT: ; use a012318; GFX90A-NEXT: ;;#ASMEND12319; GFX90A-NEXT: s_setpc_b64 s[30:31]12320;12321; GFX950-LABEL: flat_atomic_fmax_v2bf16_ret_a_a:12322; GFX950: ; %bb.0:12323; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12324; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012325; GFX950-NEXT: ;;#ASMSTART12326; GFX950-NEXT: ; def a012327; GFX950-NEXT: ;;#ASMEND12328; GFX950-NEXT: s_mov_b64 s[0:1], 012329; GFX950-NEXT: v_accvgpr_read_b32 v2, a012330; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212331; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212332; GFX950-NEXT: .LBB155_1: ; %atomicrmw.start12333; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112334; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12335; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312336; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312337; GFX950-NEXT: v_max_f32_e32 v2, v2, v412338; GFX950-NEXT: v_max_f32_e32 v6, v6, v512339; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212340; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012341; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12342; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312343; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12344; GFX950-NEXT: v_mov_b32_e32 v3, v212345; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12346; GFX950-NEXT: s_cbranch_execnz .LBB155_112347; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12348; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12349; GFX950-NEXT: v_accvgpr_write_b32 a0, v212350; GFX950-NEXT: ;;#ASMSTART12351; GFX950-NEXT: ; use a012352; GFX950-NEXT: ;;#ASMEND12353; GFX950-NEXT: s_setpc_b64 s[30:31]12354 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012355 %data = call <2 x bfloat> asm "; def $0", "=a"()12356 %result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012357 call void asm "; use $0", "a"(<2 x bfloat> %result)12358 ret void12359}12360 12361define void @flat_atomic_fmax_v2bf16_ret_av_av(ptr %ptr) #0 {12362; GFX90A-LABEL: flat_atomic_fmax_v2bf16_ret_av_av:12363; GFX90A: ; %bb.0:12364; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12365; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012366; GFX90A-NEXT: ;;#ASMSTART12367; GFX90A-NEXT: ; def v212368; GFX90A-NEXT: ;;#ASMEND12369; GFX90A-NEXT: s_mov_b64 s[6:7], 012370; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212371; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12372; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212373; GFX90A-NEXT: s_mov_b32 s9, 0x706030212374; GFX90A-NEXT: .LBB156_1: ; %atomicrmw.start12375; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112376; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12377; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312378; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312379; GFX90A-NEXT: v_max_f32_e32 v2, v2, v412380; GFX90A-NEXT: v_max_f32_e32 v6, v6, v512381; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112382; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112383; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212384; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612385; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812386; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812387; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612388; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212389; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12390; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12391; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912392; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12393; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12394; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312395; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12396; GFX90A-NEXT: v_mov_b32_e32 v3, v212397; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12398; GFX90A-NEXT: s_cbranch_execnz .LBB156_112399; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12400; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12401; GFX90A-NEXT: ;;#ASMSTART12402; GFX90A-NEXT: ; use v212403; GFX90A-NEXT: ;;#ASMEND12404; GFX90A-NEXT: s_setpc_b64 s[30:31]12405;12406; GFX950-LABEL: flat_atomic_fmax_v2bf16_ret_av_av:12407; GFX950: ; %bb.0:12408; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12409; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012410; GFX950-NEXT: ;;#ASMSTART12411; GFX950-NEXT: ; def v212412; GFX950-NEXT: ;;#ASMEND12413; GFX950-NEXT: s_mov_b64 s[0:1], 012414; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212415; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212416; GFX950-NEXT: .LBB156_1: ; %atomicrmw.start12417; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112418; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12419; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312420; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312421; GFX950-NEXT: v_max_f32_e32 v2, v2, v412422; GFX950-NEXT: v_max_f32_e32 v6, v6, v512423; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212424; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012425; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12426; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312427; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12428; GFX950-NEXT: v_mov_b32_e32 v3, v212429; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12430; GFX950-NEXT: s_cbranch_execnz .LBB156_112431; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12432; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12433; GFX950-NEXT: ;;#ASMSTART12434; GFX950-NEXT: ; use v212435; GFX950-NEXT: ;;#ASMEND12436; GFX950-NEXT: s_setpc_b64 s[30:31]12437 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012438 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12439 %result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012440 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12441 ret void12442}12443 12444define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {12445; GFX90A-LABEL: flat_atomic_fmin_v2bf16_ret_a_a:12446; GFX90A: ; %bb.0:12447; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12448; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012449; GFX90A-NEXT: ;;#ASMSTART12450; GFX90A-NEXT: ; def a012451; GFX90A-NEXT: ;;#ASMEND12452; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012453; GFX90A-NEXT: s_mov_b64 s[6:7], 012454; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212455; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12456; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212457; GFX90A-NEXT: s_mov_b32 s9, 0x706030212458; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start12459; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112460; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12461; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312462; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312463; GFX90A-NEXT: v_min_f32_e32 v2, v2, v412464; GFX90A-NEXT: v_min_f32_e32 v6, v6, v512465; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112466; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112467; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212468; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612469; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812470; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812471; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612472; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212473; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12474; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12475; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912476; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12477; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12478; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312479; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12480; GFX90A-NEXT: v_mov_b32_e32 v3, v212481; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12482; GFX90A-NEXT: s_cbranch_execnz .LBB157_112483; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12484; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12485; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212486; GFX90A-NEXT: ;;#ASMSTART12487; GFX90A-NEXT: ; use a012488; GFX90A-NEXT: ;;#ASMEND12489; GFX90A-NEXT: s_setpc_b64 s[30:31]12490;12491; GFX950-LABEL: flat_atomic_fmin_v2bf16_ret_a_a:12492; GFX950: ; %bb.0:12493; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12494; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012495; GFX950-NEXT: ;;#ASMSTART12496; GFX950-NEXT: ; def a012497; GFX950-NEXT: ;;#ASMEND12498; GFX950-NEXT: s_mov_b64 s[0:1], 012499; GFX950-NEXT: v_accvgpr_read_b32 v2, a012500; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212501; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212502; GFX950-NEXT: .LBB157_1: ; %atomicrmw.start12503; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112504; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12505; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312506; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312507; GFX950-NEXT: v_min_f32_e32 v2, v2, v412508; GFX950-NEXT: v_min_f32_e32 v6, v6, v512509; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212510; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012511; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12512; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312513; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12514; GFX950-NEXT: v_mov_b32_e32 v3, v212515; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12516; GFX950-NEXT: s_cbranch_execnz .LBB157_112517; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12518; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12519; GFX950-NEXT: v_accvgpr_write_b32 a0, v212520; GFX950-NEXT: ;;#ASMSTART12521; GFX950-NEXT: ; use a012522; GFX950-NEXT: ;;#ASMEND12523; GFX950-NEXT: s_setpc_b64 s[30:31]12524 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012525 %data = call <2 x bfloat> asm "; def $0", "=a"()12526 %result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012527 call void asm "; use $0", "a"(<2 x bfloat> %result)12528 ret void12529}12530 12531define void @flat_atomic_fmin_v2bf16_ret_av_av(ptr %ptr) #0 {12532; GFX90A-LABEL: flat_atomic_fmin_v2bf16_ret_av_av:12533; GFX90A: ; %bb.0:12534; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12535; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012536; GFX90A-NEXT: ;;#ASMSTART12537; GFX90A-NEXT: ; def v212538; GFX90A-NEXT: ;;#ASMEND12539; GFX90A-NEXT: s_mov_b64 s[6:7], 012540; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212541; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12542; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v212543; GFX90A-NEXT: s_mov_b32 s9, 0x706030212544; GFX90A-NEXT: .LBB158_1: ; %atomicrmw.start12545; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112546; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12547; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312548; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v312549; GFX90A-NEXT: v_min_f32_e32 v2, v2, v412550; GFX90A-NEXT: v_min_f32_e32 v6, v6, v512551; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 112552; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 112553; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v212554; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v612555; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s812556; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s812557; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v612558; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212559; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]12560; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc12561; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s912562; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12563; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12564; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312565; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12566; GFX90A-NEXT: v_mov_b32_e32 v3, v212567; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12568; GFX90A-NEXT: s_cbranch_execnz .LBB158_112569; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12570; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12571; GFX90A-NEXT: ;;#ASMSTART12572; GFX90A-NEXT: ; use v212573; GFX90A-NEXT: ;;#ASMEND12574; GFX90A-NEXT: s_setpc_b64 s[30:31]12575;12576; GFX950-LABEL: flat_atomic_fmin_v2bf16_ret_av_av:12577; GFX950: ; %bb.0:12578; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12579; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012580; GFX950-NEXT: ;;#ASMSTART12581; GFX950-NEXT: ; def v212582; GFX950-NEXT: ;;#ASMEND12583; GFX950-NEXT: s_mov_b64 s[0:1], 012584; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212585; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212586; GFX950-NEXT: .LBB158_1: ; %atomicrmw.start12587; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112588; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12589; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312590; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312591; GFX950-NEXT: v_min_f32_e32 v2, v2, v412592; GFX950-NEXT: v_min_f32_e32 v6, v6, v512593; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212594; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012595; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12596; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312597; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12598; GFX950-NEXT: v_mov_b32_e32 v3, v212599; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12600; GFX950-NEXT: s_cbranch_execnz .LBB158_112601; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12602; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12603; GFX950-NEXT: ;;#ASMSTART12604; GFX950-NEXT: ; use v212605; GFX950-NEXT: ;;#ASMEND12606; GFX950-NEXT: s_setpc_b64 s[30:31]12607 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012608 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12609 %result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012610 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12611 ret void12612}12613 12614define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {12615; GFX90A-LABEL: flat_atomic_fmaximum_v2bf16_ret_a_a:12616; GFX90A: ; %bb.0:12617; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12618; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012619; GFX90A-NEXT: ;;#ASMSTART12620; GFX90A-NEXT: ; def a012621; GFX90A-NEXT: ;;#ASMEND12622; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012623; GFX90A-NEXT: s_mov_b64 s[6:7], 012624; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212625; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000012626; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12627; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v212628; GFX90A-NEXT: s_mov_b32 s9, 0x706030212629; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start12630; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112631; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12632; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312633; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v312634; GFX90A-NEXT: v_max_f32_e32 v8, v2, v412635; GFX90A-NEXT: v_max_f32_e32 v9, v7, v612636; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v612637; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v412638; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]12639; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc12640; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 112641; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 112642; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v212643; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v712644; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s812645; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s812646; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v712647; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212648; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]12649; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc12650; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s912651; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12652; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12653; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312654; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12655; GFX90A-NEXT: v_mov_b32_e32 v3, v212656; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12657; GFX90A-NEXT: s_cbranch_execnz .LBB159_112658; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12659; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12660; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212661; GFX90A-NEXT: ;;#ASMSTART12662; GFX90A-NEXT: ; use a012663; GFX90A-NEXT: ;;#ASMEND12664; GFX90A-NEXT: s_setpc_b64 s[30:31]12665;12666; GFX950-LABEL: flat_atomic_fmaximum_v2bf16_ret_a_a:12667; GFX950: ; %bb.0:12668; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12669; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012670; GFX950-NEXT: ;;#ASMSTART12671; GFX950-NEXT: ; def a012672; GFX950-NEXT: ;;#ASMEND12673; GFX950-NEXT: s_mov_b64 s[0:1], 012674; GFX950-NEXT: v_accvgpr_read_b32 v2, a012675; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212676; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212677; GFX950-NEXT: .LBB159_1: ; %atomicrmw.start12678; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112679; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12680; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312681; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312682; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v412683; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v512684; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212685; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012686; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12687; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312688; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12689; GFX950-NEXT: v_mov_b32_e32 v3, v212690; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12691; GFX950-NEXT: s_cbranch_execnz .LBB159_112692; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12693; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12694; GFX950-NEXT: v_accvgpr_write_b32 a0, v212695; GFX950-NEXT: ;;#ASMSTART12696; GFX950-NEXT: ; use a012697; GFX950-NEXT: ;;#ASMEND12698; GFX950-NEXT: s_setpc_b64 s[30:31]12699 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012700 %data = call <2 x bfloat> asm "; def $0", "=a"()12701 %result = atomicrmw fmaximum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012702 call void asm "; use $0", "a"(<2 x bfloat> %result)12703 ret void12704}12705 12706define void @flat_atomic_fmaximum_v2bf16_ret_av_av(ptr %ptr) #0 {12707; GFX90A-LABEL: flat_atomic_fmaximum_v2bf16_ret_av_av:12708; GFX90A: ; %bb.0:12709; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12710; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012711; GFX90A-NEXT: ;;#ASMSTART12712; GFX90A-NEXT: ; def v212713; GFX90A-NEXT: ;;#ASMEND12714; GFX90A-NEXT: s_mov_b64 s[6:7], 012715; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212716; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000012717; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12718; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v212719; GFX90A-NEXT: s_mov_b32 s9, 0x706030212720; GFX90A-NEXT: .LBB160_1: ; %atomicrmw.start12721; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112722; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12723; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312724; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v312725; GFX90A-NEXT: v_max_f32_e32 v8, v2, v412726; GFX90A-NEXT: v_max_f32_e32 v9, v7, v612727; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v612728; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v412729; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]12730; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc12731; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 112732; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 112733; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v212734; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v712735; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s812736; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s812737; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v712738; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212739; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]12740; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc12741; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s912742; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12743; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12744; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312745; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12746; GFX90A-NEXT: v_mov_b32_e32 v3, v212747; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12748; GFX90A-NEXT: s_cbranch_execnz .LBB160_112749; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12750; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12751; GFX90A-NEXT: ;;#ASMSTART12752; GFX90A-NEXT: ; use v212753; GFX90A-NEXT: ;;#ASMEND12754; GFX90A-NEXT: s_setpc_b64 s[30:31]12755;12756; GFX950-LABEL: flat_atomic_fmaximum_v2bf16_ret_av_av:12757; GFX950: ; %bb.0:12758; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12759; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012760; GFX950-NEXT: ;;#ASMSTART12761; GFX950-NEXT: ; def v212762; GFX950-NEXT: ;;#ASMEND12763; GFX950-NEXT: s_mov_b64 s[0:1], 012764; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212765; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212766; GFX950-NEXT: .LBB160_1: ; %atomicrmw.start12767; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112768; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12769; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312770; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312771; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v412772; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v512773; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212774; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012775; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12776; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312777; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12778; GFX950-NEXT: v_mov_b32_e32 v3, v212779; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12780; GFX950-NEXT: s_cbranch_execnz .LBB160_112781; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12782; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12783; GFX950-NEXT: ;;#ASMSTART12784; GFX950-NEXT: ; use v212785; GFX950-NEXT: ;;#ASMEND12786; GFX950-NEXT: s_setpc_b64 s[30:31]12787 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012788 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12789 %result = atomicrmw fmaximum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012790 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12791 ret void12792}12793 12794define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {12795; GFX90A-LABEL: flat_atomic_fminimum_v2bf16_ret_a_a:12796; GFX90A: ; %bb.0:12797; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12798; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012799; GFX90A-NEXT: ;;#ASMSTART12800; GFX90A-NEXT: ; def a012801; GFX90A-NEXT: ;;#ASMEND12802; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012803; GFX90A-NEXT: s_mov_b64 s[6:7], 012804; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212805; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000012806; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12807; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v212808; GFX90A-NEXT: s_mov_b32 s9, 0x706030212809; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start12810; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112811; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12812; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312813; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v312814; GFX90A-NEXT: v_min_f32_e32 v8, v2, v412815; GFX90A-NEXT: v_min_f32_e32 v9, v7, v612816; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v612817; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v412818; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]12819; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc12820; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 112821; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 112822; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v212823; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v712824; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s812825; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s812826; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v712827; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212828; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]12829; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc12830; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s912831; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12832; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12833; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312834; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12835; GFX90A-NEXT: v_mov_b32_e32 v3, v212836; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12837; GFX90A-NEXT: s_cbranch_execnz .LBB161_112838; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12839; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12840; GFX90A-NEXT: v_accvgpr_write_b32 a0, v212841; GFX90A-NEXT: ;;#ASMSTART12842; GFX90A-NEXT: ; use a012843; GFX90A-NEXT: ;;#ASMEND12844; GFX90A-NEXT: s_setpc_b64 s[30:31]12845;12846; GFX950-LABEL: flat_atomic_fminimum_v2bf16_ret_a_a:12847; GFX950: ; %bb.0:12848; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12849; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012850; GFX950-NEXT: ;;#ASMSTART12851; GFX950-NEXT: ; def a012852; GFX950-NEXT: ;;#ASMEND12853; GFX950-NEXT: s_mov_b64 s[0:1], 012854; GFX950-NEXT: v_accvgpr_read_b32 v2, a012855; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212856; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212857; GFX950-NEXT: .LBB161_1: ; %atomicrmw.start12858; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112859; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12860; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312861; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312862; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v412863; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v512864; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212865; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012866; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12867; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312868; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12869; GFX950-NEXT: v_mov_b32_e32 v3, v212870; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12871; GFX950-NEXT: s_cbranch_execnz .LBB161_112872; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12873; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12874; GFX950-NEXT: v_accvgpr_write_b32 a0, v212875; GFX950-NEXT: ;;#ASMSTART12876; GFX950-NEXT: ; use a012877; GFX950-NEXT: ;;#ASMEND12878; GFX950-NEXT: s_setpc_b64 s[30:31]12879 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012880 %data = call <2 x bfloat> asm "; def $0", "=a"()12881 %result = atomicrmw fminimum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012882 call void asm "; use $0", "a"(<2 x bfloat> %result)12883 ret void12884}12885 12886define void @flat_atomic_fminimum_v2bf16_ret_av_av(ptr %ptr) #0 {12887; GFX90A-LABEL: flat_atomic_fminimum_v2bf16_ret_av_av:12888; GFX90A: ; %bb.0:12889; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12890; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:4012891; GFX90A-NEXT: ;;#ASMSTART12892; GFX90A-NEXT: ; def v212893; GFX90A-NEXT: ;;#ASMEND12894; GFX90A-NEXT: s_mov_b64 s[6:7], 012895; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v212896; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000012897; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12898; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v212899; GFX90A-NEXT: s_mov_b32 s9, 0x706030212900; GFX90A-NEXT: .LBB162_1: ; %atomicrmw.start12901; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112902; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12903; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v312904; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v312905; GFX90A-NEXT: v_min_f32_e32 v8, v2, v412906; GFX90A-NEXT: v_min_f32_e32 v9, v7, v612907; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v612908; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v412909; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]12910; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc12911; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 112912; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 112913; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v212914; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v712915; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s812916; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s812917; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v712918; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v212919; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]12920; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc12921; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s912922; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc12923; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12924; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312925; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12926; GFX90A-NEXT: v_mov_b32_e32 v3, v212927; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12928; GFX90A-NEXT: s_cbranch_execnz .LBB162_112929; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12930; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12931; GFX90A-NEXT: ;;#ASMSTART12932; GFX90A-NEXT: ; use v212933; GFX90A-NEXT: ;;#ASMEND12934; GFX90A-NEXT: s_setpc_b64 s[30:31]12935;12936; GFX950-LABEL: flat_atomic_fminimum_v2bf16_ret_av_av:12937; GFX950: ; %bb.0:12938; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12939; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:4012940; GFX950-NEXT: ;;#ASMSTART12941; GFX950-NEXT: ; def v212942; GFX950-NEXT: ;;#ASMEND12943; GFX950-NEXT: s_mov_b64 s[0:1], 012944; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v212945; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v212946; GFX950-NEXT: .LBB162_1: ; %atomicrmw.start12947; GFX950-NEXT: ; =>This Inner Loop Header: Depth=112948; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12949; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v312950; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v312951; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v412952; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v512953; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v212954; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc012955; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12956; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v312957; GFX950-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12958; GFX950-NEXT: v_mov_b32_e32 v3, v212959; GFX950-NEXT: s_andn2_b64 exec, exec, s[0:1]12960; GFX950-NEXT: s_cbranch_execnz .LBB162_112961; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end12962; GFX950-NEXT: s_or_b64 exec, exec, s[0:1]12963; GFX950-NEXT: ;;#ASMSTART12964; GFX950-NEXT: ; use v212965; GFX950-NEXT: ;;#ASMEND12966; GFX950-NEXT: s_setpc_b64 s[30:31]12967 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1012968 %data = call <2 x bfloat> asm "; def $0", "=^VA"()12969 %result = atomicrmw fminimum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !012970 call void asm "; use $0", "^VA"(<2 x bfloat> %result)12971 ret void12972}12973 12974;---------------------------------------------------------------------12975; other atomics i32, with aa+av cases using saddr12976;---------------------------------------------------------------------12977 12978define void @flat_atomic_xchg_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {12979; GFX90A-LABEL: flat_atomic_xchg_i32_saddr_ret_a_a:12980; GFX90A: ; %bb.0:12981; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12982; GFX90A-NEXT: v_mov_b32_e32 v0, s1612983; GFX90A-NEXT: v_mov_b32_e32 v1, s1712984; GFX90A-NEXT: ;;#ASMSTART12985; GFX90A-NEXT: ; def a012986; GFX90A-NEXT: ;;#ASMEND12987; GFX90A-NEXT: v_accvgpr_read_b32 v2, a012988; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc12989; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12990; GFX90A-NEXT: v_accvgpr_write_b32 a0, v012991; GFX90A-NEXT: ;;#ASMSTART12992; GFX90A-NEXT: ; use a012993; GFX90A-NEXT: ;;#ASMEND12994; GFX90A-NEXT: s_setpc_b64 s[30:31]12995;12996; GFX950-LABEL: flat_atomic_xchg_i32_saddr_ret_a_a:12997; GFX950: ; %bb.0:12998; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12999; GFX950-NEXT: v_mov_b32_e32 v0, s013000; GFX950-NEXT: v_mov_b32_e32 v1, s113001; GFX950-NEXT: ;;#ASMSTART13002; GFX950-NEXT: ; def a013003; GFX950-NEXT: ;;#ASMEND13004; GFX950-NEXT: s_nop 013005; GFX950-NEXT: v_accvgpr_read_b32 v2, a013006; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc013007; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13008; GFX950-NEXT: v_accvgpr_write_b32 a0, v013009; GFX950-NEXT: ;;#ASMSTART13010; GFX950-NEXT: ; use a013011; GFX950-NEXT: ;;#ASMEND13012; GFX950-NEXT: s_setpc_b64 s[30:31]13013 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013014 %data = call i32 asm "; def $0", "=a"()13015 %result = atomicrmw xchg ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013016 call void asm "; use $0", "a"(i32 %result)13017 ret void13018}13019 13020define void @flat_atomic_xchg_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13021; GFX90A-LABEL: flat_atomic_xchg_i32_saddr_ret_av_av:13022; GFX90A: ; %bb.0:13023; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13024; GFX90A-NEXT: v_mov_b32_e32 v0, s1613025; GFX90A-NEXT: v_mov_b32_e32 v1, s1713026; GFX90A-NEXT: ;;#ASMSTART13027; GFX90A-NEXT: ; def v213028; GFX90A-NEXT: ;;#ASMEND13029; GFX90A-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 glc13030; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13031; GFX90A-NEXT: ;;#ASMSTART13032; GFX90A-NEXT: ; use v013033; GFX90A-NEXT: ;;#ASMEND13034; GFX90A-NEXT: s_setpc_b64 s[30:31]13035;13036; GFX950-LABEL: flat_atomic_xchg_i32_saddr_ret_av_av:13037; GFX950: ; %bb.0:13038; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13039; GFX950-NEXT: v_mov_b32_e32 v0, s013040; GFX950-NEXT: v_mov_b32_e32 v1, s113041; GFX950-NEXT: ;;#ASMSTART13042; GFX950-NEXT: ; def v213043; GFX950-NEXT: ;;#ASMEND13044; GFX950-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:40 sc013045; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13046; GFX950-NEXT: ;;#ASMSTART13047; GFX950-NEXT: ; use v013048; GFX950-NEXT: ;;#ASMEND13049; GFX950-NEXT: s_setpc_b64 s[30:31]13050 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013051 %data = call i32 asm "; def $0", "=^VA"()13052 %result = atomicrmw xchg ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013053 call void asm "; use $0", "^VA"(i32 %result)13054 ret void13055}13056 13057define void @flat_atomic_add_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13058; GFX90A-LABEL: flat_atomic_add_i32_saddr_ret_a_a:13059; GFX90A: ; %bb.0:13060; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13061; GFX90A-NEXT: v_mov_b32_e32 v0, s1613062; GFX90A-NEXT: v_mov_b32_e32 v1, s1713063; GFX90A-NEXT: ;;#ASMSTART13064; GFX90A-NEXT: ; def a013065; GFX90A-NEXT: ;;#ASMEND13066; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013067; GFX90A-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 glc13068; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13069; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013070; GFX90A-NEXT: ;;#ASMSTART13071; GFX90A-NEXT: ; use a013072; GFX90A-NEXT: ;;#ASMEND13073; GFX90A-NEXT: s_setpc_b64 s[30:31]13074;13075; GFX950-LABEL: flat_atomic_add_i32_saddr_ret_a_a:13076; GFX950: ; %bb.0:13077; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13078; GFX950-NEXT: v_mov_b32_e32 v0, s013079; GFX950-NEXT: v_mov_b32_e32 v1, s113080; GFX950-NEXT: ;;#ASMSTART13081; GFX950-NEXT: ; def a013082; GFX950-NEXT: ;;#ASMEND13083; GFX950-NEXT: s_nop 013084; GFX950-NEXT: v_accvgpr_read_b32 v2, a013085; GFX950-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 sc013086; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13087; GFX950-NEXT: v_accvgpr_write_b32 a0, v013088; GFX950-NEXT: ;;#ASMSTART13089; GFX950-NEXT: ; use a013090; GFX950-NEXT: ;;#ASMEND13091; GFX950-NEXT: s_setpc_b64 s[30:31]13092 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013093 %data = call i32 asm "; def $0", "=a"()13094 %result = atomicrmw add ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013095 call void asm "; use $0", "a"(i32 %result)13096 ret void13097}13098 13099define void @flat_atomic_add_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13100; GFX90A-LABEL: flat_atomic_add_i32_saddr_ret_av_av:13101; GFX90A: ; %bb.0:13102; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13103; GFX90A-NEXT: v_mov_b32_e32 v0, s1613104; GFX90A-NEXT: v_mov_b32_e32 v1, s1713105; GFX90A-NEXT: ;;#ASMSTART13106; GFX90A-NEXT: ; def v213107; GFX90A-NEXT: ;;#ASMEND13108; GFX90A-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 glc13109; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13110; GFX90A-NEXT: ;;#ASMSTART13111; GFX90A-NEXT: ; use v013112; GFX90A-NEXT: ;;#ASMEND13113; GFX90A-NEXT: s_setpc_b64 s[30:31]13114;13115; GFX950-LABEL: flat_atomic_add_i32_saddr_ret_av_av:13116; GFX950: ; %bb.0:13117; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13118; GFX950-NEXT: v_mov_b32_e32 v0, s013119; GFX950-NEXT: v_mov_b32_e32 v1, s113120; GFX950-NEXT: ;;#ASMSTART13121; GFX950-NEXT: ; def v213122; GFX950-NEXT: ;;#ASMEND13123; GFX950-NEXT: flat_atomic_add v0, v[0:1], v2 offset:40 sc013124; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13125; GFX950-NEXT: ;;#ASMSTART13126; GFX950-NEXT: ; use v013127; GFX950-NEXT: ;;#ASMEND13128; GFX950-NEXT: s_setpc_b64 s[30:31]13129 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013130 %data = call i32 asm "; def $0", "=^VA"()13131 %result = atomicrmw add ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013132 call void asm "; use $0", "^VA"(i32 %result)13133 ret void13134}13135 13136define void @flat_atomic_sub_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13137; GFX90A-LABEL: flat_atomic_sub_i32_saddr_ret_a_a:13138; GFX90A: ; %bb.0:13139; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13140; GFX90A-NEXT: v_mov_b32_e32 v0, s1613141; GFX90A-NEXT: v_mov_b32_e32 v1, s1713142; GFX90A-NEXT: ;;#ASMSTART13143; GFX90A-NEXT: ; def a013144; GFX90A-NEXT: ;;#ASMEND13145; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013146; GFX90A-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 glc13147; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13148; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013149; GFX90A-NEXT: ;;#ASMSTART13150; GFX90A-NEXT: ; use a013151; GFX90A-NEXT: ;;#ASMEND13152; GFX90A-NEXT: s_setpc_b64 s[30:31]13153;13154; GFX950-LABEL: flat_atomic_sub_i32_saddr_ret_a_a:13155; GFX950: ; %bb.0:13156; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13157; GFX950-NEXT: v_mov_b32_e32 v0, s013158; GFX950-NEXT: v_mov_b32_e32 v1, s113159; GFX950-NEXT: ;;#ASMSTART13160; GFX950-NEXT: ; def a013161; GFX950-NEXT: ;;#ASMEND13162; GFX950-NEXT: s_nop 013163; GFX950-NEXT: v_accvgpr_read_b32 v2, a013164; GFX950-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 sc013165; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13166; GFX950-NEXT: v_accvgpr_write_b32 a0, v013167; GFX950-NEXT: ;;#ASMSTART13168; GFX950-NEXT: ; use a013169; GFX950-NEXT: ;;#ASMEND13170; GFX950-NEXT: s_setpc_b64 s[30:31]13171 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013172 %data = call i32 asm "; def $0", "=a"()13173 %result = atomicrmw sub ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013174 call void asm "; use $0", "a"(i32 %result)13175 ret void13176}13177 13178define void @flat_atomic_sub_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13179; GFX90A-LABEL: flat_atomic_sub_i32_saddr_ret_av_av:13180; GFX90A: ; %bb.0:13181; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13182; GFX90A-NEXT: v_mov_b32_e32 v0, s1613183; GFX90A-NEXT: v_mov_b32_e32 v1, s1713184; GFX90A-NEXT: ;;#ASMSTART13185; GFX90A-NEXT: ; def v213186; GFX90A-NEXT: ;;#ASMEND13187; GFX90A-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 glc13188; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13189; GFX90A-NEXT: ;;#ASMSTART13190; GFX90A-NEXT: ; use v013191; GFX90A-NEXT: ;;#ASMEND13192; GFX90A-NEXT: s_setpc_b64 s[30:31]13193;13194; GFX950-LABEL: flat_atomic_sub_i32_saddr_ret_av_av:13195; GFX950: ; %bb.0:13196; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13197; GFX950-NEXT: v_mov_b32_e32 v0, s013198; GFX950-NEXT: v_mov_b32_e32 v1, s113199; GFX950-NEXT: ;;#ASMSTART13200; GFX950-NEXT: ; def v213201; GFX950-NEXT: ;;#ASMEND13202; GFX950-NEXT: flat_atomic_sub v0, v[0:1], v2 offset:40 sc013203; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13204; GFX950-NEXT: ;;#ASMSTART13205; GFX950-NEXT: ; use v013206; GFX950-NEXT: ;;#ASMEND13207; GFX950-NEXT: s_setpc_b64 s[30:31]13208 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013209 %data = call i32 asm "; def $0", "=^VA"()13210 %result = atomicrmw sub ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013211 call void asm "; use $0", "^VA"(i32 %result)13212 ret void13213}13214 13215define void @flat_atomic_and_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13216; GFX90A-LABEL: flat_atomic_and_i32_saddr_ret_a_a:13217; GFX90A: ; %bb.0:13218; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13219; GFX90A-NEXT: v_mov_b32_e32 v0, s1613220; GFX90A-NEXT: v_mov_b32_e32 v1, s1713221; GFX90A-NEXT: ;;#ASMSTART13222; GFX90A-NEXT: ; def a013223; GFX90A-NEXT: ;;#ASMEND13224; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013225; GFX90A-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 glc13226; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13227; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013228; GFX90A-NEXT: ;;#ASMSTART13229; GFX90A-NEXT: ; use a013230; GFX90A-NEXT: ;;#ASMEND13231; GFX90A-NEXT: s_setpc_b64 s[30:31]13232;13233; GFX950-LABEL: flat_atomic_and_i32_saddr_ret_a_a:13234; GFX950: ; %bb.0:13235; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13236; GFX950-NEXT: v_mov_b32_e32 v0, s013237; GFX950-NEXT: v_mov_b32_e32 v1, s113238; GFX950-NEXT: ;;#ASMSTART13239; GFX950-NEXT: ; def a013240; GFX950-NEXT: ;;#ASMEND13241; GFX950-NEXT: s_nop 013242; GFX950-NEXT: v_accvgpr_read_b32 v2, a013243; GFX950-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 sc013244; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13245; GFX950-NEXT: v_accvgpr_write_b32 a0, v013246; GFX950-NEXT: ;;#ASMSTART13247; GFX950-NEXT: ; use a013248; GFX950-NEXT: ;;#ASMEND13249; GFX950-NEXT: s_setpc_b64 s[30:31]13250 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013251 %data = call i32 asm "; def $0", "=a"()13252 %result = atomicrmw and ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013253 call void asm "; use $0", "a"(i32 %result)13254 ret void13255}13256 13257define void @flat_atomic_and_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13258; GFX90A-LABEL: flat_atomic_and_i32_saddr_ret_av_av:13259; GFX90A: ; %bb.0:13260; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13261; GFX90A-NEXT: v_mov_b32_e32 v0, s1613262; GFX90A-NEXT: v_mov_b32_e32 v1, s1713263; GFX90A-NEXT: ;;#ASMSTART13264; GFX90A-NEXT: ; def v213265; GFX90A-NEXT: ;;#ASMEND13266; GFX90A-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 glc13267; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13268; GFX90A-NEXT: ;;#ASMSTART13269; GFX90A-NEXT: ; use v013270; GFX90A-NEXT: ;;#ASMEND13271; GFX90A-NEXT: s_setpc_b64 s[30:31]13272;13273; GFX950-LABEL: flat_atomic_and_i32_saddr_ret_av_av:13274; GFX950: ; %bb.0:13275; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13276; GFX950-NEXT: v_mov_b32_e32 v0, s013277; GFX950-NEXT: v_mov_b32_e32 v1, s113278; GFX950-NEXT: ;;#ASMSTART13279; GFX950-NEXT: ; def v213280; GFX950-NEXT: ;;#ASMEND13281; GFX950-NEXT: flat_atomic_and v0, v[0:1], v2 offset:40 sc013282; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13283; GFX950-NEXT: ;;#ASMSTART13284; GFX950-NEXT: ; use v013285; GFX950-NEXT: ;;#ASMEND13286; GFX950-NEXT: s_setpc_b64 s[30:31]13287 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013288 %data = call i32 asm "; def $0", "=^VA"()13289 %result = atomicrmw and ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013290 call void asm "; use $0", "^VA"(i32 %result)13291 ret void13292}13293 13294define void @flat_atomic_nand_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13295; GFX90A-LABEL: flat_atomic_nand_i32_saddr_ret_a_a:13296; GFX90A: ; %bb.0:13297; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13298; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]13299; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4013300; GFX90A-NEXT: ;;#ASMSTART13301; GFX90A-NEXT: ; def a013302; GFX90A-NEXT: ;;#ASMEND13303; GFX90A-NEXT: v_accvgpr_read_b32 v4, a013304; GFX90A-NEXT: s_mov_b64 s[4:5], 013305; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]13306; GFX90A-NEXT: .LBB171_1: ; %atomicrmw.start13307; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=113308; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13309; GFX90A-NEXT: v_and_b32_e32 v0, v1, v413310; GFX90A-NEXT: v_not_b32_e32 v0, v013311; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc13312; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13313; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113314; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]13315; GFX90A-NEXT: v_mov_b32_e32 v1, v013316; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]13317; GFX90A-NEXT: s_cbranch_execnz .LBB171_113318; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end13319; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]13320; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013321; GFX90A-NEXT: ;;#ASMSTART13322; GFX90A-NEXT: ; use a013323; GFX90A-NEXT: ;;#ASMEND13324; GFX90A-NEXT: s_setpc_b64 s[30:31]13325;13326; GFX950-LABEL: flat_atomic_nand_i32_saddr_ret_a_a:13327; GFX950: ; %bb.0:13328; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13329; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]13330; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4013331; GFX950-NEXT: ;;#ASMSTART13332; GFX950-NEXT: ; def a013333; GFX950-NEXT: ;;#ASMEND13334; GFX950-NEXT: s_mov_b64 s[2:3], 013335; GFX950-NEXT: v_accvgpr_read_b32 v4, a013336; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13337; GFX950-NEXT: .LBB171_1: ; %atomicrmw.start13338; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113339; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13340; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f13341; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013342; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13343; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113344; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13345; GFX950-NEXT: v_mov_b32_e32 v1, v013346; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13347; GFX950-NEXT: s_cbranch_execnz .LBB171_113348; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13349; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13350; GFX950-NEXT: v_accvgpr_write_b32 a0, v013351; GFX950-NEXT: ;;#ASMSTART13352; GFX950-NEXT: ; use a013353; GFX950-NEXT: ;;#ASMEND13354; GFX950-NEXT: s_setpc_b64 s[30:31]13355 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013356 %data = call i32 asm "; def $0", "=a"()13357 %result = atomicrmw nand ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013358 call void asm "; use $0", "a"(i32 %result)13359 ret void13360}13361 13362define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13363; GFX90A-LABEL: flat_atomic_nand_i32_saddr_ret_av_av:13364; GFX90A: ; %bb.0:13365; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13366; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]13367; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4013368; GFX90A-NEXT: s_mov_b64 s[4:5], 013369; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]13370; GFX90A-NEXT: ;;#ASMSTART13371; GFX90A-NEXT: ; def v413372; GFX90A-NEXT: ;;#ASMEND13373; GFX90A-NEXT: .LBB172_1: ; %atomicrmw.start13374; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=113375; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13376; GFX90A-NEXT: v_and_b32_e32 v0, v1, v413377; GFX90A-NEXT: v_not_b32_e32 v0, v013378; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc13379; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13380; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113381; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]13382; GFX90A-NEXT: v_mov_b32_e32 v1, v013383; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]13384; GFX90A-NEXT: s_cbranch_execnz .LBB172_113385; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end13386; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]13387; GFX90A-NEXT: ;;#ASMSTART13388; GFX90A-NEXT: ; use v013389; GFX90A-NEXT: ;;#ASMEND13390; GFX90A-NEXT: s_setpc_b64 s[30:31]13391;13392; GFX950-LABEL: flat_atomic_nand_i32_saddr_ret_av_av:13393; GFX950: ; %bb.0:13394; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13395; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]13396; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4013397; GFX950-NEXT: s_mov_b64 s[2:3], 013398; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13399; GFX950-NEXT: ;;#ASMSTART13400; GFX950-NEXT: ; def v413401; GFX950-NEXT: ;;#ASMEND13402; GFX950-NEXT: .LBB172_1: ; %atomicrmw.start13403; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113404; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13405; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f13406; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013407; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13408; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113409; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13410; GFX950-NEXT: v_mov_b32_e32 v1, v013411; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13412; GFX950-NEXT: s_cbranch_execnz .LBB172_113413; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13414; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13415; GFX950-NEXT: ;;#ASMSTART13416; GFX950-NEXT: ; use v013417; GFX950-NEXT: ;;#ASMEND13418; GFX950-NEXT: s_setpc_b64 s[30:31]13419 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013420 %data = call i32 asm "; def $0", "=^VA"()13421 %result = atomicrmw nand ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013422 call void asm "; use $0", "^VA"(i32 %result)13423 ret void13424}13425 13426define void @flat_atomic_or_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13427; GFX90A-LABEL: flat_atomic_or_i32_saddr_ret_a_a:13428; GFX90A: ; %bb.0:13429; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13430; GFX90A-NEXT: v_mov_b32_e32 v0, s1613431; GFX90A-NEXT: v_mov_b32_e32 v1, s1713432; GFX90A-NEXT: ;;#ASMSTART13433; GFX90A-NEXT: ; def a013434; GFX90A-NEXT: ;;#ASMEND13435; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013436; GFX90A-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 glc13437; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13438; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013439; GFX90A-NEXT: ;;#ASMSTART13440; GFX90A-NEXT: ; use a013441; GFX90A-NEXT: ;;#ASMEND13442; GFX90A-NEXT: s_setpc_b64 s[30:31]13443;13444; GFX950-LABEL: flat_atomic_or_i32_saddr_ret_a_a:13445; GFX950: ; %bb.0:13446; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13447; GFX950-NEXT: v_mov_b32_e32 v0, s013448; GFX950-NEXT: v_mov_b32_e32 v1, s113449; GFX950-NEXT: ;;#ASMSTART13450; GFX950-NEXT: ; def a013451; GFX950-NEXT: ;;#ASMEND13452; GFX950-NEXT: s_nop 013453; GFX950-NEXT: v_accvgpr_read_b32 v2, a013454; GFX950-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 sc013455; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13456; GFX950-NEXT: v_accvgpr_write_b32 a0, v013457; GFX950-NEXT: ;;#ASMSTART13458; GFX950-NEXT: ; use a013459; GFX950-NEXT: ;;#ASMEND13460; GFX950-NEXT: s_setpc_b64 s[30:31]13461 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013462 %data = call i32 asm "; def $0", "=a"()13463 %result = atomicrmw or ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013464 call void asm "; use $0", "a"(i32 %result)13465 ret void13466}13467 13468define void @flat_atomic_or_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13469; GFX90A-LABEL: flat_atomic_or_i32_saddr_ret_av_av:13470; GFX90A: ; %bb.0:13471; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13472; GFX90A-NEXT: v_mov_b32_e32 v0, s1613473; GFX90A-NEXT: v_mov_b32_e32 v1, s1713474; GFX90A-NEXT: ;;#ASMSTART13475; GFX90A-NEXT: ; def v213476; GFX90A-NEXT: ;;#ASMEND13477; GFX90A-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 glc13478; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13479; GFX90A-NEXT: ;;#ASMSTART13480; GFX90A-NEXT: ; use v013481; GFX90A-NEXT: ;;#ASMEND13482; GFX90A-NEXT: s_setpc_b64 s[30:31]13483;13484; GFX950-LABEL: flat_atomic_or_i32_saddr_ret_av_av:13485; GFX950: ; %bb.0:13486; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13487; GFX950-NEXT: v_mov_b32_e32 v0, s013488; GFX950-NEXT: v_mov_b32_e32 v1, s113489; GFX950-NEXT: ;;#ASMSTART13490; GFX950-NEXT: ; def v213491; GFX950-NEXT: ;;#ASMEND13492; GFX950-NEXT: flat_atomic_or v0, v[0:1], v2 offset:40 sc013493; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13494; GFX950-NEXT: ;;#ASMSTART13495; GFX950-NEXT: ; use v013496; GFX950-NEXT: ;;#ASMEND13497; GFX950-NEXT: s_setpc_b64 s[30:31]13498 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013499 %data = call i32 asm "; def $0", "=^VA"()13500 %result = atomicrmw or ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013501 call void asm "; use $0", "^VA"(i32 %result)13502 ret void13503}13504 13505define void @flat_atomic_xor_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13506; GFX90A-LABEL: flat_atomic_xor_i32_saddr_ret_a_a:13507; GFX90A: ; %bb.0:13508; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13509; GFX90A-NEXT: v_mov_b32_e32 v0, s1613510; GFX90A-NEXT: v_mov_b32_e32 v1, s1713511; GFX90A-NEXT: ;;#ASMSTART13512; GFX90A-NEXT: ; def a013513; GFX90A-NEXT: ;;#ASMEND13514; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013515; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 offset:40 glc13516; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13517; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013518; GFX90A-NEXT: ;;#ASMSTART13519; GFX90A-NEXT: ; use a013520; GFX90A-NEXT: ;;#ASMEND13521; GFX90A-NEXT: s_setpc_b64 s[30:31]13522;13523; GFX950-LABEL: flat_atomic_xor_i32_saddr_ret_a_a:13524; GFX950: ; %bb.0:13525; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13526; GFX950-NEXT: v_mov_b32_e32 v0, s013527; GFX950-NEXT: v_mov_b32_e32 v1, s113528; GFX950-NEXT: ;;#ASMSTART13529; GFX950-NEXT: ; def a013530; GFX950-NEXT: ;;#ASMEND13531; GFX950-NEXT: s_nop 013532; GFX950-NEXT: v_accvgpr_read_b32 v2, a013533; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 offset:40 sc013534; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13535; GFX950-NEXT: v_accvgpr_write_b32 a0, v013536; GFX950-NEXT: ;;#ASMSTART13537; GFX950-NEXT: ; use a013538; GFX950-NEXT: ;;#ASMEND13539; GFX950-NEXT: s_setpc_b64 s[30:31]13540 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013541 %data = call i32 asm "; def $0", "=a"()13542 %result = atomicrmw xor ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013543 call void asm "; use $0", "a"(i32 %result)13544 ret void13545}13546 13547define void @flat_atomic_xor_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13548; GFX90A-LABEL: flat_atomic_xor_i32_saddr_ret_av_av:13549; GFX90A: ; %bb.0:13550; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13551; GFX90A-NEXT: v_mov_b32_e32 v0, s1613552; GFX90A-NEXT: v_mov_b32_e32 v1, s1713553; GFX90A-NEXT: ;;#ASMSTART13554; GFX90A-NEXT: ; def v213555; GFX90A-NEXT: ;;#ASMEND13556; GFX90A-NEXT: flat_atomic_xor v0, v[0:1], v2 offset:40 glc13557; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13558; GFX90A-NEXT: ;;#ASMSTART13559; GFX90A-NEXT: ; use v013560; GFX90A-NEXT: ;;#ASMEND13561; GFX90A-NEXT: s_setpc_b64 s[30:31]13562;13563; GFX950-LABEL: flat_atomic_xor_i32_saddr_ret_av_av:13564; GFX950: ; %bb.0:13565; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13566; GFX950-NEXT: v_mov_b32_e32 v0, s013567; GFX950-NEXT: v_mov_b32_e32 v1, s113568; GFX950-NEXT: ;;#ASMSTART13569; GFX950-NEXT: ; def v213570; GFX950-NEXT: ;;#ASMEND13571; GFX950-NEXT: flat_atomic_xor v0, v[0:1], v2 offset:40 sc013572; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13573; GFX950-NEXT: ;;#ASMSTART13574; GFX950-NEXT: ; use v013575; GFX950-NEXT: ;;#ASMEND13576; GFX950-NEXT: s_setpc_b64 s[30:31]13577 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013578 %data = call i32 asm "; def $0", "=^VA"()13579 %result = atomicrmw xor ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013580 call void asm "; use $0", "^VA"(i32 %result)13581 ret void13582}13583 13584define void @flat_atomic_max_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13585; GFX90A-LABEL: flat_atomic_max_i32_saddr_ret_a_a:13586; GFX90A: ; %bb.0:13587; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13588; GFX90A-NEXT: v_mov_b32_e32 v0, s1613589; GFX90A-NEXT: v_mov_b32_e32 v1, s1713590; GFX90A-NEXT: ;;#ASMSTART13591; GFX90A-NEXT: ; def a013592; GFX90A-NEXT: ;;#ASMEND13593; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013594; GFX90A-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 glc13595; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13596; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013597; GFX90A-NEXT: ;;#ASMSTART13598; GFX90A-NEXT: ; use a013599; GFX90A-NEXT: ;;#ASMEND13600; GFX90A-NEXT: s_setpc_b64 s[30:31]13601;13602; GFX950-LABEL: flat_atomic_max_i32_saddr_ret_a_a:13603; GFX950: ; %bb.0:13604; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13605; GFX950-NEXT: v_mov_b32_e32 v0, s013606; GFX950-NEXT: v_mov_b32_e32 v1, s113607; GFX950-NEXT: ;;#ASMSTART13608; GFX950-NEXT: ; def a013609; GFX950-NEXT: ;;#ASMEND13610; GFX950-NEXT: s_nop 013611; GFX950-NEXT: v_accvgpr_read_b32 v2, a013612; GFX950-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 sc013613; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13614; GFX950-NEXT: v_accvgpr_write_b32 a0, v013615; GFX950-NEXT: ;;#ASMSTART13616; GFX950-NEXT: ; use a013617; GFX950-NEXT: ;;#ASMEND13618; GFX950-NEXT: s_setpc_b64 s[30:31]13619 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013620 %data = call i32 asm "; def $0", "=a"()13621 %result = atomicrmw max ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013622 call void asm "; use $0", "a"(i32 %result)13623 ret void13624}13625 13626define void @flat_atomic_max_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13627; GFX90A-LABEL: flat_atomic_max_i32_saddr_ret_av_av:13628; GFX90A: ; %bb.0:13629; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13630; GFX90A-NEXT: v_mov_b32_e32 v0, s1613631; GFX90A-NEXT: v_mov_b32_e32 v1, s1713632; GFX90A-NEXT: ;;#ASMSTART13633; GFX90A-NEXT: ; def v213634; GFX90A-NEXT: ;;#ASMEND13635; GFX90A-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 glc13636; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13637; GFX90A-NEXT: ;;#ASMSTART13638; GFX90A-NEXT: ; use v013639; GFX90A-NEXT: ;;#ASMEND13640; GFX90A-NEXT: s_setpc_b64 s[30:31]13641;13642; GFX950-LABEL: flat_atomic_max_i32_saddr_ret_av_av:13643; GFX950: ; %bb.0:13644; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13645; GFX950-NEXT: v_mov_b32_e32 v0, s013646; GFX950-NEXT: v_mov_b32_e32 v1, s113647; GFX950-NEXT: ;;#ASMSTART13648; GFX950-NEXT: ; def v213649; GFX950-NEXT: ;;#ASMEND13650; GFX950-NEXT: flat_atomic_smax v0, v[0:1], v2 offset:40 sc013651; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13652; GFX950-NEXT: ;;#ASMSTART13653; GFX950-NEXT: ; use v013654; GFX950-NEXT: ;;#ASMEND13655; GFX950-NEXT: s_setpc_b64 s[30:31]13656 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013657 %data = call i32 asm "; def $0", "=^VA"()13658 %result = atomicrmw max ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013659 call void asm "; use $0", "^VA"(i32 %result)13660 ret void13661}13662 13663define void @flat_atomic_min_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13664; GFX90A-LABEL: flat_atomic_min_i32_saddr_ret_a_a:13665; GFX90A: ; %bb.0:13666; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13667; GFX90A-NEXT: v_mov_b32_e32 v0, s1613668; GFX90A-NEXT: v_mov_b32_e32 v1, s1713669; GFX90A-NEXT: ;;#ASMSTART13670; GFX90A-NEXT: ; def a013671; GFX90A-NEXT: ;;#ASMEND13672; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013673; GFX90A-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 glc13674; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13675; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013676; GFX90A-NEXT: ;;#ASMSTART13677; GFX90A-NEXT: ; use a013678; GFX90A-NEXT: ;;#ASMEND13679; GFX90A-NEXT: s_setpc_b64 s[30:31]13680;13681; GFX950-LABEL: flat_atomic_min_i32_saddr_ret_a_a:13682; GFX950: ; %bb.0:13683; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13684; GFX950-NEXT: v_mov_b32_e32 v0, s013685; GFX950-NEXT: v_mov_b32_e32 v1, s113686; GFX950-NEXT: ;;#ASMSTART13687; GFX950-NEXT: ; def a013688; GFX950-NEXT: ;;#ASMEND13689; GFX950-NEXT: s_nop 013690; GFX950-NEXT: v_accvgpr_read_b32 v2, a013691; GFX950-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 sc013692; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13693; GFX950-NEXT: v_accvgpr_write_b32 a0, v013694; GFX950-NEXT: ;;#ASMSTART13695; GFX950-NEXT: ; use a013696; GFX950-NEXT: ;;#ASMEND13697; GFX950-NEXT: s_setpc_b64 s[30:31]13698 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013699 %data = call i32 asm "; def $0", "=a"()13700 %result = atomicrmw min ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013701 call void asm "; use $0", "a"(i32 %result)13702 ret void13703}13704 13705define void @flat_atomic_min_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13706; GFX90A-LABEL: flat_atomic_min_i32_saddr_ret_av_av:13707; GFX90A: ; %bb.0:13708; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13709; GFX90A-NEXT: v_mov_b32_e32 v0, s1613710; GFX90A-NEXT: v_mov_b32_e32 v1, s1713711; GFX90A-NEXT: ;;#ASMSTART13712; GFX90A-NEXT: ; def v213713; GFX90A-NEXT: ;;#ASMEND13714; GFX90A-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 glc13715; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13716; GFX90A-NEXT: ;;#ASMSTART13717; GFX90A-NEXT: ; use v013718; GFX90A-NEXT: ;;#ASMEND13719; GFX90A-NEXT: s_setpc_b64 s[30:31]13720;13721; GFX950-LABEL: flat_atomic_min_i32_saddr_ret_av_av:13722; GFX950: ; %bb.0:13723; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13724; GFX950-NEXT: v_mov_b32_e32 v0, s013725; GFX950-NEXT: v_mov_b32_e32 v1, s113726; GFX950-NEXT: ;;#ASMSTART13727; GFX950-NEXT: ; def v213728; GFX950-NEXT: ;;#ASMEND13729; GFX950-NEXT: flat_atomic_smin v0, v[0:1], v2 offset:40 sc013730; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13731; GFX950-NEXT: ;;#ASMSTART13732; GFX950-NEXT: ; use v013733; GFX950-NEXT: ;;#ASMEND13734; GFX950-NEXT: s_setpc_b64 s[30:31]13735 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013736 %data = call i32 asm "; def $0", "=^VA"()13737 %result = atomicrmw min ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013738 call void asm "; use $0", "^VA"(i32 %result)13739 ret void13740}13741 13742define void @flat_atomic_umax_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13743; GFX90A-LABEL: flat_atomic_umax_i32_saddr_ret_a_a:13744; GFX90A: ; %bb.0:13745; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13746; GFX90A-NEXT: v_mov_b32_e32 v0, s1613747; GFX90A-NEXT: v_mov_b32_e32 v1, s1713748; GFX90A-NEXT: ;;#ASMSTART13749; GFX90A-NEXT: ; def a013750; GFX90A-NEXT: ;;#ASMEND13751; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013752; GFX90A-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 glc13753; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13754; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013755; GFX90A-NEXT: ;;#ASMSTART13756; GFX90A-NEXT: ; use a013757; GFX90A-NEXT: ;;#ASMEND13758; GFX90A-NEXT: s_setpc_b64 s[30:31]13759;13760; GFX950-LABEL: flat_atomic_umax_i32_saddr_ret_a_a:13761; GFX950: ; %bb.0:13762; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13763; GFX950-NEXT: v_mov_b32_e32 v0, s013764; GFX950-NEXT: v_mov_b32_e32 v1, s113765; GFX950-NEXT: ;;#ASMSTART13766; GFX950-NEXT: ; def a013767; GFX950-NEXT: ;;#ASMEND13768; GFX950-NEXT: s_nop 013769; GFX950-NEXT: v_accvgpr_read_b32 v2, a013770; GFX950-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 sc013771; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13772; GFX950-NEXT: v_accvgpr_write_b32 a0, v013773; GFX950-NEXT: ;;#ASMSTART13774; GFX950-NEXT: ; use a013775; GFX950-NEXT: ;;#ASMEND13776; GFX950-NEXT: s_setpc_b64 s[30:31]13777 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013778 %data = call i32 asm "; def $0", "=a"()13779 %result = atomicrmw umax ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013780 call void asm "; use $0", "a"(i32 %result)13781 ret void13782}13783 13784define void @flat_atomic_umax_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13785; GFX90A-LABEL: flat_atomic_umax_i32_saddr_ret_av_av:13786; GFX90A: ; %bb.0:13787; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13788; GFX90A-NEXT: v_mov_b32_e32 v0, s1613789; GFX90A-NEXT: v_mov_b32_e32 v1, s1713790; GFX90A-NEXT: ;;#ASMSTART13791; GFX90A-NEXT: ; def v213792; GFX90A-NEXT: ;;#ASMEND13793; GFX90A-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 glc13794; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13795; GFX90A-NEXT: ;;#ASMSTART13796; GFX90A-NEXT: ; use v013797; GFX90A-NEXT: ;;#ASMEND13798; GFX90A-NEXT: s_setpc_b64 s[30:31]13799;13800; GFX950-LABEL: flat_atomic_umax_i32_saddr_ret_av_av:13801; GFX950: ; %bb.0:13802; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13803; GFX950-NEXT: v_mov_b32_e32 v0, s013804; GFX950-NEXT: v_mov_b32_e32 v1, s113805; GFX950-NEXT: ;;#ASMSTART13806; GFX950-NEXT: ; def v213807; GFX950-NEXT: ;;#ASMEND13808; GFX950-NEXT: flat_atomic_umax v0, v[0:1], v2 offset:40 sc013809; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13810; GFX950-NEXT: ;;#ASMSTART13811; GFX950-NEXT: ; use v013812; GFX950-NEXT: ;;#ASMEND13813; GFX950-NEXT: s_setpc_b64 s[30:31]13814 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013815 %data = call i32 asm "; def $0", "=^VA"()13816 %result = atomicrmw umax ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013817 call void asm "; use $0", "^VA"(i32 %result)13818 ret void13819}13820 13821define void @flat_atomic_umin_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13822; GFX90A-LABEL: flat_atomic_umin_i32_saddr_ret_a_a:13823; GFX90A: ; %bb.0:13824; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13825; GFX90A-NEXT: v_mov_b32_e32 v0, s1613826; GFX90A-NEXT: v_mov_b32_e32 v1, s1713827; GFX90A-NEXT: ;;#ASMSTART13828; GFX90A-NEXT: ; def a013829; GFX90A-NEXT: ;;#ASMEND13830; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013831; GFX90A-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 glc13832; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13833; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013834; GFX90A-NEXT: ;;#ASMSTART13835; GFX90A-NEXT: ; use a013836; GFX90A-NEXT: ;;#ASMEND13837; GFX90A-NEXT: s_setpc_b64 s[30:31]13838;13839; GFX950-LABEL: flat_atomic_umin_i32_saddr_ret_a_a:13840; GFX950: ; %bb.0:13841; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13842; GFX950-NEXT: v_mov_b32_e32 v0, s013843; GFX950-NEXT: v_mov_b32_e32 v1, s113844; GFX950-NEXT: ;;#ASMSTART13845; GFX950-NEXT: ; def a013846; GFX950-NEXT: ;;#ASMEND13847; GFX950-NEXT: s_nop 013848; GFX950-NEXT: v_accvgpr_read_b32 v2, a013849; GFX950-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 sc013850; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13851; GFX950-NEXT: v_accvgpr_write_b32 a0, v013852; GFX950-NEXT: ;;#ASMSTART13853; GFX950-NEXT: ; use a013854; GFX950-NEXT: ;;#ASMEND13855; GFX950-NEXT: s_setpc_b64 s[30:31]13856 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013857 %data = call i32 asm "; def $0", "=a"()13858 %result = atomicrmw umin ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013859 call void asm "; use $0", "a"(i32 %result)13860 ret void13861}13862 13863define void @flat_atomic_umin_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13864; GFX90A-LABEL: flat_atomic_umin_i32_saddr_ret_av_av:13865; GFX90A: ; %bb.0:13866; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13867; GFX90A-NEXT: v_mov_b32_e32 v0, s1613868; GFX90A-NEXT: v_mov_b32_e32 v1, s1713869; GFX90A-NEXT: ;;#ASMSTART13870; GFX90A-NEXT: ; def v213871; GFX90A-NEXT: ;;#ASMEND13872; GFX90A-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 glc13873; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13874; GFX90A-NEXT: ;;#ASMSTART13875; GFX90A-NEXT: ; use v013876; GFX90A-NEXT: ;;#ASMEND13877; GFX90A-NEXT: s_setpc_b64 s[30:31]13878;13879; GFX950-LABEL: flat_atomic_umin_i32_saddr_ret_av_av:13880; GFX950: ; %bb.0:13881; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13882; GFX950-NEXT: v_mov_b32_e32 v0, s013883; GFX950-NEXT: v_mov_b32_e32 v1, s113884; GFX950-NEXT: ;;#ASMSTART13885; GFX950-NEXT: ; def v213886; GFX950-NEXT: ;;#ASMEND13887; GFX950-NEXT: flat_atomic_umin v0, v[0:1], v2 offset:40 sc013888; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13889; GFX950-NEXT: ;;#ASMSTART13890; GFX950-NEXT: ; use v013891; GFX950-NEXT: ;;#ASMEND13892; GFX950-NEXT: s_setpc_b64 s[30:31]13893 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013894 %data = call i32 asm "; def $0", "=^VA"()13895 %result = atomicrmw umin ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013896 call void asm "; use $0", "^VA"(i32 %result)13897 ret void13898}13899 13900define void @flat_atomic_uinc_wrap_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13901; GFX90A-LABEL: flat_atomic_uinc_wrap_i32_saddr_ret_a_a:13902; GFX90A: ; %bb.0:13903; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13904; GFX90A-NEXT: v_mov_b32_e32 v0, s1613905; GFX90A-NEXT: v_mov_b32_e32 v1, s1713906; GFX90A-NEXT: ;;#ASMSTART13907; GFX90A-NEXT: ; def a013908; GFX90A-NEXT: ;;#ASMEND13909; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013910; GFX90A-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 glc13911; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13912; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013913; GFX90A-NEXT: ;;#ASMSTART13914; GFX90A-NEXT: ; use a013915; GFX90A-NEXT: ;;#ASMEND13916; GFX90A-NEXT: s_setpc_b64 s[30:31]13917;13918; GFX950-LABEL: flat_atomic_uinc_wrap_i32_saddr_ret_a_a:13919; GFX950: ; %bb.0:13920; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13921; GFX950-NEXT: v_mov_b32_e32 v0, s013922; GFX950-NEXT: v_mov_b32_e32 v1, s113923; GFX950-NEXT: ;;#ASMSTART13924; GFX950-NEXT: ; def a013925; GFX950-NEXT: ;;#ASMEND13926; GFX950-NEXT: s_nop 013927; GFX950-NEXT: v_accvgpr_read_b32 v2, a013928; GFX950-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 sc013929; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13930; GFX950-NEXT: v_accvgpr_write_b32 a0, v013931; GFX950-NEXT: ;;#ASMSTART13932; GFX950-NEXT: ; use a013933; GFX950-NEXT: ;;#ASMEND13934; GFX950-NEXT: s_setpc_b64 s[30:31]13935 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013936 %data = call i32 asm "; def $0", "=a"()13937 %result = atomicrmw uinc_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013938 call void asm "; use $0", "a"(i32 %result)13939 ret void13940}13941 13942define void @flat_atomic_uinc_wrap_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {13943; GFX90A-LABEL: flat_atomic_uinc_wrap_i32_saddr_ret_av_av:13944; GFX90A: ; %bb.0:13945; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13946; GFX90A-NEXT: v_mov_b32_e32 v0, s1613947; GFX90A-NEXT: v_mov_b32_e32 v1, s1713948; GFX90A-NEXT: ;;#ASMSTART13949; GFX90A-NEXT: ; def v213950; GFX90A-NEXT: ;;#ASMEND13951; GFX90A-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 glc13952; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13953; GFX90A-NEXT: ;;#ASMSTART13954; GFX90A-NEXT: ; use v013955; GFX90A-NEXT: ;;#ASMEND13956; GFX90A-NEXT: s_setpc_b64 s[30:31]13957;13958; GFX950-LABEL: flat_atomic_uinc_wrap_i32_saddr_ret_av_av:13959; GFX950: ; %bb.0:13960; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13961; GFX950-NEXT: v_mov_b32_e32 v0, s013962; GFX950-NEXT: v_mov_b32_e32 v1, s113963; GFX950-NEXT: ;;#ASMSTART13964; GFX950-NEXT: ; def v213965; GFX950-NEXT: ;;#ASMEND13966; GFX950-NEXT: flat_atomic_inc v0, v[0:1], v2 offset:40 sc013967; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13968; GFX950-NEXT: ;;#ASMSTART13969; GFX950-NEXT: ; use v013970; GFX950-NEXT: ;;#ASMEND13971; GFX950-NEXT: s_setpc_b64 s[30:31]13972 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1013973 %data = call i32 asm "; def $0", "=^VA"()13974 %result = atomicrmw uinc_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !013975 call void asm "; use $0", "^VA"(i32 %result)13976 ret void13977}13978 13979define void @flat_atomic_udec_wrap_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {13980; GFX90A-LABEL: flat_atomic_udec_wrap_i32_saddr_ret_a_a:13981; GFX90A: ; %bb.0:13982; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13983; GFX90A-NEXT: v_mov_b32_e32 v0, s1613984; GFX90A-NEXT: v_mov_b32_e32 v1, s1713985; GFX90A-NEXT: ;;#ASMSTART13986; GFX90A-NEXT: ; def a013987; GFX90A-NEXT: ;;#ASMEND13988; GFX90A-NEXT: v_accvgpr_read_b32 v2, a013989; GFX90A-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 glc13990; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13991; GFX90A-NEXT: v_accvgpr_write_b32 a0, v013992; GFX90A-NEXT: ;;#ASMSTART13993; GFX90A-NEXT: ; use a013994; GFX90A-NEXT: ;;#ASMEND13995; GFX90A-NEXT: s_setpc_b64 s[30:31]13996;13997; GFX950-LABEL: flat_atomic_udec_wrap_i32_saddr_ret_a_a:13998; GFX950: ; %bb.0:13999; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14000; GFX950-NEXT: v_mov_b32_e32 v0, s014001; GFX950-NEXT: v_mov_b32_e32 v1, s114002; GFX950-NEXT: ;;#ASMSTART14003; GFX950-NEXT: ; def a014004; GFX950-NEXT: ;;#ASMEND14005; GFX950-NEXT: s_nop 014006; GFX950-NEXT: v_accvgpr_read_b32 v2, a014007; GFX950-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 sc014008; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14009; GFX950-NEXT: v_accvgpr_write_b32 a0, v014010; GFX950-NEXT: ;;#ASMSTART14011; GFX950-NEXT: ; use a014012; GFX950-NEXT: ;;#ASMEND14013; GFX950-NEXT: s_setpc_b64 s[30:31]14014 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014015 %data = call i32 asm "; def $0", "=a"()14016 %result = atomicrmw udec_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014017 call void asm "; use $0", "a"(i32 %result)14018 ret void14019}14020 14021define void @flat_atomic_udec_wrap_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {14022; GFX90A-LABEL: flat_atomic_udec_wrap_i32_saddr_ret_av_av:14023; GFX90A: ; %bb.0:14024; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14025; GFX90A-NEXT: v_mov_b32_e32 v0, s1614026; GFX90A-NEXT: v_mov_b32_e32 v1, s1714027; GFX90A-NEXT: ;;#ASMSTART14028; GFX90A-NEXT: ; def v214029; GFX90A-NEXT: ;;#ASMEND14030; GFX90A-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 glc14031; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14032; GFX90A-NEXT: ;;#ASMSTART14033; GFX90A-NEXT: ; use v014034; GFX90A-NEXT: ;;#ASMEND14035; GFX90A-NEXT: s_setpc_b64 s[30:31]14036;14037; GFX950-LABEL: flat_atomic_udec_wrap_i32_saddr_ret_av_av:14038; GFX950: ; %bb.0:14039; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14040; GFX950-NEXT: v_mov_b32_e32 v0, s014041; GFX950-NEXT: v_mov_b32_e32 v1, s114042; GFX950-NEXT: ;;#ASMSTART14043; GFX950-NEXT: ; def v214044; GFX950-NEXT: ;;#ASMEND14045; GFX950-NEXT: flat_atomic_dec v0, v[0:1], v2 offset:40 sc014046; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14047; GFX950-NEXT: ;;#ASMSTART14048; GFX950-NEXT: ; use v014049; GFX950-NEXT: ;;#ASMEND14050; GFX950-NEXT: s_setpc_b64 s[30:31]14051 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014052 %data = call i32 asm "; def $0", "=^VA"()14053 %result = atomicrmw udec_wrap ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014054 call void asm "; use $0", "^VA"(i32 %result)14055 ret void14056}14057 14058define void @flat_atomic_usub_cond_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {14059; GFX90A-LABEL: flat_atomic_usub_cond_i32_saddr_ret_a_a:14060; GFX90A: ; %bb.0:14061; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14062; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]14063; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4014064; GFX90A-NEXT: ;;#ASMSTART14065; GFX90A-NEXT: ; def a014066; GFX90A-NEXT: ;;#ASMEND14067; GFX90A-NEXT: v_accvgpr_read_b32 v4, a014068; GFX90A-NEXT: s_mov_b64 s[4:5], 014069; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]14070; GFX90A-NEXT: .LBB189_1: ; %atomicrmw.start14071; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=114072; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14073; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v414074; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v414075; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc14076; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc14077; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14078; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114079; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]14080; GFX90A-NEXT: v_mov_b32_e32 v1, v014081; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]14082; GFX90A-NEXT: s_cbranch_execnz .LBB189_114083; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end14084; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]14085; GFX90A-NEXT: v_accvgpr_write_b32 a0, v014086; GFX90A-NEXT: ;;#ASMSTART14087; GFX90A-NEXT: ; use a014088; GFX90A-NEXT: ;;#ASMEND14089; GFX90A-NEXT: s_setpc_b64 s[30:31]14090;14091; GFX950-LABEL: flat_atomic_usub_cond_i32_saddr_ret_a_a:14092; GFX950: ; %bb.0:14093; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14094; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14095; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4014096; GFX950-NEXT: ;;#ASMSTART14097; GFX950-NEXT: ; def a014098; GFX950-NEXT: ;;#ASMEND14099; GFX950-NEXT: s_mov_b64 s[2:3], 014100; GFX950-NEXT: v_accvgpr_read_b32 v4, a014101; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14102; GFX950-NEXT: .LBB189_1: ; %atomicrmw.start14103; GFX950-NEXT: ; =>This Inner Loop Header: Depth=114104; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14105; GFX950-NEXT: v_sub_u32_e32 v0, v1, v414106; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v414107; GFX950-NEXT: s_nop 114108; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc14109; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc014110; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14111; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114112; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]14113; GFX950-NEXT: v_mov_b32_e32 v1, v014114; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]14115; GFX950-NEXT: s_cbranch_execnz .LBB189_114116; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end14117; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]14118; GFX950-NEXT: v_accvgpr_write_b32 a0, v014119; GFX950-NEXT: ;;#ASMSTART14120; GFX950-NEXT: ; use a014121; GFX950-NEXT: ;;#ASMEND14122; GFX950-NEXT: s_setpc_b64 s[30:31]14123 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014124 %data = call i32 asm "; def $0", "=a"()14125 %result = atomicrmw usub_cond ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014126 call void asm "; use $0", "a"(i32 %result)14127 ret void14128}14129 14130define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {14131; GFX90A-LABEL: flat_atomic_usub_cond_i32_saddr_ret_av_av:14132; GFX90A: ; %bb.0:14133; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14134; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]14135; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4014136; GFX90A-NEXT: s_mov_b64 s[4:5], 014137; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]14138; GFX90A-NEXT: ;;#ASMSTART14139; GFX90A-NEXT: ; def v414140; GFX90A-NEXT: ;;#ASMEND14141; GFX90A-NEXT: .LBB190_1: ; %atomicrmw.start14142; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=114143; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14144; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v414145; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v414146; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc14147; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc14148; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14149; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114150; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]14151; GFX90A-NEXT: v_mov_b32_e32 v1, v014152; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]14153; GFX90A-NEXT: s_cbranch_execnz .LBB190_114154; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end14155; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]14156; GFX90A-NEXT: ;;#ASMSTART14157; GFX90A-NEXT: ; use v014158; GFX90A-NEXT: ;;#ASMEND14159; GFX90A-NEXT: s_setpc_b64 s[30:31]14160;14161; GFX950-LABEL: flat_atomic_usub_cond_i32_saddr_ret_av_av:14162; GFX950: ; %bb.0:14163; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14164; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14165; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4014166; GFX950-NEXT: s_mov_b64 s[2:3], 014167; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14168; GFX950-NEXT: ;;#ASMSTART14169; GFX950-NEXT: ; def v414170; GFX950-NEXT: ;;#ASMEND14171; GFX950-NEXT: .LBB190_1: ; %atomicrmw.start14172; GFX950-NEXT: ; =>This Inner Loop Header: Depth=114173; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14174; GFX950-NEXT: v_sub_u32_e32 v0, v1, v414175; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v414176; GFX950-NEXT: s_nop 114177; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc14178; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc014179; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14180; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114181; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]14182; GFX950-NEXT: v_mov_b32_e32 v1, v014183; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]14184; GFX950-NEXT: s_cbranch_execnz .LBB190_114185; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end14186; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]14187; GFX950-NEXT: ;;#ASMSTART14188; GFX950-NEXT: ; use v014189; GFX950-NEXT: ;;#ASMEND14190; GFX950-NEXT: s_setpc_b64 s[30:31]14191 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014192 %data = call i32 asm "; def $0", "=^VA"()14193 %result = atomicrmw usub_cond ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014194 call void asm "; use $0", "^VA"(i32 %result)14195 ret void14196}14197 14198define void @flat_atomic_usub_sat_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {14199; GFX90A-LABEL: flat_atomic_usub_sat_i32_saddr_ret_a_a:14200; GFX90A: ; %bb.0:14201; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14202; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]14203; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4014204; GFX90A-NEXT: ;;#ASMSTART14205; GFX90A-NEXT: ; def a014206; GFX90A-NEXT: ;;#ASMEND14207; GFX90A-NEXT: v_accvgpr_read_b32 v4, a014208; GFX90A-NEXT: s_mov_b64 s[4:5], 014209; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]14210; GFX90A-NEXT: .LBB191_1: ; %atomicrmw.start14211; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=114212; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14213; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp14214; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc14215; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14216; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114217; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]14218; GFX90A-NEXT: v_mov_b32_e32 v1, v014219; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]14220; GFX90A-NEXT: s_cbranch_execnz .LBB191_114221; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end14222; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]14223; GFX90A-NEXT: v_accvgpr_write_b32 a0, v014224; GFX90A-NEXT: ;;#ASMSTART14225; GFX90A-NEXT: ; use a014226; GFX90A-NEXT: ;;#ASMEND14227; GFX90A-NEXT: s_setpc_b64 s[30:31]14228;14229; GFX950-LABEL: flat_atomic_usub_sat_i32_saddr_ret_a_a:14230; GFX950: ; %bb.0:14231; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14232; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14233; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4014234; GFX950-NEXT: ;;#ASMSTART14235; GFX950-NEXT: ; def a014236; GFX950-NEXT: ;;#ASMEND14237; GFX950-NEXT: s_mov_b64 s[2:3], 014238; GFX950-NEXT: v_accvgpr_read_b32 v4, a014239; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14240; GFX950-NEXT: .LBB191_1: ; %atomicrmw.start14241; GFX950-NEXT: ; =>This Inner Loop Header: Depth=114242; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14243; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp14244; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc014245; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14246; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114247; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]14248; GFX950-NEXT: v_mov_b32_e32 v1, v014249; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]14250; GFX950-NEXT: s_cbranch_execnz .LBB191_114251; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end14252; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]14253; GFX950-NEXT: v_accvgpr_write_b32 a0, v014254; GFX950-NEXT: ;;#ASMSTART14255; GFX950-NEXT: ; use a014256; GFX950-NEXT: ;;#ASMEND14257; GFX950-NEXT: s_setpc_b64 s[30:31]14258 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014259 %data = call i32 asm "; def $0", "=a"()14260 %result = atomicrmw usub_sat ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014261 call void asm "; use $0", "a"(i32 %result)14262 ret void14263}14264 14265define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {14266; GFX90A-LABEL: flat_atomic_usub_sat_i32_saddr_ret_av_av:14267; GFX90A: ; %bb.0:14268; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14269; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]14270; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4014271; GFX90A-NEXT: s_mov_b64 s[4:5], 014272; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]14273; GFX90A-NEXT: ;;#ASMSTART14274; GFX90A-NEXT: ; def v414275; GFX90A-NEXT: ;;#ASMEND14276; GFX90A-NEXT: .LBB192_1: ; %atomicrmw.start14277; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=114278; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14279; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp14280; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc14281; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14282; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114283; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]14284; GFX90A-NEXT: v_mov_b32_e32 v1, v014285; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]14286; GFX90A-NEXT: s_cbranch_execnz .LBB192_114287; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end14288; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]14289; GFX90A-NEXT: ;;#ASMSTART14290; GFX90A-NEXT: ; use v014291; GFX90A-NEXT: ;;#ASMEND14292; GFX90A-NEXT: s_setpc_b64 s[30:31]14293;14294; GFX950-LABEL: flat_atomic_usub_sat_i32_saddr_ret_av_av:14295; GFX950: ; %bb.0:14296; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14297; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14298; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4014299; GFX950-NEXT: s_mov_b64 s[2:3], 014300; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14301; GFX950-NEXT: ;;#ASMSTART14302; GFX950-NEXT: ; def v414303; GFX950-NEXT: ;;#ASMEND14304; GFX950-NEXT: .LBB192_1: ; %atomicrmw.start14305; GFX950-NEXT: ; =>This Inner Loop Header: Depth=114306; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14307; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp14308; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc014309; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)14310; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v114311; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]14312; GFX950-NEXT: v_mov_b32_e32 v1, v014313; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]14314; GFX950-NEXT: s_cbranch_execnz .LBB192_114315; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end14316; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]14317; GFX950-NEXT: ;;#ASMSTART14318; GFX950-NEXT: ; use v014319; GFX950-NEXT: ;;#ASMEND14320; GFX950-NEXT: s_setpc_b64 s[30:31]14321 %gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 1014322 %data = call i32 asm "; def $0", "=^VA"()14323 %result = atomicrmw usub_sat ptr %gep.0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014324 call void asm "; use $0", "^VA"(i32 %result)14325 ret void14326}14327 14328;---------------------------------------------------------------------14329; other atomics i64, with aa+av cases using saddr14330;---------------------------------------------------------------------14331 14332define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {14333; GFX90A-LABEL: flat_atomic_xchg_i64_saddr_ret_a_a:14334; GFX90A: ; %bb.0:14335; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14336; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014337; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14338; GFX90A-NEXT: s_addc_u32 s5, s17, 014339; GFX90A-NEXT: s_cmp_eq_u32 s5, s714340; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014341; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14342; GFX90A-NEXT: ;;#ASMSTART14343; GFX90A-NEXT: ; def a[0:1]14344; GFX90A-NEXT: ;;#ASMEND14345; GFX90A-NEXT: s_cbranch_vccz .LBB193_214346; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14347; GFX90A-NEXT: v_accvgpr_read_b32 v0, a014348; GFX90A-NEXT: v_accvgpr_read_b32 v1, a114349; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]14350; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc14351; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14352; GFX90A-NEXT: s_waitcnt vmcnt(0)14353; GFX90A-NEXT: v_accvgpr_write_b32 a3, v114354; GFX90A-NEXT: v_accvgpr_write_b32 a2, v014355; GFX90A-NEXT: s_cbranch_execz .LBB193_314356; GFX90A-NEXT: s_branch .LBB193_414357; GFX90A-NEXT: .LBB193_2:14358; GFX90A-NEXT: ; implicit-def: $agpr2_agpr314359; GFX90A-NEXT: .LBB193_3: ; %atomicrmw.private14360; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014361; GFX90A-NEXT: s_cselect_b32 s4, s4, -114362; GFX90A-NEXT: v_mov_b32_e32 v0, s414363; GFX90A-NEXT: buffer_load_dword a2, v0, s[0:3], 0 offen14364; GFX90A-NEXT: buffer_load_dword a3, v0, s[0:3], 0 offen offset:414365; GFX90A-NEXT: s_nop 014366; GFX90A-NEXT: buffer_store_dword a0, v0, s[0:3], 0 offen14367; GFX90A-NEXT: buffer_store_dword a1, v0, s[0:3], 0 offen offset:414368; GFX90A-NEXT: .LBB193_4: ; %atomicrmw.end14369; GFX90A-NEXT: s_waitcnt vmcnt(2)14370; GFX90A-NEXT: ;;#ASMSTART14371; GFX90A-NEXT: ; use a[2:3]14372; GFX90A-NEXT: ;;#ASMEND14373; GFX90A-NEXT: s_waitcnt vmcnt(0)14374; GFX90A-NEXT: s_setpc_b64 s[30:31]14375;14376; GFX950-LABEL: flat_atomic_xchg_i64_saddr_ret_a_a:14377; GFX950: ; %bb.0:14378; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14379; GFX950-NEXT: s_add_u32 s0, s0, 0x5014380; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14381; GFX950-NEXT: s_addc_u32 s1, s1, 014382; GFX950-NEXT: s_cmp_eq_u32 s1, s314383; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014384; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14385; GFX950-NEXT: ;;#ASMSTART14386; GFX950-NEXT: ; def a[0:1]14387; GFX950-NEXT: ;;#ASMEND14388; GFX950-NEXT: s_cbranch_vccz .LBB193_214389; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14390; GFX950-NEXT: v_accvgpr_read_b32 v0, a014391; GFX950-NEXT: v_accvgpr_read_b32 v1, a114392; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14393; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] sc014394; GFX950-NEXT: s_waitcnt lgkmcnt(0)14395; GFX950-NEXT: s_waitcnt vmcnt(0)14396; GFX950-NEXT: v_accvgpr_write_b32 a3, v114397; GFX950-NEXT: v_accvgpr_write_b32 a2, v014398; GFX950-NEXT: s_cbranch_execz .LBB193_314399; GFX950-NEXT: s_branch .LBB193_414400; GFX950-NEXT: .LBB193_2:14401; GFX950-NEXT: ; implicit-def: $agpr2_agpr314402; GFX950-NEXT: .LBB193_3: ; %atomicrmw.private14403; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014404; GFX950-NEXT: s_cselect_b32 s0, s0, -114405; GFX950-NEXT: scratch_load_dwordx2 a[2:3], off, s014406; GFX950-NEXT: s_nop 014407; GFX950-NEXT: scratch_store_dwordx2 off, a[0:1], s014408; GFX950-NEXT: .LBB193_4: ; %atomicrmw.end14409; GFX950-NEXT: s_waitcnt vmcnt(1)14410; GFX950-NEXT: ;;#ASMSTART14411; GFX950-NEXT: ; use a[2:3]14412; GFX950-NEXT: ;;#ASMEND14413; GFX950-NEXT: s_waitcnt vmcnt(0)14414; GFX950-NEXT: s_setpc_b64 s[30:31]14415 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014416 %data = call i64 asm "; def $0", "=a"()14417 %result = atomicrmw xchg ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014418 call void asm "; use $0", "a"(i64 %result)14419 ret void14420}14421 14422define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {14423; GFX90A-LABEL: flat_atomic_xchg_i64_saddr_ret_av_av:14424; GFX90A: ; %bb.0:14425; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14426; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014427; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14428; GFX90A-NEXT: s_addc_u32 s5, s17, 014429; GFX90A-NEXT: s_cmp_eq_u32 s5, s714430; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014431; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14432; GFX90A-NEXT: ;;#ASMSTART14433; GFX90A-NEXT: ; def v[2:3]14434; GFX90A-NEXT: ;;#ASMEND14435; GFX90A-NEXT: s_cbranch_vccz .LBB194_214436; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14437; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]14438; GFX90A-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc14439; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14440; GFX90A-NEXT: s_cbranch_execz .LBB194_314441; GFX90A-NEXT: s_branch .LBB194_414442; GFX90A-NEXT: .LBB194_2:14443; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr114444; GFX90A-NEXT: .LBB194_3: ; %atomicrmw.private14445; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014446; GFX90A-NEXT: s_cselect_b32 s4, s4, -114447; GFX90A-NEXT: v_mov_b32_e32 v4, s414448; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen14449; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:414450; GFX90A-NEXT: s_nop 014451; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen14452; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:414453; GFX90A-NEXT: .LBB194_4: ; %atomicrmw.end14454; GFX90A-NEXT: s_waitcnt vmcnt(0)14455; GFX90A-NEXT: ;;#ASMSTART14456; GFX90A-NEXT: ; use v[0:1]14457; GFX90A-NEXT: ;;#ASMEND14458; GFX90A-NEXT: s_setpc_b64 s[30:31]14459;14460; GFX950-LABEL: flat_atomic_xchg_i64_saddr_ret_av_av:14461; GFX950: ; %bb.0:14462; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14463; GFX950-NEXT: s_add_u32 s0, s0, 0x5014464; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14465; GFX950-NEXT: s_addc_u32 s1, s1, 014466; GFX950-NEXT: s_cmp_eq_u32 s1, s314467; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014468; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14469; GFX950-NEXT: ;;#ASMSTART14470; GFX950-NEXT: ; def v[2:3]14471; GFX950-NEXT: ;;#ASMEND14472; GFX950-NEXT: s_cbranch_vccz .LBB194_214473; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14474; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14475; GFX950-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] sc014476; GFX950-NEXT: s_waitcnt lgkmcnt(0)14477; GFX950-NEXT: s_cbranch_execz .LBB194_314478; GFX950-NEXT: s_branch .LBB194_414479; GFX950-NEXT: .LBB194_2:14480; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr114481; GFX950-NEXT: .LBB194_3: ; %atomicrmw.private14482; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014483; GFX950-NEXT: s_cselect_b32 s0, s0, -114484; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s014485; GFX950-NEXT: s_nop 014486; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s014487; GFX950-NEXT: .LBB194_4: ; %atomicrmw.end14488; GFX950-NEXT: s_waitcnt vmcnt(0)14489; GFX950-NEXT: ;;#ASMSTART14490; GFX950-NEXT: ; use v[0:1]14491; GFX950-NEXT: ;;#ASMEND14492; GFX950-NEXT: s_setpc_b64 s[30:31]14493 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014494 %data = call i64 asm "; def $0", "=^VA"()14495 %result = atomicrmw xchg ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014496 call void asm "; use $0", "^VA"(i64 %result)14497 ret void14498}14499 14500define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {14501; GFX90A-LABEL: flat_atomic_add_i64_saddr_ret_a_a:14502; GFX90A: ; %bb.0:14503; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14504; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014505; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14506; GFX90A-NEXT: s_addc_u32 s5, s17, 014507; GFX90A-NEXT: s_cmp_eq_u32 s5, s714508; GFX90A-NEXT: ;;#ASMSTART14509; GFX90A-NEXT: ; def a[0:1]14510; GFX90A-NEXT: ;;#ASMEND14511; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014512; GFX90A-NEXT: v_accvgpr_read_b32 v0, a014513; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14514; GFX90A-NEXT: v_accvgpr_read_b32 v1, a114515; GFX90A-NEXT: s_cbranch_vccz .LBB195_214516; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14517; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]14518; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc14519; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14520; GFX90A-NEXT: s_waitcnt vmcnt(0)14521; GFX90A-NEXT: v_accvgpr_write_b32 a0, v214522; GFX90A-NEXT: v_accvgpr_write_b32 a1, v314523; GFX90A-NEXT: s_cbranch_execz .LBB195_314524; GFX90A-NEXT: s_branch .LBB195_414525; GFX90A-NEXT: .LBB195_2:14526; GFX90A-NEXT: ; implicit-def: $agpr0_agpr114527; GFX90A-NEXT: .LBB195_3: ; %atomicrmw.private14528; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014529; GFX90A-NEXT: s_cselect_b32 s4, s4, -114530; GFX90A-NEXT: v_mov_b32_e32 v2, s414531; GFX90A-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen14532; GFX90A-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:414533; GFX90A-NEXT: s_waitcnt vmcnt(1)14534; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, v3, v014535; GFX90A-NEXT: v_accvgpr_write_b32 a0, v314536; GFX90A-NEXT: s_waitcnt vmcnt(0)14537; GFX90A-NEXT: v_accvgpr_write_b32 a1, v414538; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v1, vcc14539; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen14540; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:414541; GFX90A-NEXT: .LBB195_4: ; %atomicrmw.end14542; GFX90A-NEXT: ;;#ASMSTART14543; GFX90A-NEXT: ; use a[0:1]14544; GFX90A-NEXT: ;;#ASMEND14545; GFX90A-NEXT: s_waitcnt vmcnt(0)14546; GFX90A-NEXT: s_setpc_b64 s[30:31]14547;14548; GFX950-LABEL: flat_atomic_add_i64_saddr_ret_a_a:14549; GFX950: ; %bb.0:14550; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14551; GFX950-NEXT: s_add_u32 s0, s0, 0x5014552; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14553; GFX950-NEXT: s_addc_u32 s1, s1, 014554; GFX950-NEXT: s_cmp_eq_u32 s1, s314555; GFX950-NEXT: ;;#ASMSTART14556; GFX950-NEXT: ; def a[0:1]14557; GFX950-NEXT: ;;#ASMEND14558; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014559; GFX950-NEXT: v_accvgpr_read_b32 v0, a014560; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14561; GFX950-NEXT: v_accvgpr_read_b32 v1, a114562; GFX950-NEXT: s_cbranch_vccz .LBB195_214563; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14564; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14565; GFX950-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc014566; GFX950-NEXT: s_waitcnt lgkmcnt(0)14567; GFX950-NEXT: s_waitcnt vmcnt(0)14568; GFX950-NEXT: v_accvgpr_write_b32 a0, v214569; GFX950-NEXT: v_accvgpr_write_b32 a1, v314570; GFX950-NEXT: s_cbranch_execz .LBB195_314571; GFX950-NEXT: s_branch .LBB195_414572; GFX950-NEXT: .LBB195_2:14573; GFX950-NEXT: ; implicit-def: $agpr0_agpr114574; GFX950-NEXT: .LBB195_3: ; %atomicrmw.private14575; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014576; GFX950-NEXT: s_cselect_b32 s0, s0, -114577; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s014578; GFX950-NEXT: s_waitcnt vmcnt(0)14579; GFX950-NEXT: v_accvgpr_write_b32 a0, v214580; GFX950-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, v[0:1]14581; GFX950-NEXT: v_accvgpr_write_b32 a1, v314582; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s014583; GFX950-NEXT: .LBB195_4: ; %atomicrmw.end14584; GFX950-NEXT: ;;#ASMSTART14585; GFX950-NEXT: ; use a[0:1]14586; GFX950-NEXT: ;;#ASMEND14587; GFX950-NEXT: s_waitcnt vmcnt(0)14588; GFX950-NEXT: s_setpc_b64 s[30:31]14589 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014590 %data = call i64 asm "; def $0", "=a"()14591 %result = atomicrmw add ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014592 call void asm "; use $0", "a"(i64 %result)14593 ret void14594}14595 14596define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {14597; GFX90A-LABEL: flat_atomic_add_i64_saddr_ret_av_av:14598; GFX90A: ; %bb.0:14599; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14600; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014601; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14602; GFX90A-NEXT: s_addc_u32 s5, s17, 014603; GFX90A-NEXT: s_cmp_eq_u32 s5, s714604; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014605; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14606; GFX90A-NEXT: ;;#ASMSTART14607; GFX90A-NEXT: ; def v[2:3]14608; GFX90A-NEXT: ;;#ASMEND14609; GFX90A-NEXT: s_cbranch_vccz .LBB196_214610; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14611; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]14612; GFX90A-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc14613; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14614; GFX90A-NEXT: s_cbranch_execz .LBB196_314615; GFX90A-NEXT: s_branch .LBB196_414616; GFX90A-NEXT: .LBB196_2:14617; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr114618; GFX90A-NEXT: .LBB196_3: ; %atomicrmw.private14619; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014620; GFX90A-NEXT: s_cselect_b32 s4, s4, -114621; GFX90A-NEXT: v_mov_b32_e32 v4, s414622; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen14623; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:414624; GFX90A-NEXT: s_waitcnt vmcnt(0)14625; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, v0, v214626; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc14627; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen14628; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:414629; GFX90A-NEXT: .LBB196_4: ; %atomicrmw.end14630; GFX90A-NEXT: s_waitcnt vmcnt(0)14631; GFX90A-NEXT: ;;#ASMSTART14632; GFX90A-NEXT: ; use v[0:1]14633; GFX90A-NEXT: ;;#ASMEND14634; GFX90A-NEXT: s_setpc_b64 s[30:31]14635;14636; GFX950-LABEL: flat_atomic_add_i64_saddr_ret_av_av:14637; GFX950: ; %bb.0:14638; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14639; GFX950-NEXT: s_add_u32 s0, s0, 0x5014640; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14641; GFX950-NEXT: s_addc_u32 s1, s1, 014642; GFX950-NEXT: s_cmp_eq_u32 s1, s314643; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014644; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14645; GFX950-NEXT: ;;#ASMSTART14646; GFX950-NEXT: ; def v[2:3]14647; GFX950-NEXT: ;;#ASMEND14648; GFX950-NEXT: s_cbranch_vccz .LBB196_214649; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14650; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14651; GFX950-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] sc014652; GFX950-NEXT: s_waitcnt lgkmcnt(0)14653; GFX950-NEXT: s_cbranch_execz .LBB196_314654; GFX950-NEXT: s_branch .LBB196_414655; GFX950-NEXT: .LBB196_2:14656; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr114657; GFX950-NEXT: .LBB196_3: ; %atomicrmw.private14658; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014659; GFX950-NEXT: s_cselect_b32 s0, s0, -114660; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s014661; GFX950-NEXT: s_waitcnt vmcnt(0)14662; GFX950-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3]14663; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s014664; GFX950-NEXT: .LBB196_4: ; %atomicrmw.end14665; GFX950-NEXT: s_waitcnt vmcnt(0)14666; GFX950-NEXT: ;;#ASMSTART14667; GFX950-NEXT: ; use v[0:1]14668; GFX950-NEXT: ;;#ASMEND14669; GFX950-NEXT: s_setpc_b64 s[30:31]14670 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014671 %data = call i64 asm "; def $0", "=^VA"()14672 %result = atomicrmw add ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014673 call void asm "; use $0", "^VA"(i64 %result)14674 ret void14675}14676 14677define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {14678; GFX90A-LABEL: flat_atomic_sub_i64_saddr_ret_a_a:14679; GFX90A: ; %bb.0:14680; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14681; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014682; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14683; GFX90A-NEXT: s_addc_u32 s5, s17, 014684; GFX90A-NEXT: s_cmp_eq_u32 s5, s714685; GFX90A-NEXT: ;;#ASMSTART14686; GFX90A-NEXT: ; def a[0:1]14687; GFX90A-NEXT: ;;#ASMEND14688; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014689; GFX90A-NEXT: v_accvgpr_read_b32 v0, a014690; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14691; GFX90A-NEXT: v_accvgpr_read_b32 v1, a114692; GFX90A-NEXT: s_cbranch_vccz .LBB197_214693; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14694; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]14695; GFX90A-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] glc14696; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14697; GFX90A-NEXT: s_waitcnt vmcnt(0)14698; GFX90A-NEXT: v_accvgpr_write_b32 a0, v214699; GFX90A-NEXT: v_accvgpr_write_b32 a1, v314700; GFX90A-NEXT: s_cbranch_execz .LBB197_314701; GFX90A-NEXT: s_branch .LBB197_414702; GFX90A-NEXT: .LBB197_2:14703; GFX90A-NEXT: ; implicit-def: $agpr0_agpr114704; GFX90A-NEXT: .LBB197_3: ; %atomicrmw.private14705; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014706; GFX90A-NEXT: s_cselect_b32 s4, s4, -114707; GFX90A-NEXT: v_mov_b32_e32 v2, s414708; GFX90A-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen14709; GFX90A-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:414710; GFX90A-NEXT: s_waitcnt vmcnt(1)14711; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v3, v014712; GFX90A-NEXT: v_accvgpr_write_b32 a0, v314713; GFX90A-NEXT: s_waitcnt vmcnt(0)14714; GFX90A-NEXT: v_accvgpr_write_b32 a1, v414715; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v4, v1, vcc14716; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen14717; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:414718; GFX90A-NEXT: .LBB197_4: ; %atomicrmw.end14719; GFX90A-NEXT: ;;#ASMSTART14720; GFX90A-NEXT: ; use a[0:1]14721; GFX90A-NEXT: ;;#ASMEND14722; GFX90A-NEXT: s_waitcnt vmcnt(0)14723; GFX90A-NEXT: s_setpc_b64 s[30:31]14724;14725; GFX950-LABEL: flat_atomic_sub_i64_saddr_ret_a_a:14726; GFX950: ; %bb.0:14727; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14728; GFX950-NEXT: s_add_u32 s0, s0, 0x5014729; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14730; GFX950-NEXT: s_addc_u32 s1, s1, 014731; GFX950-NEXT: s_cmp_eq_u32 s1, s314732; GFX950-NEXT: ;;#ASMSTART14733; GFX950-NEXT: ; def a[0:1]14734; GFX950-NEXT: ;;#ASMEND14735; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014736; GFX950-NEXT: v_accvgpr_read_b32 v0, a014737; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14738; GFX950-NEXT: v_accvgpr_read_b32 v1, a114739; GFX950-NEXT: s_cbranch_vccz .LBB197_214740; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14741; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14742; GFX950-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] sc014743; GFX950-NEXT: s_waitcnt lgkmcnt(0)14744; GFX950-NEXT: s_waitcnt vmcnt(0)14745; GFX950-NEXT: v_accvgpr_write_b32 a0, v214746; GFX950-NEXT: v_accvgpr_write_b32 a1, v314747; GFX950-NEXT: s_cbranch_execz .LBB197_314748; GFX950-NEXT: s_branch .LBB197_414749; GFX950-NEXT: .LBB197_2:14750; GFX950-NEXT: ; implicit-def: $agpr0_agpr114751; GFX950-NEXT: .LBB197_3: ; %atomicrmw.private14752; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014753; GFX950-NEXT: s_cselect_b32 s0, s0, -114754; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s014755; GFX950-NEXT: s_waitcnt vmcnt(0)14756; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v014757; GFX950-NEXT: v_accvgpr_write_b32 a0, v214758; GFX950-NEXT: s_nop 014759; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc14760; GFX950-NEXT: v_accvgpr_write_b32 a1, v314761; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s014762; GFX950-NEXT: .LBB197_4: ; %atomicrmw.end14763; GFX950-NEXT: ;;#ASMSTART14764; GFX950-NEXT: ; use a[0:1]14765; GFX950-NEXT: ;;#ASMEND14766; GFX950-NEXT: s_waitcnt vmcnt(0)14767; GFX950-NEXT: s_setpc_b64 s[30:31]14768 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014769 %data = call i64 asm "; def $0", "=a"()14770 %result = atomicrmw sub ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014771 call void asm "; use $0", "a"(i64 %result)14772 ret void14773}14774 14775define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {14776; GFX90A-LABEL: flat_atomic_sub_i64_saddr_ret_av_av:14777; GFX90A: ; %bb.0:14778; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14779; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014780; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14781; GFX90A-NEXT: s_addc_u32 s5, s17, 014782; GFX90A-NEXT: s_cmp_eq_u32 s5, s714783; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014784; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14785; GFX90A-NEXT: ;;#ASMSTART14786; GFX90A-NEXT: ; def v[2:3]14787; GFX90A-NEXT: ;;#ASMEND14788; GFX90A-NEXT: s_cbranch_vccz .LBB198_214789; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14790; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]14791; GFX90A-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc14792; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14793; GFX90A-NEXT: s_cbranch_execz .LBB198_314794; GFX90A-NEXT: s_branch .LBB198_414795; GFX90A-NEXT: .LBB198_2:14796; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr114797; GFX90A-NEXT: .LBB198_3: ; %atomicrmw.private14798; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014799; GFX90A-NEXT: s_cselect_b32 s4, s4, -114800; GFX90A-NEXT: v_mov_b32_e32 v4, s414801; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen14802; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:414803; GFX90A-NEXT: s_waitcnt vmcnt(0)14804; GFX90A-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v214805; GFX90A-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc14806; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen14807; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:414808; GFX90A-NEXT: .LBB198_4: ; %atomicrmw.end14809; GFX90A-NEXT: s_waitcnt vmcnt(0)14810; GFX90A-NEXT: ;;#ASMSTART14811; GFX90A-NEXT: ; use v[0:1]14812; GFX90A-NEXT: ;;#ASMEND14813; GFX90A-NEXT: s_setpc_b64 s[30:31]14814;14815; GFX950-LABEL: flat_atomic_sub_i64_saddr_ret_av_av:14816; GFX950: ; %bb.0:14817; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14818; GFX950-NEXT: s_add_u32 s0, s0, 0x5014819; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14820; GFX950-NEXT: s_addc_u32 s1, s1, 014821; GFX950-NEXT: s_cmp_eq_u32 s1, s314822; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014823; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14824; GFX950-NEXT: ;;#ASMSTART14825; GFX950-NEXT: ; def v[2:3]14826; GFX950-NEXT: ;;#ASMEND14827; GFX950-NEXT: s_cbranch_vccz .LBB198_214828; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14829; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]14830; GFX950-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] sc014831; GFX950-NEXT: s_waitcnt lgkmcnt(0)14832; GFX950-NEXT: s_cbranch_execz .LBB198_314833; GFX950-NEXT: s_branch .LBB198_414834; GFX950-NEXT: .LBB198_2:14835; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr114836; GFX950-NEXT: .LBB198_3: ; %atomicrmw.private14837; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014838; GFX950-NEXT: s_cselect_b32 s0, s0, -114839; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s014840; GFX950-NEXT: s_waitcnt vmcnt(0)14841; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v214842; GFX950-NEXT: s_nop 114843; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc14844; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s014845; GFX950-NEXT: .LBB198_4: ; %atomicrmw.end14846; GFX950-NEXT: s_waitcnt vmcnt(0)14847; GFX950-NEXT: ;;#ASMSTART14848; GFX950-NEXT: ; use v[0:1]14849; GFX950-NEXT: ;;#ASMEND14850; GFX950-NEXT: s_setpc_b64 s[30:31]14851 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014852 %data = call i64 asm "; def $0", "=^VA"()14853 %result = atomicrmw sub ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014854 call void asm "; use $0", "^VA"(i64 %result)14855 ret void14856}14857 14858define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {14859; GFX90A-LABEL: flat_atomic_and_i64_saddr_ret_a_a:14860; GFX90A: ; %bb.0:14861; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14862; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014863; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14864; GFX90A-NEXT: s_addc_u32 s5, s17, 014865; GFX90A-NEXT: s_cmp_eq_u32 s5, s714866; GFX90A-NEXT: ;;#ASMSTART14867; GFX90A-NEXT: ; def a[0:1]14868; GFX90A-NEXT: ;;#ASMEND14869; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014870; GFX90A-NEXT: v_accvgpr_read_b32 v0, a014871; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14872; GFX90A-NEXT: v_accvgpr_read_b32 v1, a114873; GFX90A-NEXT: s_cbranch_vccz .LBB199_214874; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14875; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]14876; GFX90A-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] glc14877; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14878; GFX90A-NEXT: s_waitcnt vmcnt(0)14879; GFX90A-NEXT: v_accvgpr_write_b32 a0, v214880; GFX90A-NEXT: v_accvgpr_write_b32 a1, v314881; GFX90A-NEXT: s_cbranch_execz .LBB199_314882; GFX90A-NEXT: s_branch .LBB199_414883; GFX90A-NEXT: .LBB199_2:14884; GFX90A-NEXT: ; implicit-def: $agpr0_agpr114885; GFX90A-NEXT: .LBB199_3: ; %atomicrmw.private14886; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014887; GFX90A-NEXT: s_cselect_b32 s4, s4, -114888; GFX90A-NEXT: v_mov_b32_e32 v2, s414889; GFX90A-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen14890; GFX90A-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:414891; GFX90A-NEXT: s_waitcnt vmcnt(1)14892; GFX90A-NEXT: v_accvgpr_write_b32 a0, v314893; GFX90A-NEXT: s_waitcnt vmcnt(0)14894; GFX90A-NEXT: v_accvgpr_write_b32 a1, v414895; GFX90A-NEXT: v_and_b32_e32 v0, v3, v014896; GFX90A-NEXT: v_and_b32_e32 v1, v4, v114897; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen14898; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:414899; GFX90A-NEXT: .LBB199_4: ; %atomicrmw.end14900; GFX90A-NEXT: ;;#ASMSTART14901; GFX90A-NEXT: ; use a[0:1]14902; GFX90A-NEXT: ;;#ASMEND14903; GFX90A-NEXT: s_waitcnt vmcnt(0)14904; GFX90A-NEXT: s_setpc_b64 s[30:31]14905;14906; GFX950-LABEL: flat_atomic_and_i64_saddr_ret_a_a:14907; GFX950: ; %bb.0:14908; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14909; GFX950-NEXT: s_add_u32 s0, s0, 0x5014910; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base14911; GFX950-NEXT: s_addc_u32 s1, s1, 014912; GFX950-NEXT: s_cmp_eq_u32 s1, s314913; GFX950-NEXT: ;;#ASMSTART14914; GFX950-NEXT: ; def a[0:1]14915; GFX950-NEXT: ;;#ASMEND14916; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 014917; GFX950-NEXT: v_accvgpr_read_b32 v0, a014918; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]14919; GFX950-NEXT: v_accvgpr_read_b32 v1, a114920; GFX950-NEXT: s_cbranch_vccz .LBB199_214921; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global14922; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]14923; GFX950-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] sc014924; GFX950-NEXT: s_waitcnt lgkmcnt(0)14925; GFX950-NEXT: s_waitcnt vmcnt(0)14926; GFX950-NEXT: v_accvgpr_write_b32 a0, v214927; GFX950-NEXT: v_accvgpr_write_b32 a1, v314928; GFX950-NEXT: s_cbranch_execz .LBB199_314929; GFX950-NEXT: s_branch .LBB199_414930; GFX950-NEXT: .LBB199_2:14931; GFX950-NEXT: ; implicit-def: $agpr0_agpr114932; GFX950-NEXT: .LBB199_3: ; %atomicrmw.private14933; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 014934; GFX950-NEXT: s_cselect_b32 s0, s0, -114935; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s014936; GFX950-NEXT: s_waitcnt vmcnt(0)14937; GFX950-NEXT: v_accvgpr_write_b32 a0, v214938; GFX950-NEXT: v_and_b32_e32 v1, v3, v114939; GFX950-NEXT: v_and_b32_e32 v0, v2, v014940; GFX950-NEXT: v_accvgpr_write_b32 a1, v314941; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s014942; GFX950-NEXT: .LBB199_4: ; %atomicrmw.end14943; GFX950-NEXT: ;;#ASMSTART14944; GFX950-NEXT: ; use a[0:1]14945; GFX950-NEXT: ;;#ASMEND14946; GFX950-NEXT: s_waitcnt vmcnt(0)14947; GFX950-NEXT: s_setpc_b64 s[30:31]14948 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1014949 %data = call i64 asm "; def $0", "=a"()14950 %result = atomicrmw and ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !014951 call void asm "; use $0", "a"(i64 %result)14952 ret void14953}14954 14955define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {14956; GFX90A-LABEL: flat_atomic_and_i64_saddr_ret_av_av:14957; GFX90A: ; %bb.0:14958; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14959; GFX90A-NEXT: s_add_u32 s4, s16, 0x5014960; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base14961; GFX90A-NEXT: s_addc_u32 s5, s17, 014962; GFX90A-NEXT: s_cmp_eq_u32 s5, s714963; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 014964; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]14965; GFX90A-NEXT: ;;#ASMSTART14966; GFX90A-NEXT: ; def v[2:3]14967; GFX90A-NEXT: ;;#ASMEND14968; GFX90A-NEXT: s_cbranch_vccz .LBB200_214969; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global14970; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]14971; GFX90A-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc14972; GFX90A-NEXT: s_waitcnt lgkmcnt(0)14973; GFX90A-NEXT: s_cbranch_execz .LBB200_314974; GFX90A-NEXT: s_branch .LBB200_414975; GFX90A-NEXT: .LBB200_2:14976; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr114977; GFX90A-NEXT: .LBB200_3: ; %atomicrmw.private14978; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 014979; GFX90A-NEXT: s_cselect_b32 s4, s4, -114980; GFX90A-NEXT: v_mov_b32_e32 v4, s414981; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:414982; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen14983; GFX90A-NEXT: s_waitcnt vmcnt(0)14984; GFX90A-NEXT: v_and_b32_e32 v3, v1, v314985; GFX90A-NEXT: v_and_b32_e32 v2, v0, v214986; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen14987; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:414988; GFX90A-NEXT: .LBB200_4: ; %atomicrmw.end14989; GFX90A-NEXT: s_waitcnt vmcnt(0)14990; GFX90A-NEXT: ;;#ASMSTART14991; GFX90A-NEXT: ; use v[0:1]14992; GFX90A-NEXT: ;;#ASMEND14993; GFX90A-NEXT: s_setpc_b64 s[30:31]14994;14995; GFX950-LABEL: flat_atomic_and_i64_saddr_ret_av_av:14996; GFX950: ; %bb.0:14997; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14998; GFX950-NEXT: s_add_u32 s0, s0, 0x5014999; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15000; GFX950-NEXT: s_addc_u32 s1, s1, 015001; GFX950-NEXT: s_cmp_eq_u32 s1, s315002; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015003; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15004; GFX950-NEXT: ;;#ASMSTART15005; GFX950-NEXT: ; def v[2:3]15006; GFX950-NEXT: ;;#ASMEND15007; GFX950-NEXT: s_cbranch_vccz .LBB200_215008; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15009; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]15010; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] sc015011; GFX950-NEXT: s_waitcnt lgkmcnt(0)15012; GFX950-NEXT: s_cbranch_execz .LBB200_315013; GFX950-NEXT: s_branch .LBB200_415014; GFX950-NEXT: .LBB200_2:15015; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115016; GFX950-NEXT: .LBB200_3: ; %atomicrmw.private15017; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015018; GFX950-NEXT: s_cselect_b32 s0, s0, -115019; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015020; GFX950-NEXT: s_waitcnt vmcnt(0)15021; GFX950-NEXT: v_and_b32_e32 v3, v1, v315022; GFX950-NEXT: v_and_b32_e32 v2, v0, v215023; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015024; GFX950-NEXT: .LBB200_4: ; %atomicrmw.end15025; GFX950-NEXT: s_waitcnt vmcnt(0)15026; GFX950-NEXT: ;;#ASMSTART15027; GFX950-NEXT: ; use v[0:1]15028; GFX950-NEXT: ;;#ASMEND15029; GFX950-NEXT: s_setpc_b64 s[30:31]15030 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015031 %data = call i64 asm "; def $0", "=^VA"()15032 %result = atomicrmw and ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015033 call void asm "; use $0", "^VA"(i64 %result)15034 ret void15035}15036 15037define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {15038; GFX90A-LABEL: flat_atomic_nand_i64_saddr_ret_a_a:15039; GFX90A: ; %bb.0:15040; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15041; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015042; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15043; GFX90A-NEXT: s_addc_u32 s5, s17, 015044; GFX90A-NEXT: s_cmp_eq_u32 s5, s715045; GFX90A-NEXT: ;;#ASMSTART15046; GFX90A-NEXT: ; def a[0:1]15047; GFX90A-NEXT: ;;#ASMEND15048; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015049; GFX90A-NEXT: v_accvgpr_read_b32 v5, a115050; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15051; GFX90A-NEXT: v_accvgpr_read_b32 v4, a015052; GFX90A-NEXT: s_cbranch_vccz .LBB201_415053; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15054; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]15055; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]15056; GFX90A-NEXT: s_mov_b64 s[6:7], 015057; GFX90A-NEXT: .LBB201_2: ; %atomicrmw.start15058; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=115059; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15060; GFX90A-NEXT: v_and_b32_e32 v0, v3, v515061; GFX90A-NEXT: v_and_b32_e32 v8, v2, v415062; GFX90A-NEXT: v_not_b32_e32 v1, v015063; GFX90A-NEXT: v_not_b32_e32 v0, v815064; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc15065; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15066; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]15067; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]15068; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]15069; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]15070; GFX90A-NEXT: s_cbranch_execnz .LBB201_215071; GFX90A-NEXT: ; %bb.3: ; %Flow15072; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]15073; GFX90A-NEXT: v_accvgpr_write_b32 a0, v015074; GFX90A-NEXT: v_accvgpr_write_b32 a1, v115075; GFX90A-NEXT: s_branch .LBB201_615076; GFX90A-NEXT: .LBB201_4:15077; GFX90A-NEXT: ; implicit-def: $agpr0_agpr115078; GFX90A-NEXT: s_cbranch_execz .LBB201_615079; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private15080; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015081; GFX90A-NEXT: s_cselect_b32 s4, s4, -115082; GFX90A-NEXT: v_mov_b32_e32 v0, s415083; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen15084; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:415085; GFX90A-NEXT: s_waitcnt vmcnt(1)15086; GFX90A-NEXT: v_accvgpr_write_b32 a0, v115087; GFX90A-NEXT: v_and_b32_e32 v1, v1, v415088; GFX90A-NEXT: s_waitcnt vmcnt(0)15089; GFX90A-NEXT: v_accvgpr_write_b32 a1, v215090; GFX90A-NEXT: v_and_b32_e32 v2, v2, v515091; GFX90A-NEXT: v_not_b32_e32 v1, v115092; GFX90A-NEXT: v_not_b32_e32 v2, v215093; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen15094; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:415095; GFX90A-NEXT: .LBB201_6: ; %atomicrmw.phi15096; GFX90A-NEXT: ;;#ASMSTART15097; GFX90A-NEXT: ; use a[0:1]15098; GFX90A-NEXT: ;;#ASMEND15099; GFX90A-NEXT: s_waitcnt vmcnt(0)15100; GFX90A-NEXT: s_setpc_b64 s[30:31]15101;15102; GFX950-LABEL: flat_atomic_nand_i64_saddr_ret_a_a:15103; GFX950: ; %bb.0:15104; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15105; GFX950-NEXT: s_add_u32 s0, s0, 0x5015106; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15107; GFX950-NEXT: s_addc_u32 s1, s1, 015108; GFX950-NEXT: s_cmp_eq_u32 s1, s315109; GFX950-NEXT: ;;#ASMSTART15110; GFX950-NEXT: ; def a[0:1]15111; GFX950-NEXT: ;;#ASMEND15112; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015113; GFX950-NEXT: v_accvgpr_read_b32 v5, a115114; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15115; GFX950-NEXT: v_accvgpr_read_b32 v4, a015116; GFX950-NEXT: s_cbranch_vccz .LBB201_415117; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15118; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]15119; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]15120; GFX950-NEXT: s_mov_b64 s[2:3], 015121; GFX950-NEXT: .LBB201_2: ; %atomicrmw.start15122; GFX950-NEXT: ; =>This Inner Loop Header: Depth=115123; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15124; GFX950-NEXT: v_and_b32_e32 v0, v3, v515125; GFX950-NEXT: v_and_b32_e32 v8, v2, v415126; GFX950-NEXT: v_not_b32_e32 v1, v015127; GFX950-NEXT: v_not_b32_e32 v0, v815128; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc015129; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15130; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]15131; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]15132; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]15133; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]15134; GFX950-NEXT: s_cbranch_execnz .LBB201_215135; GFX950-NEXT: ; %bb.3: ; %Flow15136; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]15137; GFX950-NEXT: v_accvgpr_write_b32 a0, v015138; GFX950-NEXT: v_accvgpr_write_b32 a1, v115139; GFX950-NEXT: s_branch .LBB201_615140; GFX950-NEXT: .LBB201_4:15141; GFX950-NEXT: ; implicit-def: $agpr0_agpr115142; GFX950-NEXT: s_cbranch_execz .LBB201_615143; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private15144; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015145; GFX950-NEXT: s_cselect_b32 s0, s0, -115146; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015147; GFX950-NEXT: s_waitcnt vmcnt(0)15148; GFX950-NEXT: v_and_b32_e32 v2, v1, v515149; GFX950-NEXT: v_and_b32_e32 v4, v0, v415150; GFX950-NEXT: v_accvgpr_write_b32 a0, v015151; GFX950-NEXT: v_not_b32_e32 v3, v215152; GFX950-NEXT: v_not_b32_e32 v2, v415153; GFX950-NEXT: v_accvgpr_write_b32 a1, v115154; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015155; GFX950-NEXT: .LBB201_6: ; %atomicrmw.phi15156; GFX950-NEXT: ;;#ASMSTART15157; GFX950-NEXT: ; use a[0:1]15158; GFX950-NEXT: ;;#ASMEND15159; GFX950-NEXT: s_waitcnt vmcnt(0)15160; GFX950-NEXT: s_setpc_b64 s[30:31]15161 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015162 %data = call i64 asm "; def $0", "=a"()15163 %result = atomicrmw nand ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015164 call void asm "; use $0", "a"(i64 %result)15165 ret void15166}15167 15168define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {15169; GFX90A-LABEL: flat_atomic_nand_i64_saddr_ret_av_av:15170; GFX90A: ; %bb.0:15171; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15172; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015173; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15174; GFX90A-NEXT: s_addc_u32 s5, s17, 015175; GFX90A-NEXT: s_cmp_eq_u32 s5, s715176; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015177; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15178; GFX90A-NEXT: ;;#ASMSTART15179; GFX90A-NEXT: ; def v[4:5]15180; GFX90A-NEXT: ;;#ASMEND15181; GFX90A-NEXT: s_cbranch_vccz .LBB202_415182; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15183; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]15184; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]15185; GFX90A-NEXT: s_mov_b64 s[6:7], 015186; GFX90A-NEXT: .LBB202_2: ; %atomicrmw.start15187; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=115188; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15189; GFX90A-NEXT: v_and_b32_e32 v0, v3, v515190; GFX90A-NEXT: v_and_b32_e32 v8, v2, v415191; GFX90A-NEXT: v_not_b32_e32 v1, v015192; GFX90A-NEXT: v_not_b32_e32 v0, v815193; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc15194; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15195; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]15196; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]15197; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]15198; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]15199; GFX90A-NEXT: s_cbranch_execnz .LBB202_215200; GFX90A-NEXT: ; %bb.3: ; %Flow15201; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]15202; GFX90A-NEXT: s_branch .LBB202_615203; GFX90A-NEXT: .LBB202_4:15204; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr115205; GFX90A-NEXT: s_cbranch_execz .LBB202_615206; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private15207; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015208; GFX90A-NEXT: s_cselect_b32 s4, s4, -115209; GFX90A-NEXT: v_mov_b32_e32 v2, s415210; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:415211; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen15212; GFX90A-NEXT: s_waitcnt vmcnt(1)15213; GFX90A-NEXT: v_and_b32_e32 v3, v1, v515214; GFX90A-NEXT: s_waitcnt vmcnt(0)15215; GFX90A-NEXT: v_and_b32_e32 v4, v0, v415216; GFX90A-NEXT: v_not_b32_e32 v4, v415217; GFX90A-NEXT: v_not_b32_e32 v3, v315218; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen15219; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:415220; GFX90A-NEXT: .LBB202_6: ; %atomicrmw.phi15221; GFX90A-NEXT: ;;#ASMSTART15222; GFX90A-NEXT: ; use v[0:1]15223; GFX90A-NEXT: ;;#ASMEND15224; GFX90A-NEXT: s_waitcnt vmcnt(0)15225; GFX90A-NEXT: s_setpc_b64 s[30:31]15226;15227; GFX950-LABEL: flat_atomic_nand_i64_saddr_ret_av_av:15228; GFX950: ; %bb.0:15229; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15230; GFX950-NEXT: s_add_u32 s0, s0, 0x5015231; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15232; GFX950-NEXT: s_addc_u32 s1, s1, 015233; GFX950-NEXT: s_cmp_eq_u32 s1, s315234; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015235; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15236; GFX950-NEXT: ;;#ASMSTART15237; GFX950-NEXT: ; def v[4:5]15238; GFX950-NEXT: ;;#ASMEND15239; GFX950-NEXT: s_cbranch_vccz .LBB202_415240; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15241; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]15242; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]15243; GFX950-NEXT: s_mov_b64 s[2:3], 015244; GFX950-NEXT: .LBB202_2: ; %atomicrmw.start15245; GFX950-NEXT: ; =>This Inner Loop Header: Depth=115246; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15247; GFX950-NEXT: v_and_b32_e32 v0, v3, v515248; GFX950-NEXT: v_and_b32_e32 v8, v2, v415249; GFX950-NEXT: v_not_b32_e32 v1, v015250; GFX950-NEXT: v_not_b32_e32 v0, v815251; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc015252; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)15253; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]15254; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]15255; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]15256; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]15257; GFX950-NEXT: s_cbranch_execnz .LBB202_215258; GFX950-NEXT: ; %bb.3: ; %Flow15259; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]15260; GFX950-NEXT: s_branch .LBB202_615261; GFX950-NEXT: .LBB202_4:15262; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115263; GFX950-NEXT: s_cbranch_execz .LBB202_615264; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private15265; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015266; GFX950-NEXT: s_cselect_b32 s0, s0, -115267; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015268; GFX950-NEXT: s_waitcnt vmcnt(0)15269; GFX950-NEXT: v_and_b32_e32 v2, v1, v515270; GFX950-NEXT: v_and_b32_e32 v4, v0, v415271; GFX950-NEXT: v_not_b32_e32 v3, v215272; GFX950-NEXT: v_not_b32_e32 v2, v415273; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015274; GFX950-NEXT: .LBB202_6: ; %atomicrmw.phi15275; GFX950-NEXT: ;;#ASMSTART15276; GFX950-NEXT: ; use v[0:1]15277; GFX950-NEXT: ;;#ASMEND15278; GFX950-NEXT: s_waitcnt vmcnt(0)15279; GFX950-NEXT: s_setpc_b64 s[30:31]15280 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015281 %data = call i64 asm "; def $0", "=^VA"()15282 %result = atomicrmw nand ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015283 call void asm "; use $0", "^VA"(i64 %result)15284 ret void15285}15286 15287define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {15288; GFX90A-LABEL: flat_atomic_or_i64_saddr_ret_a_a:15289; GFX90A: ; %bb.0:15290; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15291; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015292; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15293; GFX90A-NEXT: s_addc_u32 s5, s17, 015294; GFX90A-NEXT: s_cmp_eq_u32 s5, s715295; GFX90A-NEXT: ;;#ASMSTART15296; GFX90A-NEXT: ; def a[0:1]15297; GFX90A-NEXT: ;;#ASMEND15298; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015299; GFX90A-NEXT: v_accvgpr_read_b32 v0, a015300; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15301; GFX90A-NEXT: v_accvgpr_read_b32 v1, a115302; GFX90A-NEXT: s_cbranch_vccz .LBB203_215303; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15304; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]15305; GFX90A-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] glc15306; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15307; GFX90A-NEXT: s_waitcnt vmcnt(0)15308; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215309; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315310; GFX90A-NEXT: s_cbranch_execz .LBB203_315311; GFX90A-NEXT: s_branch .LBB203_415312; GFX90A-NEXT: .LBB203_2:15313; GFX90A-NEXT: ; implicit-def: $agpr0_agpr115314; GFX90A-NEXT: .LBB203_3: ; %atomicrmw.private15315; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015316; GFX90A-NEXT: s_cselect_b32 s4, s4, -115317; GFX90A-NEXT: v_mov_b32_e32 v2, s415318; GFX90A-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen15319; GFX90A-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:415320; GFX90A-NEXT: s_waitcnt vmcnt(1)15321; GFX90A-NEXT: v_accvgpr_write_b32 a0, v315322; GFX90A-NEXT: s_waitcnt vmcnt(0)15323; GFX90A-NEXT: v_accvgpr_write_b32 a1, v415324; GFX90A-NEXT: v_or_b32_e32 v0, v3, v015325; GFX90A-NEXT: v_or_b32_e32 v1, v4, v115326; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen15327; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:415328; GFX90A-NEXT: .LBB203_4: ; %atomicrmw.end15329; GFX90A-NEXT: ;;#ASMSTART15330; GFX90A-NEXT: ; use a[0:1]15331; GFX90A-NEXT: ;;#ASMEND15332; GFX90A-NEXT: s_waitcnt vmcnt(0)15333; GFX90A-NEXT: s_setpc_b64 s[30:31]15334;15335; GFX950-LABEL: flat_atomic_or_i64_saddr_ret_a_a:15336; GFX950: ; %bb.0:15337; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15338; GFX950-NEXT: s_add_u32 s0, s0, 0x5015339; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15340; GFX950-NEXT: s_addc_u32 s1, s1, 015341; GFX950-NEXT: s_cmp_eq_u32 s1, s315342; GFX950-NEXT: ;;#ASMSTART15343; GFX950-NEXT: ; def a[0:1]15344; GFX950-NEXT: ;;#ASMEND15345; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015346; GFX950-NEXT: v_accvgpr_read_b32 v0, a015347; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15348; GFX950-NEXT: v_accvgpr_read_b32 v1, a115349; GFX950-NEXT: s_cbranch_vccz .LBB203_215350; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15351; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]15352; GFX950-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] sc015353; GFX950-NEXT: s_waitcnt lgkmcnt(0)15354; GFX950-NEXT: s_waitcnt vmcnt(0)15355; GFX950-NEXT: v_accvgpr_write_b32 a0, v215356; GFX950-NEXT: v_accvgpr_write_b32 a1, v315357; GFX950-NEXT: s_cbranch_execz .LBB203_315358; GFX950-NEXT: s_branch .LBB203_415359; GFX950-NEXT: .LBB203_2:15360; GFX950-NEXT: ; implicit-def: $agpr0_agpr115361; GFX950-NEXT: .LBB203_3: ; %atomicrmw.private15362; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015363; GFX950-NEXT: s_cselect_b32 s0, s0, -115364; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s015365; GFX950-NEXT: s_waitcnt vmcnt(0)15366; GFX950-NEXT: v_accvgpr_write_b32 a0, v215367; GFX950-NEXT: v_or_b32_e32 v1, v3, v115368; GFX950-NEXT: v_or_b32_e32 v0, v2, v015369; GFX950-NEXT: v_accvgpr_write_b32 a1, v315370; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s015371; GFX950-NEXT: .LBB203_4: ; %atomicrmw.end15372; GFX950-NEXT: ;;#ASMSTART15373; GFX950-NEXT: ; use a[0:1]15374; GFX950-NEXT: ;;#ASMEND15375; GFX950-NEXT: s_waitcnt vmcnt(0)15376; GFX950-NEXT: s_setpc_b64 s[30:31]15377 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015378 %data = call i64 asm "; def $0", "=a"()15379 %result = atomicrmw or ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015380 call void asm "; use $0", "a"(i64 %result)15381 ret void15382}15383 15384define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {15385; GFX90A-LABEL: flat_atomic_or_i64_saddr_ret_av_av:15386; GFX90A: ; %bb.0:15387; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15388; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015389; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15390; GFX90A-NEXT: s_addc_u32 s5, s17, 015391; GFX90A-NEXT: s_cmp_eq_u32 s5, s715392; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015393; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15394; GFX90A-NEXT: ;;#ASMSTART15395; GFX90A-NEXT: ; def v[2:3]15396; GFX90A-NEXT: ;;#ASMEND15397; GFX90A-NEXT: s_cbranch_vccz .LBB204_215398; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15399; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]15400; GFX90A-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc15401; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15402; GFX90A-NEXT: s_cbranch_execz .LBB204_315403; GFX90A-NEXT: s_branch .LBB204_415404; GFX90A-NEXT: .LBB204_2:15405; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr115406; GFX90A-NEXT: .LBB204_3: ; %atomicrmw.private15407; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015408; GFX90A-NEXT: s_cselect_b32 s4, s4, -115409; GFX90A-NEXT: v_mov_b32_e32 v4, s415410; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:415411; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen15412; GFX90A-NEXT: s_waitcnt vmcnt(0)15413; GFX90A-NEXT: v_or_b32_e32 v3, v1, v315414; GFX90A-NEXT: v_or_b32_e32 v2, v0, v215415; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen15416; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:415417; GFX90A-NEXT: .LBB204_4: ; %atomicrmw.end15418; GFX90A-NEXT: s_waitcnt vmcnt(0)15419; GFX90A-NEXT: ;;#ASMSTART15420; GFX90A-NEXT: ; use v[0:1]15421; GFX90A-NEXT: ;;#ASMEND15422; GFX90A-NEXT: s_setpc_b64 s[30:31]15423;15424; GFX950-LABEL: flat_atomic_or_i64_saddr_ret_av_av:15425; GFX950: ; %bb.0:15426; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15427; GFX950-NEXT: s_add_u32 s0, s0, 0x5015428; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15429; GFX950-NEXT: s_addc_u32 s1, s1, 015430; GFX950-NEXT: s_cmp_eq_u32 s1, s315431; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015432; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15433; GFX950-NEXT: ;;#ASMSTART15434; GFX950-NEXT: ; def v[2:3]15435; GFX950-NEXT: ;;#ASMEND15436; GFX950-NEXT: s_cbranch_vccz .LBB204_215437; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15438; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]15439; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] sc015440; GFX950-NEXT: s_waitcnt lgkmcnt(0)15441; GFX950-NEXT: s_cbranch_execz .LBB204_315442; GFX950-NEXT: s_branch .LBB204_415443; GFX950-NEXT: .LBB204_2:15444; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115445; GFX950-NEXT: .LBB204_3: ; %atomicrmw.private15446; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015447; GFX950-NEXT: s_cselect_b32 s0, s0, -115448; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015449; GFX950-NEXT: s_waitcnt vmcnt(0)15450; GFX950-NEXT: v_or_b32_e32 v3, v1, v315451; GFX950-NEXT: v_or_b32_e32 v2, v0, v215452; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015453; GFX950-NEXT: .LBB204_4: ; %atomicrmw.end15454; GFX950-NEXT: s_waitcnt vmcnt(0)15455; GFX950-NEXT: ;;#ASMSTART15456; GFX950-NEXT: ; use v[0:1]15457; GFX950-NEXT: ;;#ASMEND15458; GFX950-NEXT: s_setpc_b64 s[30:31]15459 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015460 %data = call i64 asm "; def $0", "=^VA"()15461 %result = atomicrmw or ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015462 call void asm "; use $0", "^VA"(i64 %result)15463 ret void15464}15465 15466define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {15467; GFX90A-LABEL: flat_atomic_xor_i64_saddr_ret_a_a:15468; GFX90A: ; %bb.0:15469; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15470; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015471; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15472; GFX90A-NEXT: s_addc_u32 s5, s17, 015473; GFX90A-NEXT: s_cmp_eq_u32 s5, s715474; GFX90A-NEXT: ;;#ASMSTART15475; GFX90A-NEXT: ; def a[0:1]15476; GFX90A-NEXT: ;;#ASMEND15477; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015478; GFX90A-NEXT: v_accvgpr_read_b32 v0, a015479; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15480; GFX90A-NEXT: v_accvgpr_read_b32 v1, a115481; GFX90A-NEXT: s_cbranch_vccz .LBB205_215482; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15483; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]15484; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] glc15485; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15486; GFX90A-NEXT: s_waitcnt vmcnt(0)15487; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215488; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315489; GFX90A-NEXT: s_cbranch_execz .LBB205_315490; GFX90A-NEXT: s_branch .LBB205_415491; GFX90A-NEXT: .LBB205_2:15492; GFX90A-NEXT: ; implicit-def: $agpr0_agpr115493; GFX90A-NEXT: .LBB205_3: ; %atomicrmw.private15494; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015495; GFX90A-NEXT: s_cselect_b32 s4, s4, -115496; GFX90A-NEXT: v_mov_b32_e32 v2, s415497; GFX90A-NEXT: buffer_load_dword v3, v2, s[0:3], 0 offen15498; GFX90A-NEXT: buffer_load_dword v4, v2, s[0:3], 0 offen offset:415499; GFX90A-NEXT: s_waitcnt vmcnt(1)15500; GFX90A-NEXT: v_accvgpr_write_b32 a0, v315501; GFX90A-NEXT: s_waitcnt vmcnt(0)15502; GFX90A-NEXT: v_accvgpr_write_b32 a1, v415503; GFX90A-NEXT: v_xor_b32_e32 v0, v3, v015504; GFX90A-NEXT: v_xor_b32_e32 v1, v4, v115505; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen15506; GFX90A-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:415507; GFX90A-NEXT: .LBB205_4: ; %atomicrmw.end15508; GFX90A-NEXT: ;;#ASMSTART15509; GFX90A-NEXT: ; use a[0:1]15510; GFX90A-NEXT: ;;#ASMEND15511; GFX90A-NEXT: s_waitcnt vmcnt(0)15512; GFX90A-NEXT: s_setpc_b64 s[30:31]15513;15514; GFX950-LABEL: flat_atomic_xor_i64_saddr_ret_a_a:15515; GFX950: ; %bb.0:15516; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15517; GFX950-NEXT: s_add_u32 s0, s0, 0x5015518; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15519; GFX950-NEXT: s_addc_u32 s1, s1, 015520; GFX950-NEXT: s_cmp_eq_u32 s1, s315521; GFX950-NEXT: ;;#ASMSTART15522; GFX950-NEXT: ; def a[0:1]15523; GFX950-NEXT: ;;#ASMEND15524; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015525; GFX950-NEXT: v_accvgpr_read_b32 v0, a015526; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15527; GFX950-NEXT: v_accvgpr_read_b32 v1, a115528; GFX950-NEXT: s_cbranch_vccz .LBB205_215529; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15530; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]15531; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] sc015532; GFX950-NEXT: s_waitcnt lgkmcnt(0)15533; GFX950-NEXT: s_waitcnt vmcnt(0)15534; GFX950-NEXT: v_accvgpr_write_b32 a0, v215535; GFX950-NEXT: v_accvgpr_write_b32 a1, v315536; GFX950-NEXT: s_cbranch_execz .LBB205_315537; GFX950-NEXT: s_branch .LBB205_415538; GFX950-NEXT: .LBB205_2:15539; GFX950-NEXT: ; implicit-def: $agpr0_agpr115540; GFX950-NEXT: .LBB205_3: ; %atomicrmw.private15541; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015542; GFX950-NEXT: s_cselect_b32 s0, s0, -115543; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s015544; GFX950-NEXT: s_waitcnt vmcnt(0)15545; GFX950-NEXT: v_accvgpr_write_b32 a0, v215546; GFX950-NEXT: v_xor_b32_e32 v1, v3, v115547; GFX950-NEXT: v_xor_b32_e32 v0, v2, v015548; GFX950-NEXT: v_accvgpr_write_b32 a1, v315549; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s015550; GFX950-NEXT: .LBB205_4: ; %atomicrmw.end15551; GFX950-NEXT: ;;#ASMSTART15552; GFX950-NEXT: ; use a[0:1]15553; GFX950-NEXT: ;;#ASMEND15554; GFX950-NEXT: s_waitcnt vmcnt(0)15555; GFX950-NEXT: s_setpc_b64 s[30:31]15556 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015557 %data = call i64 asm "; def $0", "=a"()15558 %result = atomicrmw xor ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015559 call void asm "; use $0", "a"(i64 %result)15560 ret void15561}15562 15563define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {15564; GFX90A-LABEL: flat_atomic_xor_i64_saddr_ret_av_av:15565; GFX90A: ; %bb.0:15566; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15567; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015568; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15569; GFX90A-NEXT: s_addc_u32 s5, s17, 015570; GFX90A-NEXT: s_cmp_eq_u32 s5, s715571; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015572; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15573; GFX90A-NEXT: ;;#ASMSTART15574; GFX90A-NEXT: ; def v[2:3]15575; GFX90A-NEXT: ;;#ASMEND15576; GFX90A-NEXT: s_cbranch_vccz .LBB206_215577; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15578; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]15579; GFX90A-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc15580; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15581; GFX90A-NEXT: s_cbranch_execz .LBB206_315582; GFX90A-NEXT: s_branch .LBB206_415583; GFX90A-NEXT: .LBB206_2:15584; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr115585; GFX90A-NEXT: .LBB206_3: ; %atomicrmw.private15586; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015587; GFX90A-NEXT: s_cselect_b32 s4, s4, -115588; GFX90A-NEXT: v_mov_b32_e32 v4, s415589; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:415590; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen15591; GFX90A-NEXT: s_waitcnt vmcnt(0)15592; GFX90A-NEXT: v_xor_b32_e32 v3, v1, v315593; GFX90A-NEXT: v_xor_b32_e32 v2, v0, v215594; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen15595; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:415596; GFX90A-NEXT: .LBB206_4: ; %atomicrmw.end15597; GFX90A-NEXT: s_waitcnt vmcnt(0)15598; GFX90A-NEXT: ;;#ASMSTART15599; GFX90A-NEXT: ; use v[0:1]15600; GFX90A-NEXT: ;;#ASMEND15601; GFX90A-NEXT: s_setpc_b64 s[30:31]15602;15603; GFX950-LABEL: flat_atomic_xor_i64_saddr_ret_av_av:15604; GFX950: ; %bb.0:15605; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15606; GFX950-NEXT: s_add_u32 s0, s0, 0x5015607; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15608; GFX950-NEXT: s_addc_u32 s1, s1, 015609; GFX950-NEXT: s_cmp_eq_u32 s1, s315610; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015611; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15612; GFX950-NEXT: ;;#ASMSTART15613; GFX950-NEXT: ; def v[2:3]15614; GFX950-NEXT: ;;#ASMEND15615; GFX950-NEXT: s_cbranch_vccz .LBB206_215616; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15617; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]15618; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc015619; GFX950-NEXT: s_waitcnt lgkmcnt(0)15620; GFX950-NEXT: s_cbranch_execz .LBB206_315621; GFX950-NEXT: s_branch .LBB206_415622; GFX950-NEXT: .LBB206_2:15623; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115624; GFX950-NEXT: .LBB206_3: ; %atomicrmw.private15625; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015626; GFX950-NEXT: s_cselect_b32 s0, s0, -115627; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015628; GFX950-NEXT: s_waitcnt vmcnt(0)15629; GFX950-NEXT: v_xor_b32_e32 v3, v1, v315630; GFX950-NEXT: v_xor_b32_e32 v2, v0, v215631; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015632; GFX950-NEXT: .LBB206_4: ; %atomicrmw.end15633; GFX950-NEXT: s_waitcnt vmcnt(0)15634; GFX950-NEXT: ;;#ASMSTART15635; GFX950-NEXT: ; use v[0:1]15636; GFX950-NEXT: ;;#ASMEND15637; GFX950-NEXT: s_setpc_b64 s[30:31]15638 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015639 %data = call i64 asm "; def $0", "=^VA"()15640 %result = atomicrmw xor ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015641 call void asm "; use $0", "^VA"(i64 %result)15642 ret void15643}15644 15645define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {15646; GFX90A-LABEL: flat_atomic_max_i64_saddr_ret_a_a:15647; GFX90A: ; %bb.0:15648; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15649; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015650; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15651; GFX90A-NEXT: s_addc_u32 s5, s17, 015652; GFX90A-NEXT: s_cmp_eq_u32 s5, s715653; GFX90A-NEXT: ;;#ASMSTART15654; GFX90A-NEXT: ; def a[0:1]15655; GFX90A-NEXT: ;;#ASMEND15656; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015657; GFX90A-NEXT: v_accvgpr_read_b32 v0, a015658; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15659; GFX90A-NEXT: v_accvgpr_read_b32 v1, a115660; GFX90A-NEXT: s_cbranch_vccz .LBB207_215661; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15662; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]15663; GFX90A-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] glc15664; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15665; GFX90A-NEXT: s_waitcnt vmcnt(0)15666; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215667; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315668; GFX90A-NEXT: s_cbranch_execz .LBB207_315669; GFX90A-NEXT: s_branch .LBB207_415670; GFX90A-NEXT: .LBB207_2:15671; GFX90A-NEXT: ; implicit-def: $agpr0_agpr115672; GFX90A-NEXT: .LBB207_3: ; %atomicrmw.private15673; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015674; GFX90A-NEXT: s_cselect_b32 s4, s4, -115675; GFX90A-NEXT: v_mov_b32_e32 v4, s415676; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen15677; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:415678; GFX90A-NEXT: s_waitcnt vmcnt(1)15679; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215680; GFX90A-NEXT: s_waitcnt vmcnt(0)15681; GFX90A-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[0:1]15682; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc15683; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315684; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc15685; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:415686; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen15687; GFX90A-NEXT: .LBB207_4: ; %atomicrmw.end15688; GFX90A-NEXT: ;;#ASMSTART15689; GFX90A-NEXT: ; use a[0:1]15690; GFX90A-NEXT: ;;#ASMEND15691; GFX90A-NEXT: s_waitcnt vmcnt(0)15692; GFX90A-NEXT: s_setpc_b64 s[30:31]15693;15694; GFX950-LABEL: flat_atomic_max_i64_saddr_ret_a_a:15695; GFX950: ; %bb.0:15696; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15697; GFX950-NEXT: s_add_u32 s0, s0, 0x5015698; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15699; GFX950-NEXT: s_addc_u32 s1, s1, 015700; GFX950-NEXT: s_cmp_eq_u32 s1, s315701; GFX950-NEXT: ;;#ASMSTART15702; GFX950-NEXT: ; def a[0:1]15703; GFX950-NEXT: ;;#ASMEND15704; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015705; GFX950-NEXT: v_accvgpr_read_b32 v0, a015706; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15707; GFX950-NEXT: v_accvgpr_read_b32 v1, a115708; GFX950-NEXT: s_cbranch_vccz .LBB207_215709; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15710; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]15711; GFX950-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] sc015712; GFX950-NEXT: s_waitcnt lgkmcnt(0)15713; GFX950-NEXT: s_waitcnt vmcnt(0)15714; GFX950-NEXT: v_accvgpr_write_b32 a0, v215715; GFX950-NEXT: v_accvgpr_write_b32 a1, v315716; GFX950-NEXT: s_cbranch_execz .LBB207_315717; GFX950-NEXT: s_branch .LBB207_415718; GFX950-NEXT: .LBB207_2:15719; GFX950-NEXT: ; implicit-def: $agpr0_agpr115720; GFX950-NEXT: .LBB207_3: ; %atomicrmw.private15721; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015722; GFX950-NEXT: s_cselect_b32 s0, s0, -115723; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s015724; GFX950-NEXT: s_waitcnt vmcnt(0)15725; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[0:1]15726; GFX950-NEXT: v_accvgpr_write_b32 a0, v215727; GFX950-NEXT: s_nop 015728; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc15729; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc15730; GFX950-NEXT: v_accvgpr_write_b32 a1, v315731; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s015732; GFX950-NEXT: .LBB207_4: ; %atomicrmw.end15733; GFX950-NEXT: ;;#ASMSTART15734; GFX950-NEXT: ; use a[0:1]15735; GFX950-NEXT: ;;#ASMEND15736; GFX950-NEXT: s_waitcnt vmcnt(0)15737; GFX950-NEXT: s_setpc_b64 s[30:31]15738 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015739 %data = call i64 asm "; def $0", "=a"()15740 %result = atomicrmw max ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015741 call void asm "; use $0", "a"(i64 %result)15742 ret void15743}15744 15745define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {15746; GFX90A-LABEL: flat_atomic_max_i64_saddr_ret_av_av:15747; GFX90A: ; %bb.0:15748; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15749; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015750; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15751; GFX90A-NEXT: s_addc_u32 s5, s17, 015752; GFX90A-NEXT: s_cmp_eq_u32 s5, s715753; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015754; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15755; GFX90A-NEXT: ;;#ASMSTART15756; GFX90A-NEXT: ; def v[2:3]15757; GFX90A-NEXT: ;;#ASMEND15758; GFX90A-NEXT: s_cbranch_vccz .LBB208_215759; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15760; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]15761; GFX90A-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc15762; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15763; GFX90A-NEXT: s_cbranch_execz .LBB208_315764; GFX90A-NEXT: s_branch .LBB208_415765; GFX90A-NEXT: .LBB208_2:15766; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr115767; GFX90A-NEXT: .LBB208_3: ; %atomicrmw.private15768; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015769; GFX90A-NEXT: s_cselect_b32 s4, s4, -115770; GFX90A-NEXT: v_mov_b32_e32 v4, s415771; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen15772; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:415773; GFX90A-NEXT: s_waitcnt vmcnt(0)15774; GFX90A-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]15775; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc15776; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc15777; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen15778; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:415779; GFX90A-NEXT: .LBB208_4: ; %atomicrmw.end15780; GFX90A-NEXT: s_waitcnt vmcnt(0)15781; GFX90A-NEXT: ;;#ASMSTART15782; GFX90A-NEXT: ; use v[0:1]15783; GFX90A-NEXT: ;;#ASMEND15784; GFX90A-NEXT: s_setpc_b64 s[30:31]15785;15786; GFX950-LABEL: flat_atomic_max_i64_saddr_ret_av_av:15787; GFX950: ; %bb.0:15788; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15789; GFX950-NEXT: s_add_u32 s0, s0, 0x5015790; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15791; GFX950-NEXT: s_addc_u32 s1, s1, 015792; GFX950-NEXT: s_cmp_eq_u32 s1, s315793; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015794; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15795; GFX950-NEXT: ;;#ASMSTART15796; GFX950-NEXT: ; def v[2:3]15797; GFX950-NEXT: ;;#ASMEND15798; GFX950-NEXT: s_cbranch_vccz .LBB208_215799; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15800; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]15801; GFX950-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] sc015802; GFX950-NEXT: s_waitcnt lgkmcnt(0)15803; GFX950-NEXT: s_cbranch_execz .LBB208_315804; GFX950-NEXT: s_branch .LBB208_415805; GFX950-NEXT: .LBB208_2:15806; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115807; GFX950-NEXT: .LBB208_3: ; %atomicrmw.private15808; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015809; GFX950-NEXT: s_cselect_b32 s0, s0, -115810; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015811; GFX950-NEXT: s_waitcnt vmcnt(0)15812; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]15813; GFX950-NEXT: s_nop 115814; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc15815; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc15816; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s015817; GFX950-NEXT: .LBB208_4: ; %atomicrmw.end15818; GFX950-NEXT: s_waitcnt vmcnt(0)15819; GFX950-NEXT: ;;#ASMSTART15820; GFX950-NEXT: ; use v[0:1]15821; GFX950-NEXT: ;;#ASMEND15822; GFX950-NEXT: s_setpc_b64 s[30:31]15823 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015824 %data = call i64 asm "; def $0", "=^VA"()15825 %result = atomicrmw max ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015826 call void asm "; use $0", "^VA"(i64 %result)15827 ret void15828}15829 15830define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {15831; GFX90A-LABEL: flat_atomic_min_i64_saddr_ret_a_a:15832; GFX90A: ; %bb.0:15833; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15834; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015835; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15836; GFX90A-NEXT: s_addc_u32 s5, s17, 015837; GFX90A-NEXT: s_cmp_eq_u32 s5, s715838; GFX90A-NEXT: ;;#ASMSTART15839; GFX90A-NEXT: ; def a[0:1]15840; GFX90A-NEXT: ;;#ASMEND15841; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015842; GFX90A-NEXT: v_accvgpr_read_b32 v0, a015843; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15844; GFX90A-NEXT: v_accvgpr_read_b32 v1, a115845; GFX90A-NEXT: s_cbranch_vccz .LBB209_215846; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15847; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]15848; GFX90A-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] glc15849; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15850; GFX90A-NEXT: s_waitcnt vmcnt(0)15851; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215852; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315853; GFX90A-NEXT: s_cbranch_execz .LBB209_315854; GFX90A-NEXT: s_branch .LBB209_415855; GFX90A-NEXT: .LBB209_2:15856; GFX90A-NEXT: ; implicit-def: $agpr0_agpr115857; GFX90A-NEXT: .LBB209_3: ; %atomicrmw.private15858; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015859; GFX90A-NEXT: s_cselect_b32 s4, s4, -115860; GFX90A-NEXT: v_mov_b32_e32 v4, s415861; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen15862; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:415863; GFX90A-NEXT: s_waitcnt vmcnt(1)15864; GFX90A-NEXT: v_accvgpr_write_b32 a0, v215865; GFX90A-NEXT: s_waitcnt vmcnt(0)15866; GFX90A-NEXT: v_cmp_le_i64_e32 vcc, v[2:3], v[0:1]15867; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc15868; GFX90A-NEXT: v_accvgpr_write_b32 a1, v315869; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc15870; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:415871; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen15872; GFX90A-NEXT: .LBB209_4: ; %atomicrmw.end15873; GFX90A-NEXT: ;;#ASMSTART15874; GFX90A-NEXT: ; use a[0:1]15875; GFX90A-NEXT: ;;#ASMEND15876; GFX90A-NEXT: s_waitcnt vmcnt(0)15877; GFX90A-NEXT: s_setpc_b64 s[30:31]15878;15879; GFX950-LABEL: flat_atomic_min_i64_saddr_ret_a_a:15880; GFX950: ; %bb.0:15881; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15882; GFX950-NEXT: s_add_u32 s0, s0, 0x5015883; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15884; GFX950-NEXT: s_addc_u32 s1, s1, 015885; GFX950-NEXT: s_cmp_eq_u32 s1, s315886; GFX950-NEXT: ;;#ASMSTART15887; GFX950-NEXT: ; def a[0:1]15888; GFX950-NEXT: ;;#ASMEND15889; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015890; GFX950-NEXT: v_accvgpr_read_b32 v0, a015891; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15892; GFX950-NEXT: v_accvgpr_read_b32 v1, a115893; GFX950-NEXT: s_cbranch_vccz .LBB209_215894; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15895; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]15896; GFX950-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] sc015897; GFX950-NEXT: s_waitcnt lgkmcnt(0)15898; GFX950-NEXT: s_waitcnt vmcnt(0)15899; GFX950-NEXT: v_accvgpr_write_b32 a0, v215900; GFX950-NEXT: v_accvgpr_write_b32 a1, v315901; GFX950-NEXT: s_cbranch_execz .LBB209_315902; GFX950-NEXT: s_branch .LBB209_415903; GFX950-NEXT: .LBB209_2:15904; GFX950-NEXT: ; implicit-def: $agpr0_agpr115905; GFX950-NEXT: .LBB209_3: ; %atomicrmw.private15906; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015907; GFX950-NEXT: s_cselect_b32 s0, s0, -115908; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s015909; GFX950-NEXT: s_waitcnt vmcnt(0)15910; GFX950-NEXT: v_cmp_le_i64_e32 vcc, v[2:3], v[0:1]15911; GFX950-NEXT: v_accvgpr_write_b32 a0, v215912; GFX950-NEXT: s_nop 015913; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc15914; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc15915; GFX950-NEXT: v_accvgpr_write_b32 a1, v315916; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s015917; GFX950-NEXT: .LBB209_4: ; %atomicrmw.end15918; GFX950-NEXT: ;;#ASMSTART15919; GFX950-NEXT: ; use a[0:1]15920; GFX950-NEXT: ;;#ASMEND15921; GFX950-NEXT: s_waitcnt vmcnt(0)15922; GFX950-NEXT: s_setpc_b64 s[30:31]15923 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1015924 %data = call i64 asm "; def $0", "=a"()15925 %result = atomicrmw min ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !015926 call void asm "; use $0", "a"(i64 %result)15927 ret void15928}15929 15930define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {15931; GFX90A-LABEL: flat_atomic_min_i64_saddr_ret_av_av:15932; GFX90A: ; %bb.0:15933; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15934; GFX90A-NEXT: s_add_u32 s4, s16, 0x5015935; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base15936; GFX90A-NEXT: s_addc_u32 s5, s17, 015937; GFX90A-NEXT: s_cmp_eq_u32 s5, s715938; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 015939; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]15940; GFX90A-NEXT: ;;#ASMSTART15941; GFX90A-NEXT: ; def v[2:3]15942; GFX90A-NEXT: ;;#ASMEND15943; GFX90A-NEXT: s_cbranch_vccz .LBB210_215944; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global15945; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]15946; GFX90A-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc15947; GFX90A-NEXT: s_waitcnt lgkmcnt(0)15948; GFX90A-NEXT: s_cbranch_execz .LBB210_315949; GFX90A-NEXT: s_branch .LBB210_415950; GFX90A-NEXT: .LBB210_2:15951; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr115952; GFX90A-NEXT: .LBB210_3: ; %atomicrmw.private15953; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 015954; GFX90A-NEXT: s_cselect_b32 s4, s4, -115955; GFX90A-NEXT: v_mov_b32_e32 v4, s415956; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen15957; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:415958; GFX90A-NEXT: s_waitcnt vmcnt(0)15959; GFX90A-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]15960; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc15961; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc15962; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen15963; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:415964; GFX90A-NEXT: .LBB210_4: ; %atomicrmw.end15965; GFX90A-NEXT: s_waitcnt vmcnt(0)15966; GFX90A-NEXT: ;;#ASMSTART15967; GFX90A-NEXT: ; use v[0:1]15968; GFX90A-NEXT: ;;#ASMEND15969; GFX90A-NEXT: s_setpc_b64 s[30:31]15970;15971; GFX950-LABEL: flat_atomic_min_i64_saddr_ret_av_av:15972; GFX950: ; %bb.0:15973; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15974; GFX950-NEXT: s_add_u32 s0, s0, 0x5015975; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base15976; GFX950-NEXT: s_addc_u32 s1, s1, 015977; GFX950-NEXT: s_cmp_eq_u32 s1, s315978; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 015979; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]15980; GFX950-NEXT: ;;#ASMSTART15981; GFX950-NEXT: ; def v[2:3]15982; GFX950-NEXT: ;;#ASMEND15983; GFX950-NEXT: s_cbranch_vccz .LBB210_215984; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global15985; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]15986; GFX950-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] sc015987; GFX950-NEXT: s_waitcnt lgkmcnt(0)15988; GFX950-NEXT: s_cbranch_execz .LBB210_315989; GFX950-NEXT: s_branch .LBB210_415990; GFX950-NEXT: .LBB210_2:15991; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr115992; GFX950-NEXT: .LBB210_3: ; %atomicrmw.private15993; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 015994; GFX950-NEXT: s_cselect_b32 s0, s0, -115995; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s015996; GFX950-NEXT: s_waitcnt vmcnt(0)15997; GFX950-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]15998; GFX950-NEXT: s_nop 115999; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc16000; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc16001; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s016002; GFX950-NEXT: .LBB210_4: ; %atomicrmw.end16003; GFX950-NEXT: s_waitcnt vmcnt(0)16004; GFX950-NEXT: ;;#ASMSTART16005; GFX950-NEXT: ; use v[0:1]16006; GFX950-NEXT: ;;#ASMEND16007; GFX950-NEXT: s_setpc_b64 s[30:31]16008 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016009 %data = call i64 asm "; def $0", "=^VA"()16010 %result = atomicrmw min ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016011 call void asm "; use $0", "^VA"(i64 %result)16012 ret void16013}16014 16015define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {16016; GFX90A-LABEL: flat_atomic_umax_i64_saddr_ret_a_a:16017; GFX90A: ; %bb.0:16018; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16019; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016020; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16021; GFX90A-NEXT: s_addc_u32 s5, s17, 016022; GFX90A-NEXT: s_cmp_eq_u32 s5, s716023; GFX90A-NEXT: ;;#ASMSTART16024; GFX90A-NEXT: ; def a[0:1]16025; GFX90A-NEXT: ;;#ASMEND16026; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016027; GFX90A-NEXT: v_accvgpr_read_b32 v0, a016028; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16029; GFX90A-NEXT: v_accvgpr_read_b32 v1, a116030; GFX90A-NEXT: s_cbranch_vccz .LBB211_216031; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16032; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]16033; GFX90A-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] glc16034; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16035; GFX90A-NEXT: s_waitcnt vmcnt(0)16036; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216037; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316038; GFX90A-NEXT: s_cbranch_execz .LBB211_316039; GFX90A-NEXT: s_branch .LBB211_416040; GFX90A-NEXT: .LBB211_2:16041; GFX90A-NEXT: ; implicit-def: $agpr0_agpr116042; GFX90A-NEXT: .LBB211_3: ; %atomicrmw.private16043; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016044; GFX90A-NEXT: s_cselect_b32 s4, s4, -116045; GFX90A-NEXT: v_mov_b32_e32 v4, s416046; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen16047; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:416048; GFX90A-NEXT: s_waitcnt vmcnt(1)16049; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216050; GFX90A-NEXT: s_waitcnt vmcnt(0)16051; GFX90A-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]16052; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc16053; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316054; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc16055; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:416056; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen16057; GFX90A-NEXT: .LBB211_4: ; %atomicrmw.end16058; GFX90A-NEXT: ;;#ASMSTART16059; GFX90A-NEXT: ; use a[0:1]16060; GFX90A-NEXT: ;;#ASMEND16061; GFX90A-NEXT: s_waitcnt vmcnt(0)16062; GFX90A-NEXT: s_setpc_b64 s[30:31]16063;16064; GFX950-LABEL: flat_atomic_umax_i64_saddr_ret_a_a:16065; GFX950: ; %bb.0:16066; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16067; GFX950-NEXT: s_add_u32 s0, s0, 0x5016068; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16069; GFX950-NEXT: s_addc_u32 s1, s1, 016070; GFX950-NEXT: s_cmp_eq_u32 s1, s316071; GFX950-NEXT: ;;#ASMSTART16072; GFX950-NEXT: ; def a[0:1]16073; GFX950-NEXT: ;;#ASMEND16074; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016075; GFX950-NEXT: v_accvgpr_read_b32 v0, a016076; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16077; GFX950-NEXT: v_accvgpr_read_b32 v1, a116078; GFX950-NEXT: s_cbranch_vccz .LBB211_216079; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16080; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]16081; GFX950-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] sc016082; GFX950-NEXT: s_waitcnt lgkmcnt(0)16083; GFX950-NEXT: s_waitcnt vmcnt(0)16084; GFX950-NEXT: v_accvgpr_write_b32 a0, v216085; GFX950-NEXT: v_accvgpr_write_b32 a1, v316086; GFX950-NEXT: s_cbranch_execz .LBB211_316087; GFX950-NEXT: s_branch .LBB211_416088; GFX950-NEXT: .LBB211_2:16089; GFX950-NEXT: ; implicit-def: $agpr0_agpr116090; GFX950-NEXT: .LBB211_3: ; %atomicrmw.private16091; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016092; GFX950-NEXT: s_cselect_b32 s0, s0, -116093; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s016094; GFX950-NEXT: s_waitcnt vmcnt(0)16095; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[0:1]16096; GFX950-NEXT: v_accvgpr_write_b32 a0, v216097; GFX950-NEXT: s_nop 016098; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc16099; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc16100; GFX950-NEXT: v_accvgpr_write_b32 a1, v316101; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s016102; GFX950-NEXT: .LBB211_4: ; %atomicrmw.end16103; GFX950-NEXT: ;;#ASMSTART16104; GFX950-NEXT: ; use a[0:1]16105; GFX950-NEXT: ;;#ASMEND16106; GFX950-NEXT: s_waitcnt vmcnt(0)16107; GFX950-NEXT: s_setpc_b64 s[30:31]16108 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016109 %data = call i64 asm "; def $0", "=a"()16110 %result = atomicrmw umax ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016111 call void asm "; use $0", "a"(i64 %result)16112 ret void16113}16114 16115define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {16116; GFX90A-LABEL: flat_atomic_umax_i64_saddr_ret_av_av:16117; GFX90A: ; %bb.0:16118; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16119; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016120; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16121; GFX90A-NEXT: s_addc_u32 s5, s17, 016122; GFX90A-NEXT: s_cmp_eq_u32 s5, s716123; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016124; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16125; GFX90A-NEXT: ;;#ASMSTART16126; GFX90A-NEXT: ; def v[2:3]16127; GFX90A-NEXT: ;;#ASMEND16128; GFX90A-NEXT: s_cbranch_vccz .LBB212_216129; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16130; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]16131; GFX90A-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc16132; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16133; GFX90A-NEXT: s_cbranch_execz .LBB212_316134; GFX90A-NEXT: s_branch .LBB212_416135; GFX90A-NEXT: .LBB212_2:16136; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr116137; GFX90A-NEXT: .LBB212_3: ; %atomicrmw.private16138; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016139; GFX90A-NEXT: s_cselect_b32 s4, s4, -116140; GFX90A-NEXT: v_mov_b32_e32 v4, s416141; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen16142; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:416143; GFX90A-NEXT: s_waitcnt vmcnt(0)16144; GFX90A-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]16145; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc16146; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc16147; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen16148; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:416149; GFX90A-NEXT: .LBB212_4: ; %atomicrmw.end16150; GFX90A-NEXT: s_waitcnt vmcnt(0)16151; GFX90A-NEXT: ;;#ASMSTART16152; GFX90A-NEXT: ; use v[0:1]16153; GFX90A-NEXT: ;;#ASMEND16154; GFX90A-NEXT: s_setpc_b64 s[30:31]16155;16156; GFX950-LABEL: flat_atomic_umax_i64_saddr_ret_av_av:16157; GFX950: ; %bb.0:16158; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16159; GFX950-NEXT: s_add_u32 s0, s0, 0x5016160; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16161; GFX950-NEXT: s_addc_u32 s1, s1, 016162; GFX950-NEXT: s_cmp_eq_u32 s1, s316163; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016164; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16165; GFX950-NEXT: ;;#ASMSTART16166; GFX950-NEXT: ; def v[2:3]16167; GFX950-NEXT: ;;#ASMEND16168; GFX950-NEXT: s_cbranch_vccz .LBB212_216169; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16170; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]16171; GFX950-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] sc016172; GFX950-NEXT: s_waitcnt lgkmcnt(0)16173; GFX950-NEXT: s_cbranch_execz .LBB212_316174; GFX950-NEXT: s_branch .LBB212_416175; GFX950-NEXT: .LBB212_2:16176; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr116177; GFX950-NEXT: .LBB212_3: ; %atomicrmw.private16178; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016179; GFX950-NEXT: s_cselect_b32 s0, s0, -116180; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s016181; GFX950-NEXT: s_waitcnt vmcnt(0)16182; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]16183; GFX950-NEXT: s_nop 116184; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc16185; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc16186; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s016187; GFX950-NEXT: .LBB212_4: ; %atomicrmw.end16188; GFX950-NEXT: s_waitcnt vmcnt(0)16189; GFX950-NEXT: ;;#ASMSTART16190; GFX950-NEXT: ; use v[0:1]16191; GFX950-NEXT: ;;#ASMEND16192; GFX950-NEXT: s_setpc_b64 s[30:31]16193 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016194 %data = call i64 asm "; def $0", "=^VA"()16195 %result = atomicrmw umax ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016196 call void asm "; use $0", "^VA"(i64 %result)16197 ret void16198}16199 16200define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {16201; GFX90A-LABEL: flat_atomic_umin_i64_saddr_ret_a_a:16202; GFX90A: ; %bb.0:16203; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16204; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016205; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16206; GFX90A-NEXT: s_addc_u32 s5, s17, 016207; GFX90A-NEXT: s_cmp_eq_u32 s5, s716208; GFX90A-NEXT: ;;#ASMSTART16209; GFX90A-NEXT: ; def a[0:1]16210; GFX90A-NEXT: ;;#ASMEND16211; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016212; GFX90A-NEXT: v_accvgpr_read_b32 v0, a016213; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16214; GFX90A-NEXT: v_accvgpr_read_b32 v1, a116215; GFX90A-NEXT: s_cbranch_vccz .LBB213_216216; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16217; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]16218; GFX90A-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] glc16219; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16220; GFX90A-NEXT: s_waitcnt vmcnt(0)16221; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216222; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316223; GFX90A-NEXT: s_cbranch_execz .LBB213_316224; GFX90A-NEXT: s_branch .LBB213_416225; GFX90A-NEXT: .LBB213_2:16226; GFX90A-NEXT: ; implicit-def: $agpr0_agpr116227; GFX90A-NEXT: .LBB213_3: ; %atomicrmw.private16228; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016229; GFX90A-NEXT: s_cselect_b32 s4, s4, -116230; GFX90A-NEXT: v_mov_b32_e32 v4, s416231; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen16232; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:416233; GFX90A-NEXT: s_waitcnt vmcnt(1)16234; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216235; GFX90A-NEXT: s_waitcnt vmcnt(0)16236; GFX90A-NEXT: v_cmp_le_u64_e32 vcc, v[2:3], v[0:1]16237; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc16238; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316239; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc16240; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:416241; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen16242; GFX90A-NEXT: .LBB213_4: ; %atomicrmw.end16243; GFX90A-NEXT: ;;#ASMSTART16244; GFX90A-NEXT: ; use a[0:1]16245; GFX90A-NEXT: ;;#ASMEND16246; GFX90A-NEXT: s_waitcnt vmcnt(0)16247; GFX90A-NEXT: s_setpc_b64 s[30:31]16248;16249; GFX950-LABEL: flat_atomic_umin_i64_saddr_ret_a_a:16250; GFX950: ; %bb.0:16251; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16252; GFX950-NEXT: s_add_u32 s0, s0, 0x5016253; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16254; GFX950-NEXT: s_addc_u32 s1, s1, 016255; GFX950-NEXT: s_cmp_eq_u32 s1, s316256; GFX950-NEXT: ;;#ASMSTART16257; GFX950-NEXT: ; def a[0:1]16258; GFX950-NEXT: ;;#ASMEND16259; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016260; GFX950-NEXT: v_accvgpr_read_b32 v0, a016261; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16262; GFX950-NEXT: v_accvgpr_read_b32 v1, a116263; GFX950-NEXT: s_cbranch_vccz .LBB213_216264; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16265; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]16266; GFX950-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] sc016267; GFX950-NEXT: s_waitcnt lgkmcnt(0)16268; GFX950-NEXT: s_waitcnt vmcnt(0)16269; GFX950-NEXT: v_accvgpr_write_b32 a0, v216270; GFX950-NEXT: v_accvgpr_write_b32 a1, v316271; GFX950-NEXT: s_cbranch_execz .LBB213_316272; GFX950-NEXT: s_branch .LBB213_416273; GFX950-NEXT: .LBB213_2:16274; GFX950-NEXT: ; implicit-def: $agpr0_agpr116275; GFX950-NEXT: .LBB213_3: ; %atomicrmw.private16276; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016277; GFX950-NEXT: s_cselect_b32 s0, s0, -116278; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s016279; GFX950-NEXT: s_waitcnt vmcnt(0)16280; GFX950-NEXT: v_cmp_le_u64_e32 vcc, v[2:3], v[0:1]16281; GFX950-NEXT: v_accvgpr_write_b32 a0, v216282; GFX950-NEXT: s_nop 016283; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc16284; GFX950-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc16285; GFX950-NEXT: v_accvgpr_write_b32 a1, v316286; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s016287; GFX950-NEXT: .LBB213_4: ; %atomicrmw.end16288; GFX950-NEXT: ;;#ASMSTART16289; GFX950-NEXT: ; use a[0:1]16290; GFX950-NEXT: ;;#ASMEND16291; GFX950-NEXT: s_waitcnt vmcnt(0)16292; GFX950-NEXT: s_setpc_b64 s[30:31]16293 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016294 %data = call i64 asm "; def $0", "=a"()16295 %result = atomicrmw umin ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016296 call void asm "; use $0", "a"(i64 %result)16297 ret void16298}16299 16300define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {16301; GFX90A-LABEL: flat_atomic_umin_i64_saddr_ret_av_av:16302; GFX90A: ; %bb.0:16303; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16304; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016305; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16306; GFX90A-NEXT: s_addc_u32 s5, s17, 016307; GFX90A-NEXT: s_cmp_eq_u32 s5, s716308; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016309; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16310; GFX90A-NEXT: ;;#ASMSTART16311; GFX90A-NEXT: ; def v[2:3]16312; GFX90A-NEXT: ;;#ASMEND16313; GFX90A-NEXT: s_cbranch_vccz .LBB214_216314; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16315; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]16316; GFX90A-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc16317; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16318; GFX90A-NEXT: s_cbranch_execz .LBB214_316319; GFX90A-NEXT: s_branch .LBB214_416320; GFX90A-NEXT: .LBB214_2:16321; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr116322; GFX90A-NEXT: .LBB214_3: ; %atomicrmw.private16323; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016324; GFX90A-NEXT: s_cselect_b32 s4, s4, -116325; GFX90A-NEXT: v_mov_b32_e32 v4, s416326; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen16327; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:416328; GFX90A-NEXT: s_waitcnt vmcnt(0)16329; GFX90A-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]16330; GFX90A-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc16331; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc16332; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen16333; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:416334; GFX90A-NEXT: .LBB214_4: ; %atomicrmw.end16335; GFX90A-NEXT: s_waitcnt vmcnt(0)16336; GFX90A-NEXT: ;;#ASMSTART16337; GFX90A-NEXT: ; use v[0:1]16338; GFX90A-NEXT: ;;#ASMEND16339; GFX90A-NEXT: s_setpc_b64 s[30:31]16340;16341; GFX950-LABEL: flat_atomic_umin_i64_saddr_ret_av_av:16342; GFX950: ; %bb.0:16343; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16344; GFX950-NEXT: s_add_u32 s0, s0, 0x5016345; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16346; GFX950-NEXT: s_addc_u32 s1, s1, 016347; GFX950-NEXT: s_cmp_eq_u32 s1, s316348; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016349; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16350; GFX950-NEXT: ;;#ASMSTART16351; GFX950-NEXT: ; def v[2:3]16352; GFX950-NEXT: ;;#ASMEND16353; GFX950-NEXT: s_cbranch_vccz .LBB214_216354; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16355; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]16356; GFX950-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] sc016357; GFX950-NEXT: s_waitcnt lgkmcnt(0)16358; GFX950-NEXT: s_cbranch_execz .LBB214_316359; GFX950-NEXT: s_branch .LBB214_416360; GFX950-NEXT: .LBB214_2:16361; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr116362; GFX950-NEXT: .LBB214_3: ; %atomicrmw.private16363; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016364; GFX950-NEXT: s_cselect_b32 s0, s0, -116365; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s016366; GFX950-NEXT: s_waitcnt vmcnt(0)16367; GFX950-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]16368; GFX950-NEXT: s_nop 116369; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc16370; GFX950-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc16371; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s016372; GFX950-NEXT: .LBB214_4: ; %atomicrmw.end16373; GFX950-NEXT: s_waitcnt vmcnt(0)16374; GFX950-NEXT: ;;#ASMSTART16375; GFX950-NEXT: ; use v[0:1]16376; GFX950-NEXT: ;;#ASMEND16377; GFX950-NEXT: s_setpc_b64 s[30:31]16378 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016379 %data = call i64 asm "; def $0", "=^VA"()16380 %result = atomicrmw umin ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016381 call void asm "; use $0", "^VA"(i64 %result)16382 ret void16383}16384 16385define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {16386; GFX90A-LABEL: flat_atomic_uinc_wrap_i64_saddr_ret_a_a:16387; GFX90A: ; %bb.0:16388; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16389; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016390; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16391; GFX90A-NEXT: s_addc_u32 s5, s17, 016392; GFX90A-NEXT: s_cmp_eq_u32 s5, s716393; GFX90A-NEXT: ;;#ASMSTART16394; GFX90A-NEXT: ; def a[0:1]16395; GFX90A-NEXT: ;;#ASMEND16396; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016397; GFX90A-NEXT: v_accvgpr_read_b32 v0, a016398; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16399; GFX90A-NEXT: v_accvgpr_read_b32 v1, a116400; GFX90A-NEXT: s_cbranch_vccz .LBB215_216401; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16402; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]16403; GFX90A-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] glc16404; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16405; GFX90A-NEXT: s_waitcnt vmcnt(0)16406; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216407; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316408; GFX90A-NEXT: s_cbranch_execz .LBB215_316409; GFX90A-NEXT: s_branch .LBB215_416410; GFX90A-NEXT: .LBB215_2:16411; GFX90A-NEXT: ; implicit-def: $agpr0_agpr116412; GFX90A-NEXT: .LBB215_3: ; %atomicrmw.private16413; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016414; GFX90A-NEXT: s_cselect_b32 s4, s4, -116415; GFX90A-NEXT: v_mov_b32_e32 v4, s416416; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen16417; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:416418; GFX90A-NEXT: s_waitcnt vmcnt(1)16419; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, 1, v216420; GFX90A-NEXT: s_waitcnt vmcnt(0)16421; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v3, vcc16422; GFX90A-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]16423; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216424; GFX90A-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc16425; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316426; GFX90A-NEXT: v_cndmask_b32_e32 v0, 0, v6, vcc16427; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen16428; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen offset:416429; GFX90A-NEXT: .LBB215_4: ; %atomicrmw.end16430; GFX90A-NEXT: ;;#ASMSTART16431; GFX90A-NEXT: ; use a[0:1]16432; GFX90A-NEXT: ;;#ASMEND16433; GFX90A-NEXT: s_waitcnt vmcnt(0)16434; GFX90A-NEXT: s_setpc_b64 s[30:31]16435;16436; GFX950-LABEL: flat_atomic_uinc_wrap_i64_saddr_ret_a_a:16437; GFX950: ; %bb.0:16438; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16439; GFX950-NEXT: s_add_u32 s0, s0, 0x5016440; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16441; GFX950-NEXT: s_addc_u32 s1, s1, 016442; GFX950-NEXT: s_cmp_eq_u32 s1, s316443; GFX950-NEXT: ;;#ASMSTART16444; GFX950-NEXT: ; def a[0:1]16445; GFX950-NEXT: ;;#ASMEND16446; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016447; GFX950-NEXT: v_accvgpr_read_b32 v0, a016448; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16449; GFX950-NEXT: v_accvgpr_read_b32 v1, a116450; GFX950-NEXT: s_cbranch_vccz .LBB215_216451; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16452; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]16453; GFX950-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] sc016454; GFX950-NEXT: s_waitcnt lgkmcnt(0)16455; GFX950-NEXT: s_waitcnt vmcnt(0)16456; GFX950-NEXT: v_accvgpr_write_b32 a0, v216457; GFX950-NEXT: v_accvgpr_write_b32 a1, v316458; GFX950-NEXT: s_cbranch_execz .LBB215_316459; GFX950-NEXT: s_branch .LBB215_416460; GFX950-NEXT: .LBB215_2:16461; GFX950-NEXT: ; implicit-def: $agpr0_agpr116462; GFX950-NEXT: .LBB215_3: ; %atomicrmw.private16463; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016464; GFX950-NEXT: s_cselect_b32 s0, s0, -116465; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s016466; GFX950-NEXT: s_waitcnt vmcnt(0)16467; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, 116468; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[0:1]16469; GFX950-NEXT: v_accvgpr_write_b32 a0, v216470; GFX950-NEXT: v_accvgpr_write_b32 a1, v316471; GFX950-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc16472; GFX950-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc16473; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s016474; GFX950-NEXT: .LBB215_4: ; %atomicrmw.end16475; GFX950-NEXT: ;;#ASMSTART16476; GFX950-NEXT: ; use a[0:1]16477; GFX950-NEXT: ;;#ASMEND16478; GFX950-NEXT: s_waitcnt vmcnt(0)16479; GFX950-NEXT: s_setpc_b64 s[30:31]16480 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016481 %data = call i64 asm "; def $0", "=a"()16482 %result = atomicrmw uinc_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016483 call void asm "; use $0", "a"(i64 %result)16484 ret void16485}16486 16487define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {16488; GFX90A-LABEL: flat_atomic_uinc_wrap_i64_saddr_ret_av_av:16489; GFX90A: ; %bb.0:16490; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16491; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016492; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16493; GFX90A-NEXT: s_addc_u32 s5, s17, 016494; GFX90A-NEXT: s_cmp_eq_u32 s5, s716495; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016496; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16497; GFX90A-NEXT: ;;#ASMSTART16498; GFX90A-NEXT: ; def v[2:3]16499; GFX90A-NEXT: ;;#ASMEND16500; GFX90A-NEXT: s_cbranch_vccz .LBB216_216501; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16502; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]16503; GFX90A-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc16504; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16505; GFX90A-NEXT: s_cbranch_execz .LBB216_316506; GFX90A-NEXT: s_branch .LBB216_416507; GFX90A-NEXT: .LBB216_2:16508; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr116509; GFX90A-NEXT: .LBB216_3: ; %atomicrmw.private16510; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016511; GFX90A-NEXT: s_cselect_b32 s4, s4, -116512; GFX90A-NEXT: v_mov_b32_e32 v4, s416513; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen16514; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:416515; GFX90A-NEXT: s_waitcnt vmcnt(0)16516; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, 1, v016517; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc16518; GFX90A-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]16519; GFX90A-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc16520; GFX90A-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc16521; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen16522; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen offset:416523; GFX90A-NEXT: .LBB216_4: ; %atomicrmw.end16524; GFX90A-NEXT: s_waitcnt vmcnt(0)16525; GFX90A-NEXT: ;;#ASMSTART16526; GFX90A-NEXT: ; use v[0:1]16527; GFX90A-NEXT: ;;#ASMEND16528; GFX90A-NEXT: s_setpc_b64 s[30:31]16529;16530; GFX950-LABEL: flat_atomic_uinc_wrap_i64_saddr_ret_av_av:16531; GFX950: ; %bb.0:16532; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16533; GFX950-NEXT: s_add_u32 s0, s0, 0x5016534; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16535; GFX950-NEXT: s_addc_u32 s1, s1, 016536; GFX950-NEXT: s_cmp_eq_u32 s1, s316537; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016538; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16539; GFX950-NEXT: ;;#ASMSTART16540; GFX950-NEXT: ; def v[2:3]16541; GFX950-NEXT: ;;#ASMEND16542; GFX950-NEXT: s_cbranch_vccz .LBB216_216543; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16544; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]16545; GFX950-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] sc016546; GFX950-NEXT: s_waitcnt lgkmcnt(0)16547; GFX950-NEXT: s_cbranch_execz .LBB216_316548; GFX950-NEXT: s_branch .LBB216_416549; GFX950-NEXT: .LBB216_2:16550; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr116551; GFX950-NEXT: .LBB216_3: ; %atomicrmw.private16552; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016553; GFX950-NEXT: s_cselect_b32 s0, s0, -116554; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s016555; GFX950-NEXT: s_waitcnt vmcnt(0)16556; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 116557; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]16558; GFX950-NEXT: s_nop 116559; GFX950-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc16560; GFX950-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc16561; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s016562; GFX950-NEXT: .LBB216_4: ; %atomicrmw.end16563; GFX950-NEXT: s_waitcnt vmcnt(0)16564; GFX950-NEXT: ;;#ASMSTART16565; GFX950-NEXT: ; use v[0:1]16566; GFX950-NEXT: ;;#ASMEND16567; GFX950-NEXT: s_setpc_b64 s[30:31]16568 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016569 %data = call i64 asm "; def $0", "=^VA"()16570 %result = atomicrmw uinc_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016571 call void asm "; use $0", "^VA"(i64 %result)16572 ret void16573}16574 16575define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {16576; GFX90A-LABEL: flat_atomic_udec_wrap_i64_saddr_ret_a_a:16577; GFX90A: ; %bb.0:16578; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16579; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016580; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16581; GFX90A-NEXT: s_addc_u32 s5, s17, 016582; GFX90A-NEXT: s_cmp_eq_u32 s5, s716583; GFX90A-NEXT: ;;#ASMSTART16584; GFX90A-NEXT: ; def a[0:1]16585; GFX90A-NEXT: ;;#ASMEND16586; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016587; GFX90A-NEXT: v_accvgpr_read_b32 v0, a016588; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16589; GFX90A-NEXT: v_accvgpr_read_b32 v1, a116590; GFX90A-NEXT: s_cbranch_vccz .LBB217_216591; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16592; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]16593; GFX90A-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] glc16594; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16595; GFX90A-NEXT: s_waitcnt vmcnt(0)16596; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216597; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316598; GFX90A-NEXT: s_cbranch_execz .LBB217_316599; GFX90A-NEXT: s_branch .LBB217_416600; GFX90A-NEXT: .LBB217_2:16601; GFX90A-NEXT: ; implicit-def: $agpr0_agpr116602; GFX90A-NEXT: .LBB217_3: ; %atomicrmw.private16603; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016604; GFX90A-NEXT: s_cselect_b32 s4, s4, -116605; GFX90A-NEXT: v_mov_b32_e32 v4, s416606; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen16607; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:416608; GFX90A-NEXT: s_waitcnt vmcnt(1)16609; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, -1, v216610; GFX90A-NEXT: s_waitcnt vmcnt(0)16611; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v3, vcc16612; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]16613; GFX90A-NEXT: v_cmp_gt_u64_e64 s[4:5], v[2:3], v[0:1]16614; GFX90A-NEXT: s_or_b64 vcc, vcc, s[4:5]16615; GFX90A-NEXT: v_accvgpr_write_b32 a0, v216616; GFX90A-NEXT: v_cndmask_b32_e32 v1, v6, v1, vcc16617; GFX90A-NEXT: v_accvgpr_write_b32 a1, v316618; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc16619; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:416620; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen16621; GFX90A-NEXT: .LBB217_4: ; %atomicrmw.end16622; GFX90A-NEXT: ;;#ASMSTART16623; GFX90A-NEXT: ; use a[0:1]16624; GFX90A-NEXT: ;;#ASMEND16625; GFX90A-NEXT: s_waitcnt vmcnt(0)16626; GFX90A-NEXT: s_setpc_b64 s[30:31]16627;16628; GFX950-LABEL: flat_atomic_udec_wrap_i64_saddr_ret_a_a:16629; GFX950: ; %bb.0:16630; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16631; GFX950-NEXT: s_add_u32 s0, s0, 0x5016632; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16633; GFX950-NEXT: s_addc_u32 s1, s1, 016634; GFX950-NEXT: s_cmp_eq_u32 s1, s316635; GFX950-NEXT: ;;#ASMSTART16636; GFX950-NEXT: ; def a[0:1]16637; GFX950-NEXT: ;;#ASMEND16638; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016639; GFX950-NEXT: v_accvgpr_read_b32 v0, a016640; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16641; GFX950-NEXT: v_accvgpr_read_b32 v1, a116642; GFX950-NEXT: s_cbranch_vccz .LBB217_216643; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16644; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]16645; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc016646; GFX950-NEXT: s_waitcnt lgkmcnt(0)16647; GFX950-NEXT: s_waitcnt vmcnt(0)16648; GFX950-NEXT: v_accvgpr_write_b32 a0, v216649; GFX950-NEXT: v_accvgpr_write_b32 a1, v316650; GFX950-NEXT: s_cbranch_execz .LBB217_316651; GFX950-NEXT: s_branch .LBB217_416652; GFX950-NEXT: .LBB217_2:16653; GFX950-NEXT: ; implicit-def: $agpr0_agpr116654; GFX950-NEXT: .LBB217_3: ; %atomicrmw.private16655; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016656; GFX950-NEXT: s_cselect_b32 s2, s0, -116657; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s216658; GFX950-NEXT: s_waitcnt vmcnt(0)16659; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]16660; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[0:1]16661; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[2:3], 0, -116662; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]16663; GFX950-NEXT: v_accvgpr_write_b32 a0, v216664; GFX950-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc16665; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc16666; GFX950-NEXT: v_accvgpr_write_b32 a1, v316667; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s216668; GFX950-NEXT: .LBB217_4: ; %atomicrmw.end16669; GFX950-NEXT: ;;#ASMSTART16670; GFX950-NEXT: ; use a[0:1]16671; GFX950-NEXT: ;;#ASMEND16672; GFX950-NEXT: s_waitcnt vmcnt(0)16673; GFX950-NEXT: s_setpc_b64 s[30:31]16674 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016675 %data = call i64 asm "; def $0", "=a"()16676 %result = atomicrmw udec_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016677 call void asm "; use $0", "a"(i64 %result)16678 ret void16679}16680 16681define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {16682; GFX90A-LABEL: flat_atomic_udec_wrap_i64_saddr_ret_av_av:16683; GFX90A: ; %bb.0:16684; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16685; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016686; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16687; GFX90A-NEXT: s_addc_u32 s5, s17, 016688; GFX90A-NEXT: s_cmp_eq_u32 s5, s716689; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016690; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16691; GFX90A-NEXT: ;;#ASMSTART16692; GFX90A-NEXT: ; def v[2:3]16693; GFX90A-NEXT: ;;#ASMEND16694; GFX90A-NEXT: s_cbranch_vccz .LBB218_216695; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16696; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]16697; GFX90A-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc16698; GFX90A-NEXT: s_waitcnt lgkmcnt(0)16699; GFX90A-NEXT: s_cbranch_execz .LBB218_316700; GFX90A-NEXT: s_branch .LBB218_416701; GFX90A-NEXT: .LBB218_2:16702; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr116703; GFX90A-NEXT: .LBB218_3: ; %atomicrmw.private16704; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016705; GFX90A-NEXT: s_cselect_b32 s4, s4, -116706; GFX90A-NEXT: v_mov_b32_e32 v4, s416707; GFX90A-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen16708; GFX90A-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:416709; GFX90A-NEXT: s_waitcnt vmcnt(0)16710; GFX90A-NEXT: v_add_co_u32_e32 v5, vcc, -1, v016711; GFX90A-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v1, vcc16712; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16713; GFX90A-NEXT: v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]16714; GFX90A-NEXT: s_or_b64 vcc, vcc, s[4:5]16715; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc16716; GFX90A-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc16717; GFX90A-NEXT: buffer_store_dword v2, v4, s[0:3], 0 offen16718; GFX90A-NEXT: buffer_store_dword v3, v4, s[0:3], 0 offen offset:416719; GFX90A-NEXT: .LBB218_4: ; %atomicrmw.end16720; GFX90A-NEXT: s_waitcnt vmcnt(0)16721; GFX90A-NEXT: ;;#ASMSTART16722; GFX90A-NEXT: ; use v[0:1]16723; GFX90A-NEXT: ;;#ASMEND16724; GFX90A-NEXT: s_setpc_b64 s[30:31]16725;16726; GFX950-LABEL: flat_atomic_udec_wrap_i64_saddr_ret_av_av:16727; GFX950: ; %bb.0:16728; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16729; GFX950-NEXT: s_add_u32 s0, s0, 0x5016730; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16731; GFX950-NEXT: s_addc_u32 s1, s1, 016732; GFX950-NEXT: s_cmp_eq_u32 s1, s316733; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016734; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16735; GFX950-NEXT: ;;#ASMSTART16736; GFX950-NEXT: ; def v[2:3]16737; GFX950-NEXT: ;;#ASMEND16738; GFX950-NEXT: s_cbranch_vccz .LBB218_216739; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16740; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]16741; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc016742; GFX950-NEXT: s_waitcnt lgkmcnt(0)16743; GFX950-NEXT: s_cbranch_execz .LBB218_316744; GFX950-NEXT: s_branch .LBB218_416745; GFX950-NEXT: .LBB218_2:16746; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr116747; GFX950-NEXT: .LBB218_3: ; %atomicrmw.private16748; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016749; GFX950-NEXT: s_cselect_b32 s2, s0, -116750; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s216751; GFX950-NEXT: s_waitcnt vmcnt(0)16752; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]16753; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]16754; GFX950-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -116755; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]16756; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc16757; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc16758; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s216759; GFX950-NEXT: .LBB218_4: ; %atomicrmw.end16760; GFX950-NEXT: s_waitcnt vmcnt(0)16761; GFX950-NEXT: ;;#ASMSTART16762; GFX950-NEXT: ; use v[0:1]16763; GFX950-NEXT: ;;#ASMEND16764; GFX950-NEXT: s_setpc_b64 s[30:31]16765 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016766 %data = call i64 asm "; def $0", "=^VA"()16767 %result = atomicrmw udec_wrap ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016768 call void asm "; use $0", "^VA"(i64 %result)16769 ret void16770}16771 16772define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {16773; GFX90A-LABEL: flat_atomic_usub_cond_i64_saddr_ret_a_a:16774; GFX90A: ; %bb.0:16775; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16776; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016777; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16778; GFX90A-NEXT: s_addc_u32 s5, s17, 016779; GFX90A-NEXT: s_cmp_eq_u32 s5, s716780; GFX90A-NEXT: ;;#ASMSTART16781; GFX90A-NEXT: ; def a[0:1]16782; GFX90A-NEXT: ;;#ASMEND16783; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016784; GFX90A-NEXT: v_accvgpr_read_b32 v5, a116785; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16786; GFX90A-NEXT: v_accvgpr_read_b32 v4, a016787; GFX90A-NEXT: s_cbranch_vccz .LBB219_416788; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16789; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]16790; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]16791; GFX90A-NEXT: s_mov_b64 s[6:7], 016792; GFX90A-NEXT: .LBB219_2: ; %atomicrmw.start16793; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=116794; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16795; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v416796; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc16797; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]16798; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc16799; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc16800; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc16801; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16802; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]16803; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]16804; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]16805; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]16806; GFX90A-NEXT: s_cbranch_execnz .LBB219_216807; GFX90A-NEXT: ; %bb.3: ; %Flow16808; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]16809; GFX90A-NEXT: v_accvgpr_write_b32 a0, v016810; GFX90A-NEXT: v_accvgpr_write_b32 a1, v116811; GFX90A-NEXT: s_branch .LBB219_616812; GFX90A-NEXT: .LBB219_4:16813; GFX90A-NEXT: ; implicit-def: $agpr0_agpr116814; GFX90A-NEXT: s_cbranch_execz .LBB219_616815; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private16816; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016817; GFX90A-NEXT: s_cselect_b32 s4, s4, -116818; GFX90A-NEXT: v_mov_b32_e32 v2, s416819; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen16820; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:416821; GFX90A-NEXT: s_waitcnt vmcnt(1)16822; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v416823; GFX90A-NEXT: s_waitcnt vmcnt(0)16824; GFX90A-NEXT: v_subb_co_u32_e32 v6, vcc, v1, v5, vcc16825; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]16826; GFX90A-NEXT: v_accvgpr_write_b32 a0, v016827; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc16828; GFX90A-NEXT: v_accvgpr_write_b32 a1, v116829; GFX90A-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc16830; GFX90A-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen16831; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:416832; GFX90A-NEXT: .LBB219_6: ; %atomicrmw.phi16833; GFX90A-NEXT: ;;#ASMSTART16834; GFX90A-NEXT: ; use a[0:1]16835; GFX90A-NEXT: ;;#ASMEND16836; GFX90A-NEXT: s_waitcnt vmcnt(0)16837; GFX90A-NEXT: s_setpc_b64 s[30:31]16838;16839; GFX950-LABEL: flat_atomic_usub_cond_i64_saddr_ret_a_a:16840; GFX950: ; %bb.0:16841; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16842; GFX950-NEXT: s_add_u32 s0, s0, 0x5016843; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16844; GFX950-NEXT: s_addc_u32 s1, s1, 016845; GFX950-NEXT: s_cmp_eq_u32 s1, s316846; GFX950-NEXT: ;;#ASMSTART16847; GFX950-NEXT: ; def a[0:1]16848; GFX950-NEXT: ;;#ASMEND16849; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016850; GFX950-NEXT: v_accvgpr_read_b32 v5, a116851; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16852; GFX950-NEXT: v_accvgpr_read_b32 v4, a016853; GFX950-NEXT: s_cbranch_vccz .LBB219_416854; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16855; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]16856; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]16857; GFX950-NEXT: s_mov_b64 s[2:3], 016858; GFX950-NEXT: .LBB219_2: ; %atomicrmw.start16859; GFX950-NEXT: ; =>This Inner Loop Header: Depth=116860; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16861; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v416862; GFX950-NEXT: s_nop 116863; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc16864; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]16865; GFX950-NEXT: s_nop 116866; GFX950-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc16867; GFX950-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc16868; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc016869; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16870; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]16871; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]16872; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]16873; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]16874; GFX950-NEXT: s_cbranch_execnz .LBB219_216875; GFX950-NEXT: ; %bb.3: ; %Flow16876; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]16877; GFX950-NEXT: v_accvgpr_write_b32 a0, v016878; GFX950-NEXT: v_accvgpr_write_b32 a1, v116879; GFX950-NEXT: s_branch .LBB219_616880; GFX950-NEXT: .LBB219_4:16881; GFX950-NEXT: ; implicit-def: $agpr0_agpr116882; GFX950-NEXT: s_cbranch_execz .LBB219_616883; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private16884; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 016885; GFX950-NEXT: s_cselect_b32 s0, s0, -116886; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s016887; GFX950-NEXT: s_waitcnt vmcnt(0)16888; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v416889; GFX950-NEXT: s_nop 116890; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc16891; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]16892; GFX950-NEXT: v_accvgpr_write_b32 a0, v016893; GFX950-NEXT: v_accvgpr_write_b32 a1, v116894; GFX950-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc16895; GFX950-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc16896; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s016897; GFX950-NEXT: .LBB219_6: ; %atomicrmw.phi16898; GFX950-NEXT: ;;#ASMSTART16899; GFX950-NEXT: ; use a[0:1]16900; GFX950-NEXT: ;;#ASMEND16901; GFX950-NEXT: s_waitcnt vmcnt(0)16902; GFX950-NEXT: s_setpc_b64 s[30:31]16903 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1016904 %data = call i64 asm "; def $0", "=a"()16905 %result = atomicrmw usub_cond ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !016906 call void asm "; use $0", "a"(i64 %result)16907 ret void16908}16909 16910define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {16911; GFX90A-LABEL: flat_atomic_usub_cond_i64_saddr_ret_av_av:16912; GFX90A: ; %bb.0:16913; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16914; GFX90A-NEXT: s_add_u32 s4, s16, 0x5016915; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base16916; GFX90A-NEXT: s_addc_u32 s5, s17, 016917; GFX90A-NEXT: s_cmp_eq_u32 s5, s716918; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 016919; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]16920; GFX90A-NEXT: ;;#ASMSTART16921; GFX90A-NEXT: ; def v[4:5]16922; GFX90A-NEXT: ;;#ASMEND16923; GFX90A-NEXT: s_cbranch_vccz .LBB220_416924; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global16925; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]16926; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]16927; GFX90A-NEXT: s_mov_b64 s[6:7], 016928; GFX90A-NEXT: .LBB220_2: ; %atomicrmw.start16929; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=116930; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16931; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v416932; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc16933; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]16934; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc16935; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc16936; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc16937; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16938; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]16939; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]16940; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]16941; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]16942; GFX90A-NEXT: s_cbranch_execnz .LBB220_216943; GFX90A-NEXT: ; %bb.3: ; %Flow16944; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]16945; GFX90A-NEXT: s_branch .LBB220_616946; GFX90A-NEXT: .LBB220_4:16947; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr116948; GFX90A-NEXT: s_cbranch_execz .LBB220_616949; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private16950; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 016951; GFX90A-NEXT: s_cselect_b32 s4, s4, -116952; GFX90A-NEXT: v_mov_b32_e32 v2, s416953; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen16954; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:416955; GFX90A-NEXT: s_waitcnt vmcnt(1)16956; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v416957; GFX90A-NEXT: s_waitcnt vmcnt(0)16958; GFX90A-NEXT: v_subb_co_u32_e32 v6, vcc, v1, v5, vcc16959; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]16960; GFX90A-NEXT: v_cndmask_b32_e32 v3, v0, v3, vcc16961; GFX90A-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc16962; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen16963; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:416964; GFX90A-NEXT: .LBB220_6: ; %atomicrmw.phi16965; GFX90A-NEXT: ;;#ASMSTART16966; GFX90A-NEXT: ; use v[0:1]16967; GFX90A-NEXT: ;;#ASMEND16968; GFX90A-NEXT: s_waitcnt vmcnt(0)16969; GFX90A-NEXT: s_setpc_b64 s[30:31]16970;16971; GFX950-LABEL: flat_atomic_usub_cond_i64_saddr_ret_av_av:16972; GFX950: ; %bb.0:16973; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16974; GFX950-NEXT: s_add_u32 s0, s0, 0x5016975; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base16976; GFX950-NEXT: s_addc_u32 s1, s1, 016977; GFX950-NEXT: s_cmp_eq_u32 s1, s316978; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 016979; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]16980; GFX950-NEXT: ;;#ASMSTART16981; GFX950-NEXT: ; def v[4:5]16982; GFX950-NEXT: ;;#ASMEND16983; GFX950-NEXT: s_cbranch_vccz .LBB220_416984; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global16985; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]16986; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]16987; GFX950-NEXT: s_mov_b64 s[2:3], 016988; GFX950-NEXT: .LBB220_2: ; %atomicrmw.start16989; GFX950-NEXT: ; =>This Inner Loop Header: Depth=116990; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16991; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v416992; GFX950-NEXT: s_nop 116993; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc16994; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]16995; GFX950-NEXT: s_nop 116996; GFX950-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc16997; GFX950-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc16998; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc016999; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17000; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]17001; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17002; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]17003; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17004; GFX950-NEXT: s_cbranch_execnz .LBB220_217005; GFX950-NEXT: ; %bb.3: ; %Flow17006; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17007; GFX950-NEXT: s_branch .LBB220_617008; GFX950-NEXT: .LBB220_4:17009; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr117010; GFX950-NEXT: s_cbranch_execz .LBB220_617011; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private17012; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 017013; GFX950-NEXT: s_cselect_b32 s0, s0, -117014; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s017015; GFX950-NEXT: s_waitcnt vmcnt(0)17016; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v417017; GFX950-NEXT: s_nop 117018; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc17019; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]17020; GFX950-NEXT: s_nop 117021; GFX950-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc17022; GFX950-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc17023; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s017024; GFX950-NEXT: .LBB220_6: ; %atomicrmw.phi17025; GFX950-NEXT: ;;#ASMSTART17026; GFX950-NEXT: ; use v[0:1]17027; GFX950-NEXT: ;;#ASMEND17028; GFX950-NEXT: s_waitcnt vmcnt(0)17029; GFX950-NEXT: s_setpc_b64 s[30:31]17030 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1017031 %data = call i64 asm "; def $0", "=^VA"()17032 %result = atomicrmw usub_cond ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !017033 call void asm "; use $0", "^VA"(i64 %result)17034 ret void17035}17036 17037define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {17038; GFX90A-LABEL: flat_atomic_usub_sat_i64_saddr_ret_a_a:17039; GFX90A: ; %bb.0:17040; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17041; GFX90A-NEXT: s_add_u32 s4, s16, 0x5017042; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base17043; GFX90A-NEXT: s_addc_u32 s5, s17, 017044; GFX90A-NEXT: s_cmp_eq_u32 s5, s717045; GFX90A-NEXT: ;;#ASMSTART17046; GFX90A-NEXT: ; def a[0:1]17047; GFX90A-NEXT: ;;#ASMEND17048; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017049; GFX90A-NEXT: v_accvgpr_read_b32 v5, a117050; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17051; GFX90A-NEXT: v_accvgpr_read_b32 v4, a017052; GFX90A-NEXT: s_cbranch_vccz .LBB221_417053; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global17054; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]17055; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]17056; GFX90A-NEXT: s_mov_b64 s[6:7], 017057; GFX90A-NEXT: .LBB221_2: ; %atomicrmw.start17058; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117059; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17060; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v417061; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc17062; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc17063; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc17064; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc17065; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17066; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]17067; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]17068; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]17069; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]17070; GFX90A-NEXT: s_cbranch_execnz .LBB221_217071; GFX90A-NEXT: ; %bb.3: ; %Flow17072; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]17073; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017074; GFX90A-NEXT: v_accvgpr_write_b32 a1, v117075; GFX90A-NEXT: s_branch .LBB221_617076; GFX90A-NEXT: .LBB221_4:17077; GFX90A-NEXT: ; implicit-def: $agpr0_agpr117078; GFX90A-NEXT: s_cbranch_execz .LBB221_617079; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private17080; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017081; GFX90A-NEXT: s_cselect_b32 s4, s4, -117082; GFX90A-NEXT: v_mov_b32_e32 v0, s417083; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen17084; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen offset:417085; GFX90A-NEXT: s_waitcnt vmcnt(1)17086; GFX90A-NEXT: v_accvgpr_write_b32 a0, v117087; GFX90A-NEXT: v_sub_co_u32_e32 v1, vcc, v1, v417088; GFX90A-NEXT: s_waitcnt vmcnt(0)17089; GFX90A-NEXT: v_accvgpr_write_b32 a1, v217090; GFX90A-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc17091; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc17092; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc17093; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen17094; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen offset:417095; GFX90A-NEXT: .LBB221_6: ; %atomicrmw.phi17096; GFX90A-NEXT: ;;#ASMSTART17097; GFX90A-NEXT: ; use a[0:1]17098; GFX90A-NEXT: ;;#ASMEND17099; GFX90A-NEXT: s_waitcnt vmcnt(0)17100; GFX90A-NEXT: s_setpc_b64 s[30:31]17101;17102; GFX950-LABEL: flat_atomic_usub_sat_i64_saddr_ret_a_a:17103; GFX950: ; %bb.0:17104; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17105; GFX950-NEXT: s_add_u32 s0, s0, 0x5017106; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base17107; GFX950-NEXT: s_addc_u32 s1, s1, 017108; GFX950-NEXT: s_cmp_eq_u32 s1, s317109; GFX950-NEXT: ;;#ASMSTART17110; GFX950-NEXT: ; def a[0:1]17111; GFX950-NEXT: ;;#ASMEND17112; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 017113; GFX950-NEXT: v_accvgpr_read_b32 v5, a117114; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]17115; GFX950-NEXT: v_accvgpr_read_b32 v4, a017116; GFX950-NEXT: s_cbranch_vccz .LBB221_417117; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global17118; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]17119; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]17120; GFX950-NEXT: s_mov_b64 s[2:3], 017121; GFX950-NEXT: .LBB221_2: ; %atomicrmw.start17122; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117123; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17124; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v417125; GFX950-NEXT: s_nop 117126; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc17127; GFX950-NEXT: s_nop 117128; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc17129; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc17130; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc017131; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17132; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]17133; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17134; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]17135; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17136; GFX950-NEXT: s_cbranch_execnz .LBB221_217137; GFX950-NEXT: ; %bb.3: ; %Flow17138; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17139; GFX950-NEXT: v_accvgpr_write_b32 a0, v017140; GFX950-NEXT: v_accvgpr_write_b32 a1, v117141; GFX950-NEXT: s_branch .LBB221_617142; GFX950-NEXT: .LBB221_4:17143; GFX950-NEXT: ; implicit-def: $agpr0_agpr117144; GFX950-NEXT: s_cbranch_execz .LBB221_617145; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private17146; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 017147; GFX950-NEXT: s_cselect_b32 s0, s0, -117148; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s017149; GFX950-NEXT: s_waitcnt vmcnt(0)17150; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v417151; GFX950-NEXT: s_nop 117152; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc17153; GFX950-NEXT: v_accvgpr_write_b32 a0, v017154; GFX950-NEXT: s_nop 017155; GFX950-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc17156; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc17157; GFX950-NEXT: v_accvgpr_write_b32 a1, v117158; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s017159; GFX950-NEXT: .LBB221_6: ; %atomicrmw.phi17160; GFX950-NEXT: ;;#ASMSTART17161; GFX950-NEXT: ; use a[0:1]17162; GFX950-NEXT: ;;#ASMEND17163; GFX950-NEXT: s_waitcnt vmcnt(0)17164; GFX950-NEXT: s_setpc_b64 s[30:31]17165 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1017166 %data = call i64 asm "; def $0", "=a"()17167 %result = atomicrmw usub_sat ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !017168 call void asm "; use $0", "a"(i64 %result)17169 ret void17170}17171 17172define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {17173; GFX90A-LABEL: flat_atomic_usub_sat_i64_saddr_ret_av_av:17174; GFX90A: ; %bb.0:17175; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17176; GFX90A-NEXT: s_add_u32 s4, s16, 0x5017177; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base17178; GFX90A-NEXT: s_addc_u32 s5, s17, 017179; GFX90A-NEXT: s_cmp_eq_u32 s5, s717180; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017181; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17182; GFX90A-NEXT: ;;#ASMSTART17183; GFX90A-NEXT: ; def v[4:5]17184; GFX90A-NEXT: ;;#ASMEND17185; GFX90A-NEXT: s_cbranch_vccz .LBB222_417186; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global17187; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]17188; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]17189; GFX90A-NEXT: s_mov_b64 s[6:7], 017190; GFX90A-NEXT: .LBB222_2: ; %atomicrmw.start17191; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117192; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17193; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v417194; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc17195; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc17196; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc17197; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc17198; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17199; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]17200; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]17201; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]17202; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]17203; GFX90A-NEXT: s_cbranch_execnz .LBB222_217204; GFX90A-NEXT: ; %bb.3: ; %Flow17205; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]17206; GFX90A-NEXT: s_branch .LBB222_617207; GFX90A-NEXT: .LBB222_4:17208; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr117209; GFX90A-NEXT: s_cbranch_execz .LBB222_617210; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private17211; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017212; GFX90A-NEXT: s_cselect_b32 s4, s4, -117213; GFX90A-NEXT: v_mov_b32_e32 v2, s417214; GFX90A-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen17215; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:417216; GFX90A-NEXT: s_waitcnt vmcnt(1)17217; GFX90A-NEXT: v_sub_co_u32_e32 v3, vcc, v0, v417218; GFX90A-NEXT: s_waitcnt vmcnt(0)17219; GFX90A-NEXT: v_subb_co_u32_e32 v4, vcc, v1, v5, vcc17220; GFX90A-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc17221; GFX90A-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc17222; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen17223; GFX90A-NEXT: buffer_store_dword v4, v2, s[0:3], 0 offen offset:417224; GFX90A-NEXT: .LBB222_6: ; %atomicrmw.phi17225; GFX90A-NEXT: ;;#ASMSTART17226; GFX90A-NEXT: ; use v[0:1]17227; GFX90A-NEXT: ;;#ASMEND17228; GFX90A-NEXT: s_waitcnt vmcnt(0)17229; GFX90A-NEXT: s_setpc_b64 s[30:31]17230;17231; GFX950-LABEL: flat_atomic_usub_sat_i64_saddr_ret_av_av:17232; GFX950: ; %bb.0:17233; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17234; GFX950-NEXT: s_add_u32 s0, s0, 0x5017235; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base17236; GFX950-NEXT: s_addc_u32 s1, s1, 017237; GFX950-NEXT: s_cmp_eq_u32 s1, s317238; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 017239; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]17240; GFX950-NEXT: ;;#ASMSTART17241; GFX950-NEXT: ; def v[4:5]17242; GFX950-NEXT: ;;#ASMEND17243; GFX950-NEXT: s_cbranch_vccz .LBB222_417244; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global17245; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]17246; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]17247; GFX950-NEXT: s_mov_b64 s[2:3], 017248; GFX950-NEXT: .LBB222_2: ; %atomicrmw.start17249; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117250; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17251; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v417252; GFX950-NEXT: s_nop 117253; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc17254; GFX950-NEXT: s_nop 117255; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc17256; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc17257; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc017258; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17259; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]17260; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17261; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]17262; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17263; GFX950-NEXT: s_cbranch_execnz .LBB222_217264; GFX950-NEXT: ; %bb.3: ; %Flow17265; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17266; GFX950-NEXT: s_branch .LBB222_617267; GFX950-NEXT: .LBB222_4:17268; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr117269; GFX950-NEXT: s_cbranch_execz .LBB222_617270; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private17271; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 017272; GFX950-NEXT: s_cselect_b32 s0, s0, -117273; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s017274; GFX950-NEXT: s_waitcnt vmcnt(0)17275; GFX950-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v417276; GFX950-NEXT: s_nop 117277; GFX950-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc17278; GFX950-NEXT: s_nop 117279; GFX950-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc17280; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc17281; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s017282; GFX950-NEXT: .LBB222_6: ; %atomicrmw.phi17283; GFX950-NEXT: ;;#ASMSTART17284; GFX950-NEXT: ; use v[0:1]17285; GFX950-NEXT: ;;#ASMEND17286; GFX950-NEXT: s_waitcnt vmcnt(0)17287; GFX950-NEXT: s_setpc_b64 s[30:31]17288 %gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 1017289 %data = call i64 asm "; def $0", "=^VA"()17290 %result = atomicrmw usub_sat ptr %gep.0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !017291 call void asm "; use $0", "^VA"(i64 %result)17292 ret void17293}17294 17295;---------------------------------------------------------------------17296; other atomics f32, with aa+av cases using saddr17297;---------------------------------------------------------------------17298 17299define void @flat_atomic_fadd_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {17300; GFX90A-LABEL: flat_atomic_fadd_f32_saddr_ret_a_a:17301; GFX90A: ; %bb.0:17302; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17303; GFX90A-NEXT: s_add_u32 s4, s16, 4017304; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base17305; GFX90A-NEXT: s_addc_u32 s5, s17, 017306; GFX90A-NEXT: s_cmp_eq_u32 s5, s717307; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017308; GFX90A-NEXT: ;;#ASMSTART17309; GFX90A-NEXT: ; def a017310; GFX90A-NEXT: ;;#ASMEND17311; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17312; GFX90A-NEXT: v_accvgpr_read_b32 v0, a017313; GFX90A-NEXT: s_cbranch_vccz .LBB223_417314; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private17315; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base17316; GFX90A-NEXT: s_cmp_eq_u32 s5, s717317; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017318; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17319; GFX90A-NEXT: s_cbranch_vccz .LBB223_717320; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global17321; GFX90A-NEXT: v_mov_b32_e32 v1, 017322; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc17323; GFX90A-NEXT: s_cbranch_execz .LBB223_817324; GFX90A-NEXT: ; %bb.3:17325; GFX90A-NEXT: s_waitcnt vmcnt(0)17326; GFX90A-NEXT: v_accvgpr_write_b32 a0, v117327; GFX90A-NEXT: s_cbranch_execz .LBB223_517328; GFX90A-NEXT: s_branch .LBB223_617329; GFX90A-NEXT: .LBB223_4:17330; GFX90A-NEXT: ; implicit-def: $agpr017331; GFX90A-NEXT: .LBB223_5: ; %atomicrmw.shared17332; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017333; GFX90A-NEXT: s_cselect_b32 s4, s4, -117334; GFX90A-NEXT: v_mov_b32_e32 v1, s417335; GFX90A-NEXT: ds_add_rtn_f32 v0, v1, v017336; GFX90A-NEXT: s_waitcnt lgkmcnt(0)17337; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017338; GFX90A-NEXT: .LBB223_6: ; %atomicrmw.end17339; GFX90A-NEXT: ;;#ASMSTART17340; GFX90A-NEXT: ; use a017341; GFX90A-NEXT: ;;#ASMEND17342; GFX90A-NEXT: s_waitcnt vmcnt(0)17343; GFX90A-NEXT: s_setpc_b64 s[30:31]17344; GFX90A-NEXT: .LBB223_7:17345; GFX90A-NEXT: ; implicit-def: $vgpr117346; GFX90A-NEXT: .LBB223_8: ; %atomicrmw.private17347; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017348; GFX90A-NEXT: s_cselect_b32 s6, s4, -117349; GFX90A-NEXT: s_waitcnt vmcnt(0)17350; GFX90A-NEXT: v_mov_b32_e32 v1, s617351; GFX90A-NEXT: buffer_load_dword v2, v1, s[0:3], 0 offen17352; GFX90A-NEXT: s_waitcnt vmcnt(0)17353; GFX90A-NEXT: v_add_f32_e32 v3, v2, v017354; GFX90A-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen17355; GFX90A-NEXT: v_accvgpr_write_b32 a0, v217356; GFX90A-NEXT: s_cbranch_execz .LBB223_517357; GFX90A-NEXT: s_branch .LBB223_617358;17359; GFX950-LABEL: flat_atomic_fadd_f32_saddr_ret_a_a:17360; GFX950: ; %bb.0:17361; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17362; GFX950-NEXT: v_mov_b32_e32 v0, s017363; GFX950-NEXT: v_mov_b32_e32 v1, s117364; GFX950-NEXT: ;;#ASMSTART17365; GFX950-NEXT: ; def a017366; GFX950-NEXT: ;;#ASMEND17367; GFX950-NEXT: s_nop 017368; GFX950-NEXT: v_accvgpr_read_b32 v2, a017369; GFX950-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:40 sc017370; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17371; GFX950-NEXT: v_accvgpr_write_b32 a0, v017372; GFX950-NEXT: ;;#ASMSTART17373; GFX950-NEXT: ; use a017374; GFX950-NEXT: ;;#ASMEND17375; GFX950-NEXT: s_setpc_b64 s[30:31]17376 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017377 %data = call float asm "; def $0", "=a"()17378 %result = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017379 call void asm "; use $0", "a"(float %result)17380 ret void17381}17382 17383define void @flat_atomic_fadd_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {17384; GFX90A-LABEL: flat_atomic_fadd_f32_saddr_ret_av_av:17385; GFX90A: ; %bb.0:17386; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17387; GFX90A-NEXT: s_add_u32 s4, s16, 4017388; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base17389; GFX90A-NEXT: s_addc_u32 s5, s17, 017390; GFX90A-NEXT: s_cmp_eq_u32 s5, s717391; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017392; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17393; GFX90A-NEXT: ;;#ASMSTART17394; GFX90A-NEXT: ; def v017395; GFX90A-NEXT: ;;#ASMEND17396; GFX90A-NEXT: s_cbranch_vccz .LBB224_317397; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private17398; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base17399; GFX90A-NEXT: s_cmp_eq_u32 s5, s717400; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 017401; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]17402; GFX90A-NEXT: s_cbranch_vccz .LBB224_417403; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global17404; GFX90A-NEXT: v_mov_b32_e32 v1, 017405; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc17406; GFX90A-NEXT: s_cbranch_execz .LBB224_517407; GFX90A-NEXT: s_branch .LBB224_617408; GFX90A-NEXT: .LBB224_3:17409; GFX90A-NEXT: ; implicit-def: $vgpr117410; GFX90A-NEXT: s_branch .LBB224_717411; GFX90A-NEXT: .LBB224_4:17412; GFX90A-NEXT: ; implicit-def: $vgpr117413; GFX90A-NEXT: .LBB224_5: ; %atomicrmw.private17414; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017415; GFX90A-NEXT: s_cselect_b32 s6, s4, -117416; GFX90A-NEXT: v_mov_b32_e32 v2, s617417; GFX90A-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen17418; GFX90A-NEXT: s_waitcnt vmcnt(0)17419; GFX90A-NEXT: v_add_f32_e32 v3, v1, v017420; GFX90A-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen17421; GFX90A-NEXT: .LBB224_6: ; %Flow117422; GFX90A-NEXT: s_cbranch_execnz .LBB224_817423; GFX90A-NEXT: .LBB224_7: ; %atomicrmw.shared17424; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 017425; GFX90A-NEXT: s_cselect_b32 s4, s4, -117426; GFX90A-NEXT: s_waitcnt vmcnt(0)17427; GFX90A-NEXT: v_mov_b32_e32 v1, s417428; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v017429; GFX90A-NEXT: s_waitcnt lgkmcnt(0)17430; GFX90A-NEXT: .LBB224_8: ; %atomicrmw.end17431; GFX90A-NEXT: s_waitcnt vmcnt(0)17432; GFX90A-NEXT: ;;#ASMSTART17433; GFX90A-NEXT: ; use v117434; GFX90A-NEXT: ;;#ASMEND17435; GFX90A-NEXT: s_setpc_b64 s[30:31]17436;17437; GFX950-LABEL: flat_atomic_fadd_f32_saddr_ret_av_av:17438; GFX950: ; %bb.0:17439; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17440; GFX950-NEXT: v_mov_b32_e32 v0, s017441; GFX950-NEXT: v_mov_b32_e32 v1, s117442; GFX950-NEXT: ;;#ASMSTART17443; GFX950-NEXT: ; def v217444; GFX950-NEXT: ;;#ASMEND17445; GFX950-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 offset:40 sc017446; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17447; GFX950-NEXT: ;;#ASMSTART17448; GFX950-NEXT: ; use v017449; GFX950-NEXT: ;;#ASMEND17450; GFX950-NEXT: s_setpc_b64 s[30:31]17451 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017452 %data = call float asm "; def $0", "=^VA"()17453 %result = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017454 call void asm "; use $0", "^VA"(float %result)17455 ret void17456}17457 17458define void @flat_atomic_fsub_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {17459; GFX90A-LABEL: flat_atomic_fsub_f32_saddr_ret_a_a:17460; GFX90A: ; %bb.0:17461; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17462; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17463; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017464; GFX90A-NEXT: ;;#ASMSTART17465; GFX90A-NEXT: ; def a017466; GFX90A-NEXT: ;;#ASMEND17467; GFX90A-NEXT: v_accvgpr_read_b32 v4, a017468; GFX90A-NEXT: s_mov_b64 s[4:5], 017469; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17470; GFX90A-NEXT: .LBB225_1: ; %atomicrmw.start17471; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117472; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17473; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v417474; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17475; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17476; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117477; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17478; GFX90A-NEXT: v_mov_b32_e32 v1, v017479; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17480; GFX90A-NEXT: s_cbranch_execnz .LBB225_117481; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17482; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17483; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017484; GFX90A-NEXT: ;;#ASMSTART17485; GFX90A-NEXT: ; use a017486; GFX90A-NEXT: ;;#ASMEND17487; GFX90A-NEXT: s_setpc_b64 s[30:31]17488;17489; GFX950-LABEL: flat_atomic_fsub_f32_saddr_ret_a_a:17490; GFX950: ; %bb.0:17491; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17492; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17493; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017494; GFX950-NEXT: ;;#ASMSTART17495; GFX950-NEXT: ; def a017496; GFX950-NEXT: ;;#ASMEND17497; GFX950-NEXT: s_mov_b64 s[2:3], 017498; GFX950-NEXT: v_accvgpr_read_b32 v4, a017499; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17500; GFX950-NEXT: .LBB225_1: ; %atomicrmw.start17501; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117502; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17503; GFX950-NEXT: v_sub_f32_e32 v0, v1, v417504; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017505; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17506; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117507; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17508; GFX950-NEXT: v_mov_b32_e32 v1, v017509; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17510; GFX950-NEXT: s_cbranch_execnz .LBB225_117511; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17512; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17513; GFX950-NEXT: v_accvgpr_write_b32 a0, v017514; GFX950-NEXT: ;;#ASMSTART17515; GFX950-NEXT: ; use a017516; GFX950-NEXT: ;;#ASMEND17517; GFX950-NEXT: s_setpc_b64 s[30:31]17518 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017519 %data = call float asm "; def $0", "=a"()17520 %result = atomicrmw fsub ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017521 call void asm "; use $0", "a"(float %result)17522 ret void17523}17524 17525define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {17526; GFX90A-LABEL: flat_atomic_fsub_f32_saddr_ret_av_av:17527; GFX90A: ; %bb.0:17528; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17529; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17530; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017531; GFX90A-NEXT: s_mov_b64 s[4:5], 017532; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17533; GFX90A-NEXT: ;;#ASMSTART17534; GFX90A-NEXT: ; def v417535; GFX90A-NEXT: ;;#ASMEND17536; GFX90A-NEXT: .LBB226_1: ; %atomicrmw.start17537; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117538; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17539; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v417540; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17541; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17542; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117543; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17544; GFX90A-NEXT: v_mov_b32_e32 v1, v017545; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17546; GFX90A-NEXT: s_cbranch_execnz .LBB226_117547; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17548; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17549; GFX90A-NEXT: ;;#ASMSTART17550; GFX90A-NEXT: ; use v017551; GFX90A-NEXT: ;;#ASMEND17552; GFX90A-NEXT: s_setpc_b64 s[30:31]17553;17554; GFX950-LABEL: flat_atomic_fsub_f32_saddr_ret_av_av:17555; GFX950: ; %bb.0:17556; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17557; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17558; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017559; GFX950-NEXT: s_mov_b64 s[2:3], 017560; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17561; GFX950-NEXT: ;;#ASMSTART17562; GFX950-NEXT: ; def v417563; GFX950-NEXT: ;;#ASMEND17564; GFX950-NEXT: .LBB226_1: ; %atomicrmw.start17565; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117566; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17567; GFX950-NEXT: v_sub_f32_e32 v0, v1, v417568; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017569; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17570; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117571; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17572; GFX950-NEXT: v_mov_b32_e32 v1, v017573; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17574; GFX950-NEXT: s_cbranch_execnz .LBB226_117575; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17576; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17577; GFX950-NEXT: ;;#ASMSTART17578; GFX950-NEXT: ; use v017579; GFX950-NEXT: ;;#ASMEND17580; GFX950-NEXT: s_setpc_b64 s[30:31]17581 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017582 %data = call float asm "; def $0", "=^VA"()17583 %result = atomicrmw fsub ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017584 call void asm "; use $0", "^VA"(float %result)17585 ret void17586}17587 17588define void @flat_atomic_fmax_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {17589; GFX90A-LABEL: flat_atomic_fmax_f32_saddr_ret_a_a:17590; GFX90A: ; %bb.0:17591; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17592; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17593; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017594; GFX90A-NEXT: ;;#ASMSTART17595; GFX90A-NEXT: ; def a017596; GFX90A-NEXT: ;;#ASMEND17597; GFX90A-NEXT: v_accvgpr_read_b32 v0, a017598; GFX90A-NEXT: s_mov_b64 s[4:5], 017599; GFX90A-NEXT: v_max_f32_e32 v4, v0, v017600; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17601; GFX90A-NEXT: .LBB227_1: ; %atomicrmw.start17602; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117603; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17604; GFX90A-NEXT: v_max_f32_e32 v0, v1, v117605; GFX90A-NEXT: v_max_f32_e32 v0, v0, v417606; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17607; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17608; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117609; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17610; GFX90A-NEXT: v_mov_b32_e32 v1, v017611; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17612; GFX90A-NEXT: s_cbranch_execnz .LBB227_117613; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17614; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17615; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017616; GFX90A-NEXT: ;;#ASMSTART17617; GFX90A-NEXT: ; use a017618; GFX90A-NEXT: ;;#ASMEND17619; GFX90A-NEXT: s_setpc_b64 s[30:31]17620;17621; GFX950-LABEL: flat_atomic_fmax_f32_saddr_ret_a_a:17622; GFX950: ; %bb.0:17623; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17624; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17625; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017626; GFX950-NEXT: ;;#ASMSTART17627; GFX950-NEXT: ; def a017628; GFX950-NEXT: ;;#ASMEND17629; GFX950-NEXT: s_mov_b64 s[2:3], 017630; GFX950-NEXT: v_accvgpr_read_b32 v0, a017631; GFX950-NEXT: v_max_f32_e32 v4, v0, v017632; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17633; GFX950-NEXT: .LBB227_1: ; %atomicrmw.start17634; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117635; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17636; GFX950-NEXT: v_max_f32_e32 v0, v1, v117637; GFX950-NEXT: v_max_f32_e32 v0, v0, v417638; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017639; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17640; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117641; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17642; GFX950-NEXT: v_mov_b32_e32 v1, v017643; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17644; GFX950-NEXT: s_cbranch_execnz .LBB227_117645; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17646; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17647; GFX950-NEXT: v_accvgpr_write_b32 a0, v017648; GFX950-NEXT: ;;#ASMSTART17649; GFX950-NEXT: ; use a017650; GFX950-NEXT: ;;#ASMEND17651; GFX950-NEXT: s_setpc_b64 s[30:31]17652 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017653 %data = call float asm "; def $0", "=a"()17654 %result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017655 call void asm "; use $0", "a"(float %result)17656 ret void17657}17658 17659define void @flat_atomic_fmax_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {17660; GFX90A-LABEL: flat_atomic_fmax_f32_saddr_ret_av_av:17661; GFX90A: ; %bb.0:17662; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17663; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17664; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017665; GFX90A-NEXT: ;;#ASMSTART17666; GFX90A-NEXT: ; def v017667; GFX90A-NEXT: ;;#ASMEND17668; GFX90A-NEXT: s_mov_b64 s[4:5], 017669; GFX90A-NEXT: v_max_f32_e32 v4, v0, v017670; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17671; GFX90A-NEXT: .LBB228_1: ; %atomicrmw.start17672; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117673; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17674; GFX90A-NEXT: v_max_f32_e32 v0, v1, v117675; GFX90A-NEXT: v_max_f32_e32 v0, v0, v417676; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17677; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17678; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117679; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17680; GFX90A-NEXT: v_mov_b32_e32 v1, v017681; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17682; GFX90A-NEXT: s_cbranch_execnz .LBB228_117683; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17684; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17685; GFX90A-NEXT: ;;#ASMSTART17686; GFX90A-NEXT: ; use v017687; GFX90A-NEXT: ;;#ASMEND17688; GFX90A-NEXT: s_setpc_b64 s[30:31]17689;17690; GFX950-LABEL: flat_atomic_fmax_f32_saddr_ret_av_av:17691; GFX950: ; %bb.0:17692; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17693; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17694; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017695; GFX950-NEXT: ;;#ASMSTART17696; GFX950-NEXT: ; def v017697; GFX950-NEXT: ;;#ASMEND17698; GFX950-NEXT: s_mov_b64 s[2:3], 017699; GFX950-NEXT: v_max_f32_e32 v4, v0, v017700; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17701; GFX950-NEXT: .LBB228_1: ; %atomicrmw.start17702; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117703; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17704; GFX950-NEXT: v_max_f32_e32 v0, v1, v117705; GFX950-NEXT: v_max_f32_e32 v0, v0, v417706; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017707; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17708; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117709; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17710; GFX950-NEXT: v_mov_b32_e32 v1, v017711; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17712; GFX950-NEXT: s_cbranch_execnz .LBB228_117713; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17714; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17715; GFX950-NEXT: ;;#ASMSTART17716; GFX950-NEXT: ; use v017717; GFX950-NEXT: ;;#ASMEND17718; GFX950-NEXT: s_setpc_b64 s[30:31]17719 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017720 %data = call float asm "; def $0", "=^VA"()17721 %result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017722 call void asm "; use $0", "^VA"(float %result)17723 ret void17724}17725 17726define void @flat_atomic_fmin_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {17727; GFX90A-LABEL: flat_atomic_fmin_f32_saddr_ret_a_a:17728; GFX90A: ; %bb.0:17729; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17730; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17731; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017732; GFX90A-NEXT: ;;#ASMSTART17733; GFX90A-NEXT: ; def a017734; GFX90A-NEXT: ;;#ASMEND17735; GFX90A-NEXT: v_accvgpr_read_b32 v0, a017736; GFX90A-NEXT: s_mov_b64 s[4:5], 017737; GFX90A-NEXT: v_max_f32_e32 v4, v0, v017738; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17739; GFX90A-NEXT: .LBB229_1: ; %atomicrmw.start17740; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117741; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17742; GFX90A-NEXT: v_max_f32_e32 v0, v1, v117743; GFX90A-NEXT: v_min_f32_e32 v0, v0, v417744; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17745; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17746; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117747; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17748; GFX90A-NEXT: v_mov_b32_e32 v1, v017749; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17750; GFX90A-NEXT: s_cbranch_execnz .LBB229_117751; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17752; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17753; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017754; GFX90A-NEXT: ;;#ASMSTART17755; GFX90A-NEXT: ; use a017756; GFX90A-NEXT: ;;#ASMEND17757; GFX90A-NEXT: s_setpc_b64 s[30:31]17758;17759; GFX950-LABEL: flat_atomic_fmin_f32_saddr_ret_a_a:17760; GFX950: ; %bb.0:17761; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17762; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17763; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017764; GFX950-NEXT: ;;#ASMSTART17765; GFX950-NEXT: ; def a017766; GFX950-NEXT: ;;#ASMEND17767; GFX950-NEXT: s_mov_b64 s[2:3], 017768; GFX950-NEXT: v_accvgpr_read_b32 v0, a017769; GFX950-NEXT: v_max_f32_e32 v4, v0, v017770; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17771; GFX950-NEXT: .LBB229_1: ; %atomicrmw.start17772; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117773; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17774; GFX950-NEXT: v_max_f32_e32 v0, v1, v117775; GFX950-NEXT: v_min_f32_e32 v0, v0, v417776; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017777; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17778; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117779; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17780; GFX950-NEXT: v_mov_b32_e32 v1, v017781; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17782; GFX950-NEXT: s_cbranch_execnz .LBB229_117783; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17784; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17785; GFX950-NEXT: v_accvgpr_write_b32 a0, v017786; GFX950-NEXT: ;;#ASMSTART17787; GFX950-NEXT: ; use a017788; GFX950-NEXT: ;;#ASMEND17789; GFX950-NEXT: s_setpc_b64 s[30:31]17790 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017791 %data = call float asm "; def $0", "=a"()17792 %result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017793 call void asm "; use $0", "a"(float %result)17794 ret void17795}17796 17797define void @flat_atomic_fmin_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {17798; GFX90A-LABEL: flat_atomic_fmin_f32_saddr_ret_av_av:17799; GFX90A: ; %bb.0:17800; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17801; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17802; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017803; GFX90A-NEXT: ;;#ASMSTART17804; GFX90A-NEXT: ; def v017805; GFX90A-NEXT: ;;#ASMEND17806; GFX90A-NEXT: s_mov_b64 s[4:5], 017807; GFX90A-NEXT: v_max_f32_e32 v4, v0, v017808; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17809; GFX90A-NEXT: .LBB230_1: ; %atomicrmw.start17810; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117811; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17812; GFX90A-NEXT: v_max_f32_e32 v0, v1, v117813; GFX90A-NEXT: v_min_f32_e32 v0, v0, v417814; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17815; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17816; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117817; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17818; GFX90A-NEXT: v_mov_b32_e32 v1, v017819; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17820; GFX90A-NEXT: s_cbranch_execnz .LBB230_117821; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17822; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17823; GFX90A-NEXT: ;;#ASMSTART17824; GFX90A-NEXT: ; use v017825; GFX90A-NEXT: ;;#ASMEND17826; GFX90A-NEXT: s_setpc_b64 s[30:31]17827;17828; GFX950-LABEL: flat_atomic_fmin_f32_saddr_ret_av_av:17829; GFX950: ; %bb.0:17830; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17831; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17832; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017833; GFX950-NEXT: ;;#ASMSTART17834; GFX950-NEXT: ; def v017835; GFX950-NEXT: ;;#ASMEND17836; GFX950-NEXT: s_mov_b64 s[2:3], 017837; GFX950-NEXT: v_max_f32_e32 v4, v0, v017838; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17839; GFX950-NEXT: .LBB230_1: ; %atomicrmw.start17840; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117841; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17842; GFX950-NEXT: v_max_f32_e32 v0, v1, v117843; GFX950-NEXT: v_min_f32_e32 v0, v0, v417844; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017845; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17846; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117847; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17848; GFX950-NEXT: v_mov_b32_e32 v1, v017849; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17850; GFX950-NEXT: s_cbranch_execnz .LBB230_117851; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17852; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17853; GFX950-NEXT: ;;#ASMSTART17854; GFX950-NEXT: ; use v017855; GFX950-NEXT: ;;#ASMEND17856; GFX950-NEXT: s_setpc_b64 s[30:31]17857 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017858 %data = call float asm "; def $0", "=^VA"()17859 %result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017860 call void asm "; use $0", "^VA"(float %result)17861 ret void17862}17863 17864define void @flat_atomic_fmaximum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {17865; GFX90A-LABEL: flat_atomic_fmaximum_f32_saddr_ret_a_a:17866; GFX90A: ; %bb.0:17867; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17868; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17869; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017870; GFX90A-NEXT: ;;#ASMSTART17871; GFX90A-NEXT: ; def a017872; GFX90A-NEXT: ;;#ASMEND17873; GFX90A-NEXT: v_accvgpr_read_b32 v4, a017874; GFX90A-NEXT: s_mov_b64 s[4:5], 017875; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000017876; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17877; GFX90A-NEXT: .LBB231_1: ; %atomicrmw.start17878; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117879; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17880; GFX90A-NEXT: v_max_f32_e32 v0, v1, v417881; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v417882; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc17883; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17884; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17885; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117886; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17887; GFX90A-NEXT: v_mov_b32_e32 v1, v017888; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17889; GFX90A-NEXT: s_cbranch_execnz .LBB231_117890; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17891; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17892; GFX90A-NEXT: v_accvgpr_write_b32 a0, v017893; GFX90A-NEXT: ;;#ASMSTART17894; GFX90A-NEXT: ; use a017895; GFX90A-NEXT: ;;#ASMEND17896; GFX90A-NEXT: s_setpc_b64 s[30:31]17897;17898; GFX950-LABEL: flat_atomic_fmaximum_f32_saddr_ret_a_a:17899; GFX950: ; %bb.0:17900; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17901; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17902; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017903; GFX950-NEXT: ;;#ASMSTART17904; GFX950-NEXT: ; def a017905; GFX950-NEXT: ;;#ASMEND17906; GFX950-NEXT: s_mov_b64 s[2:3], 017907; GFX950-NEXT: v_accvgpr_read_b32 v4, a017908; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17909; GFX950-NEXT: .LBB231_1: ; %atomicrmw.start17910; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117911; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17912; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v417913; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017914; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17915; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117916; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17917; GFX950-NEXT: v_mov_b32_e32 v1, v017918; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17919; GFX950-NEXT: s_cbranch_execnz .LBB231_117920; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17921; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17922; GFX950-NEXT: v_accvgpr_write_b32 a0, v017923; GFX950-NEXT: ;;#ASMSTART17924; GFX950-NEXT: ; use a017925; GFX950-NEXT: ;;#ASMEND17926; GFX950-NEXT: s_setpc_b64 s[30:31]17927 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017928 %data = call float asm "; def $0", "=a"()17929 %result = atomicrmw fmaximum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017930 call void asm "; use $0", "a"(float %result)17931 ret void17932}17933 17934define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {17935; GFX90A-LABEL: flat_atomic_fmaximum_f32_saddr_ret_av_av:17936; GFX90A: ; %bb.0:17937; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17938; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]17939; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4017940; GFX90A-NEXT: s_mov_b64 s[4:5], 017941; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000017942; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]17943; GFX90A-NEXT: ;;#ASMSTART17944; GFX90A-NEXT: ; def v417945; GFX90A-NEXT: ;;#ASMEND17946; GFX90A-NEXT: .LBB232_1: ; %atomicrmw.start17947; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=117948; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17949; GFX90A-NEXT: v_max_f32_e32 v0, v1, v417950; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v417951; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc17952; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc17953; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17954; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117955; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]17956; GFX90A-NEXT: v_mov_b32_e32 v1, v017957; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]17958; GFX90A-NEXT: s_cbranch_execnz .LBB232_117959; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end17960; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]17961; GFX90A-NEXT: ;;#ASMSTART17962; GFX90A-NEXT: ; use v017963; GFX90A-NEXT: ;;#ASMEND17964; GFX90A-NEXT: s_setpc_b64 s[30:31]17965;17966; GFX950-LABEL: flat_atomic_fmaximum_f32_saddr_ret_av_av:17967; GFX950: ; %bb.0:17968; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)17969; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]17970; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4017971; GFX950-NEXT: s_mov_b64 s[2:3], 017972; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]17973; GFX950-NEXT: ;;#ASMSTART17974; GFX950-NEXT: ; def v417975; GFX950-NEXT: ;;#ASMEND17976; GFX950-NEXT: .LBB232_1: ; %atomicrmw.start17977; GFX950-NEXT: ; =>This Inner Loop Header: Depth=117978; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17979; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v417980; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc017981; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17982; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v117983; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]17984; GFX950-NEXT: v_mov_b32_e32 v1, v017985; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]17986; GFX950-NEXT: s_cbranch_execnz .LBB232_117987; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end17988; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]17989; GFX950-NEXT: ;;#ASMSTART17990; GFX950-NEXT: ; use v017991; GFX950-NEXT: ;;#ASMEND17992; GFX950-NEXT: s_setpc_b64 s[30:31]17993 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1017994 %data = call float asm "; def $0", "=^VA"()17995 %result = atomicrmw fmaximum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !017996 call void asm "; use $0", "^VA"(float %result)17997 ret void17998}17999 18000define void @flat_atomic_fminimum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {18001; GFX90A-LABEL: flat_atomic_fminimum_f32_saddr_ret_a_a:18002; GFX90A: ; %bb.0:18003; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18004; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]18005; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4018006; GFX90A-NEXT: ;;#ASMSTART18007; GFX90A-NEXT: ; def a018008; GFX90A-NEXT: ;;#ASMEND18009; GFX90A-NEXT: v_accvgpr_read_b32 v4, a018010; GFX90A-NEXT: s_mov_b64 s[4:5], 018011; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000018012; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]18013; GFX90A-NEXT: .LBB233_1: ; %atomicrmw.start18014; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=118015; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18016; GFX90A-NEXT: v_min_f32_e32 v0, v1, v418017; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v418018; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc18019; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc18020; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18021; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v118022; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]18023; GFX90A-NEXT: v_mov_b32_e32 v1, v018024; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]18025; GFX90A-NEXT: s_cbranch_execnz .LBB233_118026; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end18027; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]18028; GFX90A-NEXT: v_accvgpr_write_b32 a0, v018029; GFX90A-NEXT: ;;#ASMSTART18030; GFX90A-NEXT: ; use a018031; GFX90A-NEXT: ;;#ASMEND18032; GFX90A-NEXT: s_setpc_b64 s[30:31]18033;18034; GFX950-LABEL: flat_atomic_fminimum_f32_saddr_ret_a_a:18035; GFX950: ; %bb.0:18036; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18037; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]18038; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4018039; GFX950-NEXT: ;;#ASMSTART18040; GFX950-NEXT: ; def a018041; GFX950-NEXT: ;;#ASMEND18042; GFX950-NEXT: s_mov_b64 s[2:3], 018043; GFX950-NEXT: v_accvgpr_read_b32 v4, a018044; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]18045; GFX950-NEXT: .LBB233_1: ; %atomicrmw.start18046; GFX950-NEXT: ; =>This Inner Loop Header: Depth=118047; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18048; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v418049; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc018050; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18051; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v118052; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]18053; GFX950-NEXT: v_mov_b32_e32 v1, v018054; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]18055; GFX950-NEXT: s_cbranch_execnz .LBB233_118056; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end18057; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]18058; GFX950-NEXT: v_accvgpr_write_b32 a0, v018059; GFX950-NEXT: ;;#ASMSTART18060; GFX950-NEXT: ; use a018061; GFX950-NEXT: ;;#ASMEND18062; GFX950-NEXT: s_setpc_b64 s[30:31]18063 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1018064 %data = call float asm "; def $0", "=a"()18065 %result = atomicrmw fminimum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !018066 call void asm "; use $0", "a"(float %result)18067 ret void18068}18069 18070define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {18071; GFX90A-LABEL: flat_atomic_fminimum_f32_saddr_ret_av_av:18072; GFX90A: ; %bb.0:18073; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18074; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]18075; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4018076; GFX90A-NEXT: s_mov_b64 s[4:5], 018077; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000018078; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]18079; GFX90A-NEXT: ;;#ASMSTART18080; GFX90A-NEXT: ; def v418081; GFX90A-NEXT: ;;#ASMEND18082; GFX90A-NEXT: .LBB234_1: ; %atomicrmw.start18083; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=118084; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18085; GFX90A-NEXT: v_min_f32_e32 v0, v1, v418086; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v418087; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc18088; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc18089; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18090; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v118091; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]18092; GFX90A-NEXT: v_mov_b32_e32 v1, v018093; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]18094; GFX90A-NEXT: s_cbranch_execnz .LBB234_118095; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end18096; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]18097; GFX90A-NEXT: ;;#ASMSTART18098; GFX90A-NEXT: ; use v018099; GFX90A-NEXT: ;;#ASMEND18100; GFX90A-NEXT: s_setpc_b64 s[30:31]18101;18102; GFX950-LABEL: flat_atomic_fminimum_f32_saddr_ret_av_av:18103; GFX950: ; %bb.0:18104; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18105; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]18106; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4018107; GFX950-NEXT: s_mov_b64 s[2:3], 018108; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]18109; GFX950-NEXT: ;;#ASMSTART18110; GFX950-NEXT: ; def v418111; GFX950-NEXT: ;;#ASMEND18112; GFX950-NEXT: .LBB234_1: ; %atomicrmw.start18113; GFX950-NEXT: ; =>This Inner Loop Header: Depth=118114; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18115; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v418116; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc018117; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18118; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v118119; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]18120; GFX950-NEXT: v_mov_b32_e32 v1, v018121; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]18122; GFX950-NEXT: s_cbranch_execnz .LBB234_118123; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end18124; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]18125; GFX950-NEXT: ;;#ASMSTART18126; GFX950-NEXT: ; use v018127; GFX950-NEXT: ;;#ASMEND18128; GFX950-NEXT: s_setpc_b64 s[30:31]18129 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1018130 %data = call float asm "; def $0", "=^VA"()18131 %result = atomicrmw fminimum ptr %gep.0, float %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !018132 call void asm "; use $0", "^VA"(float %result)18133 ret void18134}18135 18136;---------------------------------------------------------------------18137; other atomics f64, with aa+av cases using saddr18138;---------------------------------------------------------------------18139 18140define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {18141; GFX90A-LABEL: flat_atomic_fadd_f64_saddr_ret_a_a:18142; GFX90A: ; %bb.0:18143; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18144; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018145; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base18146; GFX90A-NEXT: s_addc_u32 s5, s17, 018147; GFX90A-NEXT: s_cmp_eq_u32 s5, s718148; GFX90A-NEXT: ;;#ASMSTART18149; GFX90A-NEXT: ; def a[0:1]18150; GFX90A-NEXT: ;;#ASMEND18151; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018152; GFX90A-NEXT: v_accvgpr_read_b32 v0, a018153; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18154; GFX90A-NEXT: v_accvgpr_read_b32 v1, a118155; GFX90A-NEXT: s_cbranch_vccz .LBB235_318156; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private18157; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18158; GFX90A-NEXT: s_cmp_eq_u32 s5, s718159; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018160; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18161; GFX90A-NEXT: s_cbranch_vccz .LBB235_418162; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global18163; GFX90A-NEXT: v_mov_b32_e32 v2, 018164; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc18165; GFX90A-NEXT: s_cbranch_execz .LBB235_518166; GFX90A-NEXT: s_branch .LBB235_618167; GFX90A-NEXT: .LBB235_3:18168; GFX90A-NEXT: ; implicit-def: $agpr0_agpr118169; GFX90A-NEXT: s_branch .LBB235_718170; GFX90A-NEXT: .LBB235_4:18171; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr318172; GFX90A-NEXT: .LBB235_5: ; %atomicrmw.private18173; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018174; GFX90A-NEXT: s_cselect_b32 s6, s4, -118175; GFX90A-NEXT: v_mov_b32_e32 v6, s618176; GFX90A-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen18177; GFX90A-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:418178; GFX90A-NEXT: s_waitcnt vmcnt(0)18179; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]18180; GFX90A-NEXT: buffer_store_dword v4, v6, s[0:3], 0 offen18181; GFX90A-NEXT: buffer_store_dword v5, v6, s[0:3], 0 offen offset:418182; GFX90A-NEXT: .LBB235_6: ; %Flow118183; GFX90A-NEXT: s_waitcnt vmcnt(0)18184; GFX90A-NEXT: v_accvgpr_write_b32 a0, v218185; GFX90A-NEXT: v_accvgpr_write_b32 a1, v318186; GFX90A-NEXT: s_cbranch_execnz .LBB235_818187; GFX90A-NEXT: .LBB235_7: ; %atomicrmw.shared18188; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018189; GFX90A-NEXT: s_cselect_b32 s4, s4, -118190; GFX90A-NEXT: v_mov_b32_e32 v2, s418191; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v2, v[0:1]18192; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18193; GFX90A-NEXT: v_accvgpr_write_b32 a0, v018194; GFX90A-NEXT: v_accvgpr_write_b32 a1, v118195; GFX90A-NEXT: .LBB235_8: ; %atomicrmw.end18196; GFX90A-NEXT: ;;#ASMSTART18197; GFX90A-NEXT: ; use a[0:1]18198; GFX90A-NEXT: ;;#ASMEND18199; GFX90A-NEXT: s_setpc_b64 s[30:31]18200;18201; GFX950-LABEL: flat_atomic_fadd_f64_saddr_ret_a_a:18202; GFX950: ; %bb.0:18203; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18204; GFX950-NEXT: s_add_u32 s0, s0, 0x5018205; GFX950-NEXT: s_mov_b64 s[2:3], src_shared_base18206; GFX950-NEXT: s_addc_u32 s1, s1, 018207; GFX950-NEXT: s_cmp_eq_u32 s1, s318208; GFX950-NEXT: ;;#ASMSTART18209; GFX950-NEXT: ; def a[0:1]18210; GFX950-NEXT: ;;#ASMEND18211; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018212; GFX950-NEXT: v_accvgpr_read_b32 v0, a018213; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18214; GFX950-NEXT: v_accvgpr_read_b32 v1, a118215; GFX950-NEXT: s_cbranch_vccz .LBB235_318216; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private18217; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18218; GFX950-NEXT: s_cmp_eq_u32 s1, s318219; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018220; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18221; GFX950-NEXT: s_cbranch_vccz .LBB235_418222; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global18223; GFX950-NEXT: v_mov_b32_e32 v2, 018224; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc018225; GFX950-NEXT: s_cbranch_execz .LBB235_518226; GFX950-NEXT: s_branch .LBB235_618227; GFX950-NEXT: .LBB235_3:18228; GFX950-NEXT: ; implicit-def: $agpr0_agpr118229; GFX950-NEXT: s_branch .LBB235_718230; GFX950-NEXT: .LBB235_4:18231; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr318232; GFX950-NEXT: .LBB235_5: ; %atomicrmw.private18233; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018234; GFX950-NEXT: s_cselect_b32 s2, s0, -118235; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s218236; GFX950-NEXT: s_waitcnt vmcnt(0)18237; GFX950-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]18238; GFX950-NEXT: scratch_store_dwordx2 off, v[4:5], s218239; GFX950-NEXT: .LBB235_6: ; %Flow118240; GFX950-NEXT: s_waitcnt vmcnt(0)18241; GFX950-NEXT: v_accvgpr_write_b32 a0, v218242; GFX950-NEXT: v_accvgpr_write_b32 a1, v318243; GFX950-NEXT: s_cbranch_execnz .LBB235_818244; GFX950-NEXT: .LBB235_7: ; %atomicrmw.shared18245; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018246; GFX950-NEXT: s_cselect_b32 s0, s0, -118247; GFX950-NEXT: v_mov_b32_e32 v2, s018248; GFX950-NEXT: ds_add_rtn_f64 v[0:1], v2, v[0:1]18249; GFX950-NEXT: s_waitcnt lgkmcnt(0)18250; GFX950-NEXT: v_accvgpr_write_b32 a0, v018251; GFX950-NEXT: v_accvgpr_write_b32 a1, v118252; GFX950-NEXT: .LBB235_8: ; %atomicrmw.end18253; GFX950-NEXT: ;;#ASMSTART18254; GFX950-NEXT: ; use a[0:1]18255; GFX950-NEXT: ;;#ASMEND18256; GFX950-NEXT: s_setpc_b64 s[30:31]18257 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018258 %data = call double asm "; def $0", "=a"()18259 %result = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018260 call void asm "; use $0", "a"(double %result)18261 ret void18262}18263 18264define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {18265; GFX90A-LABEL: flat_atomic_fadd_f64_saddr_ret_av_av:18266; GFX90A: ; %bb.0:18267; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18268; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018269; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base18270; GFX90A-NEXT: s_addc_u32 s5, s17, 018271; GFX90A-NEXT: s_cmp_eq_u32 s5, s718272; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018273; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18274; GFX90A-NEXT: ;;#ASMSTART18275; GFX90A-NEXT: ; def v[0:1]18276; GFX90A-NEXT: ;;#ASMEND18277; GFX90A-NEXT: s_cbranch_vccz .LBB236_318278; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private18279; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18280; GFX90A-NEXT: s_cmp_eq_u32 s5, s718281; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018282; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18283; GFX90A-NEXT: s_cbranch_vccz .LBB236_418284; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global18285; GFX90A-NEXT: v_mov_b32_e32 v2, 018286; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc18287; GFX90A-NEXT: s_cbranch_execz .LBB236_518288; GFX90A-NEXT: s_branch .LBB236_618289; GFX90A-NEXT: .LBB236_3:18290; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr318291; GFX90A-NEXT: s_branch .LBB236_718292; GFX90A-NEXT: .LBB236_4:18293; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr318294; GFX90A-NEXT: .LBB236_5: ; %atomicrmw.private18295; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018296; GFX90A-NEXT: s_cselect_b32 s6, s4, -118297; GFX90A-NEXT: v_mov_b32_e32 v6, s618298; GFX90A-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen18299; GFX90A-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:418300; GFX90A-NEXT: s_waitcnt vmcnt(0)18301; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]18302; GFX90A-NEXT: buffer_store_dword v4, v6, s[0:3], 0 offen18303; GFX90A-NEXT: buffer_store_dword v5, v6, s[0:3], 0 offen offset:418304; GFX90A-NEXT: .LBB236_6: ; %Flow118305; GFX90A-NEXT: s_cbranch_execnz .LBB236_818306; GFX90A-NEXT: .LBB236_7: ; %atomicrmw.shared18307; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018308; GFX90A-NEXT: s_cselect_b32 s4, s4, -118309; GFX90A-NEXT: s_waitcnt vmcnt(0)18310; GFX90A-NEXT: v_mov_b32_e32 v2, s418311; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]18312; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18313; GFX90A-NEXT: .LBB236_8: ; %atomicrmw.end18314; GFX90A-NEXT: s_waitcnt vmcnt(0)18315; GFX90A-NEXT: ;;#ASMSTART18316; GFX90A-NEXT: ; use v[2:3]18317; GFX90A-NEXT: ;;#ASMEND18318; GFX90A-NEXT: s_setpc_b64 s[30:31]18319;18320; GFX950-LABEL: flat_atomic_fadd_f64_saddr_ret_av_av:18321; GFX950: ; %bb.0:18322; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18323; GFX950-NEXT: s_add_u32 s0, s0, 0x5018324; GFX950-NEXT: s_mov_b64 s[2:3], src_shared_base18325; GFX950-NEXT: s_addc_u32 s1, s1, 018326; GFX950-NEXT: s_cmp_eq_u32 s1, s318327; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018328; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18329; GFX950-NEXT: ;;#ASMSTART18330; GFX950-NEXT: ; def v[0:1]18331; GFX950-NEXT: ;;#ASMEND18332; GFX950-NEXT: s_cbranch_vccz .LBB236_318333; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private18334; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18335; GFX950-NEXT: s_cmp_eq_u32 s1, s318336; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018337; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18338; GFX950-NEXT: s_cbranch_vccz .LBB236_418339; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global18340; GFX950-NEXT: v_mov_b32_e32 v2, 018341; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc018342; GFX950-NEXT: s_cbranch_execz .LBB236_518343; GFX950-NEXT: s_branch .LBB236_618344; GFX950-NEXT: .LBB236_3:18345; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr318346; GFX950-NEXT: s_branch .LBB236_718347; GFX950-NEXT: .LBB236_4:18348; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr318349; GFX950-NEXT: .LBB236_5: ; %atomicrmw.private18350; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018351; GFX950-NEXT: s_cselect_b32 s2, s0, -118352; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s218353; GFX950-NEXT: s_waitcnt vmcnt(0)18354; GFX950-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]18355; GFX950-NEXT: scratch_store_dwordx2 off, v[4:5], s218356; GFX950-NEXT: .LBB236_6: ; %Flow118357; GFX950-NEXT: s_cbranch_execnz .LBB236_818358; GFX950-NEXT: .LBB236_7: ; %atomicrmw.shared18359; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018360; GFX950-NEXT: s_cselect_b32 s0, s0, -118361; GFX950-NEXT: s_waitcnt vmcnt(0)18362; GFX950-NEXT: v_mov_b32_e32 v2, s018363; GFX950-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]18364; GFX950-NEXT: s_waitcnt lgkmcnt(0)18365; GFX950-NEXT: .LBB236_8: ; %atomicrmw.end18366; GFX950-NEXT: s_waitcnt vmcnt(0)18367; GFX950-NEXT: ;;#ASMSTART18368; GFX950-NEXT: ; use v[2:3]18369; GFX950-NEXT: ;;#ASMEND18370; GFX950-NEXT: s_setpc_b64 s[30:31]18371 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018372 %data = call double asm "; def $0", "=^VA"()18373 %result = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018374 call void asm "; use $0", "^VA"(double %result)18375 ret void18376}18377 18378define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {18379; GFX90A-LABEL: flat_atomic_fsub_f64_saddr_ret_a_a:18380; GFX90A: ; %bb.0:18381; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18382; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018383; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18384; GFX90A-NEXT: s_addc_u32 s5, s17, 018385; GFX90A-NEXT: s_cmp_eq_u32 s5, s718386; GFX90A-NEXT: ;;#ASMSTART18387; GFX90A-NEXT: ; def a[0:1]18388; GFX90A-NEXT: ;;#ASMEND18389; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018390; GFX90A-NEXT: v_accvgpr_read_b32 v5, a118391; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18392; GFX90A-NEXT: v_accvgpr_read_b32 v4, a018393; GFX90A-NEXT: s_cbranch_vccz .LBB237_418394; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18395; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]18396; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]18397; GFX90A-NEXT: s_mov_b64 s[6:7], 018398; GFX90A-NEXT: .LBB237_2: ; %atomicrmw.start18399; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=118400; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18401; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]18402; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc18403; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18404; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]18405; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]18406; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]18407; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]18408; GFX90A-NEXT: s_cbranch_execnz .LBB237_218409; GFX90A-NEXT: ; %bb.3: ; %Flow18410; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]18411; GFX90A-NEXT: v_accvgpr_write_b32 a0, v018412; GFX90A-NEXT: v_accvgpr_write_b32 a1, v118413; GFX90A-NEXT: s_branch .LBB237_618414; GFX90A-NEXT: .LBB237_4:18415; GFX90A-NEXT: ; implicit-def: $agpr0_agpr118416; GFX90A-NEXT: s_cbranch_execz .LBB237_618417; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private18418; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018419; GFX90A-NEXT: s_cselect_b32 s4, s4, -118420; GFX90A-NEXT: v_mov_b32_e32 v6, s418421; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen18422; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:418423; GFX90A-NEXT: s_waitcnt vmcnt(0)18424; GFX90A-NEXT: v_accvgpr_write_b32 a0, v018425; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], -v[4:5]18426; GFX90A-NEXT: v_accvgpr_write_b32 a1, v118427; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen18428; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:418429; GFX90A-NEXT: .LBB237_6: ; %atomicrmw.phi18430; GFX90A-NEXT: ;;#ASMSTART18431; GFX90A-NEXT: ; use a[0:1]18432; GFX90A-NEXT: ;;#ASMEND18433; GFX90A-NEXT: s_waitcnt vmcnt(0)18434; GFX90A-NEXT: s_setpc_b64 s[30:31]18435;18436; GFX950-LABEL: flat_atomic_fsub_f64_saddr_ret_a_a:18437; GFX950: ; %bb.0:18438; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18439; GFX950-NEXT: s_add_u32 s0, s0, 0x5018440; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18441; GFX950-NEXT: s_addc_u32 s1, s1, 018442; GFX950-NEXT: s_cmp_eq_u32 s1, s318443; GFX950-NEXT: ;;#ASMSTART18444; GFX950-NEXT: ; def a[0:1]18445; GFX950-NEXT: ;;#ASMEND18446; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018447; GFX950-NEXT: v_accvgpr_read_b32 v5, a118448; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18449; GFX950-NEXT: v_accvgpr_read_b32 v4, a018450; GFX950-NEXT: s_cbranch_vccz .LBB237_418451; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18452; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]18453; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]18454; GFX950-NEXT: s_mov_b64 s[2:3], 018455; GFX950-NEXT: .LBB237_2: ; %atomicrmw.start18456; GFX950-NEXT: ; =>This Inner Loop Header: Depth=118457; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18458; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]18459; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc018460; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18461; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]18462; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]18463; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]18464; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]18465; GFX950-NEXT: s_cbranch_execnz .LBB237_218466; GFX950-NEXT: ; %bb.3: ; %Flow18467; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]18468; GFX950-NEXT: v_accvgpr_write_b32 a0, v018469; GFX950-NEXT: v_accvgpr_write_b32 a1, v118470; GFX950-NEXT: s_branch .LBB237_618471; GFX950-NEXT: .LBB237_4:18472; GFX950-NEXT: ; implicit-def: $agpr0_agpr118473; GFX950-NEXT: s_cbranch_execz .LBB237_618474; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private18475; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018476; GFX950-NEXT: s_cselect_b32 s0, s0, -118477; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s018478; GFX950-NEXT: s_waitcnt vmcnt(0)18479; GFX950-NEXT: v_accvgpr_write_b32 a0, v018480; GFX950-NEXT: v_add_f64 v[2:3], v[0:1], -v[4:5]18481; GFX950-NEXT: v_accvgpr_write_b32 a1, v118482; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s018483; GFX950-NEXT: .LBB237_6: ; %atomicrmw.phi18484; GFX950-NEXT: ;;#ASMSTART18485; GFX950-NEXT: ; use a[0:1]18486; GFX950-NEXT: ;;#ASMEND18487; GFX950-NEXT: s_waitcnt vmcnt(0)18488; GFX950-NEXT: s_setpc_b64 s[30:31]18489 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018490 %data = call double asm "; def $0", "=a"()18491 %result = atomicrmw fsub ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018492 call void asm "; use $0", "a"(double %result)18493 ret void18494}18495 18496define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {18497; GFX90A-LABEL: flat_atomic_fsub_f64_saddr_ret_av_av:18498; GFX90A: ; %bb.0:18499; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18500; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018501; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18502; GFX90A-NEXT: s_addc_u32 s5, s17, 018503; GFX90A-NEXT: s_cmp_eq_u32 s5, s718504; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018505; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18506; GFX90A-NEXT: ;;#ASMSTART18507; GFX90A-NEXT: ; def v[4:5]18508; GFX90A-NEXT: ;;#ASMEND18509; GFX90A-NEXT: s_cbranch_vccz .LBB238_418510; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18511; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]18512; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]18513; GFX90A-NEXT: s_mov_b64 s[6:7], 018514; GFX90A-NEXT: .LBB238_2: ; %atomicrmw.start18515; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=118516; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18517; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]18518; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc18519; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18520; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]18521; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]18522; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]18523; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]18524; GFX90A-NEXT: s_cbranch_execnz .LBB238_218525; GFX90A-NEXT: ; %bb.3: ; %Flow18526; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]18527; GFX90A-NEXT: s_branch .LBB238_618528; GFX90A-NEXT: .LBB238_4:18529; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr118530; GFX90A-NEXT: s_cbranch_execz .LBB238_618531; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private18532; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018533; GFX90A-NEXT: s_cselect_b32 s4, s4, -118534; GFX90A-NEXT: v_mov_b32_e32 v6, s418535; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen18536; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:418537; GFX90A-NEXT: s_waitcnt vmcnt(0)18538; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], -v[4:5]18539; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen18540; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:418541; GFX90A-NEXT: .LBB238_6: ; %atomicrmw.phi18542; GFX90A-NEXT: ;;#ASMSTART18543; GFX90A-NEXT: ; use v[0:1]18544; GFX90A-NEXT: ;;#ASMEND18545; GFX90A-NEXT: s_waitcnt vmcnt(0)18546; GFX90A-NEXT: s_setpc_b64 s[30:31]18547;18548; GFX950-LABEL: flat_atomic_fsub_f64_saddr_ret_av_av:18549; GFX950: ; %bb.0:18550; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18551; GFX950-NEXT: s_add_u32 s0, s0, 0x5018552; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18553; GFX950-NEXT: s_addc_u32 s1, s1, 018554; GFX950-NEXT: s_cmp_eq_u32 s1, s318555; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018556; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18557; GFX950-NEXT: ;;#ASMSTART18558; GFX950-NEXT: ; def v[4:5]18559; GFX950-NEXT: ;;#ASMEND18560; GFX950-NEXT: s_cbranch_vccz .LBB238_418561; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18562; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]18563; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]18564; GFX950-NEXT: s_mov_b64 s[2:3], 018565; GFX950-NEXT: .LBB238_2: ; %atomicrmw.start18566; GFX950-NEXT: ; =>This Inner Loop Header: Depth=118567; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18568; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]18569; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc018570; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18571; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]18572; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]18573; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]18574; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]18575; GFX950-NEXT: s_cbranch_execnz .LBB238_218576; GFX950-NEXT: ; %bb.3: ; %Flow18577; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]18578; GFX950-NEXT: s_branch .LBB238_618579; GFX950-NEXT: .LBB238_4:18580; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr118581; GFX950-NEXT: s_cbranch_execz .LBB238_618582; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private18583; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018584; GFX950-NEXT: s_cselect_b32 s0, s0, -118585; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s018586; GFX950-NEXT: s_waitcnt vmcnt(0)18587; GFX950-NEXT: v_add_f64 v[2:3], v[0:1], -v[4:5]18588; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s018589; GFX950-NEXT: .LBB238_6: ; %atomicrmw.phi18590; GFX950-NEXT: ;;#ASMSTART18591; GFX950-NEXT: ; use v[0:1]18592; GFX950-NEXT: ;;#ASMEND18593; GFX950-NEXT: s_waitcnt vmcnt(0)18594; GFX950-NEXT: s_setpc_b64 s[30:31]18595 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018596 %data = call double asm "; def $0", "=^VA"()18597 %result = atomicrmw fsub ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018598 call void asm "; use $0", "^VA"(double %result)18599 ret void18600}18601 18602define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {18603; GFX90A-LABEL: flat_atomic_fmax_f64_saddr_ret_a_a:18604; GFX90A: ; %bb.0:18605; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18606; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018607; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18608; GFX90A-NEXT: s_addc_u32 s5, s17, 018609; GFX90A-NEXT: s_cmp_eq_u32 s5, s718610; GFX90A-NEXT: ;;#ASMSTART18611; GFX90A-NEXT: ; def a[0:1]18612; GFX90A-NEXT: ;;#ASMEND18613; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018614; GFX90A-NEXT: v_accvgpr_read_b32 v0, a018615; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18616; GFX90A-NEXT: v_accvgpr_read_b32 v1, a118617; GFX90A-NEXT: s_cbranch_vccz .LBB239_218618; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18619; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]18620; GFX90A-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] glc18621; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18622; GFX90A-NEXT: s_waitcnt vmcnt(0)18623; GFX90A-NEXT: v_accvgpr_write_b32 a0, v218624; GFX90A-NEXT: v_accvgpr_write_b32 a1, v318625; GFX90A-NEXT: s_cbranch_execz .LBB239_318626; GFX90A-NEXT: s_branch .LBB239_418627; GFX90A-NEXT: .LBB239_2:18628; GFX90A-NEXT: ; implicit-def: $agpr0_agpr118629; GFX90A-NEXT: .LBB239_3: ; %atomicrmw.private18630; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018631; GFX90A-NEXT: s_cselect_b32 s4, s4, -118632; GFX90A-NEXT: v_mov_b32_e32 v6, s418633; GFX90A-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen18634; GFX90A-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:418635; GFX90A-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]18636; GFX90A-NEXT: s_waitcnt vmcnt(0)18637; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]18638; GFX90A-NEXT: v_accvgpr_write_b32 a0, v218639; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]18640; GFX90A-NEXT: v_accvgpr_write_b32 a1, v318641; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen18642; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:418643; GFX90A-NEXT: .LBB239_4: ; %atomicrmw.end18644; GFX90A-NEXT: ;;#ASMSTART18645; GFX90A-NEXT: ; use a[0:1]18646; GFX90A-NEXT: ;;#ASMEND18647; GFX90A-NEXT: s_waitcnt vmcnt(0)18648; GFX90A-NEXT: s_setpc_b64 s[30:31]18649;18650; GFX950-LABEL: flat_atomic_fmax_f64_saddr_ret_a_a:18651; GFX950: ; %bb.0:18652; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18653; GFX950-NEXT: s_add_u32 s0, s0, 0x5018654; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18655; GFX950-NEXT: s_addc_u32 s1, s1, 018656; GFX950-NEXT: s_cmp_eq_u32 s1, s318657; GFX950-NEXT: ;;#ASMSTART18658; GFX950-NEXT: ; def a[0:1]18659; GFX950-NEXT: ;;#ASMEND18660; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018661; GFX950-NEXT: v_accvgpr_read_b32 v0, a018662; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18663; GFX950-NEXT: v_accvgpr_read_b32 v1, a118664; GFX950-NEXT: s_cbranch_vccz .LBB239_218665; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18666; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]18667; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc018668; GFX950-NEXT: s_waitcnt lgkmcnt(0)18669; GFX950-NEXT: s_waitcnt vmcnt(0)18670; GFX950-NEXT: v_accvgpr_write_b32 a0, v218671; GFX950-NEXT: v_accvgpr_write_b32 a1, v318672; GFX950-NEXT: s_cbranch_execz .LBB239_318673; GFX950-NEXT: s_branch .LBB239_418674; GFX950-NEXT: .LBB239_2:18675; GFX950-NEXT: ; implicit-def: $agpr0_agpr118676; GFX950-NEXT: .LBB239_3: ; %atomicrmw.private18677; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018678; GFX950-NEXT: s_cselect_b32 s0, s0, -118679; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s018680; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]18681; GFX950-NEXT: s_waitcnt vmcnt(0)18682; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]18683; GFX950-NEXT: v_accvgpr_write_b32 a0, v218684; GFX950-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]18685; GFX950-NEXT: v_accvgpr_write_b32 a1, v318686; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s018687; GFX950-NEXT: .LBB239_4: ; %atomicrmw.end18688; GFX950-NEXT: ;;#ASMSTART18689; GFX950-NEXT: ; use a[0:1]18690; GFX950-NEXT: ;;#ASMEND18691; GFX950-NEXT: s_waitcnt vmcnt(0)18692; GFX950-NEXT: s_setpc_b64 s[30:31]18693 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018694 %data = call double asm "; def $0", "=a"()18695 %result = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018696 call void asm "; use $0", "a"(double %result)18697 ret void18698}18699 18700define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {18701; GFX90A-LABEL: flat_atomic_fmax_f64_saddr_ret_av_av:18702; GFX90A: ; %bb.0:18703; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18704; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018705; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18706; GFX90A-NEXT: s_addc_u32 s5, s17, 018707; GFX90A-NEXT: s_cmp_eq_u32 s5, s718708; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018709; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18710; GFX90A-NEXT: ;;#ASMSTART18711; GFX90A-NEXT: ; def v[2:3]18712; GFX90A-NEXT: ;;#ASMEND18713; GFX90A-NEXT: s_cbranch_vccz .LBB240_218714; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18715; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]18716; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc18717; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18718; GFX90A-NEXT: s_cbranch_execz .LBB240_318719; GFX90A-NEXT: s_branch .LBB240_418720; GFX90A-NEXT: .LBB240_2:18721; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr118722; GFX90A-NEXT: .LBB240_3: ; %atomicrmw.private18723; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018724; GFX90A-NEXT: s_cselect_b32 s4, s4, -118725; GFX90A-NEXT: v_mov_b32_e32 v6, s418726; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen18727; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:418728; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]18729; GFX90A-NEXT: s_waitcnt vmcnt(0)18730; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]18731; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]18732; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen18733; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:418734; GFX90A-NEXT: .LBB240_4: ; %atomicrmw.end18735; GFX90A-NEXT: s_waitcnt vmcnt(0)18736; GFX90A-NEXT: ;;#ASMSTART18737; GFX90A-NEXT: ; use v[0:1]18738; GFX90A-NEXT: ;;#ASMEND18739; GFX90A-NEXT: s_setpc_b64 s[30:31]18740;18741; GFX950-LABEL: flat_atomic_fmax_f64_saddr_ret_av_av:18742; GFX950: ; %bb.0:18743; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18744; GFX950-NEXT: s_add_u32 s0, s0, 0x5018745; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18746; GFX950-NEXT: s_addc_u32 s1, s1, 018747; GFX950-NEXT: s_cmp_eq_u32 s1, s318748; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018749; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18750; GFX950-NEXT: ;;#ASMSTART18751; GFX950-NEXT: ; def v[2:3]18752; GFX950-NEXT: ;;#ASMEND18753; GFX950-NEXT: s_cbranch_vccz .LBB240_218754; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18755; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]18756; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc018757; GFX950-NEXT: s_waitcnt lgkmcnt(0)18758; GFX950-NEXT: s_cbranch_execz .LBB240_318759; GFX950-NEXT: s_branch .LBB240_418760; GFX950-NEXT: .LBB240_2:18761; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr118762; GFX950-NEXT: .LBB240_3: ; %atomicrmw.private18763; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018764; GFX950-NEXT: s_cselect_b32 s0, s0, -118765; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s018766; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]18767; GFX950-NEXT: s_waitcnt vmcnt(0)18768; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]18769; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]18770; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s018771; GFX950-NEXT: .LBB240_4: ; %atomicrmw.end18772; GFX950-NEXT: s_waitcnt vmcnt(0)18773; GFX950-NEXT: ;;#ASMSTART18774; GFX950-NEXT: ; use v[0:1]18775; GFX950-NEXT: ;;#ASMEND18776; GFX950-NEXT: s_setpc_b64 s[30:31]18777 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018778 %data = call double asm "; def $0", "=^VA"()18779 %result = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018780 call void asm "; use $0", "^VA"(double %result)18781 ret void18782}18783 18784define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {18785; GFX90A-LABEL: flat_atomic_fmin_f64_saddr_ret_a_a:18786; GFX90A: ; %bb.0:18787; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18788; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018789; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18790; GFX90A-NEXT: s_addc_u32 s5, s17, 018791; GFX90A-NEXT: s_cmp_eq_u32 s5, s718792; GFX90A-NEXT: ;;#ASMSTART18793; GFX90A-NEXT: ; def a[0:1]18794; GFX90A-NEXT: ;;#ASMEND18795; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018796; GFX90A-NEXT: v_accvgpr_read_b32 v0, a018797; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18798; GFX90A-NEXT: v_accvgpr_read_b32 v1, a118799; GFX90A-NEXT: s_cbranch_vccz .LBB241_218800; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18801; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]18802; GFX90A-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] glc18803; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18804; GFX90A-NEXT: s_waitcnt vmcnt(0)18805; GFX90A-NEXT: v_accvgpr_write_b32 a0, v218806; GFX90A-NEXT: v_accvgpr_write_b32 a1, v318807; GFX90A-NEXT: s_cbranch_execz .LBB241_318808; GFX90A-NEXT: s_branch .LBB241_418809; GFX90A-NEXT: .LBB241_2:18810; GFX90A-NEXT: ; implicit-def: $agpr0_agpr118811; GFX90A-NEXT: .LBB241_3: ; %atomicrmw.private18812; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018813; GFX90A-NEXT: s_cselect_b32 s4, s4, -118814; GFX90A-NEXT: v_mov_b32_e32 v6, s418815; GFX90A-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen18816; GFX90A-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:418817; GFX90A-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]18818; GFX90A-NEXT: s_waitcnt vmcnt(0)18819; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]18820; GFX90A-NEXT: v_accvgpr_write_b32 a0, v218821; GFX90A-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]18822; GFX90A-NEXT: v_accvgpr_write_b32 a1, v318823; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen18824; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:418825; GFX90A-NEXT: .LBB241_4: ; %atomicrmw.end18826; GFX90A-NEXT: ;;#ASMSTART18827; GFX90A-NEXT: ; use a[0:1]18828; GFX90A-NEXT: ;;#ASMEND18829; GFX90A-NEXT: s_waitcnt vmcnt(0)18830; GFX90A-NEXT: s_setpc_b64 s[30:31]18831;18832; GFX950-LABEL: flat_atomic_fmin_f64_saddr_ret_a_a:18833; GFX950: ; %bb.0:18834; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18835; GFX950-NEXT: s_add_u32 s0, s0, 0x5018836; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18837; GFX950-NEXT: s_addc_u32 s1, s1, 018838; GFX950-NEXT: s_cmp_eq_u32 s1, s318839; GFX950-NEXT: ;;#ASMSTART18840; GFX950-NEXT: ; def a[0:1]18841; GFX950-NEXT: ;;#ASMEND18842; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018843; GFX950-NEXT: v_accvgpr_read_b32 v0, a018844; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18845; GFX950-NEXT: v_accvgpr_read_b32 v1, a118846; GFX950-NEXT: s_cbranch_vccz .LBB241_218847; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18848; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]18849; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc018850; GFX950-NEXT: s_waitcnt lgkmcnt(0)18851; GFX950-NEXT: s_waitcnt vmcnt(0)18852; GFX950-NEXT: v_accvgpr_write_b32 a0, v218853; GFX950-NEXT: v_accvgpr_write_b32 a1, v318854; GFX950-NEXT: s_cbranch_execz .LBB241_318855; GFX950-NEXT: s_branch .LBB241_418856; GFX950-NEXT: .LBB241_2:18857; GFX950-NEXT: ; implicit-def: $agpr0_agpr118858; GFX950-NEXT: .LBB241_3: ; %atomicrmw.private18859; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018860; GFX950-NEXT: s_cselect_b32 s0, s0, -118861; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s018862; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]18863; GFX950-NEXT: s_waitcnt vmcnt(0)18864; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]18865; GFX950-NEXT: v_accvgpr_write_b32 a0, v218866; GFX950-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]18867; GFX950-NEXT: v_accvgpr_write_b32 a1, v318868; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s018869; GFX950-NEXT: .LBB241_4: ; %atomicrmw.end18870; GFX950-NEXT: ;;#ASMSTART18871; GFX950-NEXT: ; use a[0:1]18872; GFX950-NEXT: ;;#ASMEND18873; GFX950-NEXT: s_waitcnt vmcnt(0)18874; GFX950-NEXT: s_setpc_b64 s[30:31]18875 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018876 %data = call double asm "; def $0", "=a"()18877 %result = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018878 call void asm "; use $0", "a"(double %result)18879 ret void18880}18881 18882define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {18883; GFX90A-LABEL: flat_atomic_fmin_f64_saddr_ret_av_av:18884; GFX90A: ; %bb.0:18885; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18886; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018887; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18888; GFX90A-NEXT: s_addc_u32 s5, s17, 018889; GFX90A-NEXT: s_cmp_eq_u32 s5, s718890; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018891; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18892; GFX90A-NEXT: ;;#ASMSTART18893; GFX90A-NEXT: ; def v[2:3]18894; GFX90A-NEXT: ;;#ASMEND18895; GFX90A-NEXT: s_cbranch_vccz .LBB242_218896; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18897; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]18898; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc18899; GFX90A-NEXT: s_waitcnt lgkmcnt(0)18900; GFX90A-NEXT: s_cbranch_execz .LBB242_318901; GFX90A-NEXT: s_branch .LBB242_418902; GFX90A-NEXT: .LBB242_2:18903; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr118904; GFX90A-NEXT: .LBB242_3: ; %atomicrmw.private18905; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 018906; GFX90A-NEXT: s_cselect_b32 s4, s4, -118907; GFX90A-NEXT: v_mov_b32_e32 v6, s418908; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen18909; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:418910; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]18911; GFX90A-NEXT: s_waitcnt vmcnt(0)18912; GFX90A-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]18913; GFX90A-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]18914; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen18915; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:418916; GFX90A-NEXT: .LBB242_4: ; %atomicrmw.end18917; GFX90A-NEXT: s_waitcnt vmcnt(0)18918; GFX90A-NEXT: ;;#ASMSTART18919; GFX90A-NEXT: ; use v[0:1]18920; GFX90A-NEXT: ;;#ASMEND18921; GFX90A-NEXT: s_setpc_b64 s[30:31]18922;18923; GFX950-LABEL: flat_atomic_fmin_f64_saddr_ret_av_av:18924; GFX950: ; %bb.0:18925; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18926; GFX950-NEXT: s_add_u32 s0, s0, 0x5018927; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base18928; GFX950-NEXT: s_addc_u32 s1, s1, 018929; GFX950-NEXT: s_cmp_eq_u32 s1, s318930; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 018931; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]18932; GFX950-NEXT: ;;#ASMSTART18933; GFX950-NEXT: ; def v[2:3]18934; GFX950-NEXT: ;;#ASMEND18935; GFX950-NEXT: s_cbranch_vccz .LBB242_218936; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global18937; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]18938; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc018939; GFX950-NEXT: s_waitcnt lgkmcnt(0)18940; GFX950-NEXT: s_cbranch_execz .LBB242_318941; GFX950-NEXT: s_branch .LBB242_418942; GFX950-NEXT: .LBB242_2:18943; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr118944; GFX950-NEXT: .LBB242_3: ; %atomicrmw.private18945; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 018946; GFX950-NEXT: s_cselect_b32 s0, s0, -118947; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s018948; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]18949; GFX950-NEXT: s_waitcnt vmcnt(0)18950; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]18951; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]18952; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s018953; GFX950-NEXT: .LBB242_4: ; %atomicrmw.end18954; GFX950-NEXT: s_waitcnt vmcnt(0)18955; GFX950-NEXT: ;;#ASMSTART18956; GFX950-NEXT: ; use v[0:1]18957; GFX950-NEXT: ;;#ASMEND18958; GFX950-NEXT: s_setpc_b64 s[30:31]18959 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1018960 %data = call double asm "; def $0", "=^VA"()18961 %result = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !018962 call void asm "; use $0", "^VA"(double %result)18963 ret void18964}18965 18966define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {18967; GFX90A-LABEL: flat_atomic_fmaximum_f64_saddr_ret_a_a:18968; GFX90A: ; %bb.0:18969; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)18970; GFX90A-NEXT: s_add_u32 s4, s16, 0x5018971; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base18972; GFX90A-NEXT: s_addc_u32 s5, s17, 018973; GFX90A-NEXT: s_cmp_eq_u32 s5, s718974; GFX90A-NEXT: ;;#ASMSTART18975; GFX90A-NEXT: ; def a[0:1]18976; GFX90A-NEXT: ;;#ASMEND18977; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 018978; GFX90A-NEXT: v_accvgpr_read_b32 v5, a118979; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]18980; GFX90A-NEXT: v_accvgpr_read_b32 v4, a018981; GFX90A-NEXT: s_cbranch_vccz .LBB243_418982; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global18983; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]18984; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]18985; GFX90A-NEXT: s_mov_b64 s[6:7], 018986; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000018987; GFX90A-NEXT: .LBB243_2: ; %atomicrmw.start18988; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=118989; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18990; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]18991; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]18992; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc18993; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc18994; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc18995; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)18996; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]18997; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]18998; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]18999; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]19000; GFX90A-NEXT: s_cbranch_execnz .LBB243_219001; GFX90A-NEXT: ; %bb.3: ; %Flow19002; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]19003; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019004; GFX90A-NEXT: v_accvgpr_write_b32 a1, v119005; GFX90A-NEXT: s_branch .LBB243_619006; GFX90A-NEXT: .LBB243_4:19007; GFX90A-NEXT: ; implicit-def: $agpr0_agpr119008; GFX90A-NEXT: s_cbranch_execz .LBB243_619009; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private19010; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 019011; GFX90A-NEXT: s_cselect_b32 s4, s4, -119012; GFX90A-NEXT: v_mov_b32_e32 v6, s419013; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen19014; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:419015; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000019016; GFX90A-NEXT: s_waitcnt vmcnt(0)19017; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]19018; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19019; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019020; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19021; GFX90A-NEXT: v_accvgpr_write_b32 a1, v119022; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc19023; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen19024; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:419025; GFX90A-NEXT: .LBB243_6: ; %atomicrmw.phi19026; GFX90A-NEXT: ;;#ASMSTART19027; GFX90A-NEXT: ; use a[0:1]19028; GFX90A-NEXT: ;;#ASMEND19029; GFX90A-NEXT: s_waitcnt vmcnt(0)19030; GFX90A-NEXT: s_setpc_b64 s[30:31]19031;19032; GFX950-LABEL: flat_atomic_fmaximum_f64_saddr_ret_a_a:19033; GFX950: ; %bb.0:19034; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19035; GFX950-NEXT: s_add_u32 s0, s0, 0x5019036; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base19037; GFX950-NEXT: s_addc_u32 s1, s1, 019038; GFX950-NEXT: s_cmp_eq_u32 s1, s319039; GFX950-NEXT: ;;#ASMSTART19040; GFX950-NEXT: ; def a[0:1]19041; GFX950-NEXT: ;;#ASMEND19042; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 019043; GFX950-NEXT: v_accvgpr_read_b32 v5, a119044; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]19045; GFX950-NEXT: v_accvgpr_read_b32 v4, a019046; GFX950-NEXT: s_cbranch_vccz .LBB243_419047; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global19048; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]19049; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19050; GFX950-NEXT: s_mov_b64 s[2:3], 019051; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000019052; GFX950-NEXT: .LBB243_2: ; %atomicrmw.start19053; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119054; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19055; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]19056; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19057; GFX950-NEXT: s_nop 119058; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19059; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19060; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc019061; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19062; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19063; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19064; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]19065; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19066; GFX950-NEXT: s_cbranch_execnz .LBB243_219067; GFX950-NEXT: ; %bb.3: ; %Flow19068; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19069; GFX950-NEXT: v_accvgpr_write_b32 a0, v019070; GFX950-NEXT: v_accvgpr_write_b32 a1, v119071; GFX950-NEXT: s_branch .LBB243_619072; GFX950-NEXT: .LBB243_4:19073; GFX950-NEXT: ; implicit-def: $agpr0_agpr119074; GFX950-NEXT: s_cbranch_execz .LBB243_619075; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private19076; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 019077; GFX950-NEXT: s_cselect_b32 s0, s0, -119078; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s019079; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff8000019080; GFX950-NEXT: s_waitcnt vmcnt(0)19081; GFX950-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]19082; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19083; GFX950-NEXT: v_accvgpr_write_b32 a0, v019084; GFX950-NEXT: v_accvgpr_write_b32 a1, v119085; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc19086; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19087; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s019088; GFX950-NEXT: .LBB243_6: ; %atomicrmw.phi19089; GFX950-NEXT: ;;#ASMSTART19090; GFX950-NEXT: ; use a[0:1]19091; GFX950-NEXT: ;;#ASMEND19092; GFX950-NEXT: s_waitcnt vmcnt(0)19093; GFX950-NEXT: s_setpc_b64 s[30:31]19094 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1019095 %data = call double asm "; def $0", "=a"()19096 %result = atomicrmw fmaximum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019097 call void asm "; use $0", "a"(double %result)19098 ret void19099}19100 19101define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {19102; GFX90A-LABEL: flat_atomic_fmaximum_f64_saddr_ret_av_av:19103; GFX90A: ; %bb.0:19104; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19105; GFX90A-NEXT: s_add_u32 s4, s16, 0x5019106; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base19107; GFX90A-NEXT: s_addc_u32 s5, s17, 019108; GFX90A-NEXT: s_cmp_eq_u32 s5, s719109; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 019110; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]19111; GFX90A-NEXT: ;;#ASMSTART19112; GFX90A-NEXT: ; def v[4:5]19113; GFX90A-NEXT: ;;#ASMEND19114; GFX90A-NEXT: s_cbranch_vccz .LBB244_419115; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global19116; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]19117; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19118; GFX90A-NEXT: s_mov_b64 s[6:7], 019119; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000019120; GFX90A-NEXT: .LBB244_2: ; %atomicrmw.start19121; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119122; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19123; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]19124; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19125; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19126; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19127; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc19128; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19129; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19130; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]19131; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]19132; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]19133; GFX90A-NEXT: s_cbranch_execnz .LBB244_219134; GFX90A-NEXT: ; %bb.3: ; %Flow19135; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]19136; GFX90A-NEXT: s_branch .LBB244_619137; GFX90A-NEXT: .LBB244_4:19138; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr119139; GFX90A-NEXT: s_cbranch_execz .LBB244_619140; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private19141; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 019142; GFX90A-NEXT: s_cselect_b32 s4, s4, -119143; GFX90A-NEXT: v_mov_b32_e32 v6, s419144; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen19145; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:419146; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000019147; GFX90A-NEXT: s_waitcnt vmcnt(0)19148; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]19149; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19150; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19151; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc19152; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen19153; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:419154; GFX90A-NEXT: .LBB244_6: ; %atomicrmw.phi19155; GFX90A-NEXT: ;;#ASMSTART19156; GFX90A-NEXT: ; use v[0:1]19157; GFX90A-NEXT: ;;#ASMEND19158; GFX90A-NEXT: s_waitcnt vmcnt(0)19159; GFX90A-NEXT: s_setpc_b64 s[30:31]19160;19161; GFX950-LABEL: flat_atomic_fmaximum_f64_saddr_ret_av_av:19162; GFX950: ; %bb.0:19163; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19164; GFX950-NEXT: s_add_u32 s0, s0, 0x5019165; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base19166; GFX950-NEXT: s_addc_u32 s1, s1, 019167; GFX950-NEXT: s_cmp_eq_u32 s1, s319168; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 019169; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]19170; GFX950-NEXT: ;;#ASMSTART19171; GFX950-NEXT: ; def v[4:5]19172; GFX950-NEXT: ;;#ASMEND19173; GFX950-NEXT: s_cbranch_vccz .LBB244_419174; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global19175; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]19176; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19177; GFX950-NEXT: s_mov_b64 s[2:3], 019178; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000019179; GFX950-NEXT: .LBB244_2: ; %atomicrmw.start19180; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119181; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19182; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]19183; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19184; GFX950-NEXT: s_nop 119185; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19186; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19187; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc019188; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19189; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19190; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19191; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]19192; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19193; GFX950-NEXT: s_cbranch_execnz .LBB244_219194; GFX950-NEXT: ; %bb.3: ; %Flow19195; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19196; GFX950-NEXT: s_branch .LBB244_619197; GFX950-NEXT: .LBB244_4:19198; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr119199; GFX950-NEXT: s_cbranch_execz .LBB244_619200; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private19201; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 019202; GFX950-NEXT: s_cselect_b32 s0, s0, -119203; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s019204; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff8000019205; GFX950-NEXT: s_waitcnt vmcnt(0)19206; GFX950-NEXT: v_max_f64 v[2:3], v[0:1], v[4:5]19207; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19208; GFX950-NEXT: s_nop 119209; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc19210; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19211; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s019212; GFX950-NEXT: .LBB244_6: ; %atomicrmw.phi19213; GFX950-NEXT: ;;#ASMSTART19214; GFX950-NEXT: ; use v[0:1]19215; GFX950-NEXT: ;;#ASMEND19216; GFX950-NEXT: s_waitcnt vmcnt(0)19217; GFX950-NEXT: s_setpc_b64 s[30:31]19218 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1019219 %data = call double asm "; def $0", "=^VA"()19220 %result = atomicrmw fmaximum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019221 call void asm "; use $0", "^VA"(double %result)19222 ret void19223}19224 19225define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {19226; GFX90A-LABEL: flat_atomic_fminimum_f64_saddr_ret_a_a:19227; GFX90A: ; %bb.0:19228; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19229; GFX90A-NEXT: s_add_u32 s4, s16, 0x5019230; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base19231; GFX90A-NEXT: s_addc_u32 s5, s17, 019232; GFX90A-NEXT: s_cmp_eq_u32 s5, s719233; GFX90A-NEXT: ;;#ASMSTART19234; GFX90A-NEXT: ; def a[0:1]19235; GFX90A-NEXT: ;;#ASMEND19236; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 019237; GFX90A-NEXT: v_accvgpr_read_b32 v5, a119238; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]19239; GFX90A-NEXT: v_accvgpr_read_b32 v4, a019240; GFX90A-NEXT: s_cbranch_vccz .LBB245_419241; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global19242; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]19243; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19244; GFX90A-NEXT: s_mov_b64 s[6:7], 019245; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000019246; GFX90A-NEXT: .LBB245_2: ; %atomicrmw.start19247; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119248; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19249; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]19250; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19251; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19252; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19253; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc19254; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19255; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19256; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]19257; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]19258; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]19259; GFX90A-NEXT: s_cbranch_execnz .LBB245_219260; GFX90A-NEXT: ; %bb.3: ; %Flow19261; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]19262; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019263; GFX90A-NEXT: v_accvgpr_write_b32 a1, v119264; GFX90A-NEXT: s_branch .LBB245_619265; GFX90A-NEXT: .LBB245_4:19266; GFX90A-NEXT: ; implicit-def: $agpr0_agpr119267; GFX90A-NEXT: s_cbranch_execz .LBB245_619268; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private19269; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 019270; GFX90A-NEXT: s_cselect_b32 s4, s4, -119271; GFX90A-NEXT: v_mov_b32_e32 v6, s419272; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen19273; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:419274; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000019275; GFX90A-NEXT: s_waitcnt vmcnt(0)19276; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]19277; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19278; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019279; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19280; GFX90A-NEXT: v_accvgpr_write_b32 a1, v119281; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc19282; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen19283; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:419284; GFX90A-NEXT: .LBB245_6: ; %atomicrmw.phi19285; GFX90A-NEXT: ;;#ASMSTART19286; GFX90A-NEXT: ; use a[0:1]19287; GFX90A-NEXT: ;;#ASMEND19288; GFX90A-NEXT: s_waitcnt vmcnt(0)19289; GFX90A-NEXT: s_setpc_b64 s[30:31]19290;19291; GFX950-LABEL: flat_atomic_fminimum_f64_saddr_ret_a_a:19292; GFX950: ; %bb.0:19293; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19294; GFX950-NEXT: s_add_u32 s0, s0, 0x5019295; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base19296; GFX950-NEXT: s_addc_u32 s1, s1, 019297; GFX950-NEXT: s_cmp_eq_u32 s1, s319298; GFX950-NEXT: ;;#ASMSTART19299; GFX950-NEXT: ; def a[0:1]19300; GFX950-NEXT: ;;#ASMEND19301; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 019302; GFX950-NEXT: v_accvgpr_read_b32 v5, a119303; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]19304; GFX950-NEXT: v_accvgpr_read_b32 v4, a019305; GFX950-NEXT: s_cbranch_vccz .LBB245_419306; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global19307; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]19308; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19309; GFX950-NEXT: s_mov_b64 s[2:3], 019310; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000019311; GFX950-NEXT: .LBB245_2: ; %atomicrmw.start19312; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119313; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19314; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]19315; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19316; GFX950-NEXT: s_nop 119317; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19318; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19319; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc019320; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19321; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19322; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19323; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]19324; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19325; GFX950-NEXT: s_cbranch_execnz .LBB245_219326; GFX950-NEXT: ; %bb.3: ; %Flow19327; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19328; GFX950-NEXT: v_accvgpr_write_b32 a0, v019329; GFX950-NEXT: v_accvgpr_write_b32 a1, v119330; GFX950-NEXT: s_branch .LBB245_619331; GFX950-NEXT: .LBB245_4:19332; GFX950-NEXT: ; implicit-def: $agpr0_agpr119333; GFX950-NEXT: s_cbranch_execz .LBB245_619334; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private19335; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 019336; GFX950-NEXT: s_cselect_b32 s0, s0, -119337; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s019338; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff8000019339; GFX950-NEXT: s_waitcnt vmcnt(0)19340; GFX950-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]19341; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19342; GFX950-NEXT: v_accvgpr_write_b32 a0, v019343; GFX950-NEXT: v_accvgpr_write_b32 a1, v119344; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc19345; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19346; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s019347; GFX950-NEXT: .LBB245_6: ; %atomicrmw.phi19348; GFX950-NEXT: ;;#ASMSTART19349; GFX950-NEXT: ; use a[0:1]19350; GFX950-NEXT: ;;#ASMEND19351; GFX950-NEXT: s_waitcnt vmcnt(0)19352; GFX950-NEXT: s_setpc_b64 s[30:31]19353 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1019354 %data = call double asm "; def $0", "=a"()19355 %result = atomicrmw fminimum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019356 call void asm "; use $0", "a"(double %result)19357 ret void19358}19359 19360define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {19361; GFX90A-LABEL: flat_atomic_fminimum_f64_saddr_ret_av_av:19362; GFX90A: ; %bb.0:19363; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19364; GFX90A-NEXT: s_add_u32 s4, s16, 0x5019365; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base19366; GFX90A-NEXT: s_addc_u32 s5, s17, 019367; GFX90A-NEXT: s_cmp_eq_u32 s5, s719368; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 019369; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]19370; GFX90A-NEXT: ;;#ASMSTART19371; GFX90A-NEXT: ; def v[4:5]19372; GFX90A-NEXT: ;;#ASMEND19373; GFX90A-NEXT: s_cbranch_vccz .LBB246_419374; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global19375; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]19376; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19377; GFX90A-NEXT: s_mov_b64 s[6:7], 019378; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff8000019379; GFX90A-NEXT: .LBB246_2: ; %atomicrmw.start19380; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119381; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19382; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]19383; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19384; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19385; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19386; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc19387; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19388; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19389; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]19390; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]19391; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]19392; GFX90A-NEXT: s_cbranch_execnz .LBB246_219393; GFX90A-NEXT: ; %bb.3: ; %Flow19394; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]19395; GFX90A-NEXT: s_branch .LBB246_619396; GFX90A-NEXT: .LBB246_4:19397; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr119398; GFX90A-NEXT: s_cbranch_execz .LBB246_619399; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private19400; GFX90A-NEXT: s_cmp_lg_u64 s[4:5], 019401; GFX90A-NEXT: s_cselect_b32 s4, s4, -119402; GFX90A-NEXT: v_mov_b32_e32 v6, s419403; GFX90A-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen19404; GFX90A-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:419405; GFX90A-NEXT: v_mov_b32_e32 v7, 0x7ff8000019406; GFX90A-NEXT: s_waitcnt vmcnt(0)19407; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]19408; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19409; GFX90A-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19410; GFX90A-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc19411; GFX90A-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen19412; GFX90A-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:419413; GFX90A-NEXT: .LBB246_6: ; %atomicrmw.phi19414; GFX90A-NEXT: ;;#ASMSTART19415; GFX90A-NEXT: ; use v[0:1]19416; GFX90A-NEXT: ;;#ASMEND19417; GFX90A-NEXT: s_waitcnt vmcnt(0)19418; GFX90A-NEXT: s_setpc_b64 s[30:31]19419;19420; GFX950-LABEL: flat_atomic_fminimum_f64_saddr_ret_av_av:19421; GFX950: ; %bb.0:19422; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19423; GFX950-NEXT: s_add_u32 s0, s0, 0x5019424; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base19425; GFX950-NEXT: s_addc_u32 s1, s1, 019426; GFX950-NEXT: s_cmp_eq_u32 s1, s319427; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 019428; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]19429; GFX950-NEXT: ;;#ASMSTART19430; GFX950-NEXT: ; def v[4:5]19431; GFX950-NEXT: ;;#ASMEND19432; GFX950-NEXT: s_cbranch_vccz .LBB246_419433; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global19434; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]19435; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]19436; GFX950-NEXT: s_mov_b64 s[2:3], 019437; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff8000019438; GFX950-NEXT: .LBB246_2: ; %atomicrmw.start19439; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119440; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19441; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]19442; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]19443; GFX950-NEXT: s_nop 119444; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc19445; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc19446; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc019447; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19448; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]19449; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19450; GFX950-NEXT: v_mov_b64_e32 v[2:3], v[0:1]19451; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19452; GFX950-NEXT: s_cbranch_execnz .LBB246_219453; GFX950-NEXT: ; %bb.3: ; %Flow19454; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19455; GFX950-NEXT: s_branch .LBB246_619456; GFX950-NEXT: .LBB246_4:19457; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr119458; GFX950-NEXT: s_cbranch_execz .LBB246_619459; GFX950-NEXT: ; %bb.5: ; %atomicrmw.private19460; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 019461; GFX950-NEXT: s_cselect_b32 s0, s0, -119462; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s019463; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff8000019464; GFX950-NEXT: s_waitcnt vmcnt(0)19465; GFX950-NEXT: v_min_f64 v[2:3], v[0:1], v[4:5]19466; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[4:5]19467; GFX950-NEXT: s_nop 119468; GFX950-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc19469; GFX950-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc19470; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s019471; GFX950-NEXT: .LBB246_6: ; %atomicrmw.phi19472; GFX950-NEXT: ;;#ASMSTART19473; GFX950-NEXT: ; use v[0:1]19474; GFX950-NEXT: ;;#ASMEND19475; GFX950-NEXT: s_waitcnt vmcnt(0)19476; GFX950-NEXT: s_setpc_b64 s[30:31]19477 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1019478 %data = call double asm "; def $0", "=^VA"()19479 %result = atomicrmw fminimum ptr %gep.0, double %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019480 call void asm "; use $0", "^VA"(double %result)19481 ret void19482}19483 19484;---------------------------------------------------------------------19485; other atomics v2f16, with aa+av cases using saddr19486;---------------------------------------------------------------------19487 19488define void @flat_atomic_fadd_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {19489; GFX90A-LABEL: flat_atomic_fadd_v2f16_saddr_ret_a_a:19490; GFX90A: ; %bb.0:19491; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19492; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19493; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019494; GFX90A-NEXT: ;;#ASMSTART19495; GFX90A-NEXT: ; def a019496; GFX90A-NEXT: ;;#ASMEND19497; GFX90A-NEXT: v_accvgpr_read_b32 v4, a019498; GFX90A-NEXT: s_mov_b64 s[4:5], 019499; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19500; GFX90A-NEXT: .LBB247_1: ; %atomicrmw.start19501; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119502; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19503; GFX90A-NEXT: v_pk_add_f16 v0, v1, v419504; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19505; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19506; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119507; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19508; GFX90A-NEXT: v_mov_b32_e32 v1, v019509; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19510; GFX90A-NEXT: s_cbranch_execnz .LBB247_119511; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19512; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19513; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019514; GFX90A-NEXT: ;;#ASMSTART19515; GFX90A-NEXT: ; use a019516; GFX90A-NEXT: ;;#ASMEND19517; GFX90A-NEXT: s_setpc_b64 s[30:31]19518;19519; GFX950-LABEL: flat_atomic_fadd_v2f16_saddr_ret_a_a:19520; GFX950: ; %bb.0:19521; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19522; GFX950-NEXT: v_mov_b32_e32 v0, s019523; GFX950-NEXT: v_mov_b32_e32 v1, s119524; GFX950-NEXT: ;;#ASMSTART19525; GFX950-NEXT: ; def a019526; GFX950-NEXT: ;;#ASMEND19527; GFX950-NEXT: s_nop 019528; GFX950-NEXT: v_accvgpr_read_b32 v2, a019529; GFX950-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:40 sc019530; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19531; GFX950-NEXT: v_accvgpr_write_b32 a0, v019532; GFX950-NEXT: ;;#ASMSTART19533; GFX950-NEXT: ; use a019534; GFX950-NEXT: ;;#ASMEND19535; GFX950-NEXT: s_setpc_b64 s[30:31]19536 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019537 %data = call <2 x half> asm "; def $0", "=a"()19538 %result = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019539 call void asm "; use $0", "a"(<2 x half> %result)19540 ret void19541}19542 19543define void @flat_atomic_fadd_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {19544; GFX90A-LABEL: flat_atomic_fadd_v2f16_saddr_ret_av_av:19545; GFX90A: ; %bb.0:19546; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19547; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19548; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019549; GFX90A-NEXT: s_mov_b64 s[4:5], 019550; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19551; GFX90A-NEXT: ;;#ASMSTART19552; GFX90A-NEXT: ; def v419553; GFX90A-NEXT: ;;#ASMEND19554; GFX90A-NEXT: .LBB248_1: ; %atomicrmw.start19555; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119556; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19557; GFX90A-NEXT: v_pk_add_f16 v0, v1, v419558; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19559; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19560; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119561; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19562; GFX90A-NEXT: v_mov_b32_e32 v1, v019563; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19564; GFX90A-NEXT: s_cbranch_execnz .LBB248_119565; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19566; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19567; GFX90A-NEXT: ;;#ASMSTART19568; GFX90A-NEXT: ; use v019569; GFX90A-NEXT: ;;#ASMEND19570; GFX90A-NEXT: s_setpc_b64 s[30:31]19571;19572; GFX950-LABEL: flat_atomic_fadd_v2f16_saddr_ret_av_av:19573; GFX950: ; %bb.0:19574; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19575; GFX950-NEXT: v_mov_b32_e32 v0, s019576; GFX950-NEXT: v_mov_b32_e32 v1, s119577; GFX950-NEXT: ;;#ASMSTART19578; GFX950-NEXT: ; def v219579; GFX950-NEXT: ;;#ASMEND19580; GFX950-NEXT: flat_atomic_pk_add_f16 v0, v[0:1], v2 offset:40 sc019581; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19582; GFX950-NEXT: ;;#ASMSTART19583; GFX950-NEXT: ; use v019584; GFX950-NEXT: ;;#ASMEND19585; GFX950-NEXT: s_setpc_b64 s[30:31]19586 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019587 %data = call <2 x half> asm "; def $0", "=^VA"()19588 %result = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019589 call void asm "; use $0", "^VA"(<2 x half> %result)19590 ret void19591}19592 19593define void @flat_atomic_fsub_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {19594; GFX90A-LABEL: flat_atomic_fsub_v2f16_saddr_ret_a_a:19595; GFX90A: ; %bb.0:19596; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19597; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19598; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019599; GFX90A-NEXT: ;;#ASMSTART19600; GFX90A-NEXT: ; def a019601; GFX90A-NEXT: ;;#ASMEND19602; GFX90A-NEXT: v_accvgpr_read_b32 v4, a019603; GFX90A-NEXT: s_mov_b64 s[4:5], 019604; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19605; GFX90A-NEXT: .LBB249_1: ; %atomicrmw.start19606; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119607; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19608; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]19609; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19610; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19611; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119612; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19613; GFX90A-NEXT: v_mov_b32_e32 v1, v019614; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19615; GFX90A-NEXT: s_cbranch_execnz .LBB249_119616; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19617; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19618; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019619; GFX90A-NEXT: ;;#ASMSTART19620; GFX90A-NEXT: ; use a019621; GFX90A-NEXT: ;;#ASMEND19622; GFX90A-NEXT: s_setpc_b64 s[30:31]19623;19624; GFX950-LABEL: flat_atomic_fsub_v2f16_saddr_ret_a_a:19625; GFX950: ; %bb.0:19626; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19627; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19628; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019629; GFX950-NEXT: ;;#ASMSTART19630; GFX950-NEXT: ; def a019631; GFX950-NEXT: ;;#ASMEND19632; GFX950-NEXT: s_mov_b64 s[2:3], 019633; GFX950-NEXT: v_accvgpr_read_b32 v4, a019634; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19635; GFX950-NEXT: .LBB249_1: ; %atomicrmw.start19636; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119637; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19638; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]19639; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019640; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19641; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119642; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19643; GFX950-NEXT: v_mov_b32_e32 v1, v019644; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19645; GFX950-NEXT: s_cbranch_execnz .LBB249_119646; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19647; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19648; GFX950-NEXT: v_accvgpr_write_b32 a0, v019649; GFX950-NEXT: ;;#ASMSTART19650; GFX950-NEXT: ; use a019651; GFX950-NEXT: ;;#ASMEND19652; GFX950-NEXT: s_setpc_b64 s[30:31]19653 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019654 %data = call <2 x half> asm "; def $0", "=a"()19655 %result = atomicrmw fsub ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019656 call void asm "; use $0", "a"(<2 x half> %result)19657 ret void19658}19659 19660define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {19661; GFX90A-LABEL: flat_atomic_fsub_v2f16_saddr_ret_av_av:19662; GFX90A: ; %bb.0:19663; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19664; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19665; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019666; GFX90A-NEXT: s_mov_b64 s[4:5], 019667; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19668; GFX90A-NEXT: ;;#ASMSTART19669; GFX90A-NEXT: ; def v419670; GFX90A-NEXT: ;;#ASMEND19671; GFX90A-NEXT: .LBB250_1: ; %atomicrmw.start19672; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119673; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19674; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]19675; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19676; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19677; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119678; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19679; GFX90A-NEXT: v_mov_b32_e32 v1, v019680; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19681; GFX90A-NEXT: s_cbranch_execnz .LBB250_119682; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19683; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19684; GFX90A-NEXT: ;;#ASMSTART19685; GFX90A-NEXT: ; use v019686; GFX90A-NEXT: ;;#ASMEND19687; GFX90A-NEXT: s_setpc_b64 s[30:31]19688;19689; GFX950-LABEL: flat_atomic_fsub_v2f16_saddr_ret_av_av:19690; GFX950: ; %bb.0:19691; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19692; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19693; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019694; GFX950-NEXT: s_mov_b64 s[2:3], 019695; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19696; GFX950-NEXT: ;;#ASMSTART19697; GFX950-NEXT: ; def v419698; GFX950-NEXT: ;;#ASMEND19699; GFX950-NEXT: .LBB250_1: ; %atomicrmw.start19700; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119701; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19702; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]19703; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019704; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19705; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119706; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19707; GFX950-NEXT: v_mov_b32_e32 v1, v019708; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19709; GFX950-NEXT: s_cbranch_execnz .LBB250_119710; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19711; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19712; GFX950-NEXT: ;;#ASMSTART19713; GFX950-NEXT: ; use v019714; GFX950-NEXT: ;;#ASMEND19715; GFX950-NEXT: s_setpc_b64 s[30:31]19716 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019717 %data = call <2 x half> asm "; def $0", "=^VA"()19718 %result = atomicrmw fsub ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019719 call void asm "; use $0", "^VA"(<2 x half> %result)19720 ret void19721}19722 19723define void @flat_atomic_fmax_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {19724; GFX90A-LABEL: flat_atomic_fmax_v2f16_saddr_ret_a_a:19725; GFX90A: ; %bb.0:19726; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19727; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19728; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019729; GFX90A-NEXT: ;;#ASMSTART19730; GFX90A-NEXT: ; def a019731; GFX90A-NEXT: ;;#ASMEND19732; GFX90A-NEXT: v_accvgpr_read_b32 v0, a019733; GFX90A-NEXT: s_mov_b64 s[4:5], 019734; GFX90A-NEXT: v_pk_max_f16 v4, v0, v019735; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19736; GFX90A-NEXT: .LBB251_1: ; %atomicrmw.start19737; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119738; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19739; GFX90A-NEXT: v_pk_max_f16 v0, v1, v119740; GFX90A-NEXT: v_pk_max_f16 v0, v0, v419741; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19742; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19743; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119744; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19745; GFX90A-NEXT: v_mov_b32_e32 v1, v019746; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19747; GFX90A-NEXT: s_cbranch_execnz .LBB251_119748; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19749; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19750; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019751; GFX90A-NEXT: ;;#ASMSTART19752; GFX90A-NEXT: ; use a019753; GFX90A-NEXT: ;;#ASMEND19754; GFX90A-NEXT: s_setpc_b64 s[30:31]19755;19756; GFX950-LABEL: flat_atomic_fmax_v2f16_saddr_ret_a_a:19757; GFX950: ; %bb.0:19758; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19759; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19760; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019761; GFX950-NEXT: ;;#ASMSTART19762; GFX950-NEXT: ; def a019763; GFX950-NEXT: ;;#ASMEND19764; GFX950-NEXT: s_mov_b64 s[2:3], 019765; GFX950-NEXT: v_accvgpr_read_b32 v0, a019766; GFX950-NEXT: v_pk_max_f16 v4, v0, v019767; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19768; GFX950-NEXT: .LBB251_1: ; %atomicrmw.start19769; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119770; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19771; GFX950-NEXT: v_pk_max_f16 v0, v1, v119772; GFX950-NEXT: s_nop 019773; GFX950-NEXT: v_pk_max_f16 v0, v0, v419774; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019775; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19776; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119777; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19778; GFX950-NEXT: v_mov_b32_e32 v1, v019779; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19780; GFX950-NEXT: s_cbranch_execnz .LBB251_119781; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19782; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19783; GFX950-NEXT: v_accvgpr_write_b32 a0, v019784; GFX950-NEXT: ;;#ASMSTART19785; GFX950-NEXT: ; use a019786; GFX950-NEXT: ;;#ASMEND19787; GFX950-NEXT: s_setpc_b64 s[30:31]19788 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019789 %data = call <2 x half> asm "; def $0", "=a"()19790 %result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019791 call void asm "; use $0", "a"(<2 x half> %result)19792 ret void19793}19794 19795define void @flat_atomic_fmax_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {19796; GFX90A-LABEL: flat_atomic_fmax_v2f16_saddr_ret_av_av:19797; GFX90A: ; %bb.0:19798; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19799; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19800; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019801; GFX90A-NEXT: ;;#ASMSTART19802; GFX90A-NEXT: ; def v019803; GFX90A-NEXT: ;;#ASMEND19804; GFX90A-NEXT: s_mov_b64 s[4:5], 019805; GFX90A-NEXT: v_pk_max_f16 v4, v0, v019806; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19807; GFX90A-NEXT: .LBB252_1: ; %atomicrmw.start19808; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119809; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19810; GFX90A-NEXT: v_pk_max_f16 v0, v1, v119811; GFX90A-NEXT: v_pk_max_f16 v0, v0, v419812; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19813; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19814; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119815; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19816; GFX90A-NEXT: v_mov_b32_e32 v1, v019817; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19818; GFX90A-NEXT: s_cbranch_execnz .LBB252_119819; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19820; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19821; GFX90A-NEXT: ;;#ASMSTART19822; GFX90A-NEXT: ; use v019823; GFX90A-NEXT: ;;#ASMEND19824; GFX90A-NEXT: s_setpc_b64 s[30:31]19825;19826; GFX950-LABEL: flat_atomic_fmax_v2f16_saddr_ret_av_av:19827; GFX950: ; %bb.0:19828; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19829; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19830; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019831; GFX950-NEXT: ;;#ASMSTART19832; GFX950-NEXT: ; def v019833; GFX950-NEXT: ;;#ASMEND19834; GFX950-NEXT: s_mov_b64 s[2:3], 019835; GFX950-NEXT: v_pk_max_f16 v4, v0, v019836; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19837; GFX950-NEXT: .LBB252_1: ; %atomicrmw.start19838; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119839; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19840; GFX950-NEXT: v_pk_max_f16 v0, v1, v119841; GFX950-NEXT: s_nop 019842; GFX950-NEXT: v_pk_max_f16 v0, v0, v419843; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019844; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19845; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119846; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19847; GFX950-NEXT: v_mov_b32_e32 v1, v019848; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19849; GFX950-NEXT: s_cbranch_execnz .LBB252_119850; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19851; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19852; GFX950-NEXT: ;;#ASMSTART19853; GFX950-NEXT: ; use v019854; GFX950-NEXT: ;;#ASMEND19855; GFX950-NEXT: s_setpc_b64 s[30:31]19856 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019857 %data = call <2 x half> asm "; def $0", "=^VA"()19858 %result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019859 call void asm "; use $0", "^VA"(<2 x half> %result)19860 ret void19861}19862 19863define void @flat_atomic_fmin_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {19864; GFX90A-LABEL: flat_atomic_fmin_v2f16_saddr_ret_a_a:19865; GFX90A: ; %bb.0:19866; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19867; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19868; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019869; GFX90A-NEXT: ;;#ASMSTART19870; GFX90A-NEXT: ; def a019871; GFX90A-NEXT: ;;#ASMEND19872; GFX90A-NEXT: v_accvgpr_read_b32 v0, a019873; GFX90A-NEXT: s_mov_b64 s[4:5], 019874; GFX90A-NEXT: v_pk_max_f16 v4, v0, v019875; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19876; GFX90A-NEXT: .LBB253_1: ; %atomicrmw.start19877; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119878; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19879; GFX90A-NEXT: v_pk_max_f16 v0, v1, v119880; GFX90A-NEXT: v_pk_min_f16 v0, v0, v419881; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19882; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19883; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119884; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19885; GFX90A-NEXT: v_mov_b32_e32 v1, v019886; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19887; GFX90A-NEXT: s_cbranch_execnz .LBB253_119888; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19889; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19890; GFX90A-NEXT: v_accvgpr_write_b32 a0, v019891; GFX90A-NEXT: ;;#ASMSTART19892; GFX90A-NEXT: ; use a019893; GFX90A-NEXT: ;;#ASMEND19894; GFX90A-NEXT: s_setpc_b64 s[30:31]19895;19896; GFX950-LABEL: flat_atomic_fmin_v2f16_saddr_ret_a_a:19897; GFX950: ; %bb.0:19898; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19899; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19900; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019901; GFX950-NEXT: ;;#ASMSTART19902; GFX950-NEXT: ; def a019903; GFX950-NEXT: ;;#ASMEND19904; GFX950-NEXT: s_mov_b64 s[2:3], 019905; GFX950-NEXT: v_accvgpr_read_b32 v0, a019906; GFX950-NEXT: v_pk_max_f16 v4, v0, v019907; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19908; GFX950-NEXT: .LBB253_1: ; %atomicrmw.start19909; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119910; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19911; GFX950-NEXT: v_pk_max_f16 v0, v1, v119912; GFX950-NEXT: s_nop 019913; GFX950-NEXT: v_pk_min_f16 v0, v0, v419914; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019915; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19916; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119917; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19918; GFX950-NEXT: v_mov_b32_e32 v1, v019919; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19920; GFX950-NEXT: s_cbranch_execnz .LBB253_119921; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19922; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19923; GFX950-NEXT: v_accvgpr_write_b32 a0, v019924; GFX950-NEXT: ;;#ASMSTART19925; GFX950-NEXT: ; use a019926; GFX950-NEXT: ;;#ASMEND19927; GFX950-NEXT: s_setpc_b64 s[30:31]19928 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019929 %data = call <2 x half> asm "; def $0", "=a"()19930 %result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019931 call void asm "; use $0", "a"(<2 x half> %result)19932 ret void19933}19934 19935define void @flat_atomic_fmin_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {19936; GFX90A-LABEL: flat_atomic_fmin_v2f16_saddr_ret_av_av:19937; GFX90A: ; %bb.0:19938; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19939; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]19940; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4019941; GFX90A-NEXT: ;;#ASMSTART19942; GFX90A-NEXT: ; def v019943; GFX90A-NEXT: ;;#ASMEND19944; GFX90A-NEXT: s_mov_b64 s[4:5], 019945; GFX90A-NEXT: v_pk_max_f16 v4, v0, v019946; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]19947; GFX90A-NEXT: .LBB254_1: ; %atomicrmw.start19948; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=119949; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19950; GFX90A-NEXT: v_pk_max_f16 v0, v1, v119951; GFX90A-NEXT: v_pk_min_f16 v0, v0, v419952; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc19953; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19954; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119955; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]19956; GFX90A-NEXT: v_mov_b32_e32 v1, v019957; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]19958; GFX90A-NEXT: s_cbranch_execnz .LBB254_119959; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end19960; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]19961; GFX90A-NEXT: ;;#ASMSTART19962; GFX90A-NEXT: ; use v019963; GFX90A-NEXT: ;;#ASMEND19964; GFX90A-NEXT: s_setpc_b64 s[30:31]19965;19966; GFX950-LABEL: flat_atomic_fmin_v2f16_saddr_ret_av_av:19967; GFX950: ; %bb.0:19968; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19969; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]19970; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4019971; GFX950-NEXT: ;;#ASMSTART19972; GFX950-NEXT: ; def v019973; GFX950-NEXT: ;;#ASMEND19974; GFX950-NEXT: s_mov_b64 s[2:3], 019975; GFX950-NEXT: v_pk_max_f16 v4, v0, v019976; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]19977; GFX950-NEXT: .LBB254_1: ; %atomicrmw.start19978; GFX950-NEXT: ; =>This Inner Loop Header: Depth=119979; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19980; GFX950-NEXT: v_pk_max_f16 v0, v1, v119981; GFX950-NEXT: s_nop 019982; GFX950-NEXT: v_pk_min_f16 v0, v0, v419983; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc019984; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19985; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v119986; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]19987; GFX950-NEXT: v_mov_b32_e32 v1, v019988; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]19989; GFX950-NEXT: s_cbranch_execnz .LBB254_119990; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end19991; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]19992; GFX950-NEXT: ;;#ASMSTART19993; GFX950-NEXT: ; use v019994; GFX950-NEXT: ;;#ASMEND19995; GFX950-NEXT: s_setpc_b64 s[30:31]19996 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1019997 %data = call <2 x half> asm "; def $0", "=^VA"()19998 %result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !019999 call void asm "; use $0", "^VA"(<2 x half> %result)20000 ret void20001}20002 20003define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {20004; GFX90A-LABEL: flat_atomic_fmaximum_v2f16_saddr_ret_a_a:20005; GFX90A: ; %bb.0:20006; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20007; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20008; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020009; GFX90A-NEXT: ;;#ASMSTART20010; GFX90A-NEXT: ; def a020011; GFX90A-NEXT: ;;#ASMEND20012; GFX90A-NEXT: v_accvgpr_read_b32 v4, a020013; GFX90A-NEXT: s_mov_b64 s[6:7], 020014; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0020015; GFX90A-NEXT: s_mov_b32 s8, 0x504010020016; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20017; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start20018; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120019; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20020; GFX90A-NEXT: v_pk_max_f16 v0, v1, v420021; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_120022; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v420023; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]20024; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_120025; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s820026; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20027; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20028; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120029; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20030; GFX90A-NEXT: v_mov_b32_e32 v1, v020031; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20032; GFX90A-NEXT: s_cbranch_execnz .LBB255_120033; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20034; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20035; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020036; GFX90A-NEXT: ;;#ASMSTART20037; GFX90A-NEXT: ; use a020038; GFX90A-NEXT: ;;#ASMEND20039; GFX90A-NEXT: s_setpc_b64 s[30:31]20040;20041; GFX950-LABEL: flat_atomic_fmaximum_v2f16_saddr_ret_a_a:20042; GFX950: ; %bb.0:20043; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20044; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20045; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020046; GFX950-NEXT: ;;#ASMSTART20047; GFX950-NEXT: ; def a020048; GFX950-NEXT: ;;#ASMEND20049; GFX950-NEXT: s_mov_b64 s[2:3], 020050; GFX950-NEXT: v_accvgpr_read_b32 v4, a020051; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20052; GFX950-NEXT: .LBB255_1: ; %atomicrmw.start20053; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120054; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20055; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v420056; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020057; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20058; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120059; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20060; GFX950-NEXT: v_mov_b32_e32 v1, v020061; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20062; GFX950-NEXT: s_cbranch_execnz .LBB255_120063; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20064; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20065; GFX950-NEXT: v_accvgpr_write_b32 a0, v020066; GFX950-NEXT: ;;#ASMSTART20067; GFX950-NEXT: ; use a020068; GFX950-NEXT: ;;#ASMEND20069; GFX950-NEXT: s_setpc_b64 s[30:31]20070 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1020071 %data = call <2 x half> asm "; def $0", "=a"()20072 %result = atomicrmw fmaximum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020073 call void asm "; use $0", "a"(<2 x half> %result)20074 ret void20075}20076 20077define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {20078; GFX90A-LABEL: flat_atomic_fmaximum_v2f16_saddr_ret_av_av:20079; GFX90A: ; %bb.0:20080; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20081; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20082; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020083; GFX90A-NEXT: s_mov_b64 s[6:7], 020084; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0020085; GFX90A-NEXT: s_mov_b32 s8, 0x504010020086; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20087; GFX90A-NEXT: ;;#ASMSTART20088; GFX90A-NEXT: ; def v420089; GFX90A-NEXT: ;;#ASMEND20090; GFX90A-NEXT: .LBB256_1: ; %atomicrmw.start20091; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120092; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20093; GFX90A-NEXT: v_pk_max_f16 v0, v1, v420094; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_120095; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v420096; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]20097; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_120098; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s820099; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20100; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20101; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120102; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20103; GFX90A-NEXT: v_mov_b32_e32 v1, v020104; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20105; GFX90A-NEXT: s_cbranch_execnz .LBB256_120106; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20107; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20108; GFX90A-NEXT: ;;#ASMSTART20109; GFX90A-NEXT: ; use v020110; GFX90A-NEXT: ;;#ASMEND20111; GFX90A-NEXT: s_setpc_b64 s[30:31]20112;20113; GFX950-LABEL: flat_atomic_fmaximum_v2f16_saddr_ret_av_av:20114; GFX950: ; %bb.0:20115; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20116; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20117; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020118; GFX950-NEXT: s_mov_b64 s[2:3], 020119; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20120; GFX950-NEXT: ;;#ASMSTART20121; GFX950-NEXT: ; def v420122; GFX950-NEXT: ;;#ASMEND20123; GFX950-NEXT: .LBB256_1: ; %atomicrmw.start20124; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120125; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20126; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v420127; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020128; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20129; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120130; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20131; GFX950-NEXT: v_mov_b32_e32 v1, v020132; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20133; GFX950-NEXT: s_cbranch_execnz .LBB256_120134; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20135; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20136; GFX950-NEXT: ;;#ASMSTART20137; GFX950-NEXT: ; use v020138; GFX950-NEXT: ;;#ASMEND20139; GFX950-NEXT: s_setpc_b64 s[30:31]20140 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1020141 %data = call <2 x half> asm "; def $0", "=^VA"()20142 %result = atomicrmw fmaximum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020143 call void asm "; use $0", "^VA"(<2 x half> %result)20144 ret void20145}20146 20147define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {20148; GFX90A-LABEL: flat_atomic_fminimum_v2f16_saddr_ret_a_a:20149; GFX90A: ; %bb.0:20150; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20151; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20152; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020153; GFX90A-NEXT: ;;#ASMSTART20154; GFX90A-NEXT: ; def a020155; GFX90A-NEXT: ;;#ASMEND20156; GFX90A-NEXT: v_accvgpr_read_b32 v4, a020157; GFX90A-NEXT: s_mov_b64 s[6:7], 020158; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0020159; GFX90A-NEXT: s_mov_b32 s8, 0x504010020160; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20161; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start20162; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120163; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20164; GFX90A-NEXT: v_pk_min_f16 v0, v1, v420165; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_120166; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v420167; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]20168; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_120169; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s820170; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20171; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20172; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120173; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20174; GFX90A-NEXT: v_mov_b32_e32 v1, v020175; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20176; GFX90A-NEXT: s_cbranch_execnz .LBB257_120177; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20178; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20179; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020180; GFX90A-NEXT: ;;#ASMSTART20181; GFX90A-NEXT: ; use a020182; GFX90A-NEXT: ;;#ASMEND20183; GFX90A-NEXT: s_setpc_b64 s[30:31]20184;20185; GFX950-LABEL: flat_atomic_fminimum_v2f16_saddr_ret_a_a:20186; GFX950: ; %bb.0:20187; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20188; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20189; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020190; GFX950-NEXT: ;;#ASMSTART20191; GFX950-NEXT: ; def a020192; GFX950-NEXT: ;;#ASMEND20193; GFX950-NEXT: s_mov_b64 s[2:3], 020194; GFX950-NEXT: v_accvgpr_read_b32 v4, a020195; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20196; GFX950-NEXT: .LBB257_1: ; %atomicrmw.start20197; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120198; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20199; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v420200; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020201; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20202; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120203; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20204; GFX950-NEXT: v_mov_b32_e32 v1, v020205; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20206; GFX950-NEXT: s_cbranch_execnz .LBB257_120207; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20208; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20209; GFX950-NEXT: v_accvgpr_write_b32 a0, v020210; GFX950-NEXT: ;;#ASMSTART20211; GFX950-NEXT: ; use a020212; GFX950-NEXT: ;;#ASMEND20213; GFX950-NEXT: s_setpc_b64 s[30:31]20214 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1020215 %data = call <2 x half> asm "; def $0", "=a"()20216 %result = atomicrmw fminimum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020217 call void asm "; use $0", "a"(<2 x half> %result)20218 ret void20219}20220 20221define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {20222; GFX90A-LABEL: flat_atomic_fminimum_v2f16_saddr_ret_av_av:20223; GFX90A: ; %bb.0:20224; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20225; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20226; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020227; GFX90A-NEXT: s_mov_b64 s[6:7], 020228; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e0020229; GFX90A-NEXT: s_mov_b32 s8, 0x504010020230; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20231; GFX90A-NEXT: ;;#ASMSTART20232; GFX90A-NEXT: ; def v420233; GFX90A-NEXT: ;;#ASMEND20234; GFX90A-NEXT: .LBB258_1: ; %atomicrmw.start20235; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120236; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20237; GFX90A-NEXT: v_pk_min_f16 v0, v1, v420238; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_120239; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v420240; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]20241; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_120242; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s820243; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20244; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20245; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120246; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20247; GFX90A-NEXT: v_mov_b32_e32 v1, v020248; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20249; GFX90A-NEXT: s_cbranch_execnz .LBB258_120250; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20251; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20252; GFX90A-NEXT: ;;#ASMSTART20253; GFX90A-NEXT: ; use v020254; GFX90A-NEXT: ;;#ASMEND20255; GFX90A-NEXT: s_setpc_b64 s[30:31]20256;20257; GFX950-LABEL: flat_atomic_fminimum_v2f16_saddr_ret_av_av:20258; GFX950: ; %bb.0:20259; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20260; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20261; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020262; GFX950-NEXT: s_mov_b64 s[2:3], 020263; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20264; GFX950-NEXT: ;;#ASMSTART20265; GFX950-NEXT: ; def v420266; GFX950-NEXT: ;;#ASMEND20267; GFX950-NEXT: .LBB258_1: ; %atomicrmw.start20268; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120269; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20270; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v420271; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020272; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20273; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120274; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20275; GFX950-NEXT: v_mov_b32_e32 v1, v020276; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20277; GFX950-NEXT: s_cbranch_execnz .LBB258_120278; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20279; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20280; GFX950-NEXT: ;;#ASMSTART20281; GFX950-NEXT: ; use v020282; GFX950-NEXT: ;;#ASMEND20283; GFX950-NEXT: s_setpc_b64 s[30:31]20284 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1020285 %data = call <2 x half> asm "; def $0", "=^VA"()20286 %result = atomicrmw fminimum ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020287 call void asm "; use $0", "^VA"(<2 x half> %result)20288 ret void20289}20290 20291;---------------------------------------------------------------------20292; other atomics v2bf16, with aa+av cases using saddr20293;---------------------------------------------------------------------20294 20295define void @flat_atomic_fadd_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {20296; GFX90A-LABEL: flat_atomic_fadd_v2bf16_saddr_ret_a_a:20297; GFX90A: ; %bb.0:20298; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20299; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20300; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020301; GFX90A-NEXT: ;;#ASMSTART20302; GFX90A-NEXT: ; def a020303; GFX90A-NEXT: ;;#ASMEND20304; GFX90A-NEXT: v_accvgpr_read_b32 v0, a020305; GFX90A-NEXT: s_mov_b64 s[6:7], 020306; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020307; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20308; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020309; GFX90A-NEXT: s_mov_b32 s9, 0x706030220310; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20311; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start20312; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120313; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20314; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120315; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120316; GFX90A-NEXT: v_add_f32_e32 v0, v0, v420317; GFX90A-NEXT: v_add_f32_e32 v6, v6, v520318; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120319; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120320; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020321; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620322; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820323; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820324; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620325; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020326; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20327; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20328; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920329; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20330; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20331; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120332; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20333; GFX90A-NEXT: v_mov_b32_e32 v1, v020334; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20335; GFX90A-NEXT: s_cbranch_execnz .LBB259_120336; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20337; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20338; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020339; GFX90A-NEXT: ;;#ASMSTART20340; GFX90A-NEXT: ; use a020341; GFX90A-NEXT: ;;#ASMEND20342; GFX90A-NEXT: s_setpc_b64 s[30:31]20343;20344; GFX950-LABEL: flat_atomic_fadd_v2bf16_saddr_ret_a_a:20345; GFX950: ; %bb.0:20346; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20347; GFX950-NEXT: v_mov_b32_e32 v0, s020348; GFX950-NEXT: v_mov_b32_e32 v1, s120349; GFX950-NEXT: ;;#ASMSTART20350; GFX950-NEXT: ; def a020351; GFX950-NEXT: ;;#ASMEND20352; GFX950-NEXT: s_nop 020353; GFX950-NEXT: v_accvgpr_read_b32 v2, a020354; GFX950-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:40 sc020355; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20356; GFX950-NEXT: v_accvgpr_write_b32 a0, v020357; GFX950-NEXT: ;;#ASMSTART20358; GFX950-NEXT: ; use a020359; GFX950-NEXT: ;;#ASMEND20360; GFX950-NEXT: s_setpc_b64 s[30:31]20361 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020362 %data = call <2 x bfloat> asm "; def $0", "=a"()20363 %result = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020364 call void asm "; use $0", "a"(<2 x bfloat> %result)20365 ret void20366}20367 20368define void @flat_atomic_fadd_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {20369; GFX90A-LABEL: flat_atomic_fadd_v2bf16_saddr_ret_av_av:20370; GFX90A: ; %bb.0:20371; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20372; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20373; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020374; GFX90A-NEXT: ;;#ASMSTART20375; GFX90A-NEXT: ; def v020376; GFX90A-NEXT: ;;#ASMEND20377; GFX90A-NEXT: s_mov_b64 s[6:7], 020378; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020379; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20380; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020381; GFX90A-NEXT: s_mov_b32 s9, 0x706030220382; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20383; GFX90A-NEXT: .LBB260_1: ; %atomicrmw.start20384; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120385; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20386; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120387; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120388; GFX90A-NEXT: v_add_f32_e32 v0, v0, v420389; GFX90A-NEXT: v_add_f32_e32 v6, v6, v520390; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120391; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120392; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020393; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620394; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820395; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820396; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620397; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020398; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20399; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20400; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920401; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20402; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20403; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120404; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20405; GFX90A-NEXT: v_mov_b32_e32 v1, v020406; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20407; GFX90A-NEXT: s_cbranch_execnz .LBB260_120408; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20409; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20410; GFX90A-NEXT: ;;#ASMSTART20411; GFX90A-NEXT: ; use v020412; GFX90A-NEXT: ;;#ASMEND20413; GFX90A-NEXT: s_setpc_b64 s[30:31]20414;20415; GFX950-LABEL: flat_atomic_fadd_v2bf16_saddr_ret_av_av:20416; GFX950: ; %bb.0:20417; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20418; GFX950-NEXT: v_mov_b32_e32 v0, s020419; GFX950-NEXT: v_mov_b32_e32 v1, s120420; GFX950-NEXT: ;;#ASMSTART20421; GFX950-NEXT: ; def v220422; GFX950-NEXT: ;;#ASMEND20423; GFX950-NEXT: flat_atomic_pk_add_bf16 v0, v[0:1], v2 offset:40 sc020424; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20425; GFX950-NEXT: ;;#ASMSTART20426; GFX950-NEXT: ; use v020427; GFX950-NEXT: ;;#ASMEND20428; GFX950-NEXT: s_setpc_b64 s[30:31]20429 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020430 %data = call <2 x bfloat> asm "; def $0", "=^VA"()20431 %result = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020432 call void asm "; use $0", "^VA"(<2 x bfloat> %result)20433 ret void20434}20435 20436define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {20437; GFX90A-LABEL: flat_atomic_fsub_v2bf16_saddr_ret_a_a:20438; GFX90A: ; %bb.0:20439; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20440; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20441; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020442; GFX90A-NEXT: ;;#ASMSTART20443; GFX90A-NEXT: ; def a020444; GFX90A-NEXT: ;;#ASMEND20445; GFX90A-NEXT: v_accvgpr_read_b32 v0, a020446; GFX90A-NEXT: s_mov_b64 s[6:7], 020447; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020448; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20449; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020450; GFX90A-NEXT: s_mov_b32 s9, 0x706030220451; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20452; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start20453; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120454; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20455; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120456; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120457; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v420458; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v520459; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120460; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120461; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020462; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620463; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820464; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820465; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620466; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020467; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20468; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20469; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920470; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20471; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20472; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120473; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20474; GFX90A-NEXT: v_mov_b32_e32 v1, v020475; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20476; GFX90A-NEXT: s_cbranch_execnz .LBB261_120477; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20478; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20479; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020480; GFX90A-NEXT: ;;#ASMSTART20481; GFX90A-NEXT: ; use a020482; GFX90A-NEXT: ;;#ASMEND20483; GFX90A-NEXT: s_setpc_b64 s[30:31]20484;20485; GFX950-LABEL: flat_atomic_fsub_v2bf16_saddr_ret_a_a:20486; GFX950: ; %bb.0:20487; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20488; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20489; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020490; GFX950-NEXT: ;;#ASMSTART20491; GFX950-NEXT: ; def a020492; GFX950-NEXT: ;;#ASMEND20493; GFX950-NEXT: s_mov_b64 s[2:3], 020494; GFX950-NEXT: v_accvgpr_read_b32 v0, a020495; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020496; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020497; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20498; GFX950-NEXT: .LBB261_1: ; %atomicrmw.start20499; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120500; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20501; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120502; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120503; GFX950-NEXT: v_sub_f32_e32 v0, v0, v420504; GFX950-NEXT: v_sub_f32_e32 v6, v6, v520505; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020506; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020507; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20508; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120509; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20510; GFX950-NEXT: v_mov_b32_e32 v1, v020511; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20512; GFX950-NEXT: s_cbranch_execnz .LBB261_120513; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20514; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20515; GFX950-NEXT: v_accvgpr_write_b32 a0, v020516; GFX950-NEXT: ;;#ASMSTART20517; GFX950-NEXT: ; use a020518; GFX950-NEXT: ;;#ASMEND20519; GFX950-NEXT: s_setpc_b64 s[30:31]20520 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020521 %data = call <2 x bfloat> asm "; def $0", "=a"()20522 %result = atomicrmw fsub ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020523 call void asm "; use $0", "a"(<2 x bfloat> %result)20524 ret void20525}20526 20527define void @flat_atomic_fsub_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {20528; GFX90A-LABEL: flat_atomic_fsub_v2bf16_saddr_ret_av_av:20529; GFX90A: ; %bb.0:20530; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20531; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20532; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020533; GFX90A-NEXT: ;;#ASMSTART20534; GFX90A-NEXT: ; def v020535; GFX90A-NEXT: ;;#ASMEND20536; GFX90A-NEXT: s_mov_b64 s[6:7], 020537; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020538; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20539; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020540; GFX90A-NEXT: s_mov_b32 s9, 0x706030220541; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20542; GFX90A-NEXT: .LBB262_1: ; %atomicrmw.start20543; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120544; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20545; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120546; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120547; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v420548; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v520549; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120550; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120551; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020552; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620553; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820554; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820555; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620556; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020557; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20558; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20559; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920560; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20561; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20562; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120563; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20564; GFX90A-NEXT: v_mov_b32_e32 v1, v020565; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20566; GFX90A-NEXT: s_cbranch_execnz .LBB262_120567; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20568; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20569; GFX90A-NEXT: ;;#ASMSTART20570; GFX90A-NEXT: ; use v020571; GFX90A-NEXT: ;;#ASMEND20572; GFX90A-NEXT: s_setpc_b64 s[30:31]20573;20574; GFX950-LABEL: flat_atomic_fsub_v2bf16_saddr_ret_av_av:20575; GFX950: ; %bb.0:20576; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20577; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20578; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020579; GFX950-NEXT: ;;#ASMSTART20580; GFX950-NEXT: ; def v020581; GFX950-NEXT: ;;#ASMEND20582; GFX950-NEXT: s_mov_b64 s[2:3], 020583; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020584; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020585; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20586; GFX950-NEXT: .LBB262_1: ; %atomicrmw.start20587; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120588; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20589; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120590; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120591; GFX950-NEXT: v_sub_f32_e32 v0, v0, v420592; GFX950-NEXT: v_sub_f32_e32 v6, v6, v520593; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020594; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020595; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20596; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120597; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20598; GFX950-NEXT: v_mov_b32_e32 v1, v020599; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20600; GFX950-NEXT: s_cbranch_execnz .LBB262_120601; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20602; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20603; GFX950-NEXT: ;;#ASMSTART20604; GFX950-NEXT: ; use v020605; GFX950-NEXT: ;;#ASMEND20606; GFX950-NEXT: s_setpc_b64 s[30:31]20607 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020608 %data = call <2 x bfloat> asm "; def $0", "=^VA"()20609 %result = atomicrmw fsub ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020610 call void asm "; use $0", "^VA"(<2 x bfloat> %result)20611 ret void20612}20613 20614define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {20615; GFX90A-LABEL: flat_atomic_fmax_v2bf16_saddr_ret_a_a:20616; GFX90A: ; %bb.0:20617; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20618; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20619; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020620; GFX90A-NEXT: ;;#ASMSTART20621; GFX90A-NEXT: ; def a020622; GFX90A-NEXT: ;;#ASMEND20623; GFX90A-NEXT: v_accvgpr_read_b32 v0, a020624; GFX90A-NEXT: s_mov_b64 s[6:7], 020625; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020626; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20627; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020628; GFX90A-NEXT: s_mov_b32 s9, 0x706030220629; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20630; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start20631; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120632; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20633; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120634; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120635; GFX90A-NEXT: v_max_f32_e32 v0, v0, v420636; GFX90A-NEXT: v_max_f32_e32 v6, v6, v520637; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120638; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120639; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020640; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620641; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820642; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820643; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620644; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020645; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20646; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20647; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920648; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20649; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20650; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120651; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20652; GFX90A-NEXT: v_mov_b32_e32 v1, v020653; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20654; GFX90A-NEXT: s_cbranch_execnz .LBB263_120655; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20656; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20657; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020658; GFX90A-NEXT: ;;#ASMSTART20659; GFX90A-NEXT: ; use a020660; GFX90A-NEXT: ;;#ASMEND20661; GFX90A-NEXT: s_setpc_b64 s[30:31]20662;20663; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_ret_a_a:20664; GFX950: ; %bb.0:20665; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20666; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20667; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020668; GFX950-NEXT: ;;#ASMSTART20669; GFX950-NEXT: ; def a020670; GFX950-NEXT: ;;#ASMEND20671; GFX950-NEXT: s_mov_b64 s[2:3], 020672; GFX950-NEXT: v_accvgpr_read_b32 v0, a020673; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020674; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020675; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20676; GFX950-NEXT: .LBB263_1: ; %atomicrmw.start20677; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120678; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20679; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120680; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120681; GFX950-NEXT: v_max_f32_e32 v0, v0, v420682; GFX950-NEXT: v_max_f32_e32 v6, v6, v520683; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020684; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020685; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20686; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120687; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20688; GFX950-NEXT: v_mov_b32_e32 v1, v020689; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20690; GFX950-NEXT: s_cbranch_execnz .LBB263_120691; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20692; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20693; GFX950-NEXT: v_accvgpr_write_b32 a0, v020694; GFX950-NEXT: ;;#ASMSTART20695; GFX950-NEXT: ; use a020696; GFX950-NEXT: ;;#ASMEND20697; GFX950-NEXT: s_setpc_b64 s[30:31]20698 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020699 %data = call <2 x bfloat> asm "; def $0", "=a"()20700 %result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020701 call void asm "; use $0", "a"(<2 x bfloat> %result)20702 ret void20703}20704 20705define void @flat_atomic_fmax_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {20706; GFX90A-LABEL: flat_atomic_fmax_v2bf16_saddr_ret_av_av:20707; GFX90A: ; %bb.0:20708; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20709; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20710; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020711; GFX90A-NEXT: ;;#ASMSTART20712; GFX90A-NEXT: ; def v020713; GFX90A-NEXT: ;;#ASMEND20714; GFX90A-NEXT: s_mov_b64 s[6:7], 020715; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020716; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20717; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020718; GFX90A-NEXT: s_mov_b32 s9, 0x706030220719; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20720; GFX90A-NEXT: .LBB264_1: ; %atomicrmw.start20721; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120722; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20723; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120724; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120725; GFX90A-NEXT: v_max_f32_e32 v0, v0, v420726; GFX90A-NEXT: v_max_f32_e32 v6, v6, v520727; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120728; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120729; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020730; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620731; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820732; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820733; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620734; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020735; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20736; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20737; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920738; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20739; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20740; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120741; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20742; GFX90A-NEXT: v_mov_b32_e32 v1, v020743; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20744; GFX90A-NEXT: s_cbranch_execnz .LBB264_120745; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20746; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20747; GFX90A-NEXT: ;;#ASMSTART20748; GFX90A-NEXT: ; use v020749; GFX90A-NEXT: ;;#ASMEND20750; GFX90A-NEXT: s_setpc_b64 s[30:31]20751;20752; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_ret_av_av:20753; GFX950: ; %bb.0:20754; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20755; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20756; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020757; GFX950-NEXT: ;;#ASMSTART20758; GFX950-NEXT: ; def v020759; GFX950-NEXT: ;;#ASMEND20760; GFX950-NEXT: s_mov_b64 s[2:3], 020761; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020762; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020763; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20764; GFX950-NEXT: .LBB264_1: ; %atomicrmw.start20765; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120766; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20767; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120768; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120769; GFX950-NEXT: v_max_f32_e32 v0, v0, v420770; GFX950-NEXT: v_max_f32_e32 v6, v6, v520771; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020772; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020773; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20774; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120775; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20776; GFX950-NEXT: v_mov_b32_e32 v1, v020777; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20778; GFX950-NEXT: s_cbranch_execnz .LBB264_120779; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20780; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20781; GFX950-NEXT: ;;#ASMSTART20782; GFX950-NEXT: ; use v020783; GFX950-NEXT: ;;#ASMEND20784; GFX950-NEXT: s_setpc_b64 s[30:31]20785 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020786 %data = call <2 x bfloat> asm "; def $0", "=^VA"()20787 %result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020788 call void asm "; use $0", "^VA"(<2 x bfloat> %result)20789 ret void20790}20791 20792define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {20793; GFX90A-LABEL: flat_atomic_fmin_v2bf16_saddr_ret_a_a:20794; GFX90A: ; %bb.0:20795; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20796; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20797; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020798; GFX90A-NEXT: ;;#ASMSTART20799; GFX90A-NEXT: ; def a020800; GFX90A-NEXT: ;;#ASMEND20801; GFX90A-NEXT: v_accvgpr_read_b32 v0, a020802; GFX90A-NEXT: s_mov_b64 s[6:7], 020803; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020804; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20805; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020806; GFX90A-NEXT: s_mov_b32 s9, 0x706030220807; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20808; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start20809; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120810; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20811; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120812; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120813; GFX90A-NEXT: v_min_f32_e32 v0, v0, v420814; GFX90A-NEXT: v_min_f32_e32 v6, v6, v520815; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120816; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120817; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020818; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620819; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820820; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820821; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620822; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020823; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20824; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20825; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920826; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20827; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20828; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120829; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20830; GFX90A-NEXT: v_mov_b32_e32 v1, v020831; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20832; GFX90A-NEXT: s_cbranch_execnz .LBB265_120833; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20834; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20835; GFX90A-NEXT: v_accvgpr_write_b32 a0, v020836; GFX90A-NEXT: ;;#ASMSTART20837; GFX90A-NEXT: ; use a020838; GFX90A-NEXT: ;;#ASMEND20839; GFX90A-NEXT: s_setpc_b64 s[30:31]20840;20841; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_ret_a_a:20842; GFX950: ; %bb.0:20843; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20844; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20845; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020846; GFX950-NEXT: ;;#ASMSTART20847; GFX950-NEXT: ; def a020848; GFX950-NEXT: ;;#ASMEND20849; GFX950-NEXT: s_mov_b64 s[2:3], 020850; GFX950-NEXT: v_accvgpr_read_b32 v0, a020851; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020852; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020853; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20854; GFX950-NEXT: .LBB265_1: ; %atomicrmw.start20855; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120856; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20857; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120858; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120859; GFX950-NEXT: v_min_f32_e32 v0, v0, v420860; GFX950-NEXT: v_min_f32_e32 v6, v6, v520861; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020862; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020863; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20864; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120865; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20866; GFX950-NEXT: v_mov_b32_e32 v1, v020867; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20868; GFX950-NEXT: s_cbranch_execnz .LBB265_120869; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20870; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20871; GFX950-NEXT: v_accvgpr_write_b32 a0, v020872; GFX950-NEXT: ;;#ASMSTART20873; GFX950-NEXT: ; use a020874; GFX950-NEXT: ;;#ASMEND20875; GFX950-NEXT: s_setpc_b64 s[30:31]20876 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020877 %data = call <2 x bfloat> asm "; def $0", "=a"()20878 %result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020879 call void asm "; use $0", "a"(<2 x bfloat> %result)20880 ret void20881}20882 20883define void @flat_atomic_fmin_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {20884; GFX90A-LABEL: flat_atomic_fmin_v2bf16_saddr_ret_av_av:20885; GFX90A: ; %bb.0:20886; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20887; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20888; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020889; GFX90A-NEXT: ;;#ASMSTART20890; GFX90A-NEXT: ; def v020891; GFX90A-NEXT: ;;#ASMEND20892; GFX90A-NEXT: s_mov_b64 s[6:7], 020893; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020894; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20895; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v020896; GFX90A-NEXT: s_mov_b32 s9, 0x706030220897; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20898; GFX90A-NEXT: .LBB266_1: ; %atomicrmw.start20899; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120900; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20901; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120902; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v120903; GFX90A-NEXT: v_min_f32_e32 v0, v0, v420904; GFX90A-NEXT: v_min_f32_e32 v6, v6, v520905; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 120906; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 120907; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v020908; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v620909; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s820910; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s820911; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v620912; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v020913; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]20914; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc20915; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s920916; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc20917; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20918; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120919; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]20920; GFX90A-NEXT: v_mov_b32_e32 v1, v020921; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]20922; GFX90A-NEXT: s_cbranch_execnz .LBB266_120923; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end20924; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]20925; GFX90A-NEXT: ;;#ASMSTART20926; GFX90A-NEXT: ; use v020927; GFX90A-NEXT: ;;#ASMEND20928; GFX90A-NEXT: s_setpc_b64 s[30:31]20929;20930; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_ret_av_av:20931; GFX950: ; %bb.0:20932; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20933; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]20934; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4020935; GFX950-NEXT: ;;#ASMSTART20936; GFX950-NEXT: ; def v020937; GFX950-NEXT: ;;#ASMEND20938; GFX950-NEXT: s_mov_b64 s[2:3], 020939; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v020940; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v020941; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]20942; GFX950-NEXT: .LBB266_1: ; %atomicrmw.start20943; GFX950-NEXT: ; =>This Inner Loop Header: Depth=120944; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20945; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v120946; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v120947; GFX950-NEXT: v_min_f32_e32 v0, v0, v420948; GFX950-NEXT: v_min_f32_e32 v6, v6, v520949; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v020950; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc020951; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20952; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v120953; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]20954; GFX950-NEXT: v_mov_b32_e32 v1, v020955; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]20956; GFX950-NEXT: s_cbranch_execnz .LBB266_120957; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end20958; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]20959; GFX950-NEXT: ;;#ASMSTART20960; GFX950-NEXT: ; use v020961; GFX950-NEXT: ;;#ASMEND20962; GFX950-NEXT: s_setpc_b64 s[30:31]20963 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1020964 %data = call <2 x bfloat> asm "; def $0", "=^VA"()20965 %result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !020966 call void asm "; use $0", "^VA"(<2 x bfloat> %result)20967 ret void20968}20969 20970define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {20971; GFX90A-LABEL: flat_atomic_fmaximum_v2bf16_saddr_ret_a_a:20972; GFX90A: ; %bb.0:20973; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20974; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]20975; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4020976; GFX90A-NEXT: ;;#ASMSTART20977; GFX90A-NEXT: ; def a020978; GFX90A-NEXT: ;;#ASMEND20979; GFX90A-NEXT: v_accvgpr_read_b32 v0, a020980; GFX90A-NEXT: s_mov_b64 s[6:7], 020981; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v020982; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000020983; GFX90A-NEXT: s_movk_i32 s8, 0x7fff20984; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v020985; GFX90A-NEXT: s_mov_b32 s9, 0x706030220986; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]20987; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start20988; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=120989; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)20990; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v120991; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v120992; GFX90A-NEXT: v_max_f32_e32 v8, v0, v420993; GFX90A-NEXT: v_max_f32_e32 v9, v7, v620994; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v620995; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v420996; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]20997; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc20998; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 120999; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 121000; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v021001; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v721002; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s821003; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s821004; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v721005; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v021006; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]21007; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc21008; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s921009; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc21010; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21011; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121012; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]21013; GFX90A-NEXT: v_mov_b32_e32 v1, v021014; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]21015; GFX90A-NEXT: s_cbranch_execnz .LBB267_121016; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end21017; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]21018; GFX90A-NEXT: v_accvgpr_write_b32 a0, v021019; GFX90A-NEXT: ;;#ASMSTART21020; GFX90A-NEXT: ; use a021021; GFX90A-NEXT: ;;#ASMEND21022; GFX90A-NEXT: s_setpc_b64 s[30:31]21023;21024; GFX950-LABEL: flat_atomic_fmaximum_v2bf16_saddr_ret_a_a:21025; GFX950: ; %bb.0:21026; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21027; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]21028; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4021029; GFX950-NEXT: ;;#ASMSTART21030; GFX950-NEXT: ; def a021031; GFX950-NEXT: ;;#ASMEND21032; GFX950-NEXT: s_mov_b64 s[2:3], 021033; GFX950-NEXT: v_accvgpr_read_b32 v0, a021034; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v021035; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v021036; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]21037; GFX950-NEXT: .LBB267_1: ; %atomicrmw.start21038; GFX950-NEXT: ; =>This Inner Loop Header: Depth=121039; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21040; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v121041; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v121042; GFX950-NEXT: v_maximum3_f32 v0, v0, v4, v421043; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v521044; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v021045; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc021046; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21047; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121048; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]21049; GFX950-NEXT: v_mov_b32_e32 v1, v021050; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]21051; GFX950-NEXT: s_cbranch_execnz .LBB267_121052; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end21053; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]21054; GFX950-NEXT: v_accvgpr_write_b32 a0, v021055; GFX950-NEXT: ;;#ASMSTART21056; GFX950-NEXT: ; use a021057; GFX950-NEXT: ;;#ASMEND21058; GFX950-NEXT: s_setpc_b64 s[30:31]21059 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1021060 %data = call <2 x bfloat> asm "; def $0", "=a"()21061 %result = atomicrmw fmaximum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !021062 call void asm "; use $0", "a"(<2 x bfloat> %result)21063 ret void21064}21065 21066define void @flat_atomic_fmaximum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {21067; GFX90A-LABEL: flat_atomic_fmaximum_v2bf16_saddr_ret_av_av:21068; GFX90A: ; %bb.0:21069; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21070; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]21071; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4021072; GFX90A-NEXT: ;;#ASMSTART21073; GFX90A-NEXT: ; def v021074; GFX90A-NEXT: ;;#ASMEND21075; GFX90A-NEXT: s_mov_b64 s[6:7], 021076; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v021077; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000021078; GFX90A-NEXT: s_movk_i32 s8, 0x7fff21079; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v021080; GFX90A-NEXT: s_mov_b32 s9, 0x706030221081; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]21082; GFX90A-NEXT: .LBB268_1: ; %atomicrmw.start21083; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=121084; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21085; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v121086; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v121087; GFX90A-NEXT: v_max_f32_e32 v8, v0, v421088; GFX90A-NEXT: v_max_f32_e32 v9, v7, v621089; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v621090; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v421091; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]21092; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc21093; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 121094; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 121095; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v021096; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v721097; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s821098; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s821099; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v721100; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v021101; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]21102; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc21103; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s921104; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc21105; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21106; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121107; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]21108; GFX90A-NEXT: v_mov_b32_e32 v1, v021109; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]21110; GFX90A-NEXT: s_cbranch_execnz .LBB268_121111; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end21112; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]21113; GFX90A-NEXT: ;;#ASMSTART21114; GFX90A-NEXT: ; use v021115; GFX90A-NEXT: ;;#ASMEND21116; GFX90A-NEXT: s_setpc_b64 s[30:31]21117;21118; GFX950-LABEL: flat_atomic_fmaximum_v2bf16_saddr_ret_av_av:21119; GFX950: ; %bb.0:21120; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21121; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]21122; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4021123; GFX950-NEXT: ;;#ASMSTART21124; GFX950-NEXT: ; def v021125; GFX950-NEXT: ;;#ASMEND21126; GFX950-NEXT: s_mov_b64 s[2:3], 021127; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v021128; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v021129; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]21130; GFX950-NEXT: .LBB268_1: ; %atomicrmw.start21131; GFX950-NEXT: ; =>This Inner Loop Header: Depth=121132; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21133; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v121134; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v121135; GFX950-NEXT: v_maximum3_f32 v0, v0, v4, v421136; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v521137; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v021138; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc021139; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21140; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121141; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]21142; GFX950-NEXT: v_mov_b32_e32 v1, v021143; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]21144; GFX950-NEXT: s_cbranch_execnz .LBB268_121145; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end21146; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]21147; GFX950-NEXT: ;;#ASMSTART21148; GFX950-NEXT: ; use v021149; GFX950-NEXT: ;;#ASMEND21150; GFX950-NEXT: s_setpc_b64 s[30:31]21151 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1021152 %data = call <2 x bfloat> asm "; def $0", "=^VA"()21153 %result = atomicrmw fmaximum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !021154 call void asm "; use $0", "^VA"(<2 x bfloat> %result)21155 ret void21156}21157 21158define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {21159; GFX90A-LABEL: flat_atomic_fminimum_v2bf16_saddr_ret_a_a:21160; GFX90A: ; %bb.0:21161; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21162; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]21163; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4021164; GFX90A-NEXT: ;;#ASMSTART21165; GFX90A-NEXT: ; def a021166; GFX90A-NEXT: ;;#ASMEND21167; GFX90A-NEXT: v_accvgpr_read_b32 v0, a021168; GFX90A-NEXT: s_mov_b64 s[6:7], 021169; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v021170; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000021171; GFX90A-NEXT: s_movk_i32 s8, 0x7fff21172; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v021173; GFX90A-NEXT: s_mov_b32 s9, 0x706030221174; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]21175; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start21176; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=121177; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21178; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v121179; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v121180; GFX90A-NEXT: v_min_f32_e32 v8, v0, v421181; GFX90A-NEXT: v_min_f32_e32 v9, v7, v621182; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v621183; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v421184; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]21185; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc21186; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 121187; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 121188; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v021189; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v721190; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s821191; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s821192; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v721193; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v021194; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]21195; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc21196; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s921197; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc21198; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21199; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121200; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]21201; GFX90A-NEXT: v_mov_b32_e32 v1, v021202; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]21203; GFX90A-NEXT: s_cbranch_execnz .LBB269_121204; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end21205; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]21206; GFX90A-NEXT: v_accvgpr_write_b32 a0, v021207; GFX90A-NEXT: ;;#ASMSTART21208; GFX90A-NEXT: ; use a021209; GFX90A-NEXT: ;;#ASMEND21210; GFX90A-NEXT: s_setpc_b64 s[30:31]21211;21212; GFX950-LABEL: flat_atomic_fminimum_v2bf16_saddr_ret_a_a:21213; GFX950: ; %bb.0:21214; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21215; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]21216; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4021217; GFX950-NEXT: ;;#ASMSTART21218; GFX950-NEXT: ; def a021219; GFX950-NEXT: ;;#ASMEND21220; GFX950-NEXT: s_mov_b64 s[2:3], 021221; GFX950-NEXT: v_accvgpr_read_b32 v0, a021222; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v021223; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v021224; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]21225; GFX950-NEXT: .LBB269_1: ; %atomicrmw.start21226; GFX950-NEXT: ; =>This Inner Loop Header: Depth=121227; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21228; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v121229; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v121230; GFX950-NEXT: v_minimum3_f32 v0, v0, v4, v421231; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v521232; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v021233; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc021234; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21235; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121236; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]21237; GFX950-NEXT: v_mov_b32_e32 v1, v021238; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]21239; GFX950-NEXT: s_cbranch_execnz .LBB269_121240; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end21241; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]21242; GFX950-NEXT: v_accvgpr_write_b32 a0, v021243; GFX950-NEXT: ;;#ASMSTART21244; GFX950-NEXT: ; use a021245; GFX950-NEXT: ;;#ASMEND21246; GFX950-NEXT: s_setpc_b64 s[30:31]21247 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1021248 %data = call <2 x bfloat> asm "; def $0", "=a"()21249 %result = atomicrmw fminimum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !021250 call void asm "; use $0", "a"(<2 x bfloat> %result)21251 ret void21252}21253 21254define void @flat_atomic_fminimum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {21255; GFX90A-LABEL: flat_atomic_fminimum_v2bf16_saddr_ret_av_av:21256; GFX90A: ; %bb.0:21257; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21258; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]21259; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:4021260; GFX90A-NEXT: ;;#ASMSTART21261; GFX90A-NEXT: ; def v021262; GFX90A-NEXT: ;;#ASMEND21263; GFX90A-NEXT: s_mov_b64 s[6:7], 021264; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v021265; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc0000021266; GFX90A-NEXT: s_movk_i32 s8, 0x7fff21267; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v021268; GFX90A-NEXT: s_mov_b32 s9, 0x706030221269; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]21270; GFX90A-NEXT: .LBB270_1: ; %atomicrmw.start21271; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=121272; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21273; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v121274; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v121275; GFX90A-NEXT: v_min_f32_e32 v8, v0, v421276; GFX90A-NEXT: v_min_f32_e32 v9, v7, v621277; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v621278; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v421279; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]21280; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc21281; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 121282; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 121283; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v021284; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v721285; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s821286; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s821287; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v721288; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v021289; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]21290; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc21291; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s921292; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc21293; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21294; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121295; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]21296; GFX90A-NEXT: v_mov_b32_e32 v1, v021297; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]21298; GFX90A-NEXT: s_cbranch_execnz .LBB270_121299; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end21300; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]21301; GFX90A-NEXT: ;;#ASMSTART21302; GFX90A-NEXT: ; use v021303; GFX90A-NEXT: ;;#ASMEND21304; GFX90A-NEXT: s_setpc_b64 s[30:31]21305;21306; GFX950-LABEL: flat_atomic_fminimum_v2bf16_saddr_ret_av_av:21307; GFX950: ; %bb.0:21308; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)21309; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]21310; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:4021311; GFX950-NEXT: ;;#ASMSTART21312; GFX950-NEXT: ; def v021313; GFX950-NEXT: ;;#ASMEND21314; GFX950-NEXT: s_mov_b64 s[2:3], 021315; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v021316; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v021317; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]21318; GFX950-NEXT: .LBB270_1: ; %atomicrmw.start21319; GFX950-NEXT: ; =>This Inner Loop Header: Depth=121320; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21321; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v121322; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v121323; GFX950-NEXT: v_minimum3_f32 v0, v0, v4, v421324; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v521325; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v021326; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc021327; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21328; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v121329; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]21330; GFX950-NEXT: v_mov_b32_e32 v1, v021331; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]21332; GFX950-NEXT: s_cbranch_execnz .LBB270_121333; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end21334; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]21335; GFX950-NEXT: ;;#ASMSTART21336; GFX950-NEXT: ; use v021337; GFX950-NEXT: ;;#ASMEND21338; GFX950-NEXT: s_setpc_b64 s[30:31]21339 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1021340 %data = call <2 x bfloat> asm "; def $0", "=^VA"()21341 %result = atomicrmw fminimum ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !021342 call void asm "; use $0", "^VA"(<2 x bfloat> %result)21343 ret void21344}21345 21346attributes #0 = { nounwind "amdgpu-waves-per-eu"="10,10" }21347 21348!0 = !{}21349 21350;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:21351; CHECK: {{.*}}21352