633 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -passes=atomic-expand < %s | FileCheck -check-prefix=IR %s3; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=GCN %s4 5define i32 @load_atomic_private_seq_cst_i32(ptr addrspace(5) %ptr) {6; IR-LABEL: define i32 @load_atomic_private_seq_cst_i32(7; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {8; IR-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 49; IR-NEXT: ret i32 [[LOAD]]10;11; GCN-LABEL: load_atomic_private_seq_cst_i32:12; GCN: ; %bb.0:13; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen15; GCN-NEXT: s_waitcnt vmcnt(0)16; GCN-NEXT: s_setpc_b64 s[30:31]17 %load = load atomic i32, ptr addrspace(5) %ptr seq_cst, align 418 ret i32 %load19}20 21define i64 @load_atomic_private_seq_cst_i64(ptr addrspace(5) %ptr) {22; IR-LABEL: define i64 @load_atomic_private_seq_cst_i64(23; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {24; IR-NEXT: [[LOAD:%.*]] = load i64, ptr addrspace(5) [[PTR]], align 825; IR-NEXT: ret i64 [[LOAD]]26;27; GCN-LABEL: load_atomic_private_seq_cst_i64:28; GCN: ; %bb.0:29; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)30; GCN-NEXT: v_add_i32_e32 v1, vcc, 4, v031; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen32; GCN-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen33; GCN-NEXT: s_waitcnt vmcnt(0)34; GCN-NEXT: s_setpc_b64 s[30:31]35 %load = load atomic i64, ptr addrspace(5) %ptr seq_cst, align 836 ret i64 %load37}38 39define void @atomic_store_seq_cst_i32(ptr addrspace(5) %ptr, i32 %val) {40; IR-LABEL: define void @atomic_store_seq_cst_i32(41; IR-SAME: ptr addrspace(5) [[PTR:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {42; IR-NEXT: store i32 [[VAL]], ptr addrspace(5) [[PTR]], align 443; IR-NEXT: ret void44;45; GCN-LABEL: atomic_store_seq_cst_i32:46; GCN: ; %bb.0:47; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)48; GCN-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen49; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)50; GCN-NEXT: s_setpc_b64 s[30:31]51 store atomic i32 %val, ptr addrspace(5) %ptr seq_cst, align 452 ret void53}54 55define void @atomic_store_seq_cst_i64(ptr addrspace(5) %ptr, i64 %val) {56; IR-LABEL: define void @atomic_store_seq_cst_i64(57; IR-SAME: ptr addrspace(5) [[PTR:%.*]], i64 [[VAL:%.*]]) #[[ATTR0]] {58; IR-NEXT: store i64 [[VAL]], ptr addrspace(5) [[PTR]], align 859; IR-NEXT: ret void60;61; GCN-LABEL: atomic_store_seq_cst_i64:62; GCN: ; %bb.0:63; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)64; GCN-NEXT: v_add_i32_e32 v3, vcc, 4, v065; GCN-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen66; GCN-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen67; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)68; GCN-NEXT: s_setpc_b64 s[30:31]69 store atomic i64 %val, ptr addrspace(5) %ptr seq_cst, align 870 ret void71}72 73define i32 @load_atomic_private_seq_cst_syncscope_i32(ptr addrspace(5) %ptr) {74; IR-LABEL: define i32 @load_atomic_private_seq_cst_syncscope_i32(75; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {76; IR-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 477; IR-NEXT: ret i32 [[LOAD]]78;79; GCN-LABEL: load_atomic_private_seq_cst_syncscope_i32:80; GCN: ; %bb.0:81; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)82; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen83; GCN-NEXT: s_waitcnt vmcnt(0)84; GCN-NEXT: s_setpc_b64 s[30:31]85 %load = load atomic i32, ptr addrspace(5) %ptr syncscope("agent") seq_cst, align 486 ret i32 %load87}88 89define void @atomic_store_seq_cst_syncscope_i32(ptr addrspace(5) %ptr, i32 %val) {90; IR-LABEL: define void @atomic_store_seq_cst_syncscope_i32(91; IR-SAME: ptr addrspace(5) [[PTR:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {92; IR-NEXT: store i32 [[VAL]], ptr addrspace(5) [[PTR]], align 493; IR-NEXT: ret void94;95; GCN-LABEL: atomic_store_seq_cst_syncscope_i32:96; GCN: ; %bb.0:97; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)98; GCN-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen99; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)100; GCN-NEXT: s_setpc_b64 s[30:31]101 store atomic i32 %val, ptr addrspace(5) %ptr syncscope("agent") seq_cst, align 4102 ret void103}104 105define i32 @cmpxchg_private_i32(ptr addrspace(5) %ptr) {106; IR-LABEL: define i32 @cmpxchg_private_i32(107; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {108; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4109; IR-NEXT: [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0110; IR-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i32 1, i32 [[TMP1]]111; IR-NEXT: store i32 [[TMP3]], ptr addrspace(5) [[PTR]], align 4112; IR-NEXT: [[TMP4:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP1]], 0113; IR-NEXT: [[TMP5:%.*]] = insertvalue { i32, i1 } [[TMP4]], i1 [[TMP2]], 1114; IR-NEXT: [[RESULT_0:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0115; IR-NEXT: [[RESULT_1:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1116; IR-NEXT: store i1 [[RESULT_1]], ptr addrspace(1) poison, align 1117; IR-NEXT: ret i32 [[RESULT_0]]118;119; GCN-LABEL: cmpxchg_private_i32:120; GCN: ; %bb.0:121; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)122; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen123; GCN-NEXT: s_mov_b32 s7, 0xf000124; GCN-NEXT: s_mov_b32 s6, -1125; GCN-NEXT: s_waitcnt vmcnt(0)126; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1127; GCN-NEXT: v_cndmask_b32_e64 v2, v1, 1, vcc128; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen129; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc130; GCN-NEXT: buffer_store_byte v0, off, s[4:7], 0131; GCN-NEXT: s_waitcnt expcnt(0)132; GCN-NEXT: v_mov_b32_e32 v0, v1133; GCN-NEXT: s_waitcnt vmcnt(0)134; GCN-NEXT: s_setpc_b64 s[30:31]135 %result = cmpxchg ptr addrspace(5) %ptr, i32 0, i32 1 acq_rel monotonic136 %result.0 = extractvalue { i32, i1 } %result, 0137 %result.1 = extractvalue { i32, i1 } %result, 1138 store i1 %result.1, ptr addrspace(1) poison139 ret i32 %result.0140}141 142define i64 @cmpxchg_private_i64(ptr addrspace(5) %ptr) {143; IR-LABEL: define i64 @cmpxchg_private_i64(144; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {145; IR-NEXT: [[TMP1:%.*]] = load i64, ptr addrspace(5) [[PTR]], align 8146; IR-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 0147; IR-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 1, i64 [[TMP1]]148; IR-NEXT: store i64 [[TMP3]], ptr addrspace(5) [[PTR]], align 8149; IR-NEXT: [[TMP4:%.*]] = insertvalue { i64, i1 } poison, i64 [[TMP1]], 0150; IR-NEXT: [[TMP5:%.*]] = insertvalue { i64, i1 } [[TMP4]], i1 [[TMP2]], 1151; IR-NEXT: [[RESULT_0:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0152; IR-NEXT: [[RESULT_1:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1153; IR-NEXT: store i1 [[RESULT_1]], ptr addrspace(1) poison, align 1154; IR-NEXT: ret i64 [[RESULT_0]]155;156; GCN-LABEL: cmpxchg_private_i64:157; GCN: ; %bb.0:158; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)159; GCN-NEXT: v_mov_b32_e32 v2, v0160; GCN-NEXT: v_add_i32_e32 v3, vcc, 4, v2161; GCN-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen162; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen163; GCN-NEXT: s_mov_b32 s7, 0xf000164; GCN-NEXT: s_mov_b32 s6, -1165; GCN-NEXT: s_waitcnt vmcnt(0)166; GCN-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]167; GCN-NEXT: v_cndmask_b32_e64 v4, v1, 0, vcc168; GCN-NEXT: buffer_store_dword v4, v3, s[0:3], 0 offen169; GCN-NEXT: v_cndmask_b32_e64 v3, v0, 1, vcc170; GCN-NEXT: s_waitcnt expcnt(0)171; GCN-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc172; GCN-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen173; GCN-NEXT: buffer_store_byte v4, off, s[4:7], 0174; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)175; GCN-NEXT: s_setpc_b64 s[30:31]176 %result = cmpxchg ptr addrspace(5) %ptr, i64 0, i64 1 acq_rel monotonic177 %result.0 = extractvalue { i64, i1 } %result, 0178 %result.1 = extractvalue { i64, i1 } %result, 1179 store i1 %result.1, ptr addrspace(1) poison180 ret i64 %result.0181}182 183 184define i32 @atomicrmw_xchg_private_i32(ptr addrspace(5) %ptr) {185; IR-LABEL: define i32 @atomicrmw_xchg_private_i32(186; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {187; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4188; IR-NEXT: store i32 4, ptr addrspace(5) [[PTR]], align 4189; IR-NEXT: ret i32 [[TMP1]]190;191; GCN-LABEL: atomicrmw_xchg_private_i32:192; GCN: ; %bb.0:193; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)194; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen195; GCN-NEXT: v_mov_b32_e32 v2, 4196; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen197; GCN-NEXT: s_waitcnt vmcnt(1)198; GCN-NEXT: v_mov_b32_e32 v0, v1199; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)200; GCN-NEXT: s_setpc_b64 s[30:31]201 %result = atomicrmw xchg ptr addrspace(5) %ptr, i32 4 seq_cst202 ret i32 %result203}204 205define i32 @atomicrmw_add_private_i32(ptr addrspace(5) %ptr) {206; IR-LABEL: define i32 @atomicrmw_add_private_i32(207; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {208; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4209; IR-NEXT: [[NEW:%.*]] = add i32 [[TMP1]], 4210; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4211; IR-NEXT: ret i32 [[TMP1]]212;213; GCN-LABEL: atomicrmw_add_private_i32:214; GCN: ; %bb.0:215; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)216; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen217; GCN-NEXT: s_waitcnt vmcnt(0)218; GCN-NEXT: v_add_i32_e32 v2, vcc, 4, v1219; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen220; GCN-NEXT: v_mov_b32_e32 v0, v1221; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)222; GCN-NEXT: s_setpc_b64 s[30:31]223 %result = atomicrmw add ptr addrspace(5) %ptr, i32 4 seq_cst224 ret i32 %result225}226 227define i32 @atomicrmw_sub_private_i32(ptr addrspace(5) %ptr) {228; IR-LABEL: define i32 @atomicrmw_sub_private_i32(229; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {230; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4231; IR-NEXT: [[NEW:%.*]] = sub i32 [[TMP1]], 4232; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4233; IR-NEXT: ret i32 [[TMP1]]234;235; GCN-LABEL: atomicrmw_sub_private_i32:236; GCN: ; %bb.0:237; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)238; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen239; GCN-NEXT: s_waitcnt vmcnt(0)240; GCN-NEXT: v_add_i32_e32 v2, vcc, -4, v1241; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen242; GCN-NEXT: v_mov_b32_e32 v0, v1243; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)244; GCN-NEXT: s_setpc_b64 s[30:31]245 %result = atomicrmw sub ptr addrspace(5) %ptr, i32 4 seq_cst246 ret i32 %result247}248 249define i32 @atomicrmw_and_private_i32(ptr addrspace(5) %ptr) {250; IR-LABEL: define i32 @atomicrmw_and_private_i32(251; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {252; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4253; IR-NEXT: [[NEW:%.*]] = and i32 [[TMP1]], 4254; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4255; IR-NEXT: ret i32 [[TMP1]]256;257; GCN-LABEL: atomicrmw_and_private_i32:258; GCN: ; %bb.0:259; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)260; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen261; GCN-NEXT: s_waitcnt vmcnt(0)262; GCN-NEXT: v_and_b32_e32 v2, 4, v1263; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen264; GCN-NEXT: v_mov_b32_e32 v0, v1265; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)266; GCN-NEXT: s_setpc_b64 s[30:31]267 %result = atomicrmw and ptr addrspace(5) %ptr, i32 4 seq_cst268 ret i32 %result269}270 271define i32 @atomicrmw_nand_private_i32(ptr addrspace(5) %ptr) {272; IR-LABEL: define i32 @atomicrmw_nand_private_i32(273; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {274; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4275; IR-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 4276; IR-NEXT: [[NEW:%.*]] = xor i32 [[TMP2]], -1277; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4278; IR-NEXT: ret i32 [[TMP1]]279;280; GCN-LABEL: atomicrmw_nand_private_i32:281; GCN: ; %bb.0:282; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)283; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen284; GCN-NEXT: s_waitcnt vmcnt(0)285; GCN-NEXT: v_bfi_b32 v2, v1, -5, -1286; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen287; GCN-NEXT: v_mov_b32_e32 v0, v1288; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)289; GCN-NEXT: s_setpc_b64 s[30:31]290 %result = atomicrmw nand ptr addrspace(5) %ptr, i32 4 seq_cst291 ret i32 %result292}293 294define i32 @atomicrmw_or_private_i32(ptr addrspace(5) %ptr) {295; IR-LABEL: define i32 @atomicrmw_or_private_i32(296; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {297; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4298; IR-NEXT: [[NEW:%.*]] = or i32 [[TMP1]], 4299; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4300; IR-NEXT: ret i32 [[TMP1]]301;302; GCN-LABEL: atomicrmw_or_private_i32:303; GCN: ; %bb.0:304; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)305; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen306; GCN-NEXT: s_waitcnt vmcnt(0)307; GCN-NEXT: v_or_b32_e32 v2, 4, v1308; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen309; GCN-NEXT: v_mov_b32_e32 v0, v1310; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)311; GCN-NEXT: s_setpc_b64 s[30:31]312 %result = atomicrmw or ptr addrspace(5) %ptr, i32 4 seq_cst313 ret i32 %result314}315 316define i32 @atomicrmw_xor_private_i32(ptr addrspace(5) %ptr) {317; IR-LABEL: define i32 @atomicrmw_xor_private_i32(318; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {319; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4320; IR-NEXT: [[NEW:%.*]] = xor i32 [[TMP1]], 4321; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4322; IR-NEXT: ret i32 [[TMP1]]323;324; GCN-LABEL: atomicrmw_xor_private_i32:325; GCN: ; %bb.0:326; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)327; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen328; GCN-NEXT: s_waitcnt vmcnt(0)329; GCN-NEXT: v_xor_b32_e32 v2, 4, v1330; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen331; GCN-NEXT: v_mov_b32_e32 v0, v1332; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)333; GCN-NEXT: s_setpc_b64 s[30:31]334 %result = atomicrmw xor ptr addrspace(5) %ptr, i32 4 seq_cst335 ret i32 %result336}337 338define i32 @atomicrmw_max_private_i32(ptr addrspace(5) %ptr) {339; IR-LABEL: define i32 @atomicrmw_max_private_i32(340; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {341; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4342; IR-NEXT: [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 4343; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 4344; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4345; IR-NEXT: ret i32 [[TMP1]]346;347; GCN-LABEL: atomicrmw_max_private_i32:348; GCN: ; %bb.0:349; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)350; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen351; GCN-NEXT: s_waitcnt vmcnt(0)352; GCN-NEXT: v_max_i32_e32 v2, 4, v1353; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen354; GCN-NEXT: v_mov_b32_e32 v0, v1355; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)356; GCN-NEXT: s_setpc_b64 s[30:31]357 %result = atomicrmw max ptr addrspace(5) %ptr, i32 4 seq_cst358 ret i32 %result359}360 361define i32 @atomicrmw_min_private_i32(ptr addrspace(5) %ptr) {362; IR-LABEL: define i32 @atomicrmw_min_private_i32(363; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {364; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4365; IR-NEXT: [[TMP2:%.*]] = icmp sle i32 [[TMP1]], 4366; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 4367; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4368; IR-NEXT: ret i32 [[TMP1]]369;370; GCN-LABEL: atomicrmw_min_private_i32:371; GCN: ; %bb.0:372; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)373; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen374; GCN-NEXT: s_waitcnt vmcnt(0)375; GCN-NEXT: v_min_i32_e32 v2, 4, v1376; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen377; GCN-NEXT: v_mov_b32_e32 v0, v1378; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)379; GCN-NEXT: s_setpc_b64 s[30:31]380 %result = atomicrmw min ptr addrspace(5) %ptr, i32 4 seq_cst381 ret i32 %result382}383 384define i32 @atomicrmw_umax_private_i32(ptr addrspace(5) %ptr) {385; IR-LABEL: define i32 @atomicrmw_umax_private_i32(386; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {387; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4388; IR-NEXT: [[TMP2:%.*]] = icmp ugt i32 [[TMP1]], 4389; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 4390; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4391; IR-NEXT: ret i32 [[TMP1]]392;393; GCN-LABEL: atomicrmw_umax_private_i32:394; GCN: ; %bb.0:395; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)396; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen397; GCN-NEXT: s_waitcnt vmcnt(0)398; GCN-NEXT: v_max_u32_e32 v2, 4, v1399; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen400; GCN-NEXT: v_mov_b32_e32 v0, v1401; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)402; GCN-NEXT: s_setpc_b64 s[30:31]403 %result = atomicrmw umax ptr addrspace(5) %ptr, i32 4 seq_cst404 ret i32 %result405}406 407define i32 @atomicrmw_umin_private_i32(ptr addrspace(5) %ptr) {408; IR-LABEL: define i32 @atomicrmw_umin_private_i32(409; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {410; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4411; IR-NEXT: [[TMP2:%.*]] = icmp ule i32 [[TMP1]], 4412; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP2]], i32 [[TMP1]], i32 4413; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4414; IR-NEXT: ret i32 [[TMP1]]415;416; GCN-LABEL: atomicrmw_umin_private_i32:417; GCN: ; %bb.0:418; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)419; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen420; GCN-NEXT: s_waitcnt vmcnt(0)421; GCN-NEXT: v_min_u32_e32 v2, 4, v1422; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen423; GCN-NEXT: v_mov_b32_e32 v0, v1424; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)425; GCN-NEXT: s_setpc_b64 s[30:31]426 %result = atomicrmw umin ptr addrspace(5) %ptr, i32 4 seq_cst427 ret i32 %result428}429 430define float @atomicrmw_fadd_private_f32(ptr addrspace(5) %ptr) {431; IR-LABEL: define float @atomicrmw_fadd_private_f32(432; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {433; IR-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(5) [[PTR]], align 4434; IR-NEXT: [[NEW:%.*]] = fadd float [[TMP1]], 2.000000e+00435; IR-NEXT: store float [[NEW]], ptr addrspace(5) [[PTR]], align 4436; IR-NEXT: ret float [[TMP1]]437;438; GCN-LABEL: atomicrmw_fadd_private_f32:439; GCN: ; %bb.0:440; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)441; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen442; GCN-NEXT: s_waitcnt vmcnt(0)443; GCN-NEXT: v_add_f32_e32 v2, 2.0, v1444; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen445; GCN-NEXT: v_mov_b32_e32 v0, v1446; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)447; GCN-NEXT: s_setpc_b64 s[30:31]448 %result = atomicrmw fadd ptr addrspace(5) %ptr, float 2.0 seq_cst449 ret float %result450}451 452define bfloat @atomicrmw_fadd_private_bf16(ptr addrspace(5) %ptr) {453; IR-LABEL: define bfloat @atomicrmw_fadd_private_bf16(454; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {455; IR-NEXT: [[TMP1:%.*]] = load bfloat, ptr addrspace(5) [[PTR]], align 2456; IR-NEXT: [[NEW:%.*]] = fadd bfloat [[TMP1]], 0xR4000457; IR-NEXT: store bfloat [[NEW]], ptr addrspace(5) [[PTR]], align 2458; IR-NEXT: ret bfloat [[TMP1]]459;460; GCN-LABEL: atomicrmw_fadd_private_bf16:461; GCN: ; %bb.0:462; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)463; GCN-NEXT: buffer_load_ushort v1, v0, s[0:3], 0 offen464; GCN-NEXT: s_waitcnt vmcnt(0)465; GCN-NEXT: v_lshlrev_b32_e32 v1, 16, v1466; GCN-NEXT: v_add_f32_e32 v2, 2.0, v1467; GCN-NEXT: v_lshrrev_b32_e32 v2, 16, v2468; GCN-NEXT: buffer_store_short v2, v0, s[0:3], 0 offen469; GCN-NEXT: v_mov_b32_e32 v0, v1470; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)471; GCN-NEXT: s_setpc_b64 s[30:31]472 %result = atomicrmw fadd ptr addrspace(5) %ptr, bfloat 2.0 seq_cst473 ret bfloat %result474}475 476define float @atomicrmw_fsub_private_i32(ptr addrspace(5) %ptr, float %val) {477; IR-LABEL: define float @atomicrmw_fsub_private_i32(478; IR-SAME: ptr addrspace(5) [[PTR:%.*]], float [[VAL:%.*]]) #[[ATTR0]] {479; IR-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(5) [[PTR]], align 4480; IR-NEXT: [[NEW:%.*]] = fsub float [[TMP1]], [[VAL]]481; IR-NEXT: store float [[NEW]], ptr addrspace(5) [[PTR]], align 4482; IR-NEXT: ret float [[TMP1]]483;484; GCN-LABEL: atomicrmw_fsub_private_i32:485; GCN: ; %bb.0:486; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)487; GCN-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen488; GCN-NEXT: s_waitcnt vmcnt(0)489; GCN-NEXT: v_sub_f32_e32 v1, v2, v1490; GCN-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen491; GCN-NEXT: v_mov_b32_e32 v0, v2492; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)493; GCN-NEXT: s_setpc_b64 s[30:31]494 %result = atomicrmw fsub ptr addrspace(5) %ptr, float %val seq_cst495 ret float %result496}497 498define amdgpu_kernel void @alloca_promote_atomicrmw_private_lds_promote(ptr addrspace(1) %out, i32 %in) nounwind {499; IR-LABEL: define amdgpu_kernel void @alloca_promote_atomicrmw_private_lds_promote(500; IR-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]]) #[[ATTR1:[0-9]+]] {501; IR-NEXT: entry:502; IR-NEXT: [[TMP:%.*]] = alloca [2 x i32], align 4, addrspace(5)503; IR-NEXT: [[GEP2:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(5) [[TMP]], i32 0, i32 1504; IR-NEXT: store i32 0, ptr addrspace(5) [[TMP]], align 4505; IR-NEXT: store i32 1, ptr addrspace(5) [[GEP2]], align 4506; IR-NEXT: [[GEP3:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(5) [[TMP]], i32 0, i32 [[IN]]507; IR-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(5) [[GEP3]], align 4508; IR-NEXT: [[NEW:%.*]] = add i32 [[TMP0]], 7509; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[GEP3]], align 4510; IR-NEXT: store i32 [[TMP0]], ptr addrspace(1) [[OUT]], align 4511; IR-NEXT: ret void512;513; GCN-LABEL: alloca_promote_atomicrmw_private_lds_promote:514; GCN: ; %bb.0: ; %entry515; GCN-NEXT: s_load_dword s6, s[4:5], 0xb516; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9517; GCN-NEXT: s_mov_b32 s3, 0xf000518; GCN-NEXT: s_mov_b32 s2, -1519; GCN-NEXT: s_waitcnt lgkmcnt(0)520; GCN-NEXT: s_cmp_eq_u32 s6, 1521; GCN-NEXT: s_cselect_b32 s4, 1, 0522; GCN-NEXT: v_mov_b32_e32 v0, s4523; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0524; GCN-NEXT: s_endpgm525entry:526 %tmp = alloca [2 x i32], addrspace(5)527 %gep2 = getelementptr inbounds [2 x i32], ptr addrspace(5) %tmp, i32 0, i32 1528 store i32 0, ptr addrspace(5) %tmp529 store i32 1, ptr addrspace(5) %gep2530 %gep3 = getelementptr inbounds [2 x i32], ptr addrspace(5) %tmp, i32 0, i32 %in531 %rmw = atomicrmw add ptr addrspace(5) %gep3, i32 7 acq_rel532 store i32 %rmw, ptr addrspace(1) %out533 ret void534}535 536define amdgpu_kernel void @alloca_promote_cmpxchg_private(ptr addrspace(1) %out, i32 %in) nounwind {537; IR-LABEL: define amdgpu_kernel void @alloca_promote_cmpxchg_private(538; IR-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]]) #[[ATTR1]] {539; IR-NEXT: entry:540; IR-NEXT: [[TMP:%.*]] = alloca [2 x i32], align 4, addrspace(5)541; IR-NEXT: [[GEP2:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(5) [[TMP]], i32 0, i32 1542; IR-NEXT: store i32 0, ptr addrspace(5) [[TMP]], align 4543; IR-NEXT: store i32 1, ptr addrspace(5) [[GEP2]], align 4544; IR-NEXT: [[GEP3:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(5) [[TMP]], i32 0, i32 [[IN]]545; IR-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(5) [[GEP3]], align 4546; IR-NEXT: [[TMP1:%.*]] = icmp eq i32 [[TMP0]], 0547; IR-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i32 1, i32 [[TMP0]]548; IR-NEXT: store i32 [[TMP2]], ptr addrspace(5) [[GEP3]], align 4549; IR-NEXT: [[TMP3:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP0]], 0550; IR-NEXT: [[TMP4:%.*]] = insertvalue { i32, i1 } [[TMP3]], i1 [[TMP1]], 1551; IR-NEXT: [[VAL:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0552; IR-NEXT: store i32 [[VAL]], ptr addrspace(1) [[OUT]], align 4553; IR-NEXT: ret void554;555; GCN-LABEL: alloca_promote_cmpxchg_private:556; GCN: ; %bb.0: ; %entry557; GCN-NEXT: s_load_dword s6, s[4:5], 0xb558; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9559; GCN-NEXT: s_mov_b32 s3, 0xf000560; GCN-NEXT: s_mov_b32 s2, -1561; GCN-NEXT: s_waitcnt lgkmcnt(0)562; GCN-NEXT: s_cmp_eq_u32 s6, 1563; GCN-NEXT: s_cselect_b32 s4, 1, 0564; GCN-NEXT: v_mov_b32_e32 v0, s4565; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0566; GCN-NEXT: s_endpgm567entry:568 %tmp = alloca [2 x i32], addrspace(5)569 %gep2 = getelementptr inbounds [2 x i32], ptr addrspace(5) %tmp, i32 0, i32 1570 store i32 0, ptr addrspace(5) %tmp571 store i32 1, ptr addrspace(5) %gep2572 %gep3 = getelementptr inbounds [2 x i32], ptr addrspace(5) %tmp, i32 0, i32 %in573 %xchg = cmpxchg ptr addrspace(5) %gep3, i32 0, i32 1 acq_rel monotonic574 %val = extractvalue { i32, i1 } %xchg, 0575 store i32 %val, ptr addrspace(1) %out576 ret void577}578 579define i32 @atomicrmw_inc_private_i32(ptr addrspace(5) %ptr) {580; IR-LABEL: define i32 @atomicrmw_inc_private_i32(581; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {582; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4583; IR-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 1584; IR-NEXT: [[TMP3:%.*]] = icmp uge i32 [[TMP1]], 4585; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]586; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4587; IR-NEXT: ret i32 [[TMP1]]588;589; GCN-LABEL: atomicrmw_inc_private_i32:590; GCN: ; %bb.0:591; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)592; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen593; GCN-NEXT: s_waitcnt vmcnt(0)594; GCN-NEXT: v_add_i32_e32 v2, vcc, 1, v1595; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 4, v1596; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc597; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen598; GCN-NEXT: v_mov_b32_e32 v0, v1599; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)600; GCN-NEXT: s_setpc_b64 s[30:31]601 %result = atomicrmw uinc_wrap ptr addrspace(5) %ptr, i32 4 seq_cst602 ret i32 %result603}604 605define i32 @atomicrmw_dec_private_i32(ptr addrspace(5) %ptr) {606; IR-LABEL: define i32 @atomicrmw_dec_private_i32(607; IR-SAME: ptr addrspace(5) [[PTR:%.*]]) #[[ATTR0]] {608; IR-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[PTR]], align 4609; IR-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 1610; IR-NEXT: [[TMP3:%.*]] = icmp eq i32 [[TMP1]], 0611; IR-NEXT: [[TMP4:%.*]] = icmp ugt i32 [[TMP1]], 4612; IR-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]613; IR-NEXT: [[NEW:%.*]] = select i1 [[TMP5]], i32 4, i32 [[TMP2]]614; IR-NEXT: store i32 [[NEW]], ptr addrspace(5) [[PTR]], align 4615; IR-NEXT: ret i32 [[TMP1]]616;617; GCN-LABEL: atomicrmw_dec_private_i32:618; GCN: ; %bb.0:619; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)620; GCN-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen621; GCN-NEXT: s_waitcnt vmcnt(0)622; GCN-NEXT: v_subrev_i32_e32 v2, vcc, 1, v1623; GCN-NEXT: v_cmp_lt_u32_e64 s[4:5], 4, v1624; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5]625; GCN-NEXT: v_cndmask_b32_e64 v2, v2, 4, s[4:5]626; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen627; GCN-NEXT: v_mov_b32_e32 v0, v1628; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)629; GCN-NEXT: s_setpc_b64 s[30:31]630 %result = atomicrmw udec_wrap ptr addrspace(5) %ptr, i32 4 seq_cst631 ret i32 %result632}633