760 lines · plain
1; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,SI,SICIVI %s2; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,SICIVI,GFX89 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX89 %s4 5; GCN-LABEL: {{^}}lds_atomic_xchg_ret_i64:6; SICIVI: s_mov_b32 m07; GFX9-NOT: m08 9; GCN: ds_wrxchg_rtn_b6410; GCN: s_endpgm11define amdgpu_kernel void @lds_atomic_xchg_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {12 %result = atomicrmw xchg ptr addrspace(3) %ptr, i64 4 seq_cst13 store i64 %result, ptr addrspace(1) %out, align 814 ret void15}16 17; GCN-LABEL: {{^}}lds_atomic_xchg_ret_i64_offset:18; SICIVI: s_mov_b32 m019; GFX9-NOT: m020 21; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:3222; GCN: s_endpgm23define amdgpu_kernel void @lds_atomic_xchg_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {24 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 425 %result = atomicrmw xchg ptr addrspace(3) %gep, i64 4 seq_cst26 store i64 %result, ptr addrspace(1) %out, align 827 ret void28}29 30; GCN-LABEL: {{^}}lds_atomic_xchg_ret_f64_offset:31; SICIVI: s_mov_b32 m032; GFX9-NOT: m033 34; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:3235; GCN: s_endpgm36define amdgpu_kernel void @lds_atomic_xchg_ret_f64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {37 %gep = getelementptr double, ptr addrspace(3) %ptr, i32 438 %result = atomicrmw xchg ptr addrspace(3) %gep, double 4.0 seq_cst39 store double %result, ptr addrspace(1) %out, align 840 ret void41}42 43; GCN-LABEL: {{^}}lds_atomic_xchg_ret_pointer_offset:44; SICIVI: s_mov_b32 m045; GFX9-NOT: m046 47; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:3248; GCN: s_endpgm49define amdgpu_kernel void @lds_atomic_xchg_ret_pointer_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {50 %gep = getelementptr ptr, ptr addrspace(3) %ptr, i32 451 %result = atomicrmw xchg ptr addrspace(3) %gep, ptr null seq_cst52 store ptr %result, ptr addrspace(1) %out, align 853 ret void54}55 56; GCN-LABEL: {{^}}lds_atomic_add_ret_i64:57; SICIVI: s_mov_b32 m058; GFX9-NOT: m059 60; GCN: ds_add_rtn_u6461; GCN: s_endpgm62define amdgpu_kernel void @lds_atomic_add_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {63 %result = atomicrmw add ptr addrspace(3) %ptr, i64 4 seq_cst64 store i64 %result, ptr addrspace(1) %out, align 865 ret void66}67 68; GCN-LABEL: {{^}}lds_atomic_add_ret_i64_offset:69; SICIVI-DAG: s_mov_b32 m070; GFX9-NOT: m071 72; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb73; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c74; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 975; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 076; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]77; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:3278; GCN: buffer_store_dwordx2 [[RESULT]],79; GCN: s_endpgm80define amdgpu_kernel void @lds_atomic_add_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {81 %gep = getelementptr i64, ptr addrspace(3) %ptr, i64 482 %result = atomicrmw add ptr addrspace(3) %gep, i64 9 seq_cst83 store i64 %result, ptr addrspace(1) %out, align 884 ret void85}86 87; GCN-LABEL: {{^}}lds_atomic_add1_ret_i64:88; SICIVI-DAG: s_mov_b32 m089; GFX9-NOT: m090 91; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}92; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}93; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]94; GCN: buffer_store_dwordx2 [[RESULT]],95; GCN: s_endpgm96define amdgpu_kernel void @lds_atomic_add1_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {97 %result = atomicrmw add ptr addrspace(3) %ptr, i64 1 seq_cst98 store i64 %result, ptr addrspace(1) %out, align 899 ret void100}101 102; GCN-LABEL: {{^}}lds_atomic_add1_ret_i64_offset:103; SICIVI: s_mov_b32 m0104; GFX9-NOT: m0105 106; GCN: ds_add_rtn_u64 {{.*}} offset:32107; GCN: s_endpgm108define amdgpu_kernel void @lds_atomic_add1_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {109 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4110 %result = atomicrmw add ptr addrspace(3) %gep, i64 1 seq_cst111 store i64 %result, ptr addrspace(1) %out, align 8112 ret void113}114 115; GCN-LABEL: {{^}}lds_atomic_sub_ret_i64:116; SICIVI: s_mov_b32 m0117; GFX9-NOT: m0118 119; GCN: ds_sub_rtn_u64120; GCN: s_endpgm121define amdgpu_kernel void @lds_atomic_sub_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {122 %result = atomicrmw sub ptr addrspace(3) %ptr, i64 4 seq_cst123 store i64 %result, ptr addrspace(1) %out, align 8124 ret void125}126 127; GCN-LABEL: {{^}}lds_atomic_sub_ret_i64_offset:128; SICIVI: s_mov_b32 m0129; GFX9-NOT: m0130 131; GCN: ds_sub_rtn_u64 {{.*}} offset:32132; GCN: s_endpgm133define amdgpu_kernel void @lds_atomic_sub_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {134 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4135 %result = atomicrmw sub ptr addrspace(3) %gep, i64 4 seq_cst136 store i64 %result, ptr addrspace(1) %out, align 8137 ret void138}139 140; GCN-LABEL: {{^}}lds_atomic_sub1_ret_i64:141; SICIVI-DAG: s_mov_b32 m0142; GFX9-NOT: m0143 144; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}145; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}146; GCN: ds_sub_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]147; GCN: buffer_store_dwordx2 [[RESULT]],148; GCN: s_endpgm149define amdgpu_kernel void @lds_atomic_sub1_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {150 %result = atomicrmw sub ptr addrspace(3) %ptr, i64 1 seq_cst151 store i64 %result, ptr addrspace(1) %out, align 8152 ret void153}154 155; GCN-LABEL: {{^}}lds_atomic_sub1_ret_i64_offset:156; SICIVI: s_mov_b32 m0157; GFX9-NOT: m0158 159; GCN: ds_sub_rtn_u64 {{.*}} offset:32160; GCN: s_endpgm161define amdgpu_kernel void @lds_atomic_sub1_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {162 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4163 %result = atomicrmw sub ptr addrspace(3) %gep, i64 1 seq_cst164 store i64 %result, ptr addrspace(1) %out, align 8165 ret void166}167 168; GCN-LABEL: {{^}}lds_atomic_and_ret_i64:169; SICIVI: s_mov_b32 m0170; GFX9-NOT: m0171 172; GCN: ds_and_rtn_b64173; GCN: s_endpgm174define amdgpu_kernel void @lds_atomic_and_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {175 %result = atomicrmw and ptr addrspace(3) %ptr, i64 4 seq_cst176 store i64 %result, ptr addrspace(1) %out, align 8177 ret void178}179 180; GCN-LABEL: {{^}}lds_atomic_and_ret_i64_offset:181; SICIVI: s_mov_b32 m0182; GFX9-NOT: m0183 184; GCN: ds_and_rtn_b64 {{.*}} offset:32185; GCN: s_endpgm186define amdgpu_kernel void @lds_atomic_and_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {187 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4188 %result = atomicrmw and ptr addrspace(3) %gep, i64 4 seq_cst189 store i64 %result, ptr addrspace(1) %out, align 8190 ret void191}192 193; GCN-LABEL: {{^}}lds_atomic_or_ret_i64:194; SICIVI: s_mov_b32 m0195; GFX9-NOT: m0196 197; GCN: ds_or_rtn_b64198; GCN: s_endpgm199define amdgpu_kernel void @lds_atomic_or_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {200 %result = atomicrmw or ptr addrspace(3) %ptr, i64 4 seq_cst201 store i64 %result, ptr addrspace(1) %out, align 8202 ret void203}204 205; GCN-LABEL: {{^}}lds_atomic_or_ret_i64_offset:206; SICIVI: s_mov_b32 m0207; GFX9-NOT: m0208 209; GCN: ds_or_rtn_b64 {{.*}} offset:32210; GCN: s_endpgm211define amdgpu_kernel void @lds_atomic_or_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {212 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4213 %result = atomicrmw or ptr addrspace(3) %gep, i64 4 seq_cst214 store i64 %result, ptr addrspace(1) %out, align 8215 ret void216}217 218; GCN-LABEL: {{^}}lds_atomic_xor_ret_i64:219; SICIVI: s_mov_b32 m0220; GFX9-NOT: m0221 222; GCN: ds_xor_rtn_b64223; GCN: s_endpgm224define amdgpu_kernel void @lds_atomic_xor_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {225 %result = atomicrmw xor ptr addrspace(3) %ptr, i64 4 seq_cst226 store i64 %result, ptr addrspace(1) %out, align 8227 ret void228}229 230; GCN-LABEL: {{^}}lds_atomic_xor_ret_i64_offset:231; SICIVI: s_mov_b32 m0232; GFX9-NOT: m0233 234; GCN: ds_xor_rtn_b64 {{.*}} offset:32235; GCN: s_endpgm236define amdgpu_kernel void @lds_atomic_xor_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {237 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4238 %result = atomicrmw xor ptr addrspace(3) %gep, i64 4 seq_cst239 store i64 %result, ptr addrspace(1) %out, align 8240 ret void241}242 243; FIXME: There is no atomic nand instr244; XGCN-LABEL: {{^}}lds_atomic_nand_ret_i64:uction, so we somehow need to expand this.245; define amdgpu_kernel void @lds_atomic_nand_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {246; %result = atomicrmw nand ptr addrspace(3) %ptr, i32 4 seq_cst247; store i64 %result, ptr addrspace(1) %out, align 8248; ret void249; }250 251; GCN-LABEL: {{^}}lds_atomic_min_ret_i64:252; SICIVI: s_mov_b32 m0253; GFX9-NOT: m0254 255; GCN: ds_min_rtn_i64256; GCN: s_endpgm257define amdgpu_kernel void @lds_atomic_min_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {258 %result = atomicrmw min ptr addrspace(3) %ptr, i64 4 seq_cst259 store i64 %result, ptr addrspace(1) %out, align 8260 ret void261}262 263; GCN-LABEL: {{^}}lds_atomic_min_ret_i64_offset:264; SICIVI: s_mov_b32 m0265; GFX9-NOT: m0266 267; GCN: ds_min_rtn_i64 {{.*}} offset:32268; GCN: s_endpgm269define amdgpu_kernel void @lds_atomic_min_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {270 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4271 %result = atomicrmw min ptr addrspace(3) %gep, i64 4 seq_cst272 store i64 %result, ptr addrspace(1) %out, align 8273 ret void274}275 276; GCN-LABEL: {{^}}lds_atomic_max_ret_i64:277; SICIVI: s_mov_b32 m0278; GFX9-NOT: m0279 280; GCN: ds_max_rtn_i64281; GCN: s_endpgm282define amdgpu_kernel void @lds_atomic_max_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {283 %result = atomicrmw max ptr addrspace(3) %ptr, i64 4 seq_cst284 store i64 %result, ptr addrspace(1) %out, align 8285 ret void286}287 288; GCN-LABEL: {{^}}lds_atomic_max_ret_i64_offset:289; SICIVI: s_mov_b32 m0290; GFX9-NOT: m0291 292; GCN: ds_max_rtn_i64 {{.*}} offset:32293; GCN: s_endpgm294define amdgpu_kernel void @lds_atomic_max_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {295 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4296 %result = atomicrmw max ptr addrspace(3) %gep, i64 4 seq_cst297 store i64 %result, ptr addrspace(1) %out, align 8298 ret void299}300 301; GCN-LABEL: {{^}}lds_atomic_umin_ret_i64:302; SICIVI: s_mov_b32 m0303; GFX9-NOT: m0304 305; GCN: ds_min_rtn_u64306; GCN: s_endpgm307define amdgpu_kernel void @lds_atomic_umin_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {308 %result = atomicrmw umin ptr addrspace(3) %ptr, i64 4 seq_cst309 store i64 %result, ptr addrspace(1) %out, align 8310 ret void311}312 313; GCN-LABEL: {{^}}lds_atomic_umin_ret_i64_offset:314; SICIVI: s_mov_b32 m0315; GFX9-NOT: m0316 317; GCN: ds_min_rtn_u64 {{.*}} offset:32318; GCN: s_endpgm319define amdgpu_kernel void @lds_atomic_umin_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {320 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4321 %result = atomicrmw umin ptr addrspace(3) %gep, i64 4 seq_cst322 store i64 %result, ptr addrspace(1) %out, align 8323 ret void324}325 326; GCN-LABEL: {{^}}lds_atomic_umax_ret_i64:327; SICIVI: s_mov_b32 m0328; GFX9-NOT: m0329 330; GCN: ds_max_rtn_u64331; GCN: s_endpgm332define amdgpu_kernel void @lds_atomic_umax_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {333 %result = atomicrmw umax ptr addrspace(3) %ptr, i64 4 seq_cst334 store i64 %result, ptr addrspace(1) %out, align 8335 ret void336}337 338; GCN-LABEL: {{^}}lds_atomic_umax_ret_i64_offset:339; SICIVI: s_mov_b32 m0340; GFX9-NOT: m0341 342; GCN: ds_max_rtn_u64 {{.*}} offset:32343; GCN: s_endpgm344define amdgpu_kernel void @lds_atomic_umax_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {345 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4346 %result = atomicrmw umax ptr addrspace(3) %gep, i64 4 seq_cst347 store i64 %result, ptr addrspace(1) %out, align 8348 ret void349}350 351; GCN-LABEL: {{^}}lds_atomic_xchg_noret_i64:352; SICIVI: s_mov_b32 m0353; GFX9-NOT: m0354 355; GCN: ds_wrxchg_rtn_b64356; GCN: s_endpgm357define amdgpu_kernel void @lds_atomic_xchg_noret_i64(ptr addrspace(3) %ptr) nounwind {358 %result = atomicrmw xchg ptr addrspace(3) %ptr, i64 4 seq_cst359 ret void360}361 362; GCN-LABEL: {{^}}lds_atomic_xchg_noret_i64_offset:363; SICIVI: s_mov_b32 m0364; GFX9-NOT: m0365 366; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:32367; GCN: s_endpgm368define amdgpu_kernel void @lds_atomic_xchg_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {369 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4370 %result = atomicrmw xchg ptr addrspace(3) %gep, i64 4 seq_cst371 ret void372}373 374; GCN-LABEL: {{^}}lds_atomic_add_noret_i64:375; SICIVI: s_mov_b32 m0376; GFX9-NOT: m0377 378; GCN: ds_add_u64379; GCN: s_endpgm380define amdgpu_kernel void @lds_atomic_add_noret_i64(ptr addrspace(3) %ptr) nounwind {381 %result = atomicrmw add ptr addrspace(3) %ptr, i64 4 seq_cst382 ret void383}384 385; GCN-LABEL: {{^}}lds_atomic_add_noret_i64_offset:386; SICIVI-DAG: s_mov_b32 m0387; GFX9-NOT: m0388 389; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9390; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24391; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9392; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0393; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]394; GCN: ds_add_u64 {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]] offset:32395; GCN: s_endpgm396define amdgpu_kernel void @lds_atomic_add_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {397 %gep = getelementptr i64, ptr addrspace(3) %ptr, i64 4398 %result = atomicrmw add ptr addrspace(3) %gep, i64 9 seq_cst399 ret void400}401 402; GCN-LABEL: {{^}}lds_atomic_add1_noret_i64:403; SICIVI-DAG: s_mov_b32 m0404; GFX9-NOT: m0405 406; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}407; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}408; GCN: ds_add_u64 {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]409; GCN: s_endpgm410define amdgpu_kernel void @lds_atomic_add1_noret_i64(ptr addrspace(3) %ptr) nounwind {411 %result = atomicrmw add ptr addrspace(3) %ptr, i64 1 seq_cst412 ret void413}414 415; GCN-LABEL: {{^}}lds_atomic_add1_noret_i64_offset:416; SICIVI: s_mov_b32 m0417; GFX9-NOT: m0418 419; GCN: ds_add_u64 {{.*}} offset:32420; GCN: s_endpgm421define amdgpu_kernel void @lds_atomic_add1_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {422 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4423 %result = atomicrmw add ptr addrspace(3) %gep, i64 1 seq_cst424 ret void425}426 427; GCN-LABEL: {{^}}lds_atomic_sub_noret_i64:428; SICIVI: s_mov_b32 m0429; GFX9-NOT: m0430 431; GCN: ds_sub_u64432; GCN: s_endpgm433define amdgpu_kernel void @lds_atomic_sub_noret_i64(ptr addrspace(3) %ptr) nounwind {434 %result = atomicrmw sub ptr addrspace(3) %ptr, i64 4 seq_cst435 ret void436}437 438; GCN-LABEL: {{^}}lds_atomic_sub_noret_i64_offset:439; SICIVI: s_mov_b32 m0440; GFX9-NOT: m0441 442; GCN: ds_sub_u64 {{.*}} offset:32443; GCN: s_endpgm444define amdgpu_kernel void @lds_atomic_sub_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {445 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4446 %result = atomicrmw sub ptr addrspace(3) %gep, i64 4 seq_cst447 ret void448}449 450; GCN-LABEL: {{^}}lds_atomic_sub1_noret_i64:451; SICIVI-DAG: s_mov_b32 m0452; GFX9-NOT: m0453 454; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}455; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}456; GCN: ds_sub_u64 {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]457; GCN: s_endpgm458define amdgpu_kernel void @lds_atomic_sub1_noret_i64(ptr addrspace(3) %ptr) nounwind {459 %result = atomicrmw sub ptr addrspace(3) %ptr, i64 1 seq_cst460 ret void461}462 463; GCN-LABEL: {{^}}lds_atomic_sub1_noret_i64_offset:464; SICIVI: s_mov_b32 m0465; GFX9-NOT: m0466 467; GCN: ds_sub_u64 {{.*}} offset:32468; GCN: s_endpgm469define amdgpu_kernel void @lds_atomic_sub1_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {470 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4471 %result = atomicrmw sub ptr addrspace(3) %gep, i64 1 seq_cst472 ret void473}474 475; GCN-LABEL: {{^}}lds_atomic_and_noret_i64:476; SICIVI: s_mov_b32 m0477; GFX9-NOT: m0478 479; GCN: ds_and_b64480; GCN: s_endpgm481define amdgpu_kernel void @lds_atomic_and_noret_i64(ptr addrspace(3) %ptr) nounwind {482 %result = atomicrmw and ptr addrspace(3) %ptr, i64 4 seq_cst483 ret void484}485 486; GCN-LABEL: {{^}}lds_atomic_and_noret_i64_offset:487; SICIVI: s_mov_b32 m0488; GFX9-NOT: m0489 490; GCN: ds_and_b64 {{.*}} offset:32491; GCN: s_endpgm492define amdgpu_kernel void @lds_atomic_and_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {493 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4494 %result = atomicrmw and ptr addrspace(3) %gep, i64 4 seq_cst495 ret void496}497 498; GCN-LABEL: {{^}}lds_atomic_or_noret_i64:499; SICIVI: s_mov_b32 m0500; GFX9-NOT: m0501 502; GCN: ds_or_b64503; GCN: s_endpgm504define amdgpu_kernel void @lds_atomic_or_noret_i64(ptr addrspace(3) %ptr) nounwind {505 %result = atomicrmw or ptr addrspace(3) %ptr, i64 4 seq_cst506 ret void507}508 509; GCN-LABEL: {{^}}lds_atomic_or_noret_i64_offset:510; SICIVI: s_mov_b32 m0511; GFX9-NOT: m0512 513; GCN: ds_or_b64 {{.*}} offset:32514; GCN: s_endpgm515define amdgpu_kernel void @lds_atomic_or_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {516 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4517 %result = atomicrmw or ptr addrspace(3) %gep, i64 4 seq_cst518 ret void519}520 521; GCN-LABEL: {{^}}lds_atomic_xor_noret_i64:522; SICIVI: s_mov_b32 m0523; GFX9-NOT: m0524 525; GCN: ds_xor_b64526; GCN: s_endpgm527define amdgpu_kernel void @lds_atomic_xor_noret_i64(ptr addrspace(3) %ptr) nounwind {528 %result = atomicrmw xor ptr addrspace(3) %ptr, i64 4 seq_cst529 ret void530}531 532; GCN-LABEL: {{^}}lds_atomic_xor_noret_i64_offset:533; SICIVI: s_mov_b32 m0534; GFX9-NOT: m0535 536; GCN: ds_xor_b64 {{.*}} offset:32537; GCN: s_endpgm538define amdgpu_kernel void @lds_atomic_xor_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {539 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4540 %result = atomicrmw xor ptr addrspace(3) %gep, i64 4 seq_cst541 ret void542}543 544; FIXME: There is no atomic nand instr545; XGCN-LABEL: {{^}}lds_atomic_nand_noret_i64:uction, so we somehow need to expand this.546; define amdgpu_kernel void @lds_atomic_nand_noret_i64(ptr addrspace(3) %ptr) nounwind {547; %result = atomicrmw nand ptr addrspace(3) %ptr, i32 4 seq_cst548; ret void549; }550 551; GCN-LABEL: {{^}}lds_atomic_min_noret_i64:552; SICIVI: s_mov_b32 m0553; GFX9-NOT: m0554 555; GCN: ds_min_i64556; GCN: s_endpgm557define amdgpu_kernel void @lds_atomic_min_noret_i64(ptr addrspace(3) %ptr) nounwind {558 %result = atomicrmw min ptr addrspace(3) %ptr, i64 4 seq_cst559 ret void560}561 562; GCN-LABEL: {{^}}lds_atomic_min_noret_i64_offset:563; SICIVI: s_mov_b32 m0564; GFX9-NOT: m0565 566; GCN: ds_min_i64 {{.*}} offset:32567; GCN: s_endpgm568define amdgpu_kernel void @lds_atomic_min_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {569 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4570 %result = atomicrmw min ptr addrspace(3) %gep, i64 4 seq_cst571 ret void572}573 574; GCN-LABEL: {{^}}lds_atomic_max_noret_i64:575; SICIVI: s_mov_b32 m0576; GFX9-NOT: m0577 578; GCN: ds_max_i64579; GCN: s_endpgm580define amdgpu_kernel void @lds_atomic_max_noret_i64(ptr addrspace(3) %ptr) nounwind {581 %result = atomicrmw max ptr addrspace(3) %ptr, i64 4 seq_cst582 ret void583}584 585; GCN-LABEL: {{^}}lds_atomic_max_noret_i64_offset:586; SICIVI: s_mov_b32 m0587; GFX9-NOT: m0588 589; GCN: ds_max_i64 {{.*}} offset:32590; GCN: s_endpgm591define amdgpu_kernel void @lds_atomic_max_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {592 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4593 %result = atomicrmw max ptr addrspace(3) %gep, i64 4 seq_cst594 ret void595}596 597; GCN-LABEL: {{^}}lds_atomic_umin_noret_i64:598; SICIVI: s_mov_b32 m0599; GFX9-NOT: m0600 601; GCN: ds_min_u64602; GCN: s_endpgm603define amdgpu_kernel void @lds_atomic_umin_noret_i64(ptr addrspace(3) %ptr) nounwind {604 %result = atomicrmw umin ptr addrspace(3) %ptr, i64 4 seq_cst605 ret void606}607 608; GCN-LABEL: {{^}}lds_atomic_umin_noret_i64_offset:609; SICIVI: s_mov_b32 m0610; GFX9-NOT: m0611 612; GCN: ds_min_u64 {{.*}} offset:32613; GCN: s_endpgm614define amdgpu_kernel void @lds_atomic_umin_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {615 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4616 %result = atomicrmw umin ptr addrspace(3) %gep, i64 4 seq_cst617 ret void618}619 620; GCN-LABEL: {{^}}lds_atomic_umax_noret_i64:621; SICIVI: s_mov_b32 m0622; GFX9-NOT: m0623 624; GCN: ds_max_u64625; GCN: s_endpgm626define amdgpu_kernel void @lds_atomic_umax_noret_i64(ptr addrspace(3) %ptr) nounwind {627 %result = atomicrmw umax ptr addrspace(3) %ptr, i64 4 seq_cst628 ret void629}630 631; GCN-LABEL: {{^}}lds_atomic_umax_noret_i64_offset:632; SICIVI: s_mov_b32 m0633; GFX9-NOT: m0634 635; GCN: ds_max_u64 {{.*}} offset:32636; GCN: s_endpgm637define amdgpu_kernel void @lds_atomic_umax_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {638 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4639 %result = atomicrmw umax ptr addrspace(3) %gep, i64 4 seq_cst640 ret void641}642 643; GCN-LABEL: {{^}}lds_atomic_inc_ret_i64:644; SICIVI: s_mov_b32 m0645; GFX9-NOT: m0646 647; GCN: ds_inc_rtn_u64648; GCN: s_endpgm649define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {650 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 4 seq_cst651 store i64 %result, ptr addrspace(1) %out, align 8652 ret void653}654 655; GCN-LABEL: {{^}}lds_atomic_inc_ret_i64_offset:656; SICIVI-DAG: s_mov_b32 m0657; GFX9-NOT: m0658 659; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb660; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c661; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9662; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0663; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]664; GCN: ds_inc_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:32665; GCN: buffer_store_dwordx2 [[RESULT]],666; GCN: s_endpgm667define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {668 %gep = getelementptr i64, ptr addrspace(3) %ptr, i64 4669 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 9 seq_cst670 store i64 %result, ptr addrspace(1) %out, align 8671 ret void672}673 674; GCN-LABEL: {{^}}lds_atomic_inc1_ret_i64:675; SICIVI-DAG: s_mov_b32 m0676; GFX9-NOT: m0677 678; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}679; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}680; GCN: ds_inc_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]681; GCN: buffer_store_dwordx2 [[RESULT]],682; GCN: s_endpgm683define amdgpu_kernel void @lds_atomic_inc1_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {684 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i64 1 seq_cst685 store i64 %result, ptr addrspace(1) %out, align 8686 ret void687}688 689; GCN-LABEL: {{^}}lds_atomic_inc1_ret_i64_offset:690; SICIVI: s_mov_b32 m0691; GFX9-NOT: m0692 693; GCN: ds_inc_rtn_u64 {{.*}} offset:32694; GCN: s_endpgm695define amdgpu_kernel void @lds_atomic_inc1_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {696 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4697 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i64 1 seq_cst698 store i64 %result, ptr addrspace(1) %out, align 8699 ret void700}701 702; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64:703; SICIVI: s_mov_b32 m0704; GFX9-NOT: m0705 706; GCN: ds_dec_rtn_u64707; GCN: s_endpgm708define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {709 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 4 seq_cst710 store i64 %result, ptr addrspace(1) %out, align 8711 ret void712}713 714; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64_offset:715; SICIVI-DAG: s_mov_b32 m0716; GFX9-NOT: m0717 718; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb719; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c720; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9721; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0722; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]723; GCN: ds_dec_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:32724; GCN: buffer_store_dwordx2 [[RESULT]],725; GCN: s_endpgm726define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {727 %gep = getelementptr i64, ptr addrspace(3) %ptr, i64 4728 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 9 seq_cst729 store i64 %result, ptr addrspace(1) %out, align 8730 ret void731}732 733; GCN-LABEL: {{^}}lds_atomic_dec1_ret_i64:734; SICIVI-DAG: s_mov_b32 m0735; GFX9-NOT: m0736 737; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}738; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}739; GCN: ds_dec_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v[[[LOVDATA]]:[[HIVDATA]]]740; GCN: buffer_store_dwordx2 [[RESULT]],741; GCN: s_endpgm742define amdgpu_kernel void @lds_atomic_dec1_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {743 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 1 seq_cst744 store i64 %result, ptr addrspace(1) %out, align 8745 ret void746}747 748; GCN-LABEL: {{^}}lds_atomic_dec1_ret_i64_offset:749; SICIVI: s_mov_b32 m0750; GFX9-NOT: m0751 752; GCN: ds_dec_rtn_u64 {{.*}} offset:32753; GCN: s_endpgm754define amdgpu_kernel void @lds_atomic_dec1_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {755 %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4756 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 1 seq_cst757 store i64 %result, ptr addrspace(1) %out, align 8758 ret void759}760