831 lines · plain
1; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SICIVI,FUNC %s2; RUN: llc -mtriple=amdgcn -mcpu=bonaire -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,FUNC %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,SICIVI,FUNC %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,FUNC %s5; RUN: llc -mtriple=r600 -mcpu=redwood -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=EG,FUNC %s6 7; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_i32:8; EG: LDS_WRXCHG_RET *9 10; SICIVI-DAG: s_mov_b32 m011; GFX9-NOT: m012 13; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],14; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 415; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]16; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]17; GCN: buffer_store_dword [[RESULT]],18; GCN: s_endpgm19define amdgpu_kernel void @lds_atomic_xchg_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {20 %result = atomicrmw xchg ptr addrspace(3) %ptr, i32 4 seq_cst21 store i32 %result, ptr addrspace(1) %out, align 422 ret void23}24 25; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_i32_offset:26; SICIVI: s_mov_b32 m027; GFX9-NOT: m028 29; EG: LDS_WRXCHG_RET *30; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:1631; GCN: s_endpgm32define amdgpu_kernel void @lds_atomic_xchg_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {33 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 434 %result = atomicrmw xchg ptr addrspace(3) %gep, i32 4 seq_cst35 store i32 %result, ptr addrspace(1) %out, align 436 ret void37}38 39; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_f32_offset:40; SICIVI: s_mov_b32 m041; GFX9-NOT: m042 43; EG: LDS_WRXCHG_RET *44; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:1645; GCN: s_endpgm46define amdgpu_kernel void @lds_atomic_xchg_ret_f32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {47 %gep = getelementptr float, ptr addrspace(3) %ptr, i32 448 %result = atomicrmw xchg ptr addrspace(3) %gep, float 4.0 seq_cst49 store float %result, ptr addrspace(1) %out, align 450 ret void51}52 53; XXX - Is it really necessary to load 4 into VGPR?54; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32:55; EG: LDS_ADD_RET *56 57; SICIVI-DAG: s_mov_b32 m058; GFX9-NOT: m059 60; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],61; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 462; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]63; GCN: ds_add_rtn_u32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]64; GCN: buffer_store_dword [[RESULT]],65; GCN: s_endpgm66define amdgpu_kernel void @lds_atomic_add_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {67 %result = atomicrmw add ptr addrspace(3) %ptr, i32 4 seq_cst68 store i32 %result, ptr addrspace(1) %out, align 469 ret void70}71 72; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32_offset:73; SICIVI: s_mov_b32 m074; GFX9-NOT: m075 76; EG: LDS_ADD_RET *77; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:1678; GCN: s_endpgm79define amdgpu_kernel void @lds_atomic_add_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {80 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 481 %result = atomicrmw add ptr addrspace(3) %gep, i32 4 seq_cst82 store i32 %result, ptr addrspace(1) %out, align 483 ret void84}85 86; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32_bad_si_offset:87; SICIVI: s_mov_b32 m088; GFX9-NOT: m089 90; EG: LDS_ADD_RET *91; SI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}92; CIVI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:1693; GCN: s_endpgm94define amdgpu_kernel void @lds_atomic_add_ret_i32_bad_si_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i32 %a, i32 %b) nounwind {95 %sub = sub i32 %a, %b96 %add = add i32 %sub, 497 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 %add98 %result = atomicrmw add ptr addrspace(3) %gep, i32 4 seq_cst99 store i32 %result, ptr addrspace(1) %out, align 4100 ret void101}102 103; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32:104; EG: LDS_ADD_RET *105 106; SICIVI-DAG: s_mov_b32 m0107; GFX9-NOT: m0108 109; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}110; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]]111; GCN: s_endpgm112define amdgpu_kernel void @lds_atomic_add1_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {113 %result = atomicrmw add ptr addrspace(3) %ptr, i32 1 seq_cst114 store i32 %result, ptr addrspace(1) %out, align 4115 ret void116}117 118; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32_offset:119; EG: LDS_ADD_RET *120 121; SICIVI-DAG: s_mov_b32 m0122; GFX9-NOT: m0123 124; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}125; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]] offset:16126; GCN: s_endpgm127define amdgpu_kernel void @lds_atomic_add1_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {128 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4129 %result = atomicrmw add ptr addrspace(3) %gep, i32 1 seq_cst130 store i32 %result, ptr addrspace(1) %out, align 4131 ret void132}133 134; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32_bad_si_offset:135; SICIVI: s_mov_b32 m0136; GFX9-NOT: m0137 138; EG: LDS_ADD_RET *139; SI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}140; CIVI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16141; GCN: s_endpgm142define amdgpu_kernel void @lds_atomic_add1_ret_i32_bad_si_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i32 %a, i32 %b) nounwind {143 %sub = sub i32 %a, %b144 %add = add i32 %sub, 4145 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 %add146 %result = atomicrmw add ptr addrspace(3) %gep, i32 1 seq_cst147 store i32 %result, ptr addrspace(1) %out, align 4148 ret void149}150 151; FUNC-LABEL: {{^}}lds_atomic_sub_ret_i32:152; EG: LDS_SUB_RET *153 154; SICIVI: s_mov_b32 m0155; GFX9-NOT: m0156 157; GCN: ds_sub_rtn_u32158; GCN: s_endpgm159define amdgpu_kernel void @lds_atomic_sub_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {160 %result = atomicrmw sub ptr addrspace(3) %ptr, i32 4 seq_cst161 store i32 %result, ptr addrspace(1) %out, align 4162 ret void163}164 165; FUNC-LABEL: {{^}}lds_atomic_sub_ret_i32_offset:166; EG: LDS_SUB_RET *167 168; SICIVI: s_mov_b32 m0169; GFX9-NOT: m0170 171; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16172; GCN: s_endpgm173define amdgpu_kernel void @lds_atomic_sub_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {174 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4175 %result = atomicrmw sub ptr addrspace(3) %gep, i32 4 seq_cst176 store i32 %result, ptr addrspace(1) %out, align 4177 ret void178}179 180; FUNC-LABEL: {{^}}lds_atomic_sub1_ret_i32:181; EG: LDS_SUB_RET *182 183; SICIVI-DAG: s_mov_b32 m0184; GFX9-NOT: m0185 186; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}187; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]]188; GCN: s_endpgm189define amdgpu_kernel void @lds_atomic_sub1_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {190 %result = atomicrmw sub ptr addrspace(3) %ptr, i32 1 seq_cst191 store i32 %result, ptr addrspace(1) %out, align 4192 ret void193}194 195; FUNC-LABEL: {{^}}lds_atomic_sub1_ret_i32_offset:196; EG: LDS_SUB_RET *197 198; SICIVI-DAG: s_mov_b32 m0199; GFX9-NOT: m0200 201; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}202; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]] offset:16203; GCN: s_endpgm204define amdgpu_kernel void @lds_atomic_sub1_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {205 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4206 %result = atomicrmw sub ptr addrspace(3) %gep, i32 1 seq_cst207 store i32 %result, ptr addrspace(1) %out, align 4208 ret void209}210 211; FUNC-LABEL: {{^}}lds_atomic_and_ret_i32:212; EG: LDS_AND_RET *213 214; SICIVI-DAG: s_mov_b32 m0215; GFX9-NOT: m0216 217; GCN: ds_and_rtn_b32218; GCN: s_endpgm219define amdgpu_kernel void @lds_atomic_and_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {220 %result = atomicrmw and ptr addrspace(3) %ptr, i32 4 seq_cst221 store i32 %result, ptr addrspace(1) %out, align 4222 ret void223}224 225; FUNC-LABEL: {{^}}lds_atomic_and_ret_i32_offset:226; SICIVI: s_mov_b32 m0227; GFX9-NOT: m0228 229; EG: LDS_AND_RET *230; GCN: ds_and_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16231; GCN: s_endpgm232define amdgpu_kernel void @lds_atomic_and_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {233 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4234 %result = atomicrmw and ptr addrspace(3) %gep, i32 4 seq_cst235 store i32 %result, ptr addrspace(1) %out, align 4236 ret void237}238 239; FUNC-LABEL: {{^}}lds_atomic_or_ret_i32:240; SICIVI: s_mov_b32 m0241; GFX9-NOT: m0242 243; EG: LDS_OR_RET *244; GCN: ds_or_rtn_b32245; GCN: s_endpgm246define amdgpu_kernel void @lds_atomic_or_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {247 %result = atomicrmw or ptr addrspace(3) %ptr, i32 4 seq_cst248 store i32 %result, ptr addrspace(1) %out, align 4249 ret void250}251 252; FUNC-LABEL: {{^}}lds_atomic_or_ret_i32_offset:253; SICIVI: s_mov_b32 m0254; GFX9-NOT: m0255 256; EG: LDS_OR_RET *257; GCN: ds_or_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16258; GCN: s_endpgm259define amdgpu_kernel void @lds_atomic_or_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {260 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4261 %result = atomicrmw or ptr addrspace(3) %gep, i32 4 seq_cst262 store i32 %result, ptr addrspace(1) %out, align 4263 ret void264}265 266; FUNC-LABEL: {{^}}lds_atomic_xor_ret_i32:267; SICIVI: s_mov_b32 m0268; GFX9-NOT: m0269 270; EG: LDS_XOR_RET *271; GCN: ds_xor_rtn_b32272; GCN: s_endpgm273define amdgpu_kernel void @lds_atomic_xor_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {274 %result = atomicrmw xor ptr addrspace(3) %ptr, i32 4 seq_cst275 store i32 %result, ptr addrspace(1) %out, align 4276 ret void277}278 279; FUNC-LABEL: {{^}}lds_atomic_xor_ret_i32_offset:280; SICIVI: s_mov_b32 m0281; GFX9-NOT: m0282 283; EG: LDS_XOR_RET *284; GCN: ds_xor_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16285; GCN: s_endpgm286define amdgpu_kernel void @lds_atomic_xor_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {287 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4288 %result = atomicrmw xor ptr addrspace(3) %gep, i32 4 seq_cst289 store i32 %result, ptr addrspace(1) %out, align 4290 ret void291}292 293; FIXME: There is no atomic nand instr294; XFUNC-LABEL: {{^}}lds_atomic_nand_ret_i32:uction, so we somehow need to expand this.295; define amdgpu_kernel void @lds_atomic_nand_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {296; %result = atomicrmw nand ptr addrspace(3) %ptr, i32 4 seq_cst297; store i32 %result, ptr addrspace(1) %out, align 4298; ret void299; }300 301; FUNC-LABEL: {{^}}lds_atomic_min_ret_i32:302; SICIVI: s_mov_b32 m0303; GFX9-NOT: m0304 305; EG: LDS_MIN_INT_RET *306; GCN: ds_min_rtn_i32307; GCN: s_endpgm308define amdgpu_kernel void @lds_atomic_min_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {309 %result = atomicrmw min ptr addrspace(3) %ptr, i32 4 seq_cst310 store i32 %result, ptr addrspace(1) %out, align 4311 ret void312}313 314; FUNC-LABEL: {{^}}lds_atomic_min_ret_i32_offset:315; SICIVI: s_mov_b32 m0316; GFX9-NOT: m0317 318; EG: LDS_MIN_INT_RET *319; GCN: ds_min_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16320; GCN: s_endpgm321define amdgpu_kernel void @lds_atomic_min_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {322 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4323 %result = atomicrmw min ptr addrspace(3) %gep, i32 4 seq_cst324 store i32 %result, ptr addrspace(1) %out, align 4325 ret void326}327 328; FUNC-LABEL: {{^}}lds_atomic_max_ret_i32:329; SICIVI: s_mov_b32 m0330; GFX9-NOT: m0331 332; EG: LDS_MAX_INT_RET *333; GCN: ds_max_rtn_i32334; GCN: s_endpgm335define amdgpu_kernel void @lds_atomic_max_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {336 %result = atomicrmw max ptr addrspace(3) %ptr, i32 4 seq_cst337 store i32 %result, ptr addrspace(1) %out, align 4338 ret void339}340 341; FUNC-LABEL: {{^}}lds_atomic_max_ret_i32_offset:342; SICIVI: s_mov_b32 m0343; GFX9-NOT: m0344 345; EG: LDS_MAX_INT_RET *346; GCN: ds_max_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16347; GCN: s_endpgm348define amdgpu_kernel void @lds_atomic_max_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {349 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4350 %result = atomicrmw max ptr addrspace(3) %gep, i32 4 seq_cst351 store i32 %result, ptr addrspace(1) %out, align 4352 ret void353}354 355; FUNC-LABEL: {{^}}lds_atomic_umin_ret_i32:356; SICIVI: s_mov_b32 m0357; GFX9-NOT: m0358 359; EG: LDS_MIN_UINT_RET *360; GCN: ds_min_rtn_u32361; GCN: s_endpgm362define amdgpu_kernel void @lds_atomic_umin_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {363 %result = atomicrmw umin ptr addrspace(3) %ptr, i32 4 seq_cst364 store i32 %result, ptr addrspace(1) %out, align 4365 ret void366}367 368; FUNC-LABEL: {{^}}lds_atomic_umin_ret_i32_offset:369; SICIVI: s_mov_b32 m0370; GFX9-NOT: m0371 372; EG: LDS_MIN_UINT_RET *373; GCN: ds_min_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16374; GCN: s_endpgm375define amdgpu_kernel void @lds_atomic_umin_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {376 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4377 %result = atomicrmw umin ptr addrspace(3) %gep, i32 4 seq_cst378 store i32 %result, ptr addrspace(1) %out, align 4379 ret void380}381 382; FUNC-LABEL: {{^}}lds_atomic_umax_ret_i32:383; SICIVI: s_mov_b32 m0384; GFX9-NOT: m0385 386; EG: LDS_MAX_UINT_RET *387; GCN: ds_max_rtn_u32388; GCN: s_endpgm389define amdgpu_kernel void @lds_atomic_umax_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {390 %result = atomicrmw umax ptr addrspace(3) %ptr, i32 4 seq_cst391 store i32 %result, ptr addrspace(1) %out, align 4392 ret void393}394 395; FUNC-LABEL: {{^}}lds_atomic_umax_ret_i32_offset:396; SICIVI: s_mov_b32 m0397; GFX9-NOT: m0398 399; EG: LDS_MAX_UINT_RET *400; GCN: ds_max_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16401; GCN: s_endpgm402define amdgpu_kernel void @lds_atomic_umax_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {403 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4404 %result = atomicrmw umax ptr addrspace(3) %gep, i32 4 seq_cst405 store i32 %result, ptr addrspace(1) %out, align 4406 ret void407}408 409; FUNC-LABEL: {{^}}lds_atomic_xchg_noret_i32:410; SICIVI-DAG: s_mov_b32 m0411; GFX9-NOT: m0412 413; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],414; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4415; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]416; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]417; GCN: s_endpgm418define amdgpu_kernel void @lds_atomic_xchg_noret_i32(ptr addrspace(3) %ptr) nounwind {419 %result = atomicrmw xchg ptr addrspace(3) %ptr, i32 4 seq_cst420 ret void421}422 423; FUNC-LABEL: {{^}}lds_atomic_xchg_noret_i32_offset:424; SICIVI: s_mov_b32 m0425; GFX9-NOT: m0426 427; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16428; GCN: s_endpgm429define amdgpu_kernel void @lds_atomic_xchg_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {430 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4431 %result = atomicrmw xchg ptr addrspace(3) %gep, i32 4 seq_cst432 ret void433}434 435; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32:436; SICIVI-DAG: s_mov_b32 m0437; GFX9-NOT: m0438 439; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],440; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4441; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]442; GCN: ds_add_u32 [[VPTR]], [[DATA]]443; GCN: s_endpgm444define amdgpu_kernel void @lds_atomic_add_noret_i32(ptr addrspace(3) %ptr) nounwind {445 %result = atomicrmw add ptr addrspace(3) %ptr, i32 4 seq_cst446 ret void447}448 449; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32_offset:450; SICIVI: s_mov_b32 m0451; GFX9-NOT: m0452 453; GCN: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16454; GCN: s_endpgm455define amdgpu_kernel void @lds_atomic_add_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {456 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4457 %result = atomicrmw add ptr addrspace(3) %gep, i32 4 seq_cst458 ret void459}460 461; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32_bad_si_offset462; SICIVI: s_mov_b32 m0463; GFX9-NOT: m0464 465; SI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}}466; CIVI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16467; GCN: s_endpgm468define amdgpu_kernel void @lds_atomic_add_noret_i32_bad_si_offset(ptr addrspace(3) %ptr, i32 %a, i32 %b) nounwind {469 %sub = sub i32 %a, %b470 %add = add i32 %sub, 4471 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 %add472 %result = atomicrmw add ptr addrspace(3) %gep, i32 4 seq_cst473 ret void474}475 476; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32:477; SICIVI-DAG: s_mov_b32 m0478; GFX9-NOT: m0479 480; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}481; GCN: ds_add_u32 v{{[0-9]+}}, [[ONE]]482; GCN: s_endpgm483define amdgpu_kernel void @lds_atomic_add1_noret_i32(ptr addrspace(3) %ptr) nounwind {484 %result = atomicrmw add ptr addrspace(3) %ptr, i32 1 seq_cst485 ret void486}487 488; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32_offset:489; SICIVI-DAG: s_mov_b32 m0490; GFX9-NOT: m0491 492; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}493; GCN: ds_add_u32 v{{[0-9]+}}, [[ONE]] offset:16494; GCN: s_endpgm495define amdgpu_kernel void @lds_atomic_add1_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {496 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4497 %result = atomicrmw add ptr addrspace(3) %gep, i32 1 seq_cst498 ret void499}500 501; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32_bad_si_offset:502; SICIVI: s_mov_b32 m0503; GFX9-NOT: m0504 505; SI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}}506; CIVI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16507; GCN: s_endpgm508define amdgpu_kernel void @lds_atomic_add1_noret_i32_bad_si_offset(ptr addrspace(3) %ptr, i32 %a, i32 %b) nounwind {509 %sub = sub i32 %a, %b510 %add = add i32 %sub, 4511 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 %add512 %result = atomicrmw add ptr addrspace(3) %gep, i32 1 seq_cst513 ret void514}515 516; FUNC-LABEL: {{^}}lds_atomic_sub_noret_i32:517; SICIVI: s_mov_b32 m0518; GFX9-NOT: m0519 520; GCN: ds_sub_u32521; GCN: s_endpgm522define amdgpu_kernel void @lds_atomic_sub_noret_i32(ptr addrspace(3) %ptr) nounwind {523 %result = atomicrmw sub ptr addrspace(3) %ptr, i32 4 seq_cst524 ret void525}526 527; FUNC-LABEL: {{^}}lds_atomic_sub_noret_i32_offset:528; SICIVI: s_mov_b32 m0529; GFX9-NOT: m0530 531; GCN: ds_sub_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16532; GCN: s_endpgm533define amdgpu_kernel void @lds_atomic_sub_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {534 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4535 %result = atomicrmw sub ptr addrspace(3) %gep, i32 4 seq_cst536 ret void537}538 539; FUNC-LABEL: {{^}}lds_atomic_sub1_noret_i32:540; SICIVI-DAG: s_mov_b32 m0541; GFX9-NOT: m0542 543; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}544; GCN: ds_sub_u32 v{{[0-9]+}}, [[ONE]]545; GCN: s_endpgm546define amdgpu_kernel void @lds_atomic_sub1_noret_i32(ptr addrspace(3) %ptr) nounwind {547 %result = atomicrmw sub ptr addrspace(3) %ptr, i32 1 seq_cst548 ret void549}550 551; FUNC-LABEL: {{^}}lds_atomic_sub1_noret_i32_offset:552; SICIVI-DAG: s_mov_b32 m0553; GFX9-NOT: m0554 555; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}556; GCN: ds_sub_u32 v{{[0-9]+}}, [[ONE]] offset:16557; GCN: s_endpgm558define amdgpu_kernel void @lds_atomic_sub1_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {559 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4560 %result = atomicrmw sub ptr addrspace(3) %gep, i32 1 seq_cst561 ret void562}563 564; FUNC-LABEL: {{^}}lds_atomic_and_noret_i32:565; SICIVI: s_mov_b32 m0566; GFX9-NOT: m0567 568; GCN: ds_and_b32569; GCN: s_endpgm570define amdgpu_kernel void @lds_atomic_and_noret_i32(ptr addrspace(3) %ptr) nounwind {571 %result = atomicrmw and ptr addrspace(3) %ptr, i32 4 seq_cst572 ret void573}574 575; FUNC-LABEL: {{^}}lds_atomic_and_noret_i32_offset:576; SICIVI: s_mov_b32 m0577; GFX9-NOT: m0578 579; GCN: ds_and_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16580; GCN: s_endpgm581define amdgpu_kernel void @lds_atomic_and_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {582 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4583 %result = atomicrmw and ptr addrspace(3) %gep, i32 4 seq_cst584 ret void585}586 587; FUNC-LABEL: {{^}}lds_atomic_or_noret_i32:588; SICIVI: s_mov_b32 m0589; GFX9-NOT: m0590 591; GCN: ds_or_b32592; GCN: s_endpgm593define amdgpu_kernel void @lds_atomic_or_noret_i32(ptr addrspace(3) %ptr) nounwind {594 %result = atomicrmw or ptr addrspace(3) %ptr, i32 4 seq_cst595 ret void596}597 598; FUNC-LABEL: {{^}}lds_atomic_or_noret_i32_offset:599; SICIVI: s_mov_b32 m0600; GFX9-NOT: m0601 602; GCN: ds_or_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16603; GCN: s_endpgm604define amdgpu_kernel void @lds_atomic_or_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {605 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4606 %result = atomicrmw or ptr addrspace(3) %gep, i32 4 seq_cst607 ret void608}609 610; FUNC-LABEL: {{^}}lds_atomic_xor_noret_i32:611; SICIVI: s_mov_b32 m0612; GFX9-NOT: m0613 614; GCN: ds_xor_b32615; GCN: s_endpgm616define amdgpu_kernel void @lds_atomic_xor_noret_i32(ptr addrspace(3) %ptr) nounwind {617 %result = atomicrmw xor ptr addrspace(3) %ptr, i32 4 seq_cst618 ret void619}620 621; FUNC-LABEL: {{^}}lds_atomic_xor_noret_i32_offset:622; SICIVI: s_mov_b32 m0623; GFX9-NOT: m0624 625; GCN: ds_xor_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16626; GCN: s_endpgm627define amdgpu_kernel void @lds_atomic_xor_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {628 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4629 %result = atomicrmw xor ptr addrspace(3) %gep, i32 4 seq_cst630 ret void631}632 633; FIXME: There is no atomic nand instr634; XFUNC-LABEL: {{^}}lds_atomic_nand_noret_i32:uction, so we somehow need to expand this.635; define amdgpu_kernel void @lds_atomic_nand_noret_i32(ptr addrspace(3) %ptr) nounwind {636; %result = atomicrmw nand ptr addrspace(3) %ptr, i32 4 seq_cst637; ret void638; }639 640; FUNC-LABEL: {{^}}lds_atomic_min_noret_i32:641; SICIVI: s_mov_b32 m0642; GFX9-NOT: m0643 644; GCN: ds_min_i32645; GCN: s_endpgm646define amdgpu_kernel void @lds_atomic_min_noret_i32(ptr addrspace(3) %ptr) nounwind {647 %result = atomicrmw min ptr addrspace(3) %ptr, i32 4 seq_cst648 ret void649}650 651; FUNC-LABEL: {{^}}lds_atomic_min_noret_i32_offset:652; SICIVI: s_mov_b32 m0653; GFX9-NOT: m0654 655; GCN: ds_min_i32 v{{[0-9]+}}, v{{[0-9]+}} offset:16656; GCN: s_endpgm657define amdgpu_kernel void @lds_atomic_min_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {658 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4659 %result = atomicrmw min ptr addrspace(3) %gep, i32 4 seq_cst660 ret void661}662 663; FUNC-LABEL: {{^}}lds_atomic_max_noret_i32:664; SICIVI: s_mov_b32 m0665; GFX9-NOT: m0666 667; GCN: ds_max_i32668; GCN: s_endpgm669define amdgpu_kernel void @lds_atomic_max_noret_i32(ptr addrspace(3) %ptr) nounwind {670 %result = atomicrmw max ptr addrspace(3) %ptr, i32 4 seq_cst671 ret void672}673 674; FUNC-LABEL: {{^}}lds_atomic_max_noret_i32_offset:675; SICIVI: s_mov_b32 m0676; GFX9-NOT: m0677 678; GCN: ds_max_i32 v{{[0-9]+}}, v{{[0-9]+}} offset:16679; GCN: s_endpgm680define amdgpu_kernel void @lds_atomic_max_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {681 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4682 %result = atomicrmw max ptr addrspace(3) %gep, i32 4 seq_cst683 ret void684}685 686; FUNC-LABEL: {{^}}lds_atomic_umin_noret_i32:687; SICIVI: s_mov_b32 m0688; GFX9-NOT: m0689 690; GCN: ds_min_u32691; GCN: s_endpgm692define amdgpu_kernel void @lds_atomic_umin_noret_i32(ptr addrspace(3) %ptr) nounwind {693 %result = atomicrmw umin ptr addrspace(3) %ptr, i32 4 seq_cst694 ret void695}696 697; FUNC-LABEL: {{^}}lds_atomic_umin_noret_i32_offset:698; SICIVI: s_mov_b32 m0699; GFX9-NOT: m0700 701; GCN: ds_min_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16702; GCN: s_endpgm703define amdgpu_kernel void @lds_atomic_umin_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {704 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4705 %result = atomicrmw umin ptr addrspace(3) %gep, i32 4 seq_cst706 ret void707}708 709; FUNC-LABEL: {{^}}lds_atomic_umax_noret_i32:710; SICIVI: s_mov_b32 m0711; GFX9-NOT: m0712 713; GCN: ds_max_u32714; GCN: s_endpgm715define amdgpu_kernel void @lds_atomic_umax_noret_i32(ptr addrspace(3) %ptr) nounwind {716 %result = atomicrmw umax ptr addrspace(3) %ptr, i32 4 seq_cst717 ret void718}719 720; FUNC-LABEL: {{^}}lds_atomic_umax_noret_i32_offset:721; SICIVI: s_mov_b32 m0722; GFX9-NOT: m0723 724; GCN: ds_max_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16725; GCN: s_endpgm726define amdgpu_kernel void @lds_atomic_umax_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {727 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4728 %result = atomicrmw umax ptr addrspace(3) %gep, i32 4 seq_cst729 ret void730}731 732; FUNC-LABEL: {{^}}lds_atomic_inc_ret_i32:733; SICIVI: s_mov_b32 m0734; GFX9-NOT: m0735 736; EG: LDS_CMPST737; GCN: ds_inc_rtn_u32738; GCN: s_endpgm739define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {740 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 4 seq_cst741 store i32 %result, ptr addrspace(1) %out, align 4742 ret void743}744 745; FUNC-LABEL: {{^}}lds_atomic_inc_ret_i32_offset:746; SICIVI: s_mov_b32 m0747; GFX9-NOT: m0748 749; EG: LDS_CMPST750; GCN: ds_inc_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16751; GCN: s_endpgm752define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {753 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4754 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 4 seq_cst755 store i32 %result, ptr addrspace(1) %out, align 4756 ret void757}758 759; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i32:760; SICIVI: s_mov_b32 m0761; GFX9-NOT: m0762 763; GCN: ds_inc_u32764; GCN: s_endpgm765define amdgpu_kernel void @lds_atomic_inc_noret_i32(ptr addrspace(3) %ptr) nounwind {766 %result = atomicrmw uinc_wrap ptr addrspace(3) %ptr, i32 4 seq_cst767 ret void768}769 770; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i32_offset:771; SICIVI: s_mov_b32 m0772; GFX9-NOT: m0773 774; GCN: ds_inc_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16775; GCN: s_endpgm776define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {777 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4778 %result = atomicrmw uinc_wrap ptr addrspace(3) %gep, i32 4 seq_cst779 ret void780}781 782; FUNC-LABEL: {{^}}lds_atomic_dec_ret_i32:783; SICIVI: s_mov_b32 m0784; GFX9-NOT: m0785 786; EG: LDS_CMPST787; GCN: ds_dec_rtn_u32788; GCN: s_endpgm789define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {790 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 4 seq_cst791 store i32 %result, ptr addrspace(1) %out, align 4792 ret void793}794 795; FUNC-LABEL: {{^}}lds_atomic_dec_ret_i32_offset:796; SICIVI: s_mov_b32 m0797; GFX9-NOT: m0798 799; EG: LDS_CMPST800; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16801; GCN: s_endpgm802define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) nounwind {803 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4804 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 4 seq_cst805 store i32 %result, ptr addrspace(1) %out, align 4806 ret void807}808 809; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i32:810; SICIVI: s_mov_b32 m0811; GFX9-NOT: m0812 813; GCN: ds_dec_u32814; GCN: s_endpgm815define amdgpu_kernel void @lds_atomic_dec_noret_i32(ptr addrspace(3) %ptr) nounwind {816 %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 4 seq_cst817 ret void818}819 820; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i32_offset:821; SICIVI: s_mov_b32 m0822; GFX9-NOT: m0823 824; GCN: ds_dec_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16825; GCN: s_endpgm826define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {827 %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4828 %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 4 seq_cst829 ret void830}831