206 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12-SDAG %s3; RUN: llc -mtriple=amdgcn -global-isel=1 -global-isel-abort=2 -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12-GISEL %s4 5define amdgpu_ps float @atomic_pk_add_f16_1d_v2(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {6; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v2:7; GFX12-SDAG: ; %bb.0: ; %main_body8; GFX12-SDAG-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN9; GFX12-SDAG-NEXT: s_wait_loadcnt 0x010; GFX12-SDAG-NEXT: ; return to shader part epilog11;12; GFX12-GISEL-LABEL: atomic_pk_add_f16_1d_v2:13; GFX12-GISEL: ; %bb.0: ; %main_body14; GFX12-GISEL-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN15; GFX12-GISEL-NEXT: s_wait_loadcnt 0x016; GFX12-GISEL-NEXT: ; return to shader part epilog17main_body:18 %out = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)19 %out_float = bitcast <2 x half> %out to float20 ret float %out_float21}22 23define amdgpu_ps float @atomic_pk_add_f16_1d_v2_nt(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {24; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v2_nt:25; GFX12-SDAG: ; %bb.0: ; %main_body26; GFX12-SDAG-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN27; GFX12-SDAG-NEXT: s_wait_loadcnt 0x028; GFX12-SDAG-NEXT: ; return to shader part epilog29;30; GFX12-GISEL-LABEL: atomic_pk_add_f16_1d_v2_nt:31; GFX12-GISEL: ; %bb.0: ; %main_body32; GFX12-GISEL-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT_RETURN33; GFX12-GISEL-NEXT: s_wait_loadcnt 0x034; GFX12-GISEL-NEXT: ; return to shader part epilog35main_body:36 %out = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)37 %out_float = bitcast <2 x half> %out to float38 ret float %out_float39}40 41define amdgpu_ps float @atomic_pk_add_f16_1d_v2_noret(<8 x i32> inreg %rsrc, <2 x half> %data, i32 %s) {42; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v2_noret:43; GFX12-SDAG: ; %bb.0: ; %main_body44; GFX12-SDAG-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D45; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.046; GFX12-SDAG-NEXT: ; return to shader part epilog47;48; GFX12-GISEL-LABEL: atomic_pk_add_f16_1d_v2_noret:49; GFX12-GISEL: ; %bb.0: ; %main_body50; GFX12-GISEL-NEXT: image_atomic_pk_add_f16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D51; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.052; GFX12-GISEL-NEXT: ; return to shader part epilog53main_body:54 %unused = call <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)55 ret float 1.056}57 58define amdgpu_ps float @atomic_pk_add_f16_1d_v4(<8 x i32> inreg %rsrc, <4 x half> %data, i32 %s) {59; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v4:60; GFX12-SDAG: ; %bb.0: ; %main_body61; GFX12-SDAG-NEXT: image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN62; GFX12-SDAG-NEXT: s_wait_loadcnt 0x063; GFX12-SDAG-NEXT: ; return to shader part epilog64;65; GFX12-GISEL-LABEL: atomic_pk_add_f16_1d_v4:66; GFX12-GISEL: ; %bb.0: ; %main_body67; GFX12-GISEL-NEXT: image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN68; GFX12-GISEL-NEXT: s_wait_loadcnt 0x069; GFX12-GISEL-NEXT: ; return to shader part epilog70main_body:71 %out = call <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)72 %out_v2f32 = bitcast <4 x half> %out to <2 x float>73 %out0 = extractelement <2 x float> %out_v2f32, i32 074 ret float %out075}76 77define amdgpu_ps float @atomic_pk_add_f16_1d_v4_noret(<8 x i32> inreg %rsrc, <4 x half> %data, i32 %s) {78; GFX12-SDAG-LABEL: atomic_pk_add_f16_1d_v4_noret:79; GFX12-SDAG: ; %bb.0: ; %main_body80; GFX12-SDAG-NEXT: image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D81; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.082; GFX12-SDAG-NEXT: ; return to shader part epilog83;84; GFX12-GISEL-LABEL: atomic_pk_add_f16_1d_v4_noret:85; GFX12-GISEL: ; %bb.0: ; %main_body86; GFX12-GISEL-NEXT: image_atomic_pk_add_f16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D87; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.088; GFX12-GISEL-NEXT: ; return to shader part epilog89main_body:90 %unused = call <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)91 ret float 1.092}93 94define amdgpu_ps float @atomic_pk_add_bf16_1d_v2(<8 x i32> inreg %rsrc, <2 x bfloat> %data, i32 %s) {95; GFX12-SDAG-LABEL: atomic_pk_add_bf16_1d_v2:96; GFX12-SDAG: ; %bb.0: ; %main_body97; GFX12-SDAG-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN98; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, 099; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0100; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0101; GFX12-SDAG-NEXT: flat_store_b32 v[1:2], v0102; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.0103; GFX12-SDAG-NEXT: s_wait_dscnt 0x0104; GFX12-SDAG-NEXT: ; return to shader part epilog105;106; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v2:107; GFX12-GISEL: ; %bb.0: ; %main_body108; GFX12-GISEL-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN109; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0110; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0111; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0112; GFX12-GISEL-NEXT: flat_store_b32 v[1:2], v0113; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.0114; GFX12-GISEL-NEXT: s_wait_dscnt 0x0115; GFX12-GISEL-NEXT: ; return to shader part epilog116main_body:117 %out = call <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)118 store <2 x bfloat> %out, ptr null119 ret float 1.0120}121 122define amdgpu_ps float @atomic_pk_add_bf16_1d_v2_noret(<8 x i32> inreg %rsrc, <2 x bfloat> %data, i32 %s) {123; GFX12-SDAG-LABEL: atomic_pk_add_bf16_1d_v2_noret:124; GFX12-SDAG: ; %bb.0: ; %main_body125; GFX12-SDAG-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D126; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.0127; GFX12-SDAG-NEXT: ; return to shader part epilog128;129; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v2_noret:130; GFX12-GISEL: ; %bb.0: ; %main_body131; GFX12-GISEL-NEXT: image_atomic_pk_add_bf16 v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D132; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.0133; GFX12-GISEL-NEXT: ; return to shader part epilog134main_body:135 %unused = call <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)136 ret float 1.0137}138 139define amdgpu_ps float @atomic_pk_add_bf16_1d_v4(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {140; GFX12-SDAG-LABEL: atomic_pk_add_bf16_1d_v4:141; GFX12-SDAG: ; %bb.0: ; %main_body142; GFX12-SDAG-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN143; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, 0144; GFX12-SDAG-NEXT: v_mov_b32_e32 v3, 0145; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0146; GFX12-SDAG-NEXT: flat_store_b64 v[2:3], v[0:1]147; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.0148; GFX12-SDAG-NEXT: s_wait_dscnt 0x0149; GFX12-SDAG-NEXT: ; return to shader part epilog150;151; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4:152; GFX12-GISEL: ; %bb.0: ; %main_body153; GFX12-GISEL-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_RETURN154; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, 0155; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0156; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0157; GFX12-GISEL-NEXT: flat_store_b64 v[2:3], v[0:1]158; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.0159; GFX12-GISEL-NEXT: s_wait_dscnt 0x0160; GFX12-GISEL-NEXT: ; return to shader part epilog161main_body:162 %out = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)163 store <4 x bfloat> %out, ptr null164 ret float 1.0165}166 167define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_noret(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {168; GFX12-SDAG-LABEL: atomic_pk_add_bf16_1d_v4_noret:169; GFX12-SDAG: ; %bb.0: ; %main_body170; GFX12-SDAG-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D171; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.0172; GFX12-SDAG-NEXT: ; return to shader part epilog173;174; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4_noret:175; GFX12-GISEL: ; %bb.0: ; %main_body176; GFX12-GISEL-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D177; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.0178; GFX12-GISEL-NEXT: ; return to shader part epilog179main_body:180 %unused = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)181 ret float 1.0182}183 184define amdgpu_ps float @atomic_pk_add_bf16_1d_v4_nt(<8 x i32> inreg %rsrc, <4 x bfloat> %data, i32 %s) {185; GFX12-SDAG-LABEL: atomic_pk_add_bf16_1d_v4_nt:186; GFX12-SDAG: ; %bb.0: ; %main_body187; GFX12-SDAG-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT188; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 1.0189; GFX12-SDAG-NEXT: ; return to shader part epilog190;191; GFX12-GISEL-LABEL: atomic_pk_add_bf16_1d_v4_nt:192; GFX12-GISEL: ; %bb.0: ; %main_body193; GFX12-GISEL-NEXT: image_atomic_pk_add_bf16 v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D th:TH_ATOMIC_NT194; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, 1.0195; GFX12-GISEL-NEXT: ; return to shader part epilog196main_body:197 %unused = call <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat> %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)198 ret float 1.0199}200 201 202declare <2 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v2f16.v2f16(<2 x half>, i32, <8 x i32>, i32, i32)203declare <4 x half> @llvm.amdgcn.image.atomic.pk.add.f16.1d.v4f16.v4f16(<4 x half>, i32, <8 x i32>, i32, i32)204declare <2 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v2bf16.v2bf16(<2 x bfloat>, i32, <8 x i32>, i32, i32)205declare <4 x bfloat> @llvm.amdgcn.image.atomic.pk.add.bf16.1d.v4bf16.v4bf16(<4 x bfloat>, i32, <8 x i32>, i32, i32)206