512 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx902 < %s | FileCheck -check-prefix=GCN %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=+enable-flat-scratch < %s | FileCheck -check-prefix=GCN-SCRATCH %s4 5define amdgpu_kernel void @vector_clause(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture %arg1) {6; GCN-LABEL: vector_clause:7; GCN: ; %bb.0: ; %bb8; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249; GCN-NEXT: v_lshlrev_b32_e32 v16, 4, v010; GCN-NEXT: s_waitcnt lgkmcnt(0)11; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]12; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:1613; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:3214; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:4815; GCN-NEXT: s_waitcnt vmcnt(3)16; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]17; GCN-NEXT: s_waitcnt vmcnt(3)18; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:1619; GCN-NEXT: s_waitcnt vmcnt(3)20; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:3221; GCN-NEXT: s_waitcnt vmcnt(3)22; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:4823; GCN-NEXT: s_endpgm24;25; GCN-SCRATCH-LABEL: vector_clause:26; GCN-SCRATCH: ; %bb.0: ; %bb27; GCN-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2428; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v16, 4, v029; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)30; GCN-SCRATCH-NEXT: s_clause 0x331; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]32; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:1633; GCN-SCRATCH-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:3234; GCN-SCRATCH-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:4835; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)36; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]37; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)38; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:1639; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)40; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:3241; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)42; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:4843; GCN-SCRATCH-NEXT: s_endpgm44bb:45 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()46 %tmp2 = zext i32 %tmp to i6447 %tmp3 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp248 %tmp4 = load <4 x i32>, ptr addrspace(1) %tmp3, align 1649 %tmp5 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp250 %tmp6 = add nuw nsw i64 %tmp2, 151 %tmp7 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp652 %tmp8 = load <4 x i32>, ptr addrspace(1) %tmp7, align 1653 %tmp9 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp654 %tmp10 = add nuw nsw i64 %tmp2, 255 %tmp11 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp1056 %tmp12 = load <4 x i32>, ptr addrspace(1) %tmp11, align 1657 %tmp13 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp1058 %tmp14 = add nuw nsw i64 %tmp2, 359 %tmp15 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp1460 %tmp16 = load <4 x i32>, ptr addrspace(1) %tmp15, align 1661 %tmp17 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp1462 store <4 x i32> %tmp4, ptr addrspace(1) %tmp5, align 1663 store <4 x i32> %tmp8, ptr addrspace(1) %tmp9, align 1664 store <4 x i32> %tmp12, ptr addrspace(1) %tmp13, align 1665 store <4 x i32> %tmp16, ptr addrspace(1) %tmp17, align 1666 ret void67}68 69define amdgpu_kernel void @scalar_clause(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture %arg1) {70; GCN-LABEL: scalar_clause:71; GCN: ; %bb.0: ; %bb72; GCN-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2473; GCN-NEXT: v_mov_b32_e32 v16, 074; GCN-NEXT: s_waitcnt lgkmcnt(0)75; GCN-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x076; GCN-NEXT: s_waitcnt lgkmcnt(0)77; GCN-NEXT: v_mov_b32_e32 v0, s078; GCN-NEXT: v_mov_b32_e32 v1, s179; GCN-NEXT: v_mov_b32_e32 v2, s280; GCN-NEXT: v_mov_b32_e32 v3, s381; GCN-NEXT: v_mov_b32_e32 v4, s482; GCN-NEXT: v_mov_b32_e32 v5, s583; GCN-NEXT: v_mov_b32_e32 v6, s684; GCN-NEXT: v_mov_b32_e32 v7, s785; GCN-NEXT: v_mov_b32_e32 v8, s886; GCN-NEXT: v_mov_b32_e32 v9, s987; GCN-NEXT: v_mov_b32_e32 v10, s1088; GCN-NEXT: v_mov_b32_e32 v11, s1189; GCN-NEXT: v_mov_b32_e32 v12, s1290; GCN-NEXT: v_mov_b32_e32 v13, s1391; GCN-NEXT: v_mov_b32_e32 v14, s1492; GCN-NEXT: v_mov_b32_e32 v15, s1593; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[18:19]94; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[18:19] offset:1695; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[18:19] offset:3296; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[18:19] offset:4897; GCN-NEXT: s_endpgm98;99; GCN-SCRATCH-LABEL: scalar_clause:100; GCN-SCRATCH: ; %bb.0: ; %bb101; GCN-SCRATCH-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24102; GCN-SCRATCH-NEXT: v_mov_b32_e32 v16, 0103; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)104; GCN-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0105; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)106; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s0107; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s1108; GCN-SCRATCH-NEXT: v_mov_b32_e32 v2, s2109; GCN-SCRATCH-NEXT: v_mov_b32_e32 v3, s3110; GCN-SCRATCH-NEXT: v_mov_b32_e32 v4, s4111; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, s5112; GCN-SCRATCH-NEXT: v_mov_b32_e32 v6, s6113; GCN-SCRATCH-NEXT: v_mov_b32_e32 v7, s7114; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, s8115; GCN-SCRATCH-NEXT: v_mov_b32_e32 v9, s9116; GCN-SCRATCH-NEXT: v_mov_b32_e32 v10, s10117; GCN-SCRATCH-NEXT: v_mov_b32_e32 v11, s11118; GCN-SCRATCH-NEXT: v_mov_b32_e32 v12, s12119; GCN-SCRATCH-NEXT: v_mov_b32_e32 v13, s13120; GCN-SCRATCH-NEXT: v_mov_b32_e32 v14, s14121; GCN-SCRATCH-NEXT: v_mov_b32_e32 v15, s15122; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[18:19]123; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[18:19] offset:16124; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[18:19] offset:32125; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[18:19] offset:48126; GCN-SCRATCH-NEXT: s_endpgm127bb:128 %tmp = load <4 x i32>, ptr addrspace(1) %arg, align 16129 %tmp2 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 1130 %tmp3 = load <4 x i32>, ptr addrspace(1) %tmp2, align 16131 %tmp4 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 1132 %tmp5 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 2133 %tmp6 = load <4 x i32>, ptr addrspace(1) %tmp5, align 16134 %tmp7 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 2135 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 3136 %tmp9 = load <4 x i32>, ptr addrspace(1) %tmp8, align 16137 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 3138 store <4 x i32> %tmp, ptr addrspace(1) %arg1, align 16139 store <4 x i32> %tmp3, ptr addrspace(1) %tmp4, align 16140 store <4 x i32> %tmp6, ptr addrspace(1) %tmp7, align 16141 store <4 x i32> %tmp9, ptr addrspace(1) %tmp10, align 16142 ret void143}144 145define void @mubuf_clause(ptr addrspace(5) noalias nocapture readonly %arg, ptr addrspace(5) noalias nocapture %arg1) {146; GCN-LABEL: mubuf_clause:147; GCN: ; %bb.0: ; %bb148; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)149; GCN-NEXT: v_lshlrev_b32_e32 v2, 4, v31150; GCN-NEXT: v_and_b32_e32 v2, 0x3ff0, v2151; GCN-NEXT: v_add_u32_e32 v0, v0, v2152; GCN-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:12153; GCN-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:8154; GCN-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4155; GCN-NEXT: buffer_load_dword v6, v0, s[0:3], 0 offen156; GCN-NEXT: buffer_load_dword v7, v0, s[0:3], 0 offen offset:28157; GCN-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen offset:24158; GCN-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen offset:20159; GCN-NEXT: buffer_load_dword v10, v0, s[0:3], 0 offen offset:16160; GCN-NEXT: buffer_load_dword v11, v0, s[0:3], 0 offen offset:44161; GCN-NEXT: buffer_load_dword v12, v0, s[0:3], 0 offen offset:40162; GCN-NEXT: buffer_load_dword v13, v0, s[0:3], 0 offen offset:36163; GCN-NEXT: buffer_load_dword v14, v0, s[0:3], 0 offen offset:32164; GCN-NEXT: buffer_load_dword v15, v0, s[0:3], 0 offen offset:60165; GCN-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen offset:56166; GCN-NEXT: buffer_load_dword v17, v0, s[0:3], 0 offen offset:52167; GCN-NEXT: s_nop 0168; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:48169; GCN-NEXT: v_add_u32_e32 v1, v1, v2170; GCN-NEXT: s_waitcnt vmcnt(15)171; GCN-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen offset:12172; GCN-NEXT: s_waitcnt vmcnt(15)173; GCN-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen offset:8174; GCN-NEXT: s_waitcnt vmcnt(15)175; GCN-NEXT: buffer_store_dword v5, v1, s[0:3], 0 offen offset:4176; GCN-NEXT: s_waitcnt vmcnt(15)177; GCN-NEXT: buffer_store_dword v6, v1, s[0:3], 0 offen178; GCN-NEXT: s_waitcnt vmcnt(15)179; GCN-NEXT: buffer_store_dword v7, v1, s[0:3], 0 offen offset:28180; GCN-NEXT: s_waitcnt vmcnt(15)181; GCN-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen offset:24182; GCN-NEXT: s_waitcnt vmcnt(15)183; GCN-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen offset:20184; GCN-NEXT: s_waitcnt vmcnt(15)185; GCN-NEXT: buffer_store_dword v10, v1, s[0:3], 0 offen offset:16186; GCN-NEXT: s_waitcnt vmcnt(15)187; GCN-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen offset:44188; GCN-NEXT: s_waitcnt vmcnt(15)189; GCN-NEXT: buffer_store_dword v12, v1, s[0:3], 0 offen offset:40190; GCN-NEXT: s_waitcnt vmcnt(15)191; GCN-NEXT: buffer_store_dword v13, v1, s[0:3], 0 offen offset:36192; GCN-NEXT: s_waitcnt vmcnt(15)193; GCN-NEXT: buffer_store_dword v14, v1, s[0:3], 0 offen offset:32194; GCN-NEXT: s_waitcnt vmcnt(15)195; GCN-NEXT: buffer_store_dword v15, v1, s[0:3], 0 offen offset:60196; GCN-NEXT: s_waitcnt vmcnt(15)197; GCN-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen offset:56198; GCN-NEXT: s_waitcnt vmcnt(15)199; GCN-NEXT: buffer_store_dword v17, v1, s[0:3], 0 offen offset:52200; GCN-NEXT: s_waitcnt vmcnt(15)201; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:48202; GCN-NEXT: s_waitcnt vmcnt(0)203; GCN-NEXT: s_setpc_b64 s[30:31]204;205; GCN-SCRATCH-LABEL: mubuf_clause:206; GCN-SCRATCH: ; %bb.0: ; %bb207; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)208; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v2, 4, v31209; GCN-SCRATCH-NEXT: v_and_b32_e32 v18, 0x3ff0, v2210; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v0, v18211; GCN-SCRATCH-NEXT: s_clause 0x3212; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[2:5], v0, off213; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[6:9], v0, off offset:16214; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[10:13], v0, off offset:32215; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[14:17], v0, off offset:48216; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v1, v18217; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)218; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[2:5], off219; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)220; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16221; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)222; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32223; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)224; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48225; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]226bb:227 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()228 %base = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg, i32 %tmp229 %tmp3 = load <4 x i32>, ptr addrspace(5) %base, align 16230 %base1 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg1, i32 %tmp231 %tmp6 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base, i32 1232 %tmp7 = load <4 x i32>, ptr addrspace(5) %tmp6, align 16233 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base1, i32 1234 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base, i32 2235 %tmp11 = load <4 x i32>, ptr addrspace(5) %tmp10, align 16236 %tmp12 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base1, i32 2237 %tmp14 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base, i32 3238 %tmp15 = load <4 x i32>, ptr addrspace(5) %tmp14, align 16239 %tmp16 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %base1, i32 3240 store <4 x i32> %tmp3, ptr addrspace(5) %base1, align 16241 store <4 x i32> %tmp7, ptr addrspace(5) %tmp8, align 16242 store <4 x i32> %tmp11, ptr addrspace(5) %tmp12, align 16243 store <4 x i32> %tmp15, ptr addrspace(5) %tmp16, align 16244 ret void245}246 247define amdgpu_kernel void @vector_clause_indirect(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture readnone %arg1, ptr addrspace(1) noalias nocapture %arg2) {248; GCN-LABEL: vector_clause_indirect:249; GCN: ; %bb.0: ; %bb250; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24251; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34252; GCN-NEXT: v_lshlrev_b32_e32 v0, 3, v0253; GCN-NEXT: s_waitcnt lgkmcnt(0)254; GCN-NEXT: global_load_dwordx2 v[8:9], v0, s[0:1]255; GCN-NEXT: s_waitcnt vmcnt(0)256; GCN-NEXT: global_load_dwordx4 v[0:3], v[8:9], off257; GCN-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16258; GCN-NEXT: v_mov_b32_e32 v8, 0259; GCN-NEXT: s_waitcnt vmcnt(1)260; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[2:3]261; GCN-NEXT: s_waitcnt vmcnt(1)262; GCN-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3] offset:16263; GCN-NEXT: s_endpgm264;265; GCN-SCRATCH-LABEL: vector_clause_indirect:266; GCN-SCRATCH: ; %bb.0: ; %bb267; GCN-SCRATCH-NEXT: s_clause 0x1268; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24269; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34270; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v0, 3, v0271; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, 0272; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)273; GCN-SCRATCH-NEXT: global_load_dwordx2 v[4:5], v0, s[0:1]274; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)275; GCN-SCRATCH-NEXT: s_clause 0x1276; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v[4:5], off277; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v[4:5], off offset:16278; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)279; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[0:3], s[2:3]280; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)281; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3] offset:16282; GCN-SCRATCH-NEXT: s_endpgm283bb:284 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()285 %tmp3 = zext i32 %tmp to i64286 %tmp4 = getelementptr inbounds i64, ptr addrspace(1) %arg, i64 %tmp3287 %tmp6 = load ptr addrspace(1), ptr addrspace(1) %tmp4, align 8288 %tmp7 = load <4 x i32>, ptr addrspace(1) %tmp6, align 16289 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %tmp6, i64 1290 %tmp9 = load <4 x i32>, ptr addrspace(1) %tmp8, align 16291 store <4 x i32> %tmp7, ptr addrspace(1) %arg2, align 16292 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg2, i64 1293 store <4 x i32> %tmp9, ptr addrspace(1) %tmp10, align 16294 ret void295}296 297define void @load_global_d16_hi(ptr addrspace(1) %in, i16 %reg, ptr addrspace(1) %out) {298; GCN-LABEL: load_global_d16_hi:299; GCN: ; %bb.0: ; %entry300; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)301; GCN-NEXT: v_mov_b32_e32 v5, v2302; GCN-NEXT: global_load_short_d16_hi v5, v[0:1], off303; GCN-NEXT: s_nop 0304; GCN-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64305; GCN-NEXT: s_waitcnt vmcnt(1)306; GCN-NEXT: global_store_dword v[3:4], v5, off307; GCN-NEXT: s_waitcnt vmcnt(1)308; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128309; GCN-NEXT: s_waitcnt vmcnt(0)310; GCN-NEXT: s_setpc_b64 s[30:31]311;312; GCN-SCRATCH-LABEL: load_global_d16_hi:313; GCN-SCRATCH: ; %bb.0: ; %entry314; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)315; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2316; GCN-SCRATCH-NEXT: s_clause 0x1317; GCN-SCRATCH-NEXT: global_load_short_d16_hi v5, v[0:1], off318; GCN-SCRATCH-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64319; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)320; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off321; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)322; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128323; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]324entry:325 %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 32326 %load1 = load i16, ptr addrspace(1) %in327 %load2 = load i16, ptr addrspace(1) %gep328 %build0 = insertelement <2 x i16> poison, i16 %reg, i32 0329 %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1330 store <2 x i16> %build1, ptr addrspace(1) %out331 %build2 = insertelement <2 x i16> poison, i16 %reg, i32 0332 %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1333 %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %out, i64 32334 store <2 x i16> %build3, ptr addrspace(1) %gep2335 ret void336}337 338define void @load_global_d16_lo(ptr addrspace(1) %in, i32 %reg, ptr addrspace(1) %out) {339; GCN-LABEL: load_global_d16_lo:340; GCN: ; %bb.0: ; %entry341; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)342; GCN-NEXT: v_mov_b32_e32 v5, v2343; GCN-NEXT: global_load_short_d16 v5, v[0:1], off344; GCN-NEXT: s_nop 0345; GCN-NEXT: global_load_short_d16 v2, v[0:1], off offset:64346; GCN-NEXT: s_waitcnt vmcnt(1)347; GCN-NEXT: global_store_dword v[3:4], v5, off348; GCN-NEXT: s_waitcnt vmcnt(1)349; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128350; GCN-NEXT: s_waitcnt vmcnt(0)351; GCN-NEXT: s_setpc_b64 s[30:31]352;353; GCN-SCRATCH-LABEL: load_global_d16_lo:354; GCN-SCRATCH: ; %bb.0: ; %entry355; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)356; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2357; GCN-SCRATCH-NEXT: s_clause 0x1358; GCN-SCRATCH-NEXT: global_load_short_d16 v5, v[0:1], off359; GCN-SCRATCH-NEXT: global_load_short_d16 v2, v[0:1], off offset:64360; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)361; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off362; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)363; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128364; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]365entry:366 %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 32367 %reg.bc1 = bitcast i32 %reg to <2 x i16>368 %reg.bc2 = bitcast i32 %reg to <2 x i16>369 %load1 = load i16, ptr addrspace(1) %in370 %load2 = load i16, ptr addrspace(1) %gep371 %build1 = insertelement <2 x i16> %reg.bc1, i16 %load1, i32 0372 %build2 = insertelement <2 x i16> %reg.bc2, i16 %load2, i32 0373 %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %out, i64 32374 store <2 x i16> %build1, ptr addrspace(1) %out375 store <2 x i16> %build2, ptr addrspace(1) %gep2376 ret void377}378 379define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc) {380; GCN-LABEL: flat_scratch_load:381; GCN: ; %bb.0: ; %.entry382; GCN-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0383; GCN-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1384; GCN-NEXT: s_mov_b32 s18, -1385; GCN-NEXT: s_mov_b32 s19, 0xe00000386; GCN-NEXT: s_add_u32 s16, s16, s11387; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24388; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44389; GCN-NEXT: s_addc_u32 s17, s17, 0390; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000391; GCN-NEXT: buffer_store_dword v0, off, s[16:19], 0392; GCN-NEXT: s_waitcnt vmcnt(0)393; GCN-NEXT: s_brev_b32 s0, 1394; GCN-NEXT: s_waitcnt lgkmcnt(0)395; GCN-NEXT: v_mov_b32_e32 v0, s6396; GCN-NEXT: s_mov_b32 s3, 0397; GCN-NEXT: s_mov_b32 s1, s0398; GCN-NEXT: s_mov_b32 s2, s0399; GCN-NEXT: v_mov_b32_e32 v1, s7400; GCN-NEXT: ;;#ASMSTART401; GCN-NEXT: ;;#ASMEND402; GCN-NEXT: buffer_load_dword v2, off, s[16:19], 0403; GCN-NEXT: s_nop 0404; GCN-NEXT: image_sample v0, v[0:1], s[8:15], s[0:3] dmask:0x1405; GCN-NEXT: s_waitcnt vmcnt(0)406; GCN-NEXT: v_add_f32_e32 v0, v2, v0407; GCN-NEXT: exp mrt0 v0, off, off, off done vm408; GCN-NEXT: s_endpgm409;410; GCN-SCRATCH-LABEL: flat_scratch_load:411; GCN-SCRATCH: ; %bb.0: ; %.entry412; GCN-SCRATCH-NEXT: s_add_u32 s8, s8, s13413; GCN-SCRATCH-NEXT: s_addc_u32 s9, s9, 0414; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8415; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9416; GCN-SCRATCH-NEXT: s_clause 0x1417; GCN-SCRATCH-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x24418; GCN-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x44419; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000420; GCN-SCRATCH-NEXT: s_brev_b32 s8, 1421; GCN-SCRATCH-NEXT: s_mov_b32 s9, s8422; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off423; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0424; GCN-SCRATCH-NEXT: ;;#ASMSTART425; GCN-SCRATCH-NEXT: ;;#ASMEND426; GCN-SCRATCH-NEXT: scratch_load_dword v2, off, off427; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)428; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s10429; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s11430; GCN-SCRATCH-NEXT: s_mov_b32 s11, 0431; GCN-SCRATCH-NEXT: s_mov_b32 s10, s8432; GCN-SCRATCH-NEXT: image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D433; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)434; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v2, v0435; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm436; GCN-SCRATCH-NEXT: s_endpgm437.entry:438 %alloca = alloca float, align 4, addrspace(5)439 store volatile float 5.5, ptr addrspace(5) %alloca440 call void asm sideeffect "", ""()441 ; There was a bug with flat scratch instructions that do not not use any address registers (ST mode).442 ; To trigger, the scratch_load has to be immediately before the image_sample in MIR.443 %load = load float, ptr addrspace(5) %alloca444 %val = call <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 9, float %a, float %b, <8 x i32> %desc, <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 0>, i1 false, i32 0, i32 0)445 %val0 = extractelement <2 x float> %val, i32 0446 %valadd = fadd float %load, %val0447 call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float %valadd, float poison, float poison, float poison, i1 true, i1 true)448 ret void449}450 451define amdgpu_kernel void @flat_scratch_load_clause(float %a, float %b, <8 x i32> %desc) {452; GCN-LABEL: flat_scratch_load_clause:453; GCN: ; %bb.0: ; %.entry454; GCN-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0455; GCN-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1456; GCN-NEXT: s_mov_b32 s14, -1457; GCN-NEXT: s_mov_b32 s15, 0xe00000458; GCN-NEXT: s_add_u32 s12, s12, s11459; GCN-NEXT: s_addc_u32 s13, s13, 0460; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000461; GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0462; GCN-NEXT: s_waitcnt vmcnt(0)463; GCN-NEXT: v_mov_b32_e32 v0, 0x40d00000464; GCN-NEXT: buffer_store_dword v0, off, s[12:15], 0 offset:4465; GCN-NEXT: s_waitcnt vmcnt(0)466; GCN-NEXT: ;;#ASMSTART467; GCN-NEXT: ;;#ASMEND468; GCN-NEXT: buffer_load_dword v0, off, s[12:15], 0469; GCN-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4470; GCN-NEXT: s_waitcnt vmcnt(0)471; GCN-NEXT: v_add_f32_e32 v0, v0, v1472; GCN-NEXT: exp mrt0 v0, off, off, off done vm473; GCN-NEXT: s_endpgm474;475; GCN-SCRATCH-LABEL: flat_scratch_load_clause:476; GCN-SCRATCH: ; %bb.0: ; %.entry477; GCN-SCRATCH-NEXT: s_add_u32 s8, s8, s13478; GCN-SCRATCH-NEXT: s_addc_u32 s9, s9, 0479; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8480; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9481; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000482; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, 0x40d00000483; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off484; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0485; GCN-SCRATCH-NEXT: scratch_store_dword off, v1, off offset:4486; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0487; GCN-SCRATCH-NEXT: ;;#ASMSTART488; GCN-SCRATCH-NEXT: ;;#ASMEND489; GCN-SCRATCH-NEXT: s_clause 0x1490; GCN-SCRATCH-NEXT: scratch_load_dword v0, off, off491; GCN-SCRATCH-NEXT: scratch_load_dword v1, off, off offset:4492; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)493; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v0, v1494; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm495; GCN-SCRATCH-NEXT: s_endpgm496.entry:497 %alloca = alloca float, align 4, addrspace(5)498 %alloca2 = alloca float, align 4, addrspace(5)499 store volatile float 5.5, ptr addrspace(5) %alloca500 store volatile float 6.5, ptr addrspace(5) %alloca2501 call void asm sideeffect "", ""()502 %load0 = load float, ptr addrspace(5) %alloca503 %load1 = load float, ptr addrspace(5) %alloca2504 %valadd = fadd float %load0, %load1505 call void @llvm.amdgcn.exp.f32(i32 0, i32 1, float %valadd, float poison, float poison, float poison, i1 true, i1 true)506 ret void507}508 509declare i32 @llvm.amdgcn.workitem.id.x()510declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)511declare <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)512