408 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -misched-cluster=0 < %s | FileCheck -check-prefix=GCN %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -misched-cluster=0 -amdgpu-igrouplp-exact-solver-max-branches=250000 < %s | FileCheck -check-prefix=EXACTCUTOFF %s4 5define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_cluster(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {6; GCN-LABEL: test_sched_group_barrier_pipeline_WMMA_cluster:7; GCN: ; %bb.0: ; %entry8; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x249; GCN-NEXT: v_lshlrev_b32_e32 v0, 5, v010; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)11; GCN-NEXT: v_and_b32_e32 v40, 0x7fe0, v012; GCN-NEXT: s_waitcnt lgkmcnt(0)13; GCN-NEXT: v_add_nc_u32_e32 v32, s0, v4014; GCN-NEXT: v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v4015; GCN-NEXT: ds_load_b128 v[4:7], v32 offset:1616; GCN-NEXT: ds_load_b128 v[12:15], v32 offset:206417; GCN-NEXT: ds_load_b128 v[20:23], v32 offset:616018; GCN-NEXT: ds_load_b128 v[28:31], v32 offset:1230419; GCN-NEXT: ds_load_b128 v[36:39], v32 offset:2049620; GCN-NEXT: ds_load_b128 v[0:3], v3221; GCN-NEXT: ds_load_b128 v[8:11], v32 offset:204822; GCN-NEXT: ds_load_b128 v[16:19], v32 offset:614423; GCN-NEXT: ds_load_b128 v[24:27], v32 offset:1228824; GCN-NEXT: ds_load_b128 v[32:35], v32 offset:2048025; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)26; GCN-NEXT: s_waitcnt lgkmcnt(4)27; GCN-NEXT: v_mov_b32_e32 v47, v728; GCN-NEXT: s_waitcnt lgkmcnt(3)29; GCN-NEXT: v_mov_b32_e32 v55, v1530; GCN-NEXT: s_waitcnt lgkmcnt(2)31; GCN-NEXT: v_mov_b32_e32 v63, v2332; GCN-NEXT: s_waitcnt lgkmcnt(1)33; GCN-NEXT: v_mov_b32_e32 v71, v3134; GCN-NEXT: s_waitcnt lgkmcnt(0)35; GCN-NEXT: v_dual_mov_b32 v79, v39 :: v_dual_mov_b32 v46, v636; GCN-NEXT: v_dual_mov_b32 v45, v5 :: v_dual_mov_b32 v44, v437; GCN-NEXT: v_dual_mov_b32 v43, v3 :: v_dual_mov_b32 v42, v238; GCN-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v40, v039; GCN-NEXT: v_dual_mov_b32 v54, v14 :: v_dual_mov_b32 v53, v1340; GCN-NEXT: v_dual_mov_b32 v52, v12 :: v_dual_mov_b32 v51, v1141; GCN-NEXT: v_dual_mov_b32 v50, v10 :: v_dual_mov_b32 v49, v942; GCN-NEXT: v_mov_b32_e32 v48, v843; GCN-NEXT: v_dual_mov_b32 v62, v22 :: v_dual_mov_b32 v61, v2144; GCN-NEXT: v_dual_mov_b32 v60, v20 :: v_dual_mov_b32 v59, v1945; GCN-NEXT: v_dual_mov_b32 v58, v18 :: v_dual_mov_b32 v57, v1746; GCN-NEXT: v_mov_b32_e32 v56, v1647; GCN-NEXT: v_dual_mov_b32 v70, v30 :: v_dual_mov_b32 v69, v2948; GCN-NEXT: v_dual_mov_b32 v68, v28 :: v_dual_mov_b32 v67, v2749; GCN-NEXT: v_dual_mov_b32 v66, v26 :: v_dual_mov_b32 v65, v2550; GCN-NEXT: v_mov_b32_e32 v64, v2451; GCN-NEXT: v_dual_mov_b32 v78, v38 :: v_dual_mov_b32 v77, v3752; GCN-NEXT: v_dual_mov_b32 v76, v36 :: v_dual_mov_b32 v75, v3553; GCN-NEXT: v_dual_mov_b32 v74, v34 :: v_dual_mov_b32 v73, v3354; GCN-NEXT: v_mov_b32_e32 v72, v3255; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[40:47], v[0:7], v[0:7], v[40:47]56; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[48:55], v[8:15], v[8:15], v[48:55]57; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[56:63], v[16:23], v[16:23], v[56:63]58; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[64:71], v[24:31], v[24:31], v[64:71]59; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[72:79], v[32:39], v[32:39], v[72:79]60; GCN-NEXT: ds_store_b128 v80, v[44:47] offset:1661; GCN-NEXT: ds_store_b128 v80, v[40:43]62; GCN-NEXT: ds_store_b128 v81, v[52:55] offset:206463; GCN-NEXT: ds_store_b128 v81, v[48:51] offset:204864; GCN-NEXT: ds_store_b128 v81, v[60:63] offset:411265; GCN-NEXT: ds_store_b128 v81, v[56:59] offset:409666; GCN-NEXT: ds_store_b128 v81, v[68:71] offset:616067; GCN-NEXT: ds_store_b128 v81, v[64:67] offset:614468; GCN-NEXT: ds_store_b128 v81, v[76:79] offset:820869; GCN-NEXT: ds_store_b128 v81, v[72:75] offset:819270; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)71; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(10) SyncID(0)72; GCN-NEXT: s_endpgm73;74; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_WMMA_cluster:75; EXACTCUTOFF: ; %bb.0: ; %entry76; EXACTCUTOFF-NEXT: s_load_b64 s[0:1], s[4:5], 0x2477; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 5, v078; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)79; EXACTCUTOFF-NEXT: v_and_b32_e32 v40, 0x7fe0, v080; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)81; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v32, s0, v4082; EXACTCUTOFF-NEXT: v_dual_mov_b32 v81, s1 :: v_dual_add_nc_u32 v80, s1, v4083; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v32 offset:1684; EXACTCUTOFF-NEXT: ds_load_b128 v[12:15], v32 offset:206485; EXACTCUTOFF-NEXT: ds_load_b128 v[20:23], v32 offset:616086; EXACTCUTOFF-NEXT: ds_load_b128 v[28:31], v32 offset:1230487; EXACTCUTOFF-NEXT: ds_load_b128 v[36:39], v32 offset:2049688; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v3289; EXACTCUTOFF-NEXT: ds_load_b128 v[8:11], v32 offset:204890; EXACTCUTOFF-NEXT: ds_load_b128 v[16:19], v32 offset:614491; EXACTCUTOFF-NEXT: ds_load_b128 v[24:27], v32 offset:1228892; EXACTCUTOFF-NEXT: ds_load_b128 v[32:35], v32 offset:2048093; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(10) SyncID(0)94; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(4)95; EXACTCUTOFF-NEXT: v_mov_b32_e32 v47, v796; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(3)97; EXACTCUTOFF-NEXT: v_mov_b32_e32 v55, v1598; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(2)99; EXACTCUTOFF-NEXT: v_mov_b32_e32 v63, v23100; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1)101; EXACTCUTOFF-NEXT: v_mov_b32_e32 v71, v31102; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)103; EXACTCUTOFF-NEXT: v_dual_mov_b32 v79, v39 :: v_dual_mov_b32 v46, v6104; EXACTCUTOFF-NEXT: v_dual_mov_b32 v45, v5 :: v_dual_mov_b32 v44, v4105; EXACTCUTOFF-NEXT: v_dual_mov_b32 v43, v3 :: v_dual_mov_b32 v42, v2106; EXACTCUTOFF-NEXT: v_dual_mov_b32 v41, v1 :: v_dual_mov_b32 v40, v0107; EXACTCUTOFF-NEXT: v_dual_mov_b32 v54, v14 :: v_dual_mov_b32 v53, v13108; EXACTCUTOFF-NEXT: v_dual_mov_b32 v52, v12 :: v_dual_mov_b32 v51, v11109; EXACTCUTOFF-NEXT: v_dual_mov_b32 v50, v10 :: v_dual_mov_b32 v49, v9110; EXACTCUTOFF-NEXT: v_mov_b32_e32 v48, v8111; EXACTCUTOFF-NEXT: v_dual_mov_b32 v62, v22 :: v_dual_mov_b32 v61, v21112; EXACTCUTOFF-NEXT: v_dual_mov_b32 v60, v20 :: v_dual_mov_b32 v59, v19113; EXACTCUTOFF-NEXT: v_dual_mov_b32 v58, v18 :: v_dual_mov_b32 v57, v17114; EXACTCUTOFF-NEXT: v_mov_b32_e32 v56, v16115; EXACTCUTOFF-NEXT: v_dual_mov_b32 v70, v30 :: v_dual_mov_b32 v69, v29116; EXACTCUTOFF-NEXT: v_dual_mov_b32 v68, v28 :: v_dual_mov_b32 v67, v27117; EXACTCUTOFF-NEXT: v_dual_mov_b32 v66, v26 :: v_dual_mov_b32 v65, v25118; EXACTCUTOFF-NEXT: v_mov_b32_e32 v64, v24119; EXACTCUTOFF-NEXT: v_dual_mov_b32 v78, v38 :: v_dual_mov_b32 v77, v37120; EXACTCUTOFF-NEXT: v_dual_mov_b32 v76, v36 :: v_dual_mov_b32 v75, v35121; EXACTCUTOFF-NEXT: v_dual_mov_b32 v74, v34 :: v_dual_mov_b32 v73, v33122; EXACTCUTOFF-NEXT: v_mov_b32_e32 v72, v32123; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[40:47], v[0:7], v[0:7], v[40:47]124; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[48:55], v[8:15], v[8:15], v[48:55]125; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[56:63], v[16:23], v[16:23], v[56:63]126; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[64:71], v[24:31], v[24:31], v[64:71]127; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[72:79], v[32:39], v[32:39], v[72:79]128; EXACTCUTOFF-NEXT: ds_store_b128 v80, v[44:47] offset:16129; EXACTCUTOFF-NEXT: ds_store_b128 v80, v[40:43]130; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[52:55] offset:2064131; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[48:51] offset:2048132; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[60:63] offset:4112133; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[56:59] offset:4096134; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[68:71] offset:6160135; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[64:67] offset:6144136; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[76:79] offset:8208137; EXACTCUTOFF-NEXT: ds_store_b128 v81, v[72:75] offset:8192138; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0)139; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(10) SyncID(0)140; EXACTCUTOFF-NEXT: s_endpgm141entry:142 %idx = call i32 @llvm.amdgcn.workitem.id.x()143 %load.0.addr = getelementptr <16 x half>, ptr addrspace(3) %in, i32 %idx144 %load.0 = load <16 x half>, ptr addrspace(3) %load.0.addr145 %load.1.addr = getelementptr <16 x half>, ptr addrspace(3) %load.0.addr, i32 64146 %load.1 = load <16 x half>, ptr addrspace(3) %load.1.addr147 %load.2.addr = getelementptr <16 x half>, ptr addrspace(3) %load.1.addr, i32 128148 %load.2 = load <16 x half>, ptr addrspace(3) %load.2.addr149 %load.3.addr = getelementptr <16 x half>, ptr addrspace(3) %load.2.addr, i32 192150 %load.3 = load <16 x half>, ptr addrspace(3) %load.3.addr151 %load.4.addr = getelementptr <16 x half>, ptr addrspace(3) %load.3.addr, i32 256152 %load.4 = load <16 x half>, ptr addrspace(3) %load.4.addr153 %mai.0 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.0, <16 x half> %load.0, <16 x half> %load.0, i1 0)154 %mai.1 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.1, <16 x half> %load.1, <16 x half> %load.1, i1 0)155 %mai.2 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.2, <16 x half> %load.2, <16 x half> %load.2, i1 0)156 %mai.3 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.3, <16 x half> %load.3, <16 x half> %load.3, i1 0)157 %mai.4 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.4, <16 x half> %load.4, <16 x half> %load.4, i1 0)158 %store.0.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 %idx159 store <16 x half> %mai.0, ptr addrspace(3) %store.0.addr160 %store.1.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 64161 store <16 x half> %mai.1, ptr addrspace(3) %store.1.addr162 %store.2.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 128163 store <16 x half> %mai.2, ptr addrspace(3) %store.2.addr164 %store.3.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 192165 store <16 x half> %mai.3, ptr addrspace(3) %store.3.addr166 %store.4.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 256167 store <16 x half> %mai.4, ptr addrspace(3) %store.4.addr168 ; 10 DS read169 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 10, i32 0)170 ; 5 WMMA171 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 5, i32 0)172 ; 10 DS write173 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 10, i32 0)174 ret void175}176 177define amdgpu_kernel void @test_sched_group_barrier_pipeline_WMMA_interleave(ptr addrspace(3) noalias %in, ptr addrspace(3) noalias %out) #0 {178; GCN-LABEL: test_sched_group_barrier_pipeline_WMMA_interleave:179; GCN: ; %bb.0: ; %entry180; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24181; GCN-NEXT: v_lshlrev_b32_e32 v0, 5, v0182; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)183; GCN-NEXT: v_and_b32_e32 v16, 0x7fe0, v0184; GCN-NEXT: s_waitcnt lgkmcnt(0)185; GCN-NEXT: v_add_nc_u32_e32 v17, s0, v16186; GCN-NEXT: v_add_nc_u32_e32 v16, s1, v16187; GCN-NEXT: ds_load_b128 v[4:7], v17 offset:16188; GCN-NEXT: ds_load_b128 v[0:3], v17189; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)190; GCN-NEXT: s_waitcnt lgkmcnt(0)191; GCN-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6192; GCN-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4193; GCN-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2194; GCN-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0195; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)196; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]197; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)198; GCN-NEXT: ds_store_b128 v16, v[12:15] offset:16199; GCN-NEXT: ds_store_b128 v16, v[8:11]200; GCN-NEXT: ds_load_b128 v[4:7], v17 offset:2064201; GCN-NEXT: ds_load_b128 v[0:3], v17 offset:2048202; GCN-NEXT: v_mov_b32_e32 v16, s1203; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)204; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)205; GCN-NEXT: s_waitcnt lgkmcnt(0)206; GCN-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6207; GCN-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4208; GCN-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2209; GCN-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0210; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)211; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]212; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)213; GCN-NEXT: ds_store_b128 v16, v[12:15] offset:2064214; GCN-NEXT: ds_store_b128 v16, v[8:11] offset:2048215; GCN-NEXT: ds_load_b128 v[4:7], v17 offset:6160216; GCN-NEXT: ds_load_b128 v[0:3], v17 offset:6144217; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)218; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)219; GCN-NEXT: s_waitcnt lgkmcnt(0)220; GCN-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6221; GCN-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4222; GCN-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2223; GCN-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0224; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)225; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]226; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)227; GCN-NEXT: ds_store_b128 v16, v[12:15] offset:4112228; GCN-NEXT: ds_store_b128 v16, v[8:11] offset:4096229; GCN-NEXT: ds_load_b128 v[4:7], v17 offset:12304230; GCN-NEXT: ds_load_b128 v[0:3], v17 offset:12288231; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)232; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)233; GCN-NEXT: s_waitcnt lgkmcnt(0)234; GCN-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6235; GCN-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4236; GCN-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2237; GCN-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0238; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)239; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]240; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)241; GCN-NEXT: ds_store_b128 v16, v[12:15] offset:6160242; GCN-NEXT: ds_store_b128 v16, v[8:11] offset:6144243; GCN-NEXT: ds_load_b128 v[4:7], v17 offset:20496244; GCN-NEXT: ds_load_b128 v[0:3], v17 offset:20480245; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)246; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)247; GCN-NEXT: s_waitcnt lgkmcnt(0)248; GCN-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6249; GCN-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4250; GCN-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2251; GCN-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0252; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)253; GCN-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]254; GCN-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)255; GCN-NEXT: ds_store_b128 v16, v[12:15] offset:8208256; GCN-NEXT: ds_store_b128 v16, v[8:11] offset:8192257; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)258; GCN-NEXT: s_endpgm259;260; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_WMMA_interleave:261; EXACTCUTOFF: ; %bb.0: ; %entry262; EXACTCUTOFF-NEXT: s_load_b64 s[0:1], s[4:5], 0x24263; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 5, v0264; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)265; EXACTCUTOFF-NEXT: v_and_b32_e32 v16, 0x7fe0, v0266; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)267; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v17, s0, v16268; EXACTCUTOFF-NEXT: v_add_nc_u32_e32 v16, s1, v16269; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v17 offset:16270; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v17271; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)272; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)273; EXACTCUTOFF-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6274; EXACTCUTOFF-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4275; EXACTCUTOFF-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2276; EXACTCUTOFF-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0277; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1)278; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]279; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)280; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[12:15] offset:16281; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[8:11]282; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v17 offset:2064283; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v17 offset:2048284; EXACTCUTOFF-NEXT: v_mov_b32_e32 v16, s1285; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)286; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)287; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)288; EXACTCUTOFF-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6289; EXACTCUTOFF-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4290; EXACTCUTOFF-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2291; EXACTCUTOFF-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0292; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1)293; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]294; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)295; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[12:15] offset:2064296; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[8:11] offset:2048297; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v17 offset:6160298; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v17 offset:6144299; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)300; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)301; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)302; EXACTCUTOFF-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6303; EXACTCUTOFF-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4304; EXACTCUTOFF-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2305; EXACTCUTOFF-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0306; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1)307; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]308; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)309; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[12:15] offset:4112310; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[8:11] offset:4096311; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v17 offset:12304312; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v17 offset:12288313; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)314; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)315; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)316; EXACTCUTOFF-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6317; EXACTCUTOFF-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4318; EXACTCUTOFF-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2319; EXACTCUTOFF-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0320; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1)321; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]322; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)323; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[12:15] offset:6160324; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[8:11] offset:6144325; EXACTCUTOFF-NEXT: ds_load_b128 v[4:7], v17 offset:20496326; EXACTCUTOFF-NEXT: ds_load_b128 v[0:3], v17 offset:20480327; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)328; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(2) SyncID(0)329; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)330; EXACTCUTOFF-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6331; EXACTCUTOFF-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4332; EXACTCUTOFF-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2333; EXACTCUTOFF-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0334; EXACTCUTOFF-NEXT: s_delay_alu instid0(VALU_DEP_1)335; EXACTCUTOFF-NEXT: v_wmma_f16_16x16x16_f16 v[8:15], v[0:7], v[0:7], v[8:15]336; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0)337; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[12:15] offset:8208338; EXACTCUTOFF-NEXT: ds_store_b128 v16, v[8:11] offset:8192339; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000200) size(2) SyncID(0)340; EXACTCUTOFF-NEXT: s_endpgm341entry:342 %idx = call i32 @llvm.amdgcn.workitem.id.x()343 %load.0.addr = getelementptr <16 x half>, ptr addrspace(3) %in, i32 %idx344 %load.0 = load <16 x half>, ptr addrspace(3) %load.0.addr345 %load.1.addr = getelementptr <16 x half>, ptr addrspace(3) %load.0.addr, i32 64346 %load.1 = load <16 x half>, ptr addrspace(3) %load.1.addr347 %load.2.addr = getelementptr <16 x half>, ptr addrspace(3) %load.1.addr, i32 128348 %load.2 = load <16 x half>, ptr addrspace(3) %load.2.addr349 %load.3.addr = getelementptr <16 x half>, ptr addrspace(3) %load.2.addr, i32 192350 %load.3 = load <16 x half>, ptr addrspace(3) %load.3.addr351 %load.4.addr = getelementptr <16 x half>, ptr addrspace(3) %load.3.addr, i32 256352 %load.4 = load <16 x half>, ptr addrspace(3) %load.4.addr353 %mai.0 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.0, <16 x half> %load.0, <16 x half> %load.0, i1 0)354 %mai.1 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.1, <16 x half> %load.1, <16 x half> %load.1, i1 0)355 %mai.2 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.2, <16 x half> %load.2, <16 x half> %load.2, i1 0)356 %mai.3 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.3, <16 x half> %load.3, <16 x half> %load.3, i1 0)357 %mai.4 = call <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half> %load.4, <16 x half> %load.4, <16 x half> %load.4, i1 0)358 %store.0.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 %idx359 store <16 x half> %mai.0, ptr addrspace(3) %store.0.addr360 %store.1.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 64361 store <16 x half> %mai.1, ptr addrspace(3) %store.1.addr362 %store.2.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 128363 store <16 x half> %mai.2, ptr addrspace(3) %store.2.addr364 %store.3.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 192365 store <16 x half> %mai.3, ptr addrspace(3) %store.3.addr366 %store.4.addr = getelementptr <16 x half>, ptr addrspace(3) %out, i32 256367 store <16 x half> %mai.4, ptr addrspace(3) %store.4.addr368 ; 2 DS read369 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 2, i32 0)370 ; 1 WMMA371 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)372 ; 2 DS write373 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 2, i32 0)374 ; 2 DS read375 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 2, i32 0)376 ; 1 WMMA377 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)378 ; 2 DS write379 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 2, i32 0)380 ; 2 DS read381 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 2, i32 0)382 ; 1 WMMA383 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)384 ; 2 DS write385 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 2, i32 0)386 ; 2 DS read387 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 2, i32 0)388 ; 1 WMMA389 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)390 ; 2 DS write391 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 2, i32 0)392 ; 2 DS read393 call void @llvm.amdgcn.sched.group.barrier(i32 256, i32 2, i32 0)394 ; 1 WMMA395 call void @llvm.amdgcn.sched.group.barrier(i32 8, i32 1, i32 0)396 ; 2 DS write397 call void @llvm.amdgcn.sched.group.barrier(i32 512, i32 2, i32 0)398 ret void399}400 401declare i32 @llvm.amdgcn.workitem.id.x() #2402declare void @llvm.amdgcn.sched.group.barrier(i32, i32, i32) #1403declare <16 x half> @llvm.amdgcn.wmma.f16.16x16x16.f16(<16 x half>, <16 x half> , <16 x half>, i1 immarg) #1404 405attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,32" }406attributes #1 = { nounwind }407attributes #2 = { nounwind readnone speculatable }408