737 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX12503; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -global-isel -global-isel-abort=2 < %s | FileCheck %s --check-prefix=GISEL4 5define amdgpu_ps void @test_swmmac_f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {6; GFX1250-LABEL: test_swmmac_f32_16x16x64_bf16:7; GFX1250: ; %bb.0: ; %bb8; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19; GFX1250-NEXT: global_load_b32 v32, v[32:33], off10; GFX1250-NEXT: s_wait_loadcnt 0x011; GFX1250-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 index_key:112; GFX1250-NEXT: s_clause 0x113; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:1614; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off15; GFX1250-NEXT: s_endpgm16;17; GISEL-LABEL: test_swmmac_f32_16x16x64_bf16:18; GISEL: ; %bb.0: ; %bb19; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 120; GISEL-NEXT: global_load_b32 v32, v[32:33], off21; GISEL-NEXT: s_wait_loadcnt 0x022; GISEL-NEXT: v_swmmac_f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 index_key:123; GISEL-NEXT: s_clause 0x124; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:1625; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off26; GISEL-NEXT: s_endpgm27bb:28 %IndexVecPacked = load i32, ptr addrspace(1) %IndexVecPtr, align 429 %IndexVec = bitcast i32 %IndexVecPacked to <2 x i16>30 %Index = extractelement <2 x i16> %IndexVec, i32 131 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)32 store <8 x float> %res, ptr addrspace(1) %out33 ret void34}35 36define amdgpu_ps void @test_swmmac_bf16_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x bfloat> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {37; GFX1250-LABEL: test_swmmac_bf16_16x16x64_bf16:38; GFX1250: ; %bb.0: ; %bb39; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 140; GFX1250-NEXT: global_load_b32 v28, v[28:29], off41; GFX1250-NEXT: s_wait_loadcnt 0x042; GFX1250-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 index_key:143; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off44; GFX1250-NEXT: s_endpgm45;46; GISEL-LABEL: test_swmmac_bf16_16x16x64_bf16:47; GISEL: ; %bb.0: ; %bb48; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 149; GISEL-NEXT: global_load_b32 v28, v[28:29], off50; GISEL-NEXT: s_wait_loadcnt 0x051; GISEL-NEXT: v_swmmac_bf16_16x16x64_bf16 v[24:27], v[0:7], v[8:23], v28 index_key:152; GISEL-NEXT: global_store_b128 v[30:31], v[24:27], off53; GISEL-NEXT: s_endpgm54bb:55 %IndexVecPacked = load i32, ptr addrspace(1) %IndexVecPtr, align 456 %IndexVec = bitcast i32 %IndexVecPacked to <2 x i16>57 %Index = extractelement <2 x i16> %IndexVec, i32 158 %res = call <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x bfloat> %C, i16 %Index, i1 false, i1 false)59 store <8 x bfloat> %res, ptr addrspace(1) %out60 ret void61}62 63define amdgpu_ps void @test_swmmac_bf16f32_16x16x64_bf16(<16 x bfloat> %A, <32 x bfloat> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {64; GFX1250-LABEL: test_swmmac_bf16f32_16x16x64_bf16:65; GFX1250: ; %bb.0: ; %bb66; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 167; GFX1250-NEXT: global_load_b32 v32, v[32:33], off68; GFX1250-NEXT: s_wait_loadcnt 0x069; GFX1250-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 index_key:170; GFX1250-NEXT: s_clause 0x171; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:1672; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off73; GFX1250-NEXT: s_endpgm74;75; GISEL-LABEL: test_swmmac_bf16f32_16x16x64_bf16:76; GISEL: ; %bb.0: ; %bb77; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 178; GISEL-NEXT: global_load_b32 v32, v[32:33], off79; GISEL-NEXT: s_wait_loadcnt 0x080; GISEL-NEXT: v_swmmac_bf16f32_16x16x64_bf16 v[24:31], v[0:7], v[8:23], v32 index_key:181; GISEL-NEXT: s_clause 0x182; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:1683; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off84; GISEL-NEXT: s_endpgm85bb:86 %IndexVecPacked = load i32, ptr addrspace(1) %IndexVecPtr, align 487 %IndexVec = bitcast i32 %IndexVecPacked to <2 x i16>88 %Index = extractelement <2 x i16> %IndexVec, i32 189 %res = call <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1 0, <16 x bfloat> %A, i1 0, <32 x bfloat> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)90 store <8 x float> %res, ptr addrspace(1) %out91 ret void92}93 94define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {95; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i32_index:96; GFX1250: ; %bb.0: ; %bb97; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 198; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off99; GFX1250-NEXT: s_wait_loadcnt 0x0100; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1101; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off102; GFX1250-NEXT: s_clause 0x1103; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16104; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off105; GFX1250-NEXT: s_endpgm106;107; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i32_index:108; GISEL: ; %bb.0: ; %bb109; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1110; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off111; GISEL-NEXT: s_wait_loadcnt 0x0112; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1113; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off114; GISEL-NEXT: s_clause 0x1115; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off116; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16117; GISEL-NEXT: s_endpgm118bb:119 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8120 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr121 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>122 %Index = extractelement <2 x i32> %IndexVec, i32 1123 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %Index, i1 false, i1 false)124 store <8 x float> %res, ptr addrspace(1) %out125 ret void126}127 128define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {129; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i64_index:130; GFX1250: ; %bb.0: ; %bb131; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1132; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off133; GFX1250-NEXT: s_wait_loadcnt 0x0134; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1135; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off136; GFX1250-NEXT: s_clause 0x1137; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16138; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off139; GFX1250-NEXT: s_endpgm140;141; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_fp8_i64_index:142; GISEL: ; %bb.0: ; %bb143; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1144; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off145; GISEL-NEXT: s_wait_loadcnt 0x0146; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1147; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off148; GISEL-NEXT: s_clause 0x1149; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off150; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16151; GISEL-NEXT: s_endpgm152bb:153 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8154 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr155 %Index = lshr i64 %IndexVecPacked, 32156 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 false)157 store <8 x float> %res, ptr addrspace(1) %out158 ret void159}160 161define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {162; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i32_index:163; GFX1250: ; %bb.0: ; %bb164; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1165; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off166; GFX1250-NEXT: s_wait_loadcnt 0x0167; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1168; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off169; GFX1250-NEXT: s_clause 0x1170; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16171; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off172; GFX1250-NEXT: s_endpgm173;174; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i32_index:175; GISEL: ; %bb.0: ; %bb176; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1177; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off178; GISEL-NEXT: s_wait_loadcnt 0x0179; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1180; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off181; GISEL-NEXT: s_clause 0x1182; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off183; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16184; GISEL-NEXT: s_endpgm185bb:186 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8187 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr188 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>189 %Index = extractelement <2 x i32> %IndexVec, i32 1190 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %Index, i1 false, i1 false)191 store <8 x float> %res, ptr addrspace(1) %out192 ret void193}194 195define amdgpu_ps void @test_swmmac_f32_16x16x128_fp8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {196; GFX1250-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i64_index:197; GFX1250: ; %bb.0: ; %bb198; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1199; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off200; GFX1250-NEXT: s_wait_loadcnt 0x0201; GFX1250-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1202; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off203; GFX1250-NEXT: s_clause 0x1204; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16205; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off206; GFX1250-NEXT: s_endpgm207;208; GISEL-LABEL: test_swmmac_f32_16x16x128_fp8_bf8_i64_index:209; GISEL: ; %bb.0: ; %bb210; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1211; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off212; GISEL-NEXT: s_wait_loadcnt 0x0213; GISEL-NEXT: v_swmmac_f32_16x16x128_fp8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1214; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off215; GISEL-NEXT: s_clause 0x1216; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off217; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16218; GISEL-NEXT: s_endpgm219bb:220 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8221 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr222 %Index = lshr i64 %IndexVecPacked, 32223 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 false)224 store <8 x float> %res, ptr addrspace(1) %out225 ret void226}227 228define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {229; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i32_index:230; GFX1250: ; %bb.0: ; %bb231; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1232; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off233; GFX1250-NEXT: s_wait_loadcnt 0x0234; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1235; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off236; GFX1250-NEXT: s_clause 0x1237; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16238; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off239; GFX1250-NEXT: s_endpgm240;241; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i32_index:242; GISEL: ; %bb.0: ; %bb243; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1244; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off245; GISEL-NEXT: s_wait_loadcnt 0x0246; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1247; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off248; GISEL-NEXT: s_clause 0x1249; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off250; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16251; GISEL-NEXT: s_endpgm252bb:253 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8254 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr255 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>256 %Index = extractelement <2 x i32> %IndexVec, i32 1257 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %Index, i1 false, i1 false)258 store <8 x float> %res, ptr addrspace(1) %out259 ret void260}261 262define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {263; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i64_index:264; GFX1250: ; %bb.0: ; %bb265; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1266; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off267; GFX1250-NEXT: s_wait_loadcnt 0x0268; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1269; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off270; GFX1250-NEXT: s_clause 0x1271; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16272; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off273; GFX1250-NEXT: s_endpgm274;275; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_fp8_i64_index:276; GISEL: ; %bb.0: ; %bb277; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1278; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off279; GISEL-NEXT: s_wait_loadcnt 0x0280; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_fp8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1281; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off282; GISEL-NEXT: s_clause 0x1283; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off284; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16285; GISEL-NEXT: s_endpgm286bb:287 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8288 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr289 %Index = lshr i64 %IndexVecPacked, 32290 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 false)291 store <8 x float> %res, ptr addrspace(1) %out292 ret void293}294 295define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {296; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i32_index:297; GFX1250: ; %bb.0: ; %bb298; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1299; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off300; GFX1250-NEXT: s_wait_loadcnt 0x0301; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1302; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off303; GFX1250-NEXT: s_clause 0x1304; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16305; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off306; GFX1250-NEXT: s_endpgm307;308; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i32_index:309; GISEL: ; %bb.0: ; %bb310; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1311; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off312; GISEL-NEXT: s_wait_loadcnt 0x0313; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1314; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off315; GISEL-NEXT: s_clause 0x1316; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off317; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16318; GISEL-NEXT: s_endpgm319bb:320 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8321 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr322 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>323 %Index = extractelement <2 x i32> %IndexVec, i32 1324 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i32 %Index, i1 false, i1 false)325 store <8 x float> %res, ptr addrspace(1) %out326 ret void327}328 329define amdgpu_ps void @test_swmmac_f32_16x16x128_bf8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {330; GFX1250-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i64_index:331; GFX1250: ; %bb.0: ; %bb332; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1333; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off334; GFX1250-NEXT: s_wait_loadcnt 0x0335; GFX1250-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1336; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off337; GFX1250-NEXT: s_clause 0x1338; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16339; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off340; GFX1250-NEXT: s_endpgm341;342; GISEL-LABEL: test_swmmac_f32_16x16x128_bf8_bf8_i64_index:343; GISEL: ; %bb.0: ; %bb344; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1345; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off346; GISEL-NEXT: s_wait_loadcnt 0x0347; GISEL-NEXT: v_swmmac_f32_16x16x128_bf8_bf8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1348; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off349; GISEL-NEXT: s_clause 0x1350; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off351; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16352; GISEL-NEXT: s_endpgm353bb:354 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8355 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr356 %Index = lshr i64 %IndexVecPacked, 32357 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x float> %C, i64 %Index, i1 false, i1 false)358 store <8 x float> %res, ptr addrspace(1) %out359 ret void360}361 362define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {363; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i32_index:364; GFX1250: ; %bb.0: ; %bb365; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1366; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off367; GFX1250-NEXT: s_wait_loadcnt 0x0368; GFX1250-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1369; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off370; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off371; GFX1250-NEXT: s_endpgm372;373; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i32_index:374; GISEL: ; %bb.0: ; %bb375; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1376; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off377; GISEL-NEXT: s_wait_loadcnt 0x0378; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1379; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off380; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off381; GISEL-NEXT: s_endpgm382bb:383 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8384 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr385 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>386 %Index = extractelement <2 x i32> %IndexVec, i32 1387 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i32 %Index, i1 false, i1 false)388 store <8 x half> %res, ptr addrspace(1) %out389 ret void390}391 392define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {393; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i64_index:394; GFX1250: ; %bb.0: ; %bb395; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1396; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off397; GFX1250-NEXT: s_wait_loadcnt 0x0398; GFX1250-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1399; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off400; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off401; GFX1250-NEXT: s_endpgm402;403; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_fp8_i64_index:404; GISEL: ; %bb.0: ; %bb405; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1406; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off407; GISEL-NEXT: s_wait_loadcnt 0x0408; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1409; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off410; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off411; GISEL-NEXT: s_endpgm412bb:413 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8414 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr415 %Index = lshr i64 %IndexVecPacked, 32416 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i64 %Index, i1 false, i1 false)417 store <8 x half> %res, ptr addrspace(1) %out418 ret void419}420 421define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {422; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i32_index:423; GFX1250: ; %bb.0: ; %bb424; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1425; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off426; GFX1250-NEXT: s_wait_loadcnt 0x0427; GFX1250-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1428; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off429; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off430; GFX1250-NEXT: s_endpgm431;432; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i32_index:433; GISEL: ; %bb.0: ; %bb434; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1435; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off436; GISEL-NEXT: s_wait_loadcnt 0x0437; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1438; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off439; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off440; GISEL-NEXT: s_endpgm441bb:442 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 4443 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr444 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>445 %Index = extractelement <2 x i32> %IndexVec, i32 1446 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i32 %Index, i1 false, i1 false)447 store <8 x half> %res, ptr addrspace(1) %out448 ret void449}450 451define amdgpu_ps void @test_swmmac_f16_16x16x128_fp8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {452; GFX1250-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i64_index:453; GFX1250: ; %bb.0: ; %bb454; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1455; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off456; GFX1250-NEXT: s_wait_loadcnt 0x0457; GFX1250-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1458; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off459; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off460; GFX1250-NEXT: s_endpgm461;462; GISEL-LABEL: test_swmmac_f16_16x16x128_fp8_bf8_i64_index:463; GISEL: ; %bb.0: ; %bb464; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1465; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off466; GISEL-NEXT: s_wait_loadcnt 0x0467; GISEL-NEXT: v_swmmac_f16_16x16x128_fp8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1468; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off469; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off470; GISEL-NEXT: s_endpgm471bb:472 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 4473 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr474 %Index = lshr i64 %IndexVecPacked, 32475 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i64 %Index, i1 false, i1 false)476 store <8 x half> %res, ptr addrspace(1) %out477 ret void478}479 480define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {481; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i32_index:482; GFX1250: ; %bb.0: ; %bb483; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1484; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off485; GFX1250-NEXT: s_wait_loadcnt 0x0486; GFX1250-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1487; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off488; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off489; GFX1250-NEXT: s_endpgm490;491; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i32_index:492; GISEL: ; %bb.0: ; %bb493; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1494; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off495; GISEL-NEXT: s_wait_loadcnt 0x0496; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1497; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off498; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off499; GISEL-NEXT: s_endpgm500bb:501 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8502 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr503 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>504 %Index = extractelement <2 x i32> %IndexVec, i32 1505 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i32 %Index, i1 false, i1 false)506 store <8 x half> %res, ptr addrspace(1) %out507 ret void508}509 510define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_fp8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {511; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i64_index:512; GFX1250: ; %bb.0: ; %bb513; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1514; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off515; GFX1250-NEXT: s_wait_loadcnt 0x0516; GFX1250-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1517; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off518; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off519; GFX1250-NEXT: s_endpgm520;521; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_fp8_i64_index:522; GISEL: ; %bb.0: ; %bb523; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1524; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off525; GISEL-NEXT: s_wait_loadcnt 0x0526; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_fp8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1527; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off528; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off529; GISEL-NEXT: s_endpgm530bb:531 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8532 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr533 %Index = lshr i64 %IndexVecPacked, 32534 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i64 %Index, i1 false, i1 false)535 store <8 x half> %res, ptr addrspace(1) %out536 ret void537}538 539define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {540; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i32_index:541; GFX1250: ; %bb.0: ; %bb542; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1543; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off544; GFX1250-NEXT: s_wait_loadcnt 0x0545; GFX1250-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1546; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off547; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off548; GFX1250-NEXT: s_endpgm549;550; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i32_index:551; GISEL: ; %bb.0: ; %bb552; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1553; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off554; GISEL-NEXT: s_wait_loadcnt 0x0555; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1556; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off557; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off558; GISEL-NEXT: s_endpgm559bb:560 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8561 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr562 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>563 %Index = extractelement <2 x i32> %IndexVec, i32 1564 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i32(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i32 %Index, i1 false, i1 false)565 store <8 x half> %res, ptr addrspace(1) %out566 ret void567}568 569define amdgpu_ps void @test_swmmac_f16_16x16x128_bf8_bf8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {570; GFX1250-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i64_index:571; GFX1250: ; %bb.0: ; %bb572; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1573; GFX1250-NEXT: global_load_b64 v[28:29], v[28:29], off574; GFX1250-NEXT: s_wait_loadcnt 0x0575; GFX1250-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1576; GFX1250-NEXT: global_store_b64 v[30:31], v[28:29], off577; GFX1250-NEXT: global_store_b128 v[32:33], v[24:27], off578; GFX1250-NEXT: s_endpgm579;580; GISEL-LABEL: test_swmmac_f16_16x16x128_bf8_bf8_i64_index:581; GISEL: ; %bb.0: ; %bb582; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1583; GISEL-NEXT: global_load_b64 v[28:29], v[28:29], off584; GISEL-NEXT: s_wait_loadcnt 0x0585; GISEL-NEXT: v_swmmac_f16_16x16x128_bf8_bf8 v[24:27], v[0:7], v[8:23], v[28:29] index_key:1586; GISEL-NEXT: global_store_b64 v[30:31], v[28:29], off587; GISEL-NEXT: global_store_b128 v[32:33], v[24:27], off588; GISEL-NEXT: s_endpgm589bb:590 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8591 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr592 %Index = lshr i64 %IndexVecPacked, 32593 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i64(<8 x i32> %A, <16 x i32> %B, <8 x half> %C, i64 %Index, i1 false, i1 false)594 store <8 x half> %res, ptr addrspace(1) %out595 ret void596}597 598define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i32_index(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {599; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_i32_index:600; GFX1250: ; %bb.0: ; %bb601; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1602; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off603; GFX1250-NEXT: s_wait_loadcnt 0x0604; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1605; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off606; GFX1250-NEXT: s_clause 0x1607; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16608; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off609; GFX1250-NEXT: s_endpgm610;611; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_i32_index:612; GISEL: ; %bb.0: ; %bb613; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1614; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off615; GISEL-NEXT: s_wait_loadcnt 0x0616; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1617; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off618; GISEL-NEXT: s_clause 0x1619; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off620; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16621; GISEL-NEXT: s_endpgm622bb:623 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8624 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr625 %IndexVec = bitcast i64 %IndexVecPacked to <2 x i32>626 %Index = extractelement <2 x i32> %IndexVec, i32 1627 %res = call <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i32(i1 0, <8 x i32> %A, i1 0, <16 x i32> %B, <8 x i32> %C, i32 %Index, i1 false, i1 false)628 store <8 x i32> %res, ptr addrspace(1) %out629 ret void630}631 632define amdgpu_ps void @test_swmmac_i32_16x16x128_iu8_i64_index(<8 x i32> %A, <16 x i32> %B, <8 x i32> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %IndexVecOutPtr, ptr addrspace(1) %out) {633; GFX1250-LABEL: test_swmmac_i32_16x16x128_iu8_i64_index:634; GFX1250: ; %bb.0: ; %bb635; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1636; GFX1250-NEXT: global_load_b64 v[32:33], v[32:33], off637; GFX1250-NEXT: s_wait_loadcnt 0x0638; GFX1250-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1639; GFX1250-NEXT: global_store_b64 v[34:35], v[32:33], off640; GFX1250-NEXT: s_clause 0x1641; GFX1250-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16642; GFX1250-NEXT: global_store_b128 v[36:37], v[24:27], off643; GFX1250-NEXT: s_endpgm644;645; GISEL-LABEL: test_swmmac_i32_16x16x128_iu8_i64_index:646; GISEL: ; %bb.0: ; %bb647; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1648; GISEL-NEXT: global_load_b64 v[32:33], v[32:33], off649; GISEL-NEXT: s_wait_loadcnt 0x0650; GISEL-NEXT: v_swmmac_i32_16x16x128_iu8 v[24:31], v[0:7], v[8:23], v[32:33] index_key:1651; GISEL-NEXT: global_store_b64 v[34:35], v[32:33], off652; GISEL-NEXT: s_clause 0x1653; GISEL-NEXT: global_store_b128 v[36:37], v[24:27], off654; GISEL-NEXT: global_store_b128 v[36:37], v[28:31], off offset:16655; GISEL-NEXT: s_endpgm656bb:657 %IndexVecPacked = load i64, ptr addrspace(1) %IndexVecPtr, align 8658 store i64 %IndexVecPacked, ptr addrspace(1) %IndexVecOutPtr659 %Index = lshr i64 %IndexVecPacked, 32660 %res = call <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i64(i1 0, <8 x i32> %A, i1 0, <16 x i32> %B, <8 x i32> %C, i64 %Index, i1 false, i1 false)661 store <8 x i32> %res, ptr addrspace(1) %out662 ret void663}664 665define amdgpu_ps void @test_swmmac_f32_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x float> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {666; GFX1250-LABEL: test_swmmac_f32_16x16x64_f16:667; GFX1250: ; %bb.0: ; %bb668; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1669; GFX1250-NEXT: global_load_b32 v32, v[32:33], off670; GFX1250-NEXT: s_wait_loadcnt 0x0671; GFX1250-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 index_key:1672; GFX1250-NEXT: s_clause 0x1673; GFX1250-NEXT: global_store_b128 v[34:35], v[28:31], off offset:16674; GFX1250-NEXT: global_store_b128 v[34:35], v[24:27], off675; GFX1250-NEXT: s_endpgm676;677; GISEL-LABEL: test_swmmac_f32_16x16x64_f16:678; GISEL: ; %bb.0: ; %bb679; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1680; GISEL-NEXT: global_load_b32 v32, v[32:33], off681; GISEL-NEXT: s_wait_loadcnt 0x0682; GISEL-NEXT: v_swmmac_f32_16x16x64_f16 v[24:31], v[0:7], v[8:23], v32 index_key:1683; GISEL-NEXT: s_clause 0x1684; GISEL-NEXT: global_store_b128 v[34:35], v[24:27], off685; GISEL-NEXT: global_store_b128 v[34:35], v[28:31], off offset:16686; GISEL-NEXT: s_endpgm687bb:688 %IndexVecPacked = load i32, ptr addrspace(1) %IndexVecPtr, align 4689 %IndexVec = bitcast i32 %IndexVecPacked to <2 x i16>690 %Index = extractelement <2 x i16> %IndexVec, i32 1691 %res = call <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 0, <32 x half> %B, <8 x float> %C, i16 %Index, i1 false, i1 false)692 store <8 x float> %res, ptr addrspace(1) %out693 ret void694}695 696define amdgpu_ps void @test_swmmac_f16_16x16x64_f16(<16 x half> %A, <32 x half> %B, <8 x half> %C, ptr addrspace(1) %IndexVecPtr, ptr addrspace(1) %out) {697; GFX1250-LABEL: test_swmmac_f16_16x16x64_f16:698; GFX1250: ; %bb.0: ; %bb699; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1700; GFX1250-NEXT: global_load_b32 v28, v[28:29], off701; GFX1250-NEXT: s_wait_loadcnt 0x0702; GFX1250-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 index_key:1703; GFX1250-NEXT: global_store_b128 v[30:31], v[24:27], off704; GFX1250-NEXT: s_endpgm705;706; GISEL-LABEL: test_swmmac_f16_16x16x64_f16:707; GISEL: ; %bb.0: ; %bb708; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1709; GISEL-NEXT: global_load_b32 v28, v[28:29], off710; GISEL-NEXT: s_wait_loadcnt 0x0711; GISEL-NEXT: v_swmmac_f16_16x16x64_f16 v[24:27], v[0:7], v[8:23], v28 index_key:1712; GISEL-NEXT: global_store_b128 v[30:31], v[24:27], off713; GISEL-NEXT: s_endpgm714bb:715 %IndexVecPacked = load i32, ptr addrspace(1) %IndexVecPtr, align 4716 %IndexVec = bitcast i32 %IndexVecPacked to <2 x i16>717 %Index = extractelement <2 x i16> %IndexVec, i32 1718 %res = call <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1 0, <16 x half> %A, i1 0, <32 x half> %B, <8 x half> %C, i16 %Index, i1 false, i1 false)719 store <8 x half> %res, ptr addrspace(1) %out720 ret void721}722 723declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)724declare <8 x bfloat> @llvm.amdgcn.swmmac.bf16.16x16x64.bf16.v8bf16.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x bfloat>, i16, i1, i1)725declare <8 x float> @llvm.amdgcn.swmmac.bf16f32.16x16x64.bf16.v8f32.v16bf16.v32bf16.i16(i1, <16 x bfloat>, i1, <32 x bfloat>, <8 x float>, i16, i1, i1)726declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.fp8.v8f32.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x float>, i32, i1, i1)727declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.fp8.bf8.v8f32.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x float>, i32, i1, i1)728declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.fp8.v8f32.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x float>, i32, i1, i1)729declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x128.bf8.bf8.v8f32.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x float>, i32, i1, i1)730declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.fp8.v8f16.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x half>, i32, i1, i1)731declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.fp8.bf8.v8f16.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x half>, i32, i1, i1)732declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.fp8.v8f16.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x half>, i32, i1, i1)733declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x128.bf8.bf8.v8f16.v8i32.v16i32.i32(<8 x i32>, <16 x i32>, <8 x half>, i32, i1, i1)734declare <8 x i32> @llvm.amdgcn.swmmac.i32.16x16x128.iu8.v8i32.v8i32.v16i32.i32(i1 immarg, <8 x i32>, i1 immarg, <16 x i32>, <8 x i32>, i32, i1, i1)735declare <8 x float> @llvm.amdgcn.swmmac.f32.16x16x64.f16.v8f32.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x float>, i16, i1, i1)736declare <8 x half> @llvm.amdgcn.swmmac.f16.16x16x64.f16.v8f16.v16f16.v32f16.i16(i1, <16 x half>, i1, <32 x half>, <8 x half>, i16, i1, i1)737