1222 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s3; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s4 5; Test using saddr addressing mode of flat_*store_* instructions.6 7define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {8; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr:9; GFX1250: ; %bb.0:10; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111; GFX1250-NEXT: flat_load_b32 v0, v[0:1]12; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3]14; GFX1250-NEXT: s_endpgm15 %voffset = load i32, ptr %voffset.ptr16 %zext.offset = zext i32 %voffset to i6417 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset18 store i8 %data, ptr %gep019 ret void20}21 22; Maximum positive offset on gfx1023define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {24; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:25; GFX1250: ; %bb.0:26; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 127; GFX1250-NEXT: flat_load_b32 v0, v[0:1]28; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x029; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:204730; GFX1250-NEXT: s_endpgm31 %voffset = load i32, ptr %voffset.ptr32 %zext.offset = zext i32 %voffset to i6433 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset34 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 204735 store i8 %data, ptr %gep136 ret void37}38 39; Maximum negative offset on gfx1040define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {41; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:42; GFX1250: ; %bb.0:43; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 144; GFX1250-NEXT: flat_load_b32 v0, v[0:1]45; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x046; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:-204847; GFX1250-NEXT: s_endpgm48 %voffset = load i32, ptr %voffset.ptr49 %zext.offset = zext i32 %voffset to i6450 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset51 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -204852 store i8 %data, ptr %gep153 ret void54}55 56; --------------------------------------------------------------------------------57; Uniformity edge cases58; --------------------------------------------------------------------------------59 60@ptr.in.lds = internal addrspace(3) global ptr poison61 62; Base pointer is uniform, but also in VGPRs63define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %data) {64; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:65; GFX1250-SDAG: ; %bb.0:66; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 167; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 068; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v269; GFX1250-SDAG-NEXT: s_wait_dscnt 0x070; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v271; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v372; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1]73; GFX1250-SDAG-NEXT: s_endpgm74;75; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:76; GFX1250-GISEL: ; %bb.0:77; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 178; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 079; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v280; GFX1250-GISEL-NEXT: s_wait_dscnt 0x081; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v082; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)83; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo84; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v185; GFX1250-GISEL-NEXT: s_endpgm86 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds87 %zext.offset = zext i32 %voffset to i6488 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset89 store i8 %data, ptr %gep090 ret void91}92 93; Base pointer is uniform, but also in VGPRs, with imm offset94define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset, i8 %data) {95; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:96; GFX1250-SDAG: ; %bb.0:97; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 198; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 099; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2100; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0101; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2102; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3103; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1] offset:-120104; GFX1250-SDAG-NEXT: s_endpgm105;106; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:107; GFX1250-GISEL: ; %bb.0:108; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1109; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0110; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2111; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0112; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0113; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)114; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo115; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v1 offset:-120116; GFX1250-GISEL-NEXT: s_endpgm117 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds118 %zext.offset = zext i32 %voffset to i64119 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset120 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -120121 store i8 %data, ptr %gep1122 ret void123}124 125; --------------------------------------------------------------------------------126; Stress various type stores127; --------------------------------------------------------------------------------128 129define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, i16 %data) {130; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr:131; GFX1250: ; %bb.0:132; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1133; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]134; GFX1250-NEXT: s_endpgm135 %zext.offset = zext i32 %voffset to i64136 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset137 store i16 %data, ptr %gep0138 ret void139}140 141define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i16 %data) {142; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:143; GFX1250: ; %bb.0:144; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1145; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128146; GFX1250-NEXT: s_endpgm147 %zext.offset = zext i32 %voffset to i64148 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset149 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128150 store i16 %data, ptr %gep1151 ret void152}153 154define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, half %data) {155; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr:156; GFX1250: ; %bb.0:157; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1158; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]159; GFX1250-NEXT: s_endpgm160 %zext.offset = zext i32 %voffset to i64161 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset162 store half %data, ptr %gep0163 ret void164}165 166define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, half %data) {167; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:168; GFX1250: ; %bb.0:169; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1170; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128171; GFX1250-NEXT: s_endpgm172 %zext.offset = zext i32 %voffset to i64173 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset174 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128175 store half %data, ptr %gep1176 ret void177}178 179define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {180; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr:181; GFX1250: ; %bb.0:182; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1183; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]184; GFX1250-NEXT: s_endpgm185 %zext.offset = zext i32 %voffset to i64186 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset187 store i32 %data, ptr %gep0188 ret void189}190 191define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {192; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128:193; GFX1250: ; %bb.0:194; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1195; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128196; GFX1250-NEXT: s_endpgm197 %zext.offset = zext i32 %voffset to i64198 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset199 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128200 store i32 %data, ptr %gep1201 ret void202}203 204define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, float %data) {205; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr:206; GFX1250: ; %bb.0:207; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1208; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]209; GFX1250-NEXT: s_endpgm210 %zext.offset = zext i32 %voffset to i64211 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset212 store float %data, ptr %gep0213 ret void214}215 216define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, float %data) {217; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128:218; GFX1250: ; %bb.0:219; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1220; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128221; GFX1250-NEXT: s_endpgm222 %zext.offset = zext i32 %voffset to i64223 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset224 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128225 store float %data, ptr %gep1226 ret void227}228 229define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {230; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr:231; GFX1250: ; %bb.0:232; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1233; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]234; GFX1250-NEXT: s_endpgm235 %zext.offset = zext i32 %voffset to i64236 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset237 store ptr addrspace(3) %data, ptr %gep0238 ret void239}240 241define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {242; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128:243; GFX1250: ; %bb.0:244; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1245; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128246; GFX1250-NEXT: s_endpgm247 %zext.offset = zext i32 %voffset to i64248 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset249 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128250 store ptr addrspace(3) %data, ptr %gep1251 ret void252}253 254define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) {255; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr:256; GFX1250-SDAG: ; %bb.0:257; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1258; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1259; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]260; GFX1250-SDAG-NEXT: s_endpgm261;262; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr:263; GFX1250-GISEL: ; %bb.0:264; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1265; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2266; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]267; GFX1250-GISEL-NEXT: s_endpgm268 %zext.offset = zext i32 %voffset to i64269 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset270 store i64 %data, ptr %gep0271 ret void272}273 274define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {275; GFX1250-SDAG-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128:276; GFX1250-SDAG: ; %bb.0:277; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1278; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1279; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128280; GFX1250-SDAG-NEXT: s_endpgm281;282; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128:283; GFX1250-GISEL: ; %bb.0:284; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1285; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2286; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128287; GFX1250-GISEL-NEXT: s_endpgm288 %zext.offset = zext i32 %voffset to i64289 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset290 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128291 store i64 %data, ptr %gep1292 ret void293}294 295define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, double %data) {296; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr:297; GFX1250-SDAG: ; %bb.0:298; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1299; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1300; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]301; GFX1250-SDAG-NEXT: s_endpgm302;303; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr:304; GFX1250-GISEL: ; %bb.0:305; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1306; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2307; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]308; GFX1250-GISEL-NEXT: s_endpgm309 %zext.offset = zext i32 %voffset to i64310 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset311 store double %data, ptr %gep0312 ret void313}314 315define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, double %data) {316; GFX1250-SDAG-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128:317; GFX1250-SDAG: ; %bb.0:318; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1319; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1320; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128321; GFX1250-SDAG-NEXT: s_endpgm322;323; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128:324; GFX1250-GISEL: ; %bb.0:325; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1326; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2327; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128328; GFX1250-GISEL-NEXT: s_endpgm329 %zext.offset = zext i32 %voffset to i64330 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset331 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128332 store double %data, ptr %gep1333 ret void334}335 336define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) {337; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr:338; GFX1250-SDAG: ; %bb.0:339; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1340; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1341; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]342; GFX1250-SDAG-NEXT: s_endpgm343;344; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr:345; GFX1250-GISEL: ; %bb.0:346; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1347; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2348; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]349; GFX1250-GISEL-NEXT: s_endpgm350 %zext.offset = zext i32 %voffset to i64351 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset352 store <2 x i32> %data, ptr %gep0353 ret void354}355 356define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i32> %data) {357; GFX1250-SDAG-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128:358; GFX1250-SDAG: ; %bb.0:359; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1360; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1361; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128362; GFX1250-SDAG-NEXT: s_endpgm363;364; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128:365; GFX1250-GISEL: ; %bb.0:366; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1367; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2368; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128369; GFX1250-GISEL-NEXT: s_endpgm370 %zext.offset = zext i32 %voffset to i64371 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset372 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128373 store <2 x i32> %data, ptr %gep1374 ret void375}376 377define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x float> %data) {378; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr:379; GFX1250-SDAG: ; %bb.0:380; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1381; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1382; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]383; GFX1250-SDAG-NEXT: s_endpgm384;385; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr:386; GFX1250-GISEL: ; %bb.0:387; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1388; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2389; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]390; GFX1250-GISEL-NEXT: s_endpgm391 %zext.offset = zext i32 %voffset to i64392 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset393 store <2 x float> %data, ptr %gep0394 ret void395}396 397define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x float> %data) {398; GFX1250-SDAG-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128:399; GFX1250-SDAG: ; %bb.0:400; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1401; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1402; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128403; GFX1250-SDAG-NEXT: s_endpgm404;405; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128:406; GFX1250-GISEL: ; %bb.0:407; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1408; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2409; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128410; GFX1250-GISEL-NEXT: s_endpgm411 %zext.offset = zext i32 %voffset to i64412 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset413 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128414 store <2 x float> %data, ptr %gep1415 ret void416}417 418define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) {419; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr:420; GFX1250-SDAG: ; %bb.0:421; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1422; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1423; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]424; GFX1250-SDAG-NEXT: s_endpgm425;426; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr:427; GFX1250-GISEL: ; %bb.0:428; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1429; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2430; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]431; GFX1250-GISEL-NEXT: s_endpgm432 %zext.offset = zext i32 %voffset to i64433 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset434 store <4 x i16> %data, ptr %gep0435 ret void436}437 438define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i16> %data) {439; GFX1250-SDAG-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128:440; GFX1250-SDAG: ; %bb.0:441; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1442; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1443; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128444; GFX1250-SDAG-NEXT: s_endpgm445;446; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128:447; GFX1250-GISEL: ; %bb.0:448; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1449; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2450; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128451; GFX1250-GISEL-NEXT: s_endpgm452 %zext.offset = zext i32 %voffset to i64453 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset454 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128455 store <4 x i16> %data, ptr %gep1456 ret void457}458 459define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x half> %data) {460; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr:461; GFX1250-SDAG: ; %bb.0:462; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1463; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1464; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]465; GFX1250-SDAG-NEXT: s_endpgm466;467; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr:468; GFX1250-GISEL: ; %bb.0:469; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1470; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2471; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]472; GFX1250-GISEL-NEXT: s_endpgm473 %zext.offset = zext i32 %voffset to i64474 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset475 store <4 x half> %data, ptr %gep0476 ret void477}478 479define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x half> %data) {480; GFX1250-SDAG-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128:481; GFX1250-SDAG: ; %bb.0:482; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1483; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1484; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128485; GFX1250-SDAG-NEXT: s_endpgm486;487; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128:488; GFX1250-GISEL: ; %bb.0:489; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1490; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2491; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128492; GFX1250-GISEL-NEXT: s_endpgm493 %zext.offset = zext i32 %voffset to i64494 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset495 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128496 store <4 x half> %data, ptr %gep1497 ret void498}499 500define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr %data) {501; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr:502; GFX1250-SDAG: ; %bb.0:503; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1504; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1505; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]506; GFX1250-SDAG-NEXT: s_endpgm507;508; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr:509; GFX1250-GISEL: ; %bb.0:510; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1511; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2512; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]513; GFX1250-GISEL-NEXT: s_endpgm514 %zext.offset = zext i32 %voffset to i64515 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset516 store ptr %data, ptr %gep0517 ret void518}519 520define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr %data) {521; GFX1250-SDAG-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128:522; GFX1250-SDAG: ; %bb.0:523; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1524; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1525; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128526; GFX1250-SDAG-NEXT: s_endpgm527;528; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128:529; GFX1250-GISEL: ; %bb.0:530; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1531; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2532; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128533; GFX1250-GISEL-NEXT: s_endpgm534 %zext.offset = zext i32 %voffset to i64535 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset536 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128537 store ptr %data, ptr %gep1538 ret void539}540 541define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) {542; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr:543; GFX1250-SDAG: ; %bb.0:544; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1545; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2546; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1547; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]548; GFX1250-SDAG-NEXT: s_endpgm549;550; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr:551; GFX1250-GISEL: ; %bb.0:552; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1553; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2554; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3555; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]556; GFX1250-GISEL-NEXT: s_endpgm557 %zext.offset = zext i32 %voffset to i64558 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset559 store <3 x i32> %data, ptr %gep0560 ret void561}562 563define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x i32> %data) {564; GFX1250-SDAG-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128:565; GFX1250-SDAG: ; %bb.0:566; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1567; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2568; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1569; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128570; GFX1250-SDAG-NEXT: s_endpgm571;572; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128:573; GFX1250-GISEL: ; %bb.0:574; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1575; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2576; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3577; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128578; GFX1250-GISEL-NEXT: s_endpgm579 %zext.offset = zext i32 %voffset to i64580 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset581 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128582 store <3 x i32> %data, ptr %gep1583 ret void584}585 586define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <3 x float> %data) {587; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr:588; GFX1250-SDAG: ; %bb.0:589; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1590; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2591; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1592; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]593; GFX1250-SDAG-NEXT: s_endpgm594;595; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr:596; GFX1250-GISEL: ; %bb.0:597; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1598; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2599; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3600; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]601; GFX1250-GISEL-NEXT: s_endpgm602 %zext.offset = zext i32 %voffset to i64603 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset604 store <3 x float> %data, ptr %gep0605 ret void606}607 608define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <3 x float> %data) {609; GFX1250-SDAG-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128:610; GFX1250-SDAG: ; %bb.0:611; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1612; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2613; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1614; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128615; GFX1250-SDAG-NEXT: s_endpgm616;617; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128:618; GFX1250-GISEL: ; %bb.0:619; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1620; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2621; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3622; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128623; GFX1250-GISEL-NEXT: s_endpgm624 %zext.offset = zext i32 %voffset to i64625 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset626 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128627 store <3 x float> %data, ptr %gep1628 ret void629}630 631define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) {632; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr:633; GFX1250-SDAG: ; %bb.0:634; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1635; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2636; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1637; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]638; GFX1250-SDAG-NEXT: s_endpgm639;640; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr:641; GFX1250-GISEL: ; %bb.0:642; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1643; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2644; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3645; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]646; GFX1250-GISEL-NEXT: s_endpgm647 %zext.offset = zext i32 %voffset to i64648 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset649 store <6 x i16> %data, ptr %gep0650 ret void651}652 653define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x i16> %data) {654; GFX1250-SDAG-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128:655; GFX1250-SDAG: ; %bb.0:656; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1657; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2658; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1659; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128660; GFX1250-SDAG-NEXT: s_endpgm661;662; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128:663; GFX1250-GISEL: ; %bb.0:664; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1665; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2666; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3667; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128668; GFX1250-GISEL-NEXT: s_endpgm669 %zext.offset = zext i32 %voffset to i64670 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset671 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128672 store <6 x i16> %data, ptr %gep1673 ret void674}675 676define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <6 x half> %data) {677; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr:678; GFX1250-SDAG: ; %bb.0:679; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1680; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2681; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1682; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]683; GFX1250-SDAG-NEXT: s_endpgm684;685; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr:686; GFX1250-GISEL: ; %bb.0:687; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1688; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2689; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3690; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]691; GFX1250-GISEL-NEXT: s_endpgm692 %zext.offset = zext i32 %voffset to i64693 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset694 store <6 x half> %data, ptr %gep0695 ret void696}697 698define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <6 x half> %data) {699; GFX1250-SDAG-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128:700; GFX1250-SDAG: ; %bb.0:701; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1702; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2703; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1704; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128705; GFX1250-SDAG-NEXT: s_endpgm706;707; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128:708; GFX1250-GISEL: ; %bb.0:709; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1710; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2711; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3712; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128713; GFX1250-GISEL-NEXT: s_endpgm714 %zext.offset = zext i32 %voffset to i64715 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset716 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128717 store <6 x half> %data, ptr %gep1718 ret void719}720 721define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) {722; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr:723; GFX1250-SDAG: ; %bb.0:724; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1725; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3726; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1727; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]728; GFX1250-SDAG-NEXT: s_endpgm729;730; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr:731; GFX1250-GISEL: ; %bb.0:732; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1733; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2734; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4735; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]736; GFX1250-GISEL-NEXT: s_endpgm737 %zext.offset = zext i32 %voffset to i64738 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset739 store <4 x i32> %data, ptr %gep0740 ret void741}742 743define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x i32> %data) {744; GFX1250-SDAG-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128:745; GFX1250-SDAG: ; %bb.0:746; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1747; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3748; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1749; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128750; GFX1250-SDAG-NEXT: s_endpgm751;752; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128:753; GFX1250-GISEL: ; %bb.0:754; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1755; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2756; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4757; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128758; GFX1250-GISEL-NEXT: s_endpgm759 %zext.offset = zext i32 %voffset to i64760 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset761 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128762 store <4 x i32> %data, ptr %gep1763 ret void764}765 766define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x float> %data) {767; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr:768; GFX1250-SDAG: ; %bb.0:769; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1770; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3771; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1772; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]773; GFX1250-SDAG-NEXT: s_endpgm774;775; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr:776; GFX1250-GISEL: ; %bb.0:777; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1778; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2779; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4780; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]781; GFX1250-GISEL-NEXT: s_endpgm782 %zext.offset = zext i32 %voffset to i64783 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset784 store <4 x float> %data, ptr %gep0785 ret void786}787 788define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x float> %data) {789; GFX1250-SDAG-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128:790; GFX1250-SDAG: ; %bb.0:791; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1792; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3793; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1794; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128795; GFX1250-SDAG-NEXT: s_endpgm796;797; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128:798; GFX1250-GISEL: ; %bb.0:799; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1800; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2801; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4802; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128803; GFX1250-GISEL-NEXT: s_endpgm804 %zext.offset = zext i32 %voffset to i64805 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset806 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128807 store <4 x float> %data, ptr %gep1808 ret void809}810 811define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) {812; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr:813; GFX1250-SDAG: ; %bb.0:814; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1815; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3816; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1817; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]818; GFX1250-SDAG-NEXT: s_endpgm819;820; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr:821; GFX1250-GISEL: ; %bb.0:822; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1823; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2824; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4825; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]826; GFX1250-GISEL-NEXT: s_endpgm827 %zext.offset = zext i32 %voffset to i64828 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset829 store <2 x i64> %data, ptr %gep0830 ret void831}832 833define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i64> %data) {834; GFX1250-SDAG-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128:835; GFX1250-SDAG: ; %bb.0:836; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1837; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3838; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1839; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128840; GFX1250-SDAG-NEXT: s_endpgm841;842; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128:843; GFX1250-GISEL: ; %bb.0:844; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1845; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2846; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4847; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128848; GFX1250-GISEL-NEXT: s_endpgm849 %zext.offset = zext i32 %voffset to i64850 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset851 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128852 store <2 x i64> %data, ptr %gep1853 ret void854}855 856define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x double> %data) {857; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr:858; GFX1250-SDAG: ; %bb.0:859; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1860; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3861; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1862; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]863; GFX1250-SDAG-NEXT: s_endpgm864;865; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr:866; GFX1250-GISEL: ; %bb.0:867; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1868; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2869; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4870; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]871; GFX1250-GISEL-NEXT: s_endpgm872 %zext.offset = zext i32 %voffset to i64873 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset874 store <2 x double> %data, ptr %gep0875 ret void876}877 878define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x double> %data) {879; GFX1250-SDAG-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128:880; GFX1250-SDAG: ; %bb.0:881; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1882; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3883; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1884; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128885; GFX1250-SDAG-NEXT: s_endpgm886;887; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128:888; GFX1250-GISEL: ; %bb.0:889; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1890; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2891; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4892; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128893; GFX1250-GISEL-NEXT: s_endpgm894 %zext.offset = zext i32 %voffset to i64895 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset896 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128897 store <2 x double> %data, ptr %gep1898 ret void899}900 901define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) {902; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr:903; GFX1250-SDAG: ; %bb.0:904; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1905; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3906; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1907; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]908; GFX1250-SDAG-NEXT: s_endpgm909;910; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr:911; GFX1250-GISEL: ; %bb.0:912; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1913; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2914; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4915; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]916; GFX1250-GISEL-NEXT: s_endpgm917 %zext.offset = zext i32 %voffset to i64918 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset919 store <8 x i16> %data, ptr %gep0920 ret void921}922 923define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x i16> %data) {924; GFX1250-SDAG-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128:925; GFX1250-SDAG: ; %bb.0:926; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1927; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3928; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1929; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128930; GFX1250-SDAG-NEXT: s_endpgm931;932; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128:933; GFX1250-GISEL: ; %bb.0:934; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1935; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2936; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4937; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128938; GFX1250-GISEL-NEXT: s_endpgm939 %zext.offset = zext i32 %voffset to i64940 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset941 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128942 store <8 x i16> %data, ptr %gep1943 ret void944}945 946define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, <8 x half> %data) {947; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr:948; GFX1250-SDAG: ; %bb.0:949; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1950; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3951; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1952; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]953; GFX1250-SDAG-NEXT: s_endpgm954;955; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr:956; GFX1250-GISEL: ; %bb.0:957; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1958; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2959; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4960; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]961; GFX1250-GISEL-NEXT: s_endpgm962 %zext.offset = zext i32 %voffset to i64963 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset964 store <8 x half> %data, ptr %gep0965 ret void966}967 968define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <8 x half> %data) {969; GFX1250-SDAG-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128:970; GFX1250-SDAG: ; %bb.0:971; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1972; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3973; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1974; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128975; GFX1250-SDAG-NEXT: s_endpgm976;977; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128:978; GFX1250-GISEL: ; %bb.0:979; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1980; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2981; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4982; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128983; GFX1250-GISEL-NEXT: s_endpgm984 %zext.offset = zext i32 %voffset to i64985 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset986 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128987 store <8 x half> %data, ptr %gep1988 ret void989}990 991define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) {992; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr:993; GFX1250-SDAG: ; %bb.0:994; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1995; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3996; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1997; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]998; GFX1250-SDAG-NEXT: s_endpgm999;1000; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr:1001; GFX1250-GISEL: ; %bb.0:1002; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11003; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v21004; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v41005; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]1006; GFX1250-GISEL-NEXT: s_endpgm1007 %zext.offset = zext i32 %voffset to i641008 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1009 store <2 x ptr> %data, ptr %gep01010 ret void1011}1012 1013define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x ptr> %data) {1014; GFX1250-SDAG-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128:1015; GFX1250-SDAG: ; %bb.0:1016; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11017; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v31018; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11019; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-1281020; GFX1250-SDAG-NEXT: s_endpgm1021;1022; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128:1023; GFX1250-GISEL: ; %bb.0:1024; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11025; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v21026; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v41027; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-1281028; GFX1250-GISEL-NEXT: s_endpgm1029 %zext.offset = zext i32 %voffset to i641030 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1031 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281032 store <2 x ptr> %data, ptr %gep11033 ret void1034}1035 1036define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) {1037; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr:1038; GFX1250-SDAG: ; %bb.0:1039; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11040; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v31041; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11042; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]1043; GFX1250-SDAG-NEXT: s_endpgm1044;1045; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr:1046; GFX1250-GISEL: ; %bb.0:1047; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11048; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v21049; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v41050; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]1051; GFX1250-GISEL-NEXT: s_endpgm1052 %zext.offset = zext i32 %voffset to i641053 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1054 store <4 x ptr addrspace(3)> %data, ptr %gep01055 ret void1056}1057 1058define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <4 x ptr addrspace(3)> %data) {1059; GFX1250-SDAG-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128:1060; GFX1250-SDAG: ; %bb.0:1061; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11062; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v31063; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11064; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-1281065; GFX1250-SDAG-NEXT: s_endpgm1066;1067; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128:1068; GFX1250-GISEL: ; %bb.0:1069; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11070; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v21071; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v41072; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-1281073; GFX1250-GISEL-NEXT: s_endpgm1074 %zext.offset = zext i32 %voffset to i641075 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1076 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281077 store <4 x ptr addrspace(3)> %data, ptr %gep11078 ret void1079}1080 1081; --------------------------------------------------------------------------------1082; Atomic store1083; --------------------------------------------------------------------------------1084 1085define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {1086; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr:1087; GFX1250: ; %bb.0:1088; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11089; GFX1250-NEXT: global_wb scope:SCOPE_SYS1090; GFX1250-NEXT: s_wait_storecnt 0x01091; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS1092; GFX1250-NEXT: s_endpgm1093 %zext.offset = zext i32 %voffset to i641094 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1095 store atomic i32 %data, ptr %gep0 seq_cst, align 41096 ret void1097}1098 1099define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {1100; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128:1101; GFX1250: ; %bb.0:1102; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11103; GFX1250-NEXT: global_wb scope:SCOPE_SYS1104; GFX1250-NEXT: s_wait_storecnt 0x01105; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SYS1106; GFX1250-NEXT: s_endpgm1107 %zext.offset = zext i32 %voffset to i641108 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1109 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281110 store atomic i32 %data, ptr %gep1 seq_cst, align 41111 ret void1112}1113 1114define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %voffset, i64 %data) {1115; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr:1116; GFX1250-SDAG: ; %bb.0:1117; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11118; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11119; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS1120; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01121; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] scope:SCOPE_SYS1122; GFX1250-SDAG-NEXT: s_endpgm1123;1124; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr:1125; GFX1250-GISEL: ; %bb.0:1126; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11127; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v21128; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS1129; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01130; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] scope:SCOPE_SYS1131; GFX1250-GISEL-NEXT: s_endpgm1132 %zext.offset = zext i32 %voffset to i641133 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1134 store atomic i64 %data, ptr %gep0 seq_cst, align 81135 ret void1136}1137 1138define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {1139; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128:1140; GFX1250-SDAG: ; %bb.0:1141; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11142; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11143; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS1144; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01145; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS1146; GFX1250-SDAG-NEXT: s_endpgm1147;1148; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128:1149; GFX1250-GISEL: ; %bb.0:1150; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11151; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v21152; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS1153; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01154; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_SYS1155; GFX1250-GISEL-NEXT: s_endpgm1156 %zext.offset = zext i32 %voffset to i641157 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1158 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281159 store atomic i64 %data, ptr %gep1 seq_cst, align 81160 ret void1161}1162 1163; --------------------------------------------------------------------------------1164; D16 HI store (hi 16)1165; --------------------------------------------------------------------------------1166 1167define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {1168; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:1169; GFX1250: ; %bb.0:1170; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11171; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]1172; GFX1250-NEXT: s_endpgm1173 %zext.offset = zext i32 %voffset to i641174 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1175 %data.hi = extractelement <2 x i16> %data, i32 11176 store i16 %data.hi, ptr %gep01177 ret void1178}1179 1180define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {1181; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:1182; GFX1250: ; %bb.0:1183; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11184; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-1281185; GFX1250-NEXT: s_endpgm1186 %zext.offset = zext i32 %voffset to i641187 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1188 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281189 %data.hi = extractelement <2 x i16> %data, i32 11190 store i16 %data.hi, ptr %gep11191 ret void1192}1193 1194define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {1195; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr:1196; GFX1250: ; %bb.0:1197; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11198; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3]1199; GFX1250-NEXT: s_endpgm1200 %zext.offset = zext i32 %voffset to i641201 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1202 %data.hi = extractelement <2 x i16> %data, i32 11203 %data.hi.trunc = trunc i16 %data.hi to i81204 store i8 %data.hi.trunc, ptr %gep01205 ret void1206}1207 1208define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {1209; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128:1210; GFX1250: ; %bb.0:1211; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11212; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3] offset:-1281213; GFX1250-NEXT: s_endpgm1214 %zext.offset = zext i32 %voffset to i641215 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1216 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281217 %data.hi = extractelement <2 x i16> %data, i32 11218 %data.hi.trunc = trunc i16 %data.hi to i81219 store i8 %data.hi.trunc, ptr %gep11220 ret void1221}1222