1366 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX11 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX12 %s5 6; vgpr offset7 8define amdgpu_ps void @test_scratch_load_i8_zext_v(ptr addrspace(5) %in, ptr %out) {9; GFX10-LABEL: test_scratch_load_i8_zext_v:10; GFX10: ; %bb.0:11; GFX10-NEXT: s_add_u32 s0, s0, s212; GFX10-NEXT: s_addc_u32 s1, s1, 013; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s014; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s115; GFX10-NEXT: scratch_load_ubyte v0, v0, off offset:116; GFX10-NEXT: s_waitcnt vmcnt(0)17; GFX10-NEXT: flat_store_dword v[1:2], v018; GFX10-NEXT: s_endpgm19;20; GFX11-LABEL: test_scratch_load_i8_zext_v:21; GFX11: ; %bb.0:22; GFX11-NEXT: scratch_load_u8 v0, v0, off offset:123; GFX11-NEXT: s_waitcnt vmcnt(0)24; GFX11-NEXT: flat_store_b32 v[1:2], v025; GFX11-NEXT: s_endpgm26;27; GFX12-LABEL: test_scratch_load_i8_zext_v:28; GFX12: ; %bb.0:29; GFX12-NEXT: scratch_load_u8 v0, v0, off offset:130; GFX12-NEXT: s_wait_loadcnt 0x031; GFX12-NEXT: flat_store_b32 v[1:2], v032; GFX12-NEXT: s_endpgm33 %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 134 %load = load i8, ptr addrspace(5) %gep, align 435 %ext = zext i8 %load to i3236 store i32 %ext, ptr %out, align 437 ret void38}39 40define amdgpu_ps void @test_scratch_load_i8_sext_v(ptr addrspace(5) %in, ptr %out) {41; GFX10-LABEL: test_scratch_load_i8_sext_v:42; GFX10: ; %bb.0:43; GFX10-NEXT: s_add_u32 s0, s0, s244; GFX10-NEXT: s_addc_u32 s1, s1, 045; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s046; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s147; GFX10-NEXT: scratch_load_sbyte v0, v0, off offset:148; GFX10-NEXT: s_waitcnt vmcnt(0)49; GFX10-NEXT: flat_store_dword v[1:2], v050; GFX10-NEXT: s_endpgm51;52; GFX11-LABEL: test_scratch_load_i8_sext_v:53; GFX11: ; %bb.0:54; GFX11-NEXT: scratch_load_i8 v0, v0, off offset:155; GFX11-NEXT: s_waitcnt vmcnt(0)56; GFX11-NEXT: flat_store_b32 v[1:2], v057; GFX11-NEXT: s_endpgm58;59; GFX12-LABEL: test_scratch_load_i8_sext_v:60; GFX12: ; %bb.0:61; GFX12-NEXT: scratch_load_i8 v0, v0, off offset:162; GFX12-NEXT: s_wait_loadcnt 0x063; GFX12-NEXT: flat_store_b32 v[1:2], v064; GFX12-NEXT: s_endpgm65 %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 166 %load = load i8, ptr addrspace(5) %gep, align 467 %ext = sext i8 %load to i3268 store i32 %ext, ptr %out, align 469 ret void70}71 72define amdgpu_ps void @test_scratch_load_i16_zext_v(ptr addrspace(5) %in, ptr %out) {73; GFX10-LABEL: test_scratch_load_i16_zext_v:74; GFX10: ; %bb.0:75; GFX10-NEXT: s_add_u32 s0, s0, s276; GFX10-NEXT: s_addc_u32 s1, s1, 077; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s078; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s179; GFX10-NEXT: scratch_load_ushort v0, v0, off offset:280; GFX10-NEXT: s_waitcnt vmcnt(0)81; GFX10-NEXT: flat_store_dword v[1:2], v082; GFX10-NEXT: s_endpgm83;84; GFX11-LABEL: test_scratch_load_i16_zext_v:85; GFX11: ; %bb.0:86; GFX11-NEXT: scratch_load_u16 v0, v0, off offset:287; GFX11-NEXT: s_waitcnt vmcnt(0)88; GFX11-NEXT: flat_store_b32 v[1:2], v089; GFX11-NEXT: s_endpgm90;91; GFX12-LABEL: test_scratch_load_i16_zext_v:92; GFX12: ; %bb.0:93; GFX12-NEXT: scratch_load_u16 v0, v0, off offset:294; GFX12-NEXT: s_wait_loadcnt 0x095; GFX12-NEXT: flat_store_b32 v[1:2], v096; GFX12-NEXT: s_endpgm97 %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 198 %load = load i16, ptr addrspace(5) %gep, align 499 %ext = zext i16 %load to i32100 store i32 %ext, ptr %out, align 4101 ret void102}103 104define amdgpu_ps void @test_scratch_load_i16_sext_v(ptr addrspace(5) %in, ptr %out) {105; GFX10-LABEL: test_scratch_load_i16_sext_v:106; GFX10: ; %bb.0:107; GFX10-NEXT: s_add_u32 s0, s0, s2108; GFX10-NEXT: s_addc_u32 s1, s1, 0109; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0110; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1111; GFX10-NEXT: scratch_load_sshort v0, v0, off offset:2112; GFX10-NEXT: s_waitcnt vmcnt(0)113; GFX10-NEXT: flat_store_dword v[1:2], v0114; GFX10-NEXT: s_endpgm115;116; GFX11-LABEL: test_scratch_load_i16_sext_v:117; GFX11: ; %bb.0:118; GFX11-NEXT: scratch_load_i16 v0, v0, off offset:2119; GFX11-NEXT: s_waitcnt vmcnt(0)120; GFX11-NEXT: flat_store_b32 v[1:2], v0121; GFX11-NEXT: s_endpgm122;123; GFX12-LABEL: test_scratch_load_i16_sext_v:124; GFX12: ; %bb.0:125; GFX12-NEXT: scratch_load_i16 v0, v0, off offset:2126; GFX12-NEXT: s_wait_loadcnt 0x0127; GFX12-NEXT: flat_store_b32 v[1:2], v0128; GFX12-NEXT: s_endpgm129 %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1130 %load = load i16, ptr addrspace(5) %gep, align 4131 %ext = sext i16 %load to i32132 store i32 %ext, ptr %out, align 4133 ret void134}135 136define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_v(ptr addrspace(5) %in, ptr %out) {137; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_lo_v:138; GFX10: ; %bb.0: ; %bb139; GFX10-NEXT: s_add_u32 s0, s0, s2140; GFX10-NEXT: s_addc_u32 s1, s1, 0141; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0142; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1143; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0144; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff0000145; GFX10-NEXT: scratch_load_ubyte_d16 v3, v0, off146; GFX10-NEXT: s_waitcnt vmcnt(0)147; GFX10-NEXT: flat_store_dword v[1:2], v3148; GFX10-NEXT: s_endpgm149;150; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_lo_v:151; GFX11: ; %bb.0: ; %bb152; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_add_nc_u32 v0, 1, v0153; GFX11-NEXT: scratch_load_d16_u8 v3, v0, off154; GFX11-NEXT: s_waitcnt vmcnt(0)155; GFX11-NEXT: flat_store_b32 v[1:2], v3156; GFX11-NEXT: s_endpgm157;158; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_lo_v:159; GFX12: ; %bb.0: ; %bb160; GFX12-NEXT: v_mov_b32_e32 v3, 0xffff0000161; GFX12-NEXT: scratch_load_d16_u8 v3, v0, off offset:1162; GFX12-NEXT: s_wait_loadcnt 0x0163; GFX12-NEXT: flat_store_b32 v[1:2], v3164; GFX12-NEXT: s_endpgm165bb:166 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1167 %load_lo = load i8, ptr addrspace(5) %gep168 %ext = zext i8 %load_lo to i16169 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 0170 store <2 x i16> %result, ptr %out, align 4171 ret void172}173 174define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_v(ptr addrspace(5) %in, ptr %out) {175; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_lo_v:176; GFX10: ; %bb.0: ; %bb177; GFX10-NEXT: s_add_u32 s0, s0, s2178; GFX10-NEXT: s_addc_u32 s1, s1, 0179; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0180; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1181; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0182; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff0000183; GFX10-NEXT: scratch_load_sbyte_d16 v3, v0, off184; GFX10-NEXT: s_waitcnt vmcnt(0)185; GFX10-NEXT: flat_store_dword v[1:2], v3186; GFX10-NEXT: s_endpgm187;188; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_lo_v:189; GFX11: ; %bb.0: ; %bb190; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_add_nc_u32 v0, 1, v0191; GFX11-NEXT: scratch_load_d16_i8 v3, v0, off192; GFX11-NEXT: s_waitcnt vmcnt(0)193; GFX11-NEXT: flat_store_b32 v[1:2], v3194; GFX11-NEXT: s_endpgm195;196; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_lo_v:197; GFX12: ; %bb.0: ; %bb198; GFX12-NEXT: v_mov_b32_e32 v3, 0xffff0000199; GFX12-NEXT: scratch_load_d16_i8 v3, v0, off offset:1200; GFX12-NEXT: s_wait_loadcnt 0x0201; GFX12-NEXT: flat_store_b32 v[1:2], v3202; GFX12-NEXT: s_endpgm203bb:204 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1205 %load_lo = load i8, ptr addrspace(5) %gep206 %ext = sext i8 %load_lo to i16207 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 0208 store <2 x i16> %result, ptr %out, align 4209 ret void210}211 212define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_v(ptr addrspace(5) %in, ptr %out) {213; GFX10-LABEL: test_scratch_load_i16_to_d16_lo_v:214; GFX10: ; %bb.0: ; %bb215; GFX10-NEXT: s_add_u32 s0, s0, s2216; GFX10-NEXT: s_addc_u32 s1, s1, 0217; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0218; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1219; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0220; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff0000221; GFX10-NEXT: scratch_load_short_d16 v3, v0, off222; GFX10-NEXT: s_waitcnt vmcnt(0)223; GFX10-NEXT: flat_store_dword v[1:2], v3224; GFX10-NEXT: s_endpgm225;226; GFX11-LABEL: test_scratch_load_i16_to_d16_lo_v:227; GFX11: ; %bb.0: ; %bb228; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_add_nc_u32 v0, 2, v0229; GFX11-NEXT: scratch_load_d16_b16 v3, v0, off230; GFX11-NEXT: s_waitcnt vmcnt(0)231; GFX11-NEXT: flat_store_b32 v[1:2], v3232; GFX11-NEXT: s_endpgm233;234; GFX12-LABEL: test_scratch_load_i16_to_d16_lo_v:235; GFX12: ; %bb.0: ; %bb236; GFX12-NEXT: v_mov_b32_e32 v3, 0xffff0000237; GFX12-NEXT: scratch_load_d16_b16 v3, v0, off offset:2238; GFX12-NEXT: s_wait_loadcnt 0x0239; GFX12-NEXT: flat_store_b32 v[1:2], v3240; GFX12-NEXT: s_endpgm241bb:242 %gep = getelementptr i16, ptr addrspace(5) %in, i64 1243 %load_lo = load i16, ptr addrspace(5) %gep244 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 0245 store <2 x i16> %result, ptr %out, align 4246 ret void247}248 249 250define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_v(ptr addrspace(5) %in, ptr %out) {251; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_hi_v:252; GFX10: ; %bb.0: ; %bb253; GFX10-NEXT: s_add_u32 s0, s0, s2254; GFX10-NEXT: s_addc_u32 s1, s1, 0255; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0256; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1257; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0258; GFX10-NEXT: v_mov_b32_e32 v3, -1259; GFX10-NEXT: scratch_load_ubyte_d16_hi v3, v0, off260; GFX10-NEXT: s_waitcnt vmcnt(0)261; GFX10-NEXT: flat_store_dword v[1:2], v3262; GFX10-NEXT: s_endpgm263;264; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_hi_v:265; GFX11: ; %bb.0: ; %bb266; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0267; GFX11-NEXT: v_mov_b16_e32 v3.l, -1268; GFX11-NEXT: scratch_load_d16_hi_u8 v3, v0, off269; GFX11-NEXT: s_waitcnt vmcnt(0)270; GFX11-NEXT: flat_store_b32 v[1:2], v3271; GFX11-NEXT: s_endpgm272;273; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_hi_v:274; GFX12: ; %bb.0: ; %bb275; GFX12-NEXT: v_mov_b32_e32 v3, -1276; GFX12-NEXT: scratch_load_d16_hi_u8 v3, v0, off offset:1277; GFX12-NEXT: s_wait_loadcnt 0x0278; GFX12-NEXT: flat_store_b32 v[1:2], v3279; GFX12-NEXT: s_endpgm280bb:281 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1282 %load_lo = load i8, ptr addrspace(5) %gep283 %ext = zext i8 %load_lo to i16284 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 1285 store <2 x i16> %result, ptr %out, align 4286 ret void287}288 289define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_v(ptr addrspace(5) %in, ptr %out) {290; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_hi_v:291; GFX10: ; %bb.0: ; %bb292; GFX10-NEXT: s_add_u32 s0, s0, s2293; GFX10-NEXT: s_addc_u32 s1, s1, 0294; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0295; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1296; GFX10-NEXT: v_add_nc_u32_e32 v0, 1, v0297; GFX10-NEXT: v_mov_b32_e32 v3, -1298; GFX10-NEXT: scratch_load_sbyte_d16_hi v3, v0, off299; GFX10-NEXT: s_waitcnt vmcnt(0)300; GFX10-NEXT: flat_store_dword v[1:2], v3301; GFX10-NEXT: s_endpgm302;303; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_hi_v:304; GFX11: ; %bb.0: ; %bb305; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0306; GFX11-NEXT: v_mov_b16_e32 v3.l, -1307; GFX11-NEXT: scratch_load_d16_hi_i8 v3, v0, off308; GFX11-NEXT: s_waitcnt vmcnt(0)309; GFX11-NEXT: flat_store_b32 v[1:2], v3310; GFX11-NEXT: s_endpgm311;312; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_hi_v:313; GFX12: ; %bb.0: ; %bb314; GFX12-NEXT: v_mov_b32_e32 v3, -1315; GFX12-NEXT: scratch_load_d16_hi_i8 v3, v0, off offset:1316; GFX12-NEXT: s_wait_loadcnt 0x0317; GFX12-NEXT: flat_store_b32 v[1:2], v3318; GFX12-NEXT: s_endpgm319bb:320 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1321 %load_lo = load i8, ptr addrspace(5) %gep322 %ext = sext i8 %load_lo to i16323 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 1324 store <2 x i16> %result, ptr %out, align 4325 ret void326}327 328define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_v(ptr addrspace(5) %in, ptr %out) {329; GFX10-LABEL: test_scratch_load_i16_to_d16_hi_v:330; GFX10: ; %bb.0: ; %bb331; GFX10-NEXT: s_add_u32 s0, s0, s2332; GFX10-NEXT: s_addc_u32 s1, s1, 0333; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0334; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1335; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0336; GFX10-NEXT: v_mov_b32_e32 v3, -1337; GFX10-NEXT: scratch_load_short_d16_hi v3, v0, off338; GFX10-NEXT: s_waitcnt vmcnt(0)339; GFX10-NEXT: flat_store_dword v[1:2], v3340; GFX10-NEXT: s_endpgm341;342; GFX11-LABEL: test_scratch_load_i16_to_d16_hi_v:343; GFX11: ; %bb.0: ; %bb344; GFX11-NEXT: v_add_nc_u32_e32 v0, 2, v0345; GFX11-NEXT: v_mov_b16_e32 v3.l, -1346; GFX11-NEXT: scratch_load_d16_hi_b16 v3, v0, off347; GFX11-NEXT: s_waitcnt vmcnt(0)348; GFX11-NEXT: flat_store_b32 v[1:2], v3349; GFX11-NEXT: s_endpgm350;351; GFX12-LABEL: test_scratch_load_i16_to_d16_hi_v:352; GFX12: ; %bb.0: ; %bb353; GFX12-NEXT: v_mov_b32_e32 v3, -1354; GFX12-NEXT: scratch_load_d16_hi_b16 v3, v0, off offset:2355; GFX12-NEXT: s_wait_loadcnt 0x0356; GFX12-NEXT: flat_store_b32 v[1:2], v3357; GFX12-NEXT: s_endpgm358bb:359 %gep = getelementptr i16, ptr addrspace(5) %in, i64 1360 %load_lo = load i16, ptr addrspace(5) %gep361 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 1362 store <2 x i16> %result, ptr %out, align 4363 ret void364}365 366define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_v(ptr %in, ptr addrspace(5) %out) {367; GFX10-LABEL: test_scratch_store_b8_from_d16_hi_v:368; GFX10: ; %bb.0: ; %bb369; GFX10-NEXT: s_add_u32 s0, s0, s2370; GFX10-NEXT: s_addc_u32 s1, s1, 0371; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0372; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1373; GFX10-NEXT: flat_load_dword v0, v[0:1]374; GFX10-NEXT: v_add_nc_u32_e32 v1, 4, v2375; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)376; GFX10-NEXT: scratch_store_byte_d16_hi v1, v0, off377; GFX10-NEXT: s_endpgm378;379; GFX11-LABEL: test_scratch_store_b8_from_d16_hi_v:380; GFX11: ; %bb.0: ; %bb381; GFX11-NEXT: flat_load_b32 v0, v[0:1]382; GFX11-NEXT: v_add_nc_u32_e32 v1, 4, v2383; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)384; GFX11-NEXT: scratch_store_d16_hi_b8 v1, v0, off385; GFX11-NEXT: s_endpgm386;387; GFX12-LABEL: test_scratch_store_b8_from_d16_hi_v:388; GFX12: ; %bb.0: ; %bb389; GFX12-NEXT: flat_load_b32 v0, v[0:1]390; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0391; GFX12-NEXT: scratch_store_d16_hi_b8 v2, v0, off offset:4392; GFX12-NEXT: s_endpgm393bb:394 %load = load <4 x i8>, ptr %in395 %element = extractelement <4 x i8> %load, i32 2396 %gep = getelementptr <4 x i8>, ptr addrspace(5) %out, i64 1397 store i8 %element, ptr addrspace(5) %gep, align 4398 ret void399}400 401define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_v(ptr %in, ptr addrspace(5) %out) {402; GFX10-LABEL: test_scratch_store_b16_from_d16_hi_v:403; GFX10: ; %bb.0: ; %bb404; GFX10-NEXT: s_add_u32 s0, s0, s2405; GFX10-NEXT: s_addc_u32 s1, s1, 0406; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0407; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1408; GFX10-NEXT: flat_load_dword v0, v[0:1]409; GFX10-NEXT: v_add_nc_u32_e32 v1, 2, v2410; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)411; GFX10-NEXT: scratch_store_short_d16_hi v1, v0, off412; GFX10-NEXT: s_endpgm413;414; GFX11-LABEL: test_scratch_store_b16_from_d16_hi_v:415; GFX11: ; %bb.0: ; %bb416; GFX11-NEXT: flat_load_b32 v0, v[0:1]417; GFX11-NEXT: v_add_nc_u32_e32 v1, 2, v2418; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)419; GFX11-NEXT: scratch_store_d16_hi_b16 v1, v0, off420; GFX11-NEXT: s_endpgm421;422; GFX12-LABEL: test_scratch_store_b16_from_d16_hi_v:423; GFX12: ; %bb.0: ; %bb424; GFX12-NEXT: flat_load_b32 v0, v[0:1]425; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0426; GFX12-NEXT: scratch_store_d16_hi_b16 v2, v0, off offset:2427; GFX12-NEXT: s_endpgm428bb:429 %load = load <2 x i16>, ptr %in430 %element = extractelement <2 x i16> %load, i32 1431 %gep = getelementptr <2 x i8>, ptr addrspace(5) %out, i64 1432 store i16 %element, ptr addrspace(5) %gep, align 4433 ret void434}435 436 437 438 439; sgpr offset440 441define amdgpu_ps void @test_scratch_load_i8_zext_s(ptr addrspace(5) inreg %in, ptr %out) {442; GFX10-LABEL: test_scratch_load_i8_zext_s:443; GFX10: ; %bb.0:444; GFX10-NEXT: s_add_u32 s0, s0, s3445; GFX10-NEXT: s_addc_u32 s1, s1, 0446; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0447; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1448; GFX10-NEXT: scratch_load_ubyte v2, off, s2 offset:1449; GFX10-NEXT: s_waitcnt vmcnt(0)450; GFX10-NEXT: flat_store_dword v[0:1], v2451; GFX10-NEXT: s_endpgm452;453; GFX11-LABEL: test_scratch_load_i8_zext_s:454; GFX11: ; %bb.0:455; GFX11-NEXT: scratch_load_u8 v2, off, s0 offset:1456; GFX11-NEXT: s_waitcnt vmcnt(0)457; GFX11-NEXT: flat_store_b32 v[0:1], v2458; GFX11-NEXT: s_endpgm459;460; GFX12-LABEL: test_scratch_load_i8_zext_s:461; GFX12: ; %bb.0:462; GFX12-NEXT: scratch_load_u8 v2, off, s0 offset:1463; GFX12-NEXT: s_wait_loadcnt 0x0464; GFX12-NEXT: flat_store_b32 v[0:1], v2465; GFX12-NEXT: s_endpgm466 %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1467 %load = load i8, ptr addrspace(5) %gep, align 4468 %ext = zext i8 %load to i32469 store i32 %ext, ptr %out, align 4470 ret void471}472 473define amdgpu_ps void @test_scratch_load_i8_sext_s(ptr addrspace(5) inreg %in, ptr %out) {474; GFX10-LABEL: test_scratch_load_i8_sext_s:475; GFX10: ; %bb.0:476; GFX10-NEXT: s_add_u32 s0, s0, s3477; GFX10-NEXT: s_addc_u32 s1, s1, 0478; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0479; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1480; GFX10-NEXT: scratch_load_sbyte v2, off, s2 offset:1481; GFX10-NEXT: s_waitcnt vmcnt(0)482; GFX10-NEXT: flat_store_dword v[0:1], v2483; GFX10-NEXT: s_endpgm484;485; GFX11-LABEL: test_scratch_load_i8_sext_s:486; GFX11: ; %bb.0:487; GFX11-NEXT: scratch_load_i8 v2, off, s0 offset:1488; GFX11-NEXT: s_waitcnt vmcnt(0)489; GFX11-NEXT: flat_store_b32 v[0:1], v2490; GFX11-NEXT: s_endpgm491;492; GFX12-LABEL: test_scratch_load_i8_sext_s:493; GFX12: ; %bb.0:494; GFX12-NEXT: scratch_load_i8 v2, off, s0 offset:1495; GFX12-NEXT: s_wait_loadcnt 0x0496; GFX12-NEXT: flat_store_b32 v[0:1], v2497; GFX12-NEXT: s_endpgm498 %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1499 %load = load i8, ptr addrspace(5) %gep, align 4500 %ext = sext i8 %load to i32501 store i32 %ext, ptr %out, align 4502 ret void503}504 505define amdgpu_ps void @test_scratch_load_i16_zext_s(ptr addrspace(5) inreg %in, ptr %out) {506; GFX10-LABEL: test_scratch_load_i16_zext_s:507; GFX10: ; %bb.0:508; GFX10-NEXT: s_add_u32 s0, s0, s3509; GFX10-NEXT: s_addc_u32 s1, s1, 0510; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0511; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1512; GFX10-NEXT: scratch_load_ushort v2, off, s2 offset:2513; GFX10-NEXT: s_waitcnt vmcnt(0)514; GFX10-NEXT: flat_store_dword v[0:1], v2515; GFX10-NEXT: s_endpgm516;517; GFX11-LABEL: test_scratch_load_i16_zext_s:518; GFX11: ; %bb.0:519; GFX11-NEXT: scratch_load_u16 v2, off, s0 offset:2520; GFX11-NEXT: s_waitcnt vmcnt(0)521; GFX11-NEXT: flat_store_b32 v[0:1], v2522; GFX11-NEXT: s_endpgm523;524; GFX12-LABEL: test_scratch_load_i16_zext_s:525; GFX12: ; %bb.0:526; GFX12-NEXT: scratch_load_u16 v2, off, s0 offset:2527; GFX12-NEXT: s_wait_loadcnt 0x0528; GFX12-NEXT: flat_store_b32 v[0:1], v2529; GFX12-NEXT: s_endpgm530 %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1531 %load = load i16, ptr addrspace(5) %gep, align 4532 %ext = zext i16 %load to i32533 store i32 %ext, ptr %out, align 4534 ret void535}536 537define amdgpu_ps void @test_scratch_load_i16_sext_s(ptr addrspace(5) inreg %in, ptr %out) {538; GFX10-LABEL: test_scratch_load_i16_sext_s:539; GFX10: ; %bb.0:540; GFX10-NEXT: s_add_u32 s0, s0, s3541; GFX10-NEXT: s_addc_u32 s1, s1, 0542; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0543; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1544; GFX10-NEXT: scratch_load_sshort v2, off, s2 offset:2545; GFX10-NEXT: s_waitcnt vmcnt(0)546; GFX10-NEXT: flat_store_dword v[0:1], v2547; GFX10-NEXT: s_endpgm548;549; GFX11-LABEL: test_scratch_load_i16_sext_s:550; GFX11: ; %bb.0:551; GFX11-NEXT: scratch_load_i16 v2, off, s0 offset:2552; GFX11-NEXT: s_waitcnt vmcnt(0)553; GFX11-NEXT: flat_store_b32 v[0:1], v2554; GFX11-NEXT: s_endpgm555;556; GFX12-LABEL: test_scratch_load_i16_sext_s:557; GFX12: ; %bb.0:558; GFX12-NEXT: scratch_load_i16 v2, off, s0 offset:2559; GFX12-NEXT: s_wait_loadcnt 0x0560; GFX12-NEXT: flat_store_b32 v[0:1], v2561; GFX12-NEXT: s_endpgm562 %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1563 %load = load i16, ptr addrspace(5) %gep, align 4564 %ext = sext i16 %load to i32565 store i32 %ext, ptr %out, align 4566 ret void567}568 569define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_s(ptr addrspace(5) inreg %in, ptr %out) {570; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_lo_s:571; GFX10: ; %bb.0: ; %bb572; GFX10-NEXT: s_add_u32 s0, s0, s3573; GFX10-NEXT: s_addc_u32 s1, s1, 0574; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0575; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1576; GFX10-NEXT: v_mov_b32_e32 v2, 0xffff0000577; GFX10-NEXT: s_add_i32 s2, s2, 1578; GFX10-NEXT: scratch_load_ubyte_d16 v2, off, s2579; GFX10-NEXT: s_waitcnt vmcnt(0)580; GFX10-NEXT: flat_store_dword v[0:1], v2581; GFX10-NEXT: s_endpgm582;583; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_lo_s:584; GFX11: ; %bb.0: ; %bb585; GFX11-NEXT: v_mov_b32_e32 v2, 0xffff0000586; GFX11-NEXT: s_add_i32 s0, s0, 1587; GFX11-NEXT: scratch_load_d16_u8 v2, off, s0588; GFX11-NEXT: s_waitcnt vmcnt(0)589; GFX11-NEXT: flat_store_b32 v[0:1], v2590; GFX11-NEXT: s_endpgm591;592; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_lo_s:593; GFX12: ; %bb.0: ; %bb594; GFX12-NEXT: v_mov_b32_e32 v2, 0xffff0000595; GFX12-NEXT: scratch_load_d16_u8 v2, off, s0 offset:1596; GFX12-NEXT: s_wait_loadcnt 0x0597; GFX12-NEXT: flat_store_b32 v[0:1], v2598; GFX12-NEXT: s_endpgm599bb:600 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1601 %load_lo = load i8, ptr addrspace(5) %gep602 %ext = zext i8 %load_lo to i16603 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 0604 store <2 x i16> %result, ptr %out, align 4605 ret void606}607 608define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_s(ptr addrspace(5) inreg %in, ptr %out) {609; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_lo_s:610; GFX10: ; %bb.0: ; %bb611; GFX10-NEXT: s_add_u32 s0, s0, s3612; GFX10-NEXT: s_addc_u32 s1, s1, 0613; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0614; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1615; GFX10-NEXT: v_mov_b32_e32 v2, 0xffff0000616; GFX10-NEXT: s_add_i32 s2, s2, 1617; GFX10-NEXT: scratch_load_sbyte_d16 v2, off, s2618; GFX10-NEXT: s_waitcnt vmcnt(0)619; GFX10-NEXT: flat_store_dword v[0:1], v2620; GFX10-NEXT: s_endpgm621;622; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_lo_s:623; GFX11: ; %bb.0: ; %bb624; GFX11-NEXT: v_mov_b32_e32 v2, 0xffff0000625; GFX11-NEXT: s_add_i32 s0, s0, 1626; GFX11-NEXT: scratch_load_d16_i8 v2, off, s0627; GFX11-NEXT: s_waitcnt vmcnt(0)628; GFX11-NEXT: flat_store_b32 v[0:1], v2629; GFX11-NEXT: s_endpgm630;631; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_lo_s:632; GFX12: ; %bb.0: ; %bb633; GFX12-NEXT: v_mov_b32_e32 v2, 0xffff0000634; GFX12-NEXT: scratch_load_d16_i8 v2, off, s0 offset:1635; GFX12-NEXT: s_wait_loadcnt 0x0636; GFX12-NEXT: flat_store_b32 v[0:1], v2637; GFX12-NEXT: s_endpgm638bb:639 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1640 %load_lo = load i8, ptr addrspace(5) %gep641 %ext = sext i8 %load_lo to i16642 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 0643 store <2 x i16> %result, ptr %out, align 4644 ret void645}646 647define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_s(ptr addrspace(5) inreg %in, ptr %out) {648; GFX10-LABEL: test_scratch_load_i16_to_d16_lo_s:649; GFX10: ; %bb.0: ; %bb650; GFX10-NEXT: s_add_u32 s0, s0, s3651; GFX10-NEXT: s_addc_u32 s1, s1, 0652; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0653; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1654; GFX10-NEXT: v_mov_b32_e32 v2, 0xffff0000655; GFX10-NEXT: s_add_i32 s2, s2, 2656; GFX10-NEXT: scratch_load_short_d16 v2, off, s2657; GFX10-NEXT: s_waitcnt vmcnt(0)658; GFX10-NEXT: flat_store_dword v[0:1], v2659; GFX10-NEXT: s_endpgm660;661; GFX11-LABEL: test_scratch_load_i16_to_d16_lo_s:662; GFX11: ; %bb.0: ; %bb663; GFX11-NEXT: v_mov_b32_e32 v2, 0xffff0000664; GFX11-NEXT: s_add_i32 s0, s0, 2665; GFX11-NEXT: scratch_load_d16_b16 v2, off, s0666; GFX11-NEXT: s_waitcnt vmcnt(0)667; GFX11-NEXT: flat_store_b32 v[0:1], v2668; GFX11-NEXT: s_endpgm669;670; GFX12-LABEL: test_scratch_load_i16_to_d16_lo_s:671; GFX12: ; %bb.0: ; %bb672; GFX12-NEXT: v_mov_b32_e32 v2, 0xffff0000673; GFX12-NEXT: scratch_load_d16_b16 v2, off, s0 offset:2674; GFX12-NEXT: s_wait_loadcnt 0x0675; GFX12-NEXT: flat_store_b32 v[0:1], v2676; GFX12-NEXT: s_endpgm677bb:678 %gep = getelementptr i16, ptr addrspace(5) %in, i64 1679 %load_lo = load i16, ptr addrspace(5) %gep680 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 0681 store <2 x i16> %result, ptr %out, align 4682 ret void683}684 685 686define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_s(ptr addrspace(5) inreg %in, ptr %out) {687; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_hi_s:688; GFX10: ; %bb.0: ; %bb689; GFX10-NEXT: s_add_u32 s0, s0, s3690; GFX10-NEXT: s_addc_u32 s1, s1, 0691; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0692; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1693; GFX10-NEXT: v_mov_b32_e32 v2, -1694; GFX10-NEXT: s_add_i32 s2, s2, 1695; GFX10-NEXT: scratch_load_ubyte_d16_hi v2, off, s2696; GFX10-NEXT: s_waitcnt vmcnt(0)697; GFX10-NEXT: flat_store_dword v[0:1], v2698; GFX10-NEXT: s_endpgm699;700; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_hi_s:701; GFX11: ; %bb.0: ; %bb702; GFX11-NEXT: v_mov_b16_e32 v2.l, -1703; GFX11-NEXT: s_add_i32 s0, s0, 1704; GFX11-NEXT: scratch_load_d16_hi_u8 v2, off, s0705; GFX11-NEXT: s_waitcnt vmcnt(0)706; GFX11-NEXT: flat_store_b32 v[0:1], v2707; GFX11-NEXT: s_endpgm708;709; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_hi_s:710; GFX12: ; %bb.0: ; %bb711; GFX12-NEXT: v_mov_b32_e32 v2, -1712; GFX12-NEXT: scratch_load_d16_hi_u8 v2, off, s0 offset:1713; GFX12-NEXT: s_wait_loadcnt 0x0714; GFX12-NEXT: flat_store_b32 v[0:1], v2715; GFX12-NEXT: s_endpgm716bb:717 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1718 %load_lo = load i8, ptr addrspace(5) %gep719 %ext = zext i8 %load_lo to i16720 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 1721 store <2 x i16> %result, ptr %out, align 4722 ret void723}724 725define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_s(ptr addrspace(5) inreg %in, ptr %out) {726; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_hi_s:727; GFX10: ; %bb.0: ; %bb728; GFX10-NEXT: s_add_u32 s0, s0, s3729; GFX10-NEXT: s_addc_u32 s1, s1, 0730; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0731; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1732; GFX10-NEXT: v_mov_b32_e32 v2, -1733; GFX10-NEXT: s_add_i32 s2, s2, 1734; GFX10-NEXT: scratch_load_sbyte_d16_hi v2, off, s2735; GFX10-NEXT: s_waitcnt vmcnt(0)736; GFX10-NEXT: flat_store_dword v[0:1], v2737; GFX10-NEXT: s_endpgm738;739; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_hi_s:740; GFX11: ; %bb.0: ; %bb741; GFX11-NEXT: v_mov_b16_e32 v2.l, -1742; GFX11-NEXT: s_add_i32 s0, s0, 1743; GFX11-NEXT: scratch_load_d16_hi_i8 v2, off, s0744; GFX11-NEXT: s_waitcnt vmcnt(0)745; GFX11-NEXT: flat_store_b32 v[0:1], v2746; GFX11-NEXT: s_endpgm747;748; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_hi_s:749; GFX12: ; %bb.0: ; %bb750; GFX12-NEXT: v_mov_b32_e32 v2, -1751; GFX12-NEXT: scratch_load_d16_hi_i8 v2, off, s0 offset:1752; GFX12-NEXT: s_wait_loadcnt 0x0753; GFX12-NEXT: flat_store_b32 v[0:1], v2754; GFX12-NEXT: s_endpgm755bb:756 %gep = getelementptr i8, ptr addrspace(5) %in, i64 1757 %load_lo = load i8, ptr addrspace(5) %gep758 %ext = sext i8 %load_lo to i16759 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 1760 store <2 x i16> %result, ptr %out, align 4761 ret void762}763 764define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_s(ptr addrspace(5) inreg %in, ptr %out) {765; GFX10-LABEL: test_scratch_load_i16_to_d16_hi_s:766; GFX10: ; %bb.0: ; %bb767; GFX10-NEXT: s_add_u32 s0, s0, s3768; GFX10-NEXT: s_addc_u32 s1, s1, 0769; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0770; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1771; GFX10-NEXT: v_mov_b32_e32 v2, -1772; GFX10-NEXT: s_add_i32 s2, s2, 2773; GFX10-NEXT: scratch_load_short_d16_hi v2, off, s2774; GFX10-NEXT: s_waitcnt vmcnt(0)775; GFX10-NEXT: flat_store_dword v[0:1], v2776; GFX10-NEXT: s_endpgm777;778; GFX11-LABEL: test_scratch_load_i16_to_d16_hi_s:779; GFX11: ; %bb.0: ; %bb780; GFX11-NEXT: v_mov_b16_e32 v2.l, -1781; GFX11-NEXT: s_add_i32 s0, s0, 2782; GFX11-NEXT: scratch_load_d16_hi_b16 v2, off, s0783; GFX11-NEXT: s_waitcnt vmcnt(0)784; GFX11-NEXT: flat_store_b32 v[0:1], v2785; GFX11-NEXT: s_endpgm786;787; GFX12-LABEL: test_scratch_load_i16_to_d16_hi_s:788; GFX12: ; %bb.0: ; %bb789; GFX12-NEXT: v_mov_b32_e32 v2, -1790; GFX12-NEXT: scratch_load_d16_hi_b16 v2, off, s0 offset:2791; GFX12-NEXT: s_wait_loadcnt 0x0792; GFX12-NEXT: flat_store_b32 v[0:1], v2793; GFX12-NEXT: s_endpgm794bb:795 %gep = getelementptr i16, ptr addrspace(5) %in, i64 1796 %load_lo = load i16, ptr addrspace(5) %gep797 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 1798 store <2 x i16> %result, ptr %out, align 4799 ret void800}801 802define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_s(ptr %in, ptr addrspace(5) inreg %out) {803; GFX10-LABEL: test_scratch_store_b8_from_d16_hi_s:804; GFX10: ; %bb.0: ; %bb805; GFX10-NEXT: s_add_u32 s0, s0, s3806; GFX10-NEXT: s_addc_u32 s1, s1, 0807; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0808; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1809; GFX10-NEXT: flat_load_dword v0, v[0:1]810; GFX10-NEXT: s_add_i32 s2, s2, 4811; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)812; GFX10-NEXT: scratch_store_byte_d16_hi off, v0, s2813; GFX10-NEXT: s_endpgm814;815; GFX11-LABEL: test_scratch_store_b8_from_d16_hi_s:816; GFX11: ; %bb.0: ; %bb817; GFX11-NEXT: flat_load_b32 v0, v[0:1]818; GFX11-NEXT: s_add_i32 s0, s0, 4819; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)820; GFX11-NEXT: scratch_store_d16_hi_b8 off, v0, s0821; GFX11-NEXT: s_endpgm822;823; GFX12-LABEL: test_scratch_store_b8_from_d16_hi_s:824; GFX12: ; %bb.0: ; %bb825; GFX12-NEXT: flat_load_b32 v0, v[0:1]826; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0827; GFX12-NEXT: scratch_store_d16_hi_b8 off, v0, s0 offset:4828; GFX12-NEXT: s_endpgm829bb:830 %load = load <4 x i8>, ptr %in831 %element = extractelement <4 x i8> %load, i32 2832 %gep = getelementptr <4 x i8>, ptr addrspace(5) %out, i64 1833 store i8 %element, ptr addrspace(5) %gep, align 4834 ret void835}836 837define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_s(ptr %in, ptr addrspace(5) inreg %out) {838; GFX10-LABEL: test_scratch_store_b16_from_d16_hi_s:839; GFX10: ; %bb.0: ; %bb840; GFX10-NEXT: s_add_u32 s0, s0, s3841; GFX10-NEXT: s_addc_u32 s1, s1, 0842; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0843; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1844; GFX10-NEXT: flat_load_dword v0, v[0:1]845; GFX10-NEXT: s_add_i32 s2, s2, 2846; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)847; GFX10-NEXT: scratch_store_short_d16_hi off, v0, s2848; GFX10-NEXT: s_endpgm849;850; GFX11-LABEL: test_scratch_store_b16_from_d16_hi_s:851; GFX11: ; %bb.0: ; %bb852; GFX11-NEXT: flat_load_b32 v0, v[0:1]853; GFX11-NEXT: s_add_i32 s0, s0, 2854; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)855; GFX11-NEXT: scratch_store_d16_hi_b16 off, v0, s0856; GFX11-NEXT: s_endpgm857;858; GFX12-LABEL: test_scratch_store_b16_from_d16_hi_s:859; GFX12: ; %bb.0: ; %bb860; GFX12-NEXT: flat_load_b32 v0, v[0:1]861; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0862; GFX12-NEXT: scratch_store_d16_hi_b16 off, v0, s0 offset:2863; GFX12-NEXT: s_endpgm864bb:865 %load = load <2 x i16>, ptr %in866 %element = extractelement <2 x i16> %load, i32 1867 %gep = getelementptr <2 x i8>, ptr addrspace(5) %out, i64 1868 store i16 %element, ptr addrspace(5) %gep, align 4869 ret void870}871 872 873 874 875; sgpr + vgpr offset876 877define amdgpu_ps void @test_scratch_load_i8_zext_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {878; GFX10-LABEL: test_scratch_load_i8_zext_svs:879; GFX10: ; %bb.0:880; GFX10-NEXT: s_add_u32 s0, s0, s3881; GFX10-NEXT: s_addc_u32 s1, s1, 0882; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0883; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1884; GFX10-NEXT: v_lshl_add_u32 v0, v0, 2, s2885; GFX10-NEXT: scratch_load_ubyte v0, v0, off offset:1886; GFX10-NEXT: s_waitcnt vmcnt(0)887; GFX10-NEXT: flat_store_dword v[1:2], v0888; GFX10-NEXT: s_endpgm889;890; GFX11-LABEL: test_scratch_load_i8_zext_svs:891; GFX11: ; %bb.0:892; GFX11-NEXT: v_lshl_add_u32 v0, v0, 2, s0893; GFX11-NEXT: scratch_load_u8 v0, v0, off offset:1894; GFX11-NEXT: s_waitcnt vmcnt(0)895; GFX11-NEXT: flat_store_b32 v[1:2], v0896; GFX11-NEXT: s_endpgm897;898; GFX12-LABEL: test_scratch_load_i8_zext_svs:899; GFX12: ; %bb.0:900; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0901; GFX12-NEXT: scratch_load_u8 v0, v0, s0 offset:1902; GFX12-NEXT: s_wait_loadcnt 0x0903; GFX12-NEXT: flat_store_b32 v[1:2], v0904; GFX12-NEXT: s_endpgm905 %voffset4 = mul i32 %voffset, 4906 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4907 %gep = getelementptr inbounds i8, ptr addrspace(5) %gep0, i32 1908 %load = load i8, ptr addrspace(5) %gep, align 4909 %ext = zext i8 %load to i32910 store i32 %ext, ptr %out, align 4911 ret void912}913 914define amdgpu_ps void @test_scratch_load_i8_sext_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {915; GFX10-LABEL: test_scratch_load_i8_sext_svs:916; GFX10: ; %bb.0:917; GFX10-NEXT: s_add_u32 s0, s0, s3918; GFX10-NEXT: s_addc_u32 s1, s1, 0919; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0920; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1921; GFX10-NEXT: v_lshl_add_u32 v0, v0, 2, s2922; GFX10-NEXT: scratch_load_sbyte v0, v0, off offset:1923; GFX10-NEXT: s_waitcnt vmcnt(0)924; GFX10-NEXT: flat_store_dword v[1:2], v0925; GFX10-NEXT: s_endpgm926;927; GFX11-LABEL: test_scratch_load_i8_sext_svs:928; GFX11: ; %bb.0:929; GFX11-NEXT: v_lshl_add_u32 v0, v0, 2, s0930; GFX11-NEXT: scratch_load_i8 v0, v0, off offset:1931; GFX11-NEXT: s_waitcnt vmcnt(0)932; GFX11-NEXT: flat_store_b32 v[1:2], v0933; GFX11-NEXT: s_endpgm934;935; GFX12-LABEL: test_scratch_load_i8_sext_svs:936; GFX12: ; %bb.0:937; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0938; GFX12-NEXT: scratch_load_i8 v0, v0, s0 offset:1939; GFX12-NEXT: s_wait_loadcnt 0x0940; GFX12-NEXT: flat_store_b32 v[1:2], v0941; GFX12-NEXT: s_endpgm942 %voffset4 = mul i32 %voffset, 4943 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4944 %gep = getelementptr inbounds i8, ptr addrspace(5) %gep0, i32 1945 %load = load i8, ptr addrspace(5) %gep, align 4946 %ext = sext i8 %load to i32947 store i32 %ext, ptr %out, align 4948 ret void949}950 951define amdgpu_ps void @test_scratch_load_i16_zext_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {952; GFX10-LABEL: test_scratch_load_i16_zext_svs:953; GFX10: ; %bb.0:954; GFX10-NEXT: s_add_u32 s0, s0, s3955; GFX10-NEXT: s_addc_u32 s1, s1, 0956; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0957; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1958; GFX10-NEXT: v_lshl_add_u32 v0, v0, 2, s2959; GFX10-NEXT: scratch_load_ushort v0, v0, off offset:2960; GFX10-NEXT: s_waitcnt vmcnt(0)961; GFX10-NEXT: flat_store_dword v[1:2], v0962; GFX10-NEXT: s_endpgm963;964; GFX11-LABEL: test_scratch_load_i16_zext_svs:965; GFX11: ; %bb.0:966; GFX11-NEXT: v_lshl_add_u32 v0, v0, 2, s0967; GFX11-NEXT: scratch_load_u16 v0, v0, off offset:2968; GFX11-NEXT: s_waitcnt vmcnt(0)969; GFX11-NEXT: flat_store_b32 v[1:2], v0970; GFX11-NEXT: s_endpgm971;972; GFX12-LABEL: test_scratch_load_i16_zext_svs:973; GFX12: ; %bb.0:974; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v0975; GFX12-NEXT: scratch_load_u16 v0, v0, s0 offset:2976; GFX12-NEXT: s_wait_loadcnt 0x0977; GFX12-NEXT: flat_store_b32 v[1:2], v0978; GFX12-NEXT: s_endpgm979 %voffset4 = mul i32 %voffset, 4980 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4981 %gep = getelementptr inbounds i16, ptr addrspace(5) %gep0, i32 1982 %load = load i16, ptr addrspace(5) %gep, align 4983 %ext = zext i16 %load to i32984 store i32 %ext, ptr %out, align 4985 ret void986}987 988define amdgpu_ps void @test_scratch_load_i16_sext_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {989; GFX10-LABEL: test_scratch_load_i16_sext_svs:990; GFX10: ; %bb.0:991; GFX10-NEXT: s_add_u32 s0, s0, s3992; GFX10-NEXT: s_addc_u32 s1, s1, 0993; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0994; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1995; GFX10-NEXT: v_lshl_add_u32 v0, v0, 2, s2996; GFX10-NEXT: scratch_load_sshort v0, v0, off offset:2997; GFX10-NEXT: s_waitcnt vmcnt(0)998; GFX10-NEXT: flat_store_dword v[1:2], v0999; GFX10-NEXT: s_endpgm1000;1001; GFX11-LABEL: test_scratch_load_i16_sext_svs:1002; GFX11: ; %bb.0:1003; GFX11-NEXT: v_lshl_add_u32 v0, v0, 2, s01004; GFX11-NEXT: scratch_load_i16 v0, v0, off offset:21005; GFX11-NEXT: s_waitcnt vmcnt(0)1006; GFX11-NEXT: flat_store_b32 v[1:2], v01007; GFX11-NEXT: s_endpgm1008;1009; GFX12-LABEL: test_scratch_load_i16_sext_svs:1010; GFX12: ; %bb.0:1011; GFX12-NEXT: v_lshlrev_b32_e32 v0, 2, v01012; GFX12-NEXT: scratch_load_i16 v0, v0, s0 offset:21013; GFX12-NEXT: s_wait_loadcnt 0x01014; GFX12-NEXT: flat_store_b32 v[1:2], v01015; GFX12-NEXT: s_endpgm1016 %voffset4 = mul i32 %voffset, 41017 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41018 %gep = getelementptr inbounds i16, ptr addrspace(5) %gep0, i32 11019 %load = load i16, ptr addrspace(5) %gep, align 41020 %ext = sext i16 %load to i321021 store i32 %ext, ptr %out, align 41022 ret void1023}1024 1025define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1026; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_lo_svs:1027; GFX10: ; %bb.0: ; %bb1028; GFX10-NEXT: s_add_u32 s0, s0, s31029; GFX10-NEXT: s_addc_u32 s1, s1, 01030; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01031; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11032; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01033; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff00001034; GFX10-NEXT: v_add3_u32 v0, s2, v0, 11035; GFX10-NEXT: scratch_load_ubyte_d16 v3, v0, off1036; GFX10-NEXT: s_waitcnt vmcnt(0)1037; GFX10-NEXT: flat_store_dword v[1:2], v31038; GFX10-NEXT: s_endpgm1039;1040; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_lo_svs:1041; GFX11: ; %bb.0: ; %bb1042; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01043; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1044; GFX11-NEXT: v_add3_u32 v0, s0, v0, 11045; GFX11-NEXT: scratch_load_d16_u8 v3, v0, off1046; GFX11-NEXT: s_waitcnt vmcnt(0)1047; GFX11-NEXT: flat_store_b32 v[1:2], v31048; GFX11-NEXT: s_endpgm1049;1050; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_lo_svs:1051; GFX12: ; %bb.0: ; %bb1052; GFX12-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01053; GFX12-NEXT: scratch_load_d16_u8 v3, v0, s0 offset:11054; GFX12-NEXT: s_wait_loadcnt 0x01055; GFX12-NEXT: flat_store_b32 v[1:2], v31056; GFX12-NEXT: s_endpgm1057bb:1058 %voffset4 = mul i32 %voffset, 41059 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41060 %gep = getelementptr i8, ptr addrspace(5) %gep0, i64 11061 %load_lo = load i8, ptr addrspace(5) %gep1062 %ext = zext i8 %load_lo to i161063 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 01064 store <2 x i16> %result, ptr %out, align 41065 ret void1066}1067 1068define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1069; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_lo_svs:1070; GFX10: ; %bb.0: ; %bb1071; GFX10-NEXT: s_add_u32 s0, s0, s31072; GFX10-NEXT: s_addc_u32 s1, s1, 01073; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01074; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11075; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01076; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff00001077; GFX10-NEXT: v_add3_u32 v0, s2, v0, 11078; GFX10-NEXT: scratch_load_sbyte_d16 v3, v0, off1079; GFX10-NEXT: s_waitcnt vmcnt(0)1080; GFX10-NEXT: flat_store_dword v[1:2], v31081; GFX10-NEXT: s_endpgm1082;1083; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_lo_svs:1084; GFX11: ; %bb.0: ; %bb1085; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01086; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1087; GFX11-NEXT: v_add3_u32 v0, s0, v0, 11088; GFX11-NEXT: scratch_load_d16_i8 v3, v0, off1089; GFX11-NEXT: s_waitcnt vmcnt(0)1090; GFX11-NEXT: flat_store_b32 v[1:2], v31091; GFX11-NEXT: s_endpgm1092;1093; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_lo_svs:1094; GFX12: ; %bb.0: ; %bb1095; GFX12-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01096; GFX12-NEXT: scratch_load_d16_i8 v3, v0, s0 offset:11097; GFX12-NEXT: s_wait_loadcnt 0x01098; GFX12-NEXT: flat_store_b32 v[1:2], v31099; GFX12-NEXT: s_endpgm1100bb:1101 %voffset4 = mul i32 %voffset, 41102 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41103 %gep = getelementptr i8, ptr addrspace(5) %gep0, i64 11104 %load_lo = load i8, ptr addrspace(5) %gep1105 %ext = sext i8 %load_lo to i161106 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 01107 store <2 x i16> %result, ptr %out, align 41108 ret void1109}1110 1111define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1112; GFX10-LABEL: test_scratch_load_i16_to_d16_lo_svs:1113; GFX10: ; %bb.0: ; %bb1114; GFX10-NEXT: s_add_u32 s0, s0, s31115; GFX10-NEXT: s_addc_u32 s1, s1, 01116; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01117; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11118; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01119; GFX10-NEXT: v_mov_b32_e32 v3, 0xffff00001120; GFX10-NEXT: v_add3_u32 v0, s2, v0, 21121; GFX10-NEXT: scratch_load_short_d16 v3, v0, off1122; GFX10-NEXT: s_waitcnt vmcnt(0)1123; GFX10-NEXT: flat_store_dword v[1:2], v31124; GFX10-NEXT: s_endpgm1125;1126; GFX11-LABEL: test_scratch_load_i16_to_d16_lo_svs:1127; GFX11: ; %bb.0: ; %bb1128; GFX11-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01129; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1130; GFX11-NEXT: v_add3_u32 v0, s0, v0, 21131; GFX11-NEXT: scratch_load_d16_b16 v3, v0, off1132; GFX11-NEXT: s_waitcnt vmcnt(0)1133; GFX11-NEXT: flat_store_b32 v[1:2], v31134; GFX11-NEXT: s_endpgm1135;1136; GFX12-LABEL: test_scratch_load_i16_to_d16_lo_svs:1137; GFX12: ; %bb.0: ; %bb1138; GFX12-NEXT: v_dual_mov_b32 v3, 0xffff0000 :: v_dual_lshlrev_b32 v0, 2, v01139; GFX12-NEXT: scratch_load_d16_b16 v3, v0, s0 offset:21140; GFX12-NEXT: s_wait_loadcnt 0x01141; GFX12-NEXT: flat_store_b32 v[1:2], v31142; GFX12-NEXT: s_endpgm1143bb:1144 %voffset4 = mul i32 %voffset, 41145 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41146 %gep = getelementptr i16, ptr addrspace(5) %gep0, i64 11147 %load_lo = load i16, ptr addrspace(5) %gep1148 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 01149 store <2 x i16> %result, ptr %out, align 41150 ret void1151}1152 1153 1154define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1155; GFX10-LABEL: test_scratch_load_i8_zext_to_d16_hi_svs:1156; GFX10: ; %bb.0: ; %bb1157; GFX10-NEXT: s_add_u32 s0, s0, s31158; GFX10-NEXT: s_addc_u32 s1, s1, 01159; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01160; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11161; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01162; GFX10-NEXT: v_mov_b32_e32 v3, -11163; GFX10-NEXT: v_add3_u32 v0, s2, v0, 11164; GFX10-NEXT: scratch_load_ubyte_d16_hi v3, v0, off1165; GFX10-NEXT: s_waitcnt vmcnt(0)1166; GFX10-NEXT: flat_store_dword v[1:2], v31167; GFX10-NEXT: s_endpgm1168;1169; GFX11-LABEL: test_scratch_load_i8_zext_to_d16_hi_svs:1170; GFX11: ; %bb.0: ; %bb1171; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v01172; GFX11-NEXT: v_mov_b16_e32 v3.l, -11173; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)1174; GFX11-NEXT: v_add3_u32 v0, s0, v0, 11175; GFX11-NEXT: scratch_load_d16_hi_u8 v3, v0, off1176; GFX11-NEXT: s_waitcnt vmcnt(0)1177; GFX11-NEXT: flat_store_b32 v[1:2], v31178; GFX11-NEXT: s_endpgm1179;1180; GFX12-LABEL: test_scratch_load_i8_zext_to_d16_hi_svs:1181; GFX12: ; %bb.0: ; %bb1182; GFX12-NEXT: v_dual_mov_b32 v3, -1 :: v_dual_lshlrev_b32 v0, 2, v01183; GFX12-NEXT: scratch_load_d16_hi_u8 v3, v0, s0 offset:11184; GFX12-NEXT: s_wait_loadcnt 0x01185; GFX12-NEXT: flat_store_b32 v[1:2], v31186; GFX12-NEXT: s_endpgm1187bb:1188 %voffset4 = mul i32 %voffset, 41189 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41190 %gep = getelementptr i8, ptr addrspace(5) %gep0, i64 11191 %load_lo = load i8, ptr addrspace(5) %gep1192 %ext = zext i8 %load_lo to i161193 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 11194 store <2 x i16> %result, ptr %out, align 41195 ret void1196}1197 1198define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1199; GFX10-LABEL: test_scratch_load_i8_sext_to_d16_hi_svs:1200; GFX10: ; %bb.0: ; %bb1201; GFX10-NEXT: s_add_u32 s0, s0, s31202; GFX10-NEXT: s_addc_u32 s1, s1, 01203; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01204; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11205; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01206; GFX10-NEXT: v_mov_b32_e32 v3, -11207; GFX10-NEXT: v_add3_u32 v0, s2, v0, 11208; GFX10-NEXT: scratch_load_sbyte_d16_hi v3, v0, off1209; GFX10-NEXT: s_waitcnt vmcnt(0)1210; GFX10-NEXT: flat_store_dword v[1:2], v31211; GFX10-NEXT: s_endpgm1212;1213; GFX11-LABEL: test_scratch_load_i8_sext_to_d16_hi_svs:1214; GFX11: ; %bb.0: ; %bb1215; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v01216; GFX11-NEXT: v_mov_b16_e32 v3.l, -11217; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)1218; GFX11-NEXT: v_add3_u32 v0, s0, v0, 11219; GFX11-NEXT: scratch_load_d16_hi_i8 v3, v0, off1220; GFX11-NEXT: s_waitcnt vmcnt(0)1221; GFX11-NEXT: flat_store_b32 v[1:2], v31222; GFX11-NEXT: s_endpgm1223;1224; GFX12-LABEL: test_scratch_load_i8_sext_to_d16_hi_svs:1225; GFX12: ; %bb.0: ; %bb1226; GFX12-NEXT: v_dual_mov_b32 v3, -1 :: v_dual_lshlrev_b32 v0, 2, v01227; GFX12-NEXT: scratch_load_d16_hi_i8 v3, v0, s0 offset:11228; GFX12-NEXT: s_wait_loadcnt 0x01229; GFX12-NEXT: flat_store_b32 v[1:2], v31230; GFX12-NEXT: s_endpgm1231bb:1232 %voffset4 = mul i32 %voffset, 41233 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41234 %gep = getelementptr i8, ptr addrspace(5) %gep0, i64 11235 %load_lo = load i8, ptr addrspace(5) %gep1236 %ext = sext i8 %load_lo to i161237 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %ext, i32 11238 store <2 x i16> %result, ptr %out, align 41239 ret void1240}1241 1242define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_svs(ptr addrspace(5) inreg %in, i32 %voffset, ptr %out) {1243; GFX10-LABEL: test_scratch_load_i16_to_d16_hi_svs:1244; GFX10: ; %bb.0: ; %bb1245; GFX10-NEXT: s_add_u32 s0, s0, s31246; GFX10-NEXT: s_addc_u32 s1, s1, 01247; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01248; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11249; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v01250; GFX10-NEXT: v_mov_b32_e32 v3, -11251; GFX10-NEXT: v_add3_u32 v0, s2, v0, 21252; GFX10-NEXT: scratch_load_short_d16_hi v3, v0, off1253; GFX10-NEXT: s_waitcnt vmcnt(0)1254; GFX10-NEXT: flat_store_dword v[1:2], v31255; GFX10-NEXT: s_endpgm1256;1257; GFX11-LABEL: test_scratch_load_i16_to_d16_hi_svs:1258; GFX11: ; %bb.0: ; %bb1259; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v01260; GFX11-NEXT: v_mov_b16_e32 v3.l, -11261; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)1262; GFX11-NEXT: v_add3_u32 v0, s0, v0, 21263; GFX11-NEXT: scratch_load_d16_hi_b16 v3, v0, off1264; GFX11-NEXT: s_waitcnt vmcnt(0)1265; GFX11-NEXT: flat_store_b32 v[1:2], v31266; GFX11-NEXT: s_endpgm1267;1268; GFX12-LABEL: test_scratch_load_i16_to_d16_hi_svs:1269; GFX12: ; %bb.0: ; %bb1270; GFX12-NEXT: v_dual_mov_b32 v3, -1 :: v_dual_lshlrev_b32 v0, 2, v01271; GFX12-NEXT: scratch_load_d16_hi_b16 v3, v0, s0 offset:21272; GFX12-NEXT: s_wait_loadcnt 0x01273; GFX12-NEXT: flat_store_b32 v[1:2], v31274; GFX12-NEXT: s_endpgm1275bb:1276 %voffset4 = mul i32 %voffset, 41277 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset41278 %gep = getelementptr i16, ptr addrspace(5) %gep0, i64 11279 %load_lo = load i16, ptr addrspace(5) %gep1280 %result = insertelement <2 x i16> <i16 -1, i16 -1>, i16 %load_lo, i32 11281 store <2 x i16> %result, ptr %out, align 41282 ret void1283}1284 1285define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_svs(ptr %in, ptr addrspace(5) inreg %out, i32 %voffset) {1286; GFX10-LABEL: test_scratch_store_b8_from_d16_hi_svs:1287; GFX10: ; %bb.0: ; %bb1288; GFX10-NEXT: s_add_u32 s0, s0, s31289; GFX10-NEXT: s_addc_u32 s1, s1, 01290; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01291; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11292; GFX10-NEXT: flat_load_dword v0, v[0:1]1293; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v21294; GFX10-NEXT: v_add3_u32 v1, s2, v1, 41295; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1296; GFX10-NEXT: scratch_store_byte_d16_hi v1, v0, off1297; GFX10-NEXT: s_endpgm1298;1299; GFX11-LABEL: test_scratch_store_b8_from_d16_hi_svs:1300; GFX11: ; %bb.0: ; %bb1301; GFX11-NEXT: flat_load_b32 v0, v[0:1]1302; GFX11-NEXT: v_lshlrev_b32_e32 v1, 2, v21303; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1304; GFX11-NEXT: v_add3_u32 v1, s0, v1, 41305; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1306; GFX11-NEXT: scratch_store_d16_hi_b8 v1, v0, off1307; GFX11-NEXT: s_endpgm1308;1309; GFX12-LABEL: test_scratch_store_b8_from_d16_hi_svs:1310; GFX12: ; %bb.0: ; %bb1311; GFX12-NEXT: flat_load_b32 v0, v[0:1]1312; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v21313; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01314; GFX12-NEXT: scratch_store_d16_hi_b8 v1, v0, s0 offset:41315; GFX12-NEXT: s_endpgm1316bb:1317 %load = load <4 x i8>, ptr %in1318 %element = extractelement <4 x i8> %load, i32 21319 %voffset4 = mul i32 %voffset, 41320 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %out, i32 %voffset41321 %gep = getelementptr <4 x i8>, ptr addrspace(5) %gep0, i64 11322 store i8 %element, ptr addrspace(5) %gep, align 41323 ret void1324}1325 1326define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_svs(ptr %in, ptr addrspace(5) inreg %out, i32 %voffset) {1327; GFX10-LABEL: test_scratch_store_b16_from_d16_hi_svs:1328; GFX10: ; %bb.0: ; %bb1329; GFX10-NEXT: s_add_u32 s0, s0, s31330; GFX10-NEXT: s_addc_u32 s1, s1, 01331; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s01332; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s11333; GFX10-NEXT: flat_load_dword v0, v[0:1]1334; GFX10-NEXT: v_lshlrev_b32_e32 v1, 2, v21335; GFX10-NEXT: v_add3_u32 v1, s2, v1, 21336; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1337; GFX10-NEXT: scratch_store_short_d16_hi v1, v0, off1338; GFX10-NEXT: s_endpgm1339;1340; GFX11-LABEL: test_scratch_store_b16_from_d16_hi_svs:1341; GFX11: ; %bb.0: ; %bb1342; GFX11-NEXT: flat_load_b32 v0, v[0:1]1343; GFX11-NEXT: v_lshlrev_b32_e32 v1, 2, v21344; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1345; GFX11-NEXT: v_add3_u32 v1, s0, v1, 21346; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1347; GFX11-NEXT: scratch_store_d16_hi_b16 v1, v0, off1348; GFX11-NEXT: s_endpgm1349;1350; GFX12-LABEL: test_scratch_store_b16_from_d16_hi_svs:1351; GFX12: ; %bb.0: ; %bb1352; GFX12-NEXT: flat_load_b32 v0, v[0:1]1353; GFX12-NEXT: v_lshlrev_b32_e32 v1, 2, v21354; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01355; GFX12-NEXT: scratch_store_d16_hi_b16 v1, v0, s0 offset:21356; GFX12-NEXT: s_endpgm1357bb:1358 %load = load <2 x i16>, ptr %in1359 %element = extractelement <2 x i16> %load, i32 11360 %voffset4 = mul i32 %voffset, 41361 %gep0 = getelementptr inbounds i8, ptr addrspace(5) %out, i32 %voffset41362 %gep = getelementptr <2 x i8>, ptr addrspace(5) %gep0, i64 11363 store i16 %element, ptr addrspace(5) %gep, align 41364 ret void1365}1366