1154 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx900 -mattr=-unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,ALIGNED,ALIGNED-SDAG3; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 -mattr=-unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,ALIGNED,ALIGNED-GISEL4; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx900 -mattr=+unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,UNALIGNED,UNALIGNED-SDAG5; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 -mattr=+unaligned-access-mode < %s | FileCheck %s -check-prefixes=GCN,UNALIGNED,UNALIGNED-GISEL6 7; FixMe: need to decide to move multiple instructions to vgpr that will be folded by isel pattern8 9define amdgpu_kernel void @ds1align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {10; GCN-LABEL: ds1align1:11; GCN: ; %bb.0:12; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2413; GCN-NEXT: s_waitcnt lgkmcnt(0)14; GCN-NEXT: v_mov_b32_e32 v0, s015; GCN-NEXT: ds_read_u8 v0, v016; GCN-NEXT: v_mov_b32_e32 v1, s117; GCN-NEXT: s_waitcnt lgkmcnt(0)18; GCN-NEXT: ds_write_b8 v1, v019; GCN-NEXT: s_endpgm20 %val = load i8, ptr addrspace(3) %in, align 121 store i8 %val, ptr addrspace(3) %out, align 122 ret void23}24 25define amdgpu_kernel void @ds2align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {26; ALIGNED-SDAG-LABEL: ds2align1:27; ALIGNED-SDAG: ; %bb.0:28; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2429; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)30; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s031; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v032; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:133; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v2, s134; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)35; ALIGNED-SDAG-NEXT: ds_write_b8 v2, v136; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)37; ALIGNED-SDAG-NEXT: ds_write_b8 v2, v0 offset:138; ALIGNED-SDAG-NEXT: s_endpgm39;40; ALIGNED-GISEL-LABEL: ds2align1:41; ALIGNED-GISEL: ; %bb.0:42; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2443; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)44; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s045; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v046; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:147; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v2, s148; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)49; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v150; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)51; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s1, v052; ALIGNED-GISEL-NEXT: s_lshl_b32 s1, s1, 853; ALIGNED-GISEL-NEXT: s_or_b32 s0, s1, s054; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s055; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s056; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s1, 857; ALIGNED-GISEL-NEXT: ds_write_b8 v2, v058; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s059; ALIGNED-GISEL-NEXT: ds_write_b8 v2, v0 offset:160; ALIGNED-GISEL-NEXT: s_endpgm61;62; UNALIGNED-LABEL: ds2align1:63; UNALIGNED: ; %bb.0:64; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2465; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)66; UNALIGNED-NEXT: v_mov_b32_e32 v0, s067; UNALIGNED-NEXT: ds_read_u16 v0, v068; UNALIGNED-NEXT: v_mov_b32_e32 v1, s169; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)70; UNALIGNED-NEXT: ds_write_b16 v1, v071; UNALIGNED-NEXT: s_endpgm72 %val = load i16, ptr addrspace(3) %in, align 173 store i16 %val, ptr addrspace(3) %out, align 174 ret void75}76 77define amdgpu_kernel void @ds2align2(ptr addrspace(3) %in, ptr addrspace(3) %out) {78; GCN-LABEL: ds2align2:79; GCN: ; %bb.0:80; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2481; GCN-NEXT: s_waitcnt lgkmcnt(0)82; GCN-NEXT: v_mov_b32_e32 v0, s083; GCN-NEXT: ds_read_u16 v0, v084; GCN-NEXT: v_mov_b32_e32 v1, s185; GCN-NEXT: s_waitcnt lgkmcnt(0)86; GCN-NEXT: ds_write_b16 v1, v087; GCN-NEXT: s_endpgm88 %val = load i16, ptr addrspace(3) %in, align 289 store i16 %val, ptr addrspace(3) %out, align 290 ret void91}92 93define amdgpu_kernel void @ds4align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {94; ALIGNED-SDAG-LABEL: ds4align1:95; ALIGNED-SDAG: ; %bb.0:96; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2497; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)98; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s099; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v0100; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0 offset:1101; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2102; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:3103; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v4, s1104; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)105; ALIGNED-SDAG-NEXT: ds_write_b8 v4, v1106; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)107; ALIGNED-SDAG-NEXT: ds_write_b8 v4, v2 offset:1108; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)109; ALIGNED-SDAG-NEXT: ds_write_b8 v4, v3 offset:2110; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)111; ALIGNED-SDAG-NEXT: ds_write_b8 v4, v0 offset:3112; ALIGNED-SDAG-NEXT: s_endpgm113;114; ALIGNED-GISEL-LABEL: ds4align1:115; ALIGNED-GISEL: ; %bb.0:116; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24117; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)118; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0119; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0120; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:1121; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2122; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:3123; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)124; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1125; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)126; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2127; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)128; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3129; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)130; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0131; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8132; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24133; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16134; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0135; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s3136; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0137; ALIGNED-GISEL-NEXT: s_and_b32 s3, 0xffff, s0138; ALIGNED-GISEL-NEXT: s_lshr_b32 s3, s3, 8139; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0140; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1141; ALIGNED-GISEL-NEXT: s_lshr_b32 s2, s0, 16142; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0143; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3144; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:1145; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s0, 24146; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2147; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2148; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0149; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3150; ALIGNED-GISEL-NEXT: s_endpgm151;152; UNALIGNED-LABEL: ds4align1:153; UNALIGNED: ; %bb.0:154; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24155; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)156; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0157; UNALIGNED-NEXT: ds_read_b32 v0, v0158; UNALIGNED-NEXT: v_mov_b32_e32 v1, s1159; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)160; UNALIGNED-NEXT: ds_write_b32 v1, v0161; UNALIGNED-NEXT: s_endpgm162 %val = load i32, ptr addrspace(3) %in, align 1163 store i32 %val, ptr addrspace(3) %out, align 1164 ret void165}166 167define amdgpu_kernel void @ds4align2(ptr addrspace(3) %in, ptr addrspace(3) %out) {168; ALIGNED-SDAG-LABEL: ds4align2:169; ALIGNED-SDAG: ; %bb.0:170; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24171; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)172; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0173; ALIGNED-SDAG-NEXT: ds_read_u16 v1, v0174; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:2175; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v2, s1176; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)177; ALIGNED-SDAG-NEXT: ds_write_b16 v2, v1178; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)179; ALIGNED-SDAG-NEXT: ds_write_b16 v2, v0 offset:2180; ALIGNED-SDAG-NEXT: s_endpgm181;182; ALIGNED-GISEL-LABEL: ds4align2:183; ALIGNED-GISEL: ; %bb.0:184; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24185; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)186; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0187; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0188; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:2189; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v2, s1190; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)191; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1192; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)193; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s1, v0194; ALIGNED-GISEL-NEXT: s_lshl_b32 s1, s1, 16195; ALIGNED-GISEL-NEXT: s_or_b32 s0, s1, s0196; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s0, 16197; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0198; ALIGNED-GISEL-NEXT: ds_write_b16 v2, v0199; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1200; ALIGNED-GISEL-NEXT: ds_write_b16 v2, v0 offset:2201; ALIGNED-GISEL-NEXT: s_endpgm202;203; UNALIGNED-LABEL: ds4align2:204; UNALIGNED: ; %bb.0:205; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24206; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)207; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0208; UNALIGNED-NEXT: ds_read_b32 v0, v0209; UNALIGNED-NEXT: v_mov_b32_e32 v1, s1210; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)211; UNALIGNED-NEXT: ds_write_b32 v1, v0212; UNALIGNED-NEXT: s_endpgm213 %val = load i32, ptr addrspace(3) %in, align 2214 store i32 %val, ptr addrspace(3) %out, align 2215 ret void216}217 218define amdgpu_kernel void @ds4align4(ptr addrspace(3) %in, ptr addrspace(3) %out) {219; GCN-LABEL: ds4align4:220; GCN: ; %bb.0:221; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24222; GCN-NEXT: s_waitcnt lgkmcnt(0)223; GCN-NEXT: v_mov_b32_e32 v0, s0224; GCN-NEXT: ds_read_b32 v0, v0225; GCN-NEXT: v_mov_b32_e32 v1, s1226; GCN-NEXT: s_waitcnt lgkmcnt(0)227; GCN-NEXT: ds_write_b32 v1, v0228; GCN-NEXT: s_endpgm229 %val = load i32, ptr addrspace(3) %in, align 4230 store i32 %val, ptr addrspace(3) %out, align 4231 ret void232}233 234define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {235; ALIGNED-SDAG-LABEL: ds8align1:236; ALIGNED-SDAG: ; %bb.0:237; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24238; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)239; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0240; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0241; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:1242; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:2243; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4244; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5245; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:3246; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:6247; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:7248; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1249; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)250; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v5 offset:4251; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)252; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v6 offset:5253; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v2254; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v3 offset:1255; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v4 offset:2256; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)257; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v7 offset:3258; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)259; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v8 offset:6260; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)261; ALIGNED-SDAG-NEXT: ds_write_b8 v1, v0 offset:7262; ALIGNED-SDAG-NEXT: s_endpgm263;264; ALIGNED-GISEL-LABEL: ds8align1:265; ALIGNED-GISEL: ; %bb.0:266; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24267; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)268; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0269; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0270; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:1271; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2272; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3273; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4274; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5275; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6276; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:7277; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)278; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2279; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1280; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8281; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0282; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)283; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3284; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)285; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4286; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24287; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16288; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2289; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)290; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6291; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0292; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5293; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8294; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2295; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)296; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7297; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)298; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0299; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24300; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16301; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3302; ALIGNED-GISEL-NEXT: s_and_b32 s4, 0xffff, s0303; ALIGNED-GISEL-NEXT: s_lshr_b32 s4, s4, 8304; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0305; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1306; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2307; ALIGNED-GISEL-NEXT: s_lshr_b32 s3, s0, 16308; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0309; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4310; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:1311; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s0, 24312; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3313; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2314; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0315; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2316; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3317; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8318; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2319; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 16320; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:4321; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1322; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:5323; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s2, 24324; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0325; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:6326; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1327; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:7328; ALIGNED-GISEL-NEXT: s_endpgm329;330; UNALIGNED-LABEL: ds8align1:331; UNALIGNED: ; %bb.0:332; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24333; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)334; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0335; UNALIGNED-NEXT: ds_read_b64 v[0:1], v0336; UNALIGNED-NEXT: v_mov_b32_e32 v2, s1337; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)338; UNALIGNED-NEXT: ds_write_b64 v2, v[0:1]339; UNALIGNED-NEXT: s_endpgm340 %val = load <2 x i32>, ptr addrspace(3) %in, align 1341 store <2 x i32> %val, ptr addrspace(3) %out, align 1342 ret void343}344 345define amdgpu_kernel void @ds8align2(ptr addrspace(3) %in, ptr addrspace(3) %out) {346; ALIGNED-SDAG-LABEL: ds8align2:347; ALIGNED-SDAG: ; %bb.0:348; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24349; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)350; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0351; ALIGNED-SDAG-NEXT: ds_read_u16 v1, v0 offset:4352; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0353; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:2354; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:6355; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v4, s1356; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)357; ALIGNED-SDAG-NEXT: ds_write_b16 v4, v1 offset:4358; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)359; ALIGNED-SDAG-NEXT: ds_write_b16 v4, v2360; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)361; ALIGNED-SDAG-NEXT: ds_write_b16 v4, v3 offset:2362; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)363; ALIGNED-SDAG-NEXT: ds_write_b16 v4, v0 offset:6364; ALIGNED-SDAG-NEXT: s_endpgm365;366; ALIGNED-GISEL-LABEL: ds8align2:367; ALIGNED-GISEL: ; %bb.0:368; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24369; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)370; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0371; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0372; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:2373; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4374; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:6375; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)376; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1377; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)378; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2379; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16380; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)381; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0382; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3383; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16384; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0385; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s3386; ALIGNED-GISEL-NEXT: s_lshr_b32 s3, s0, 16387; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0388; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1389; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0390; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3391; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:2392; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 16393; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2394; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:4395; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0396; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:6397; ALIGNED-GISEL-NEXT: s_endpgm398;399; UNALIGNED-LABEL: ds8align2:400; UNALIGNED: ; %bb.0:401; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24402; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)403; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0404; UNALIGNED-NEXT: ds_read_b64 v[0:1], v0405; UNALIGNED-NEXT: v_mov_b32_e32 v2, s1406; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)407; UNALIGNED-NEXT: ds_write_b64 v2, v[0:1]408; UNALIGNED-NEXT: s_endpgm409 %val = load <2 x i32>, ptr addrspace(3) %in, align 2410 store <2 x i32> %val, ptr addrspace(3) %out, align 2411 ret void412}413 414define amdgpu_kernel void @ds8align4(ptr addrspace(3) %in, ptr addrspace(3) %out) {415; GCN-LABEL: ds8align4:416; GCN: ; %bb.0:417; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24418; GCN-NEXT: s_waitcnt lgkmcnt(0)419; GCN-NEXT: v_mov_b32_e32 v0, s0420; GCN-NEXT: ds_read2_b32 v[0:1], v0 offset1:1421; GCN-NEXT: v_mov_b32_e32 v2, s1422; GCN-NEXT: s_waitcnt lgkmcnt(0)423; GCN-NEXT: ds_write2_b32 v2, v0, v1 offset1:1424; GCN-NEXT: s_endpgm425 %val = load <2 x i32>, ptr addrspace(3) %in, align 4426 store <2 x i32> %val, ptr addrspace(3) %out, align 4427 ret void428}429 430define amdgpu_kernel void @ds8align8(ptr addrspace(3) %in, ptr addrspace(3) %out) {431; GCN-LABEL: ds8align8:432; GCN: ; %bb.0:433; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24434; GCN-NEXT: s_waitcnt lgkmcnt(0)435; GCN-NEXT: v_mov_b32_e32 v0, s0436; GCN-NEXT: ds_read_b64 v[0:1], v0437; GCN-NEXT: v_mov_b32_e32 v2, s1438; GCN-NEXT: s_waitcnt lgkmcnt(0)439; GCN-NEXT: ds_write_b64 v2, v[0:1]440; GCN-NEXT: s_endpgm441 %val = load <2 x i32>, ptr addrspace(3) %in, align 8442 store <2 x i32> %val, ptr addrspace(3) %out, align 8443 ret void444}445 446define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {447; ALIGNED-SDAG-LABEL: ds12align1:448; ALIGNED-SDAG: ; %bb.0:449; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24450; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)451; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0452; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v0453; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0 offset:1454; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2455; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:3456; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4457; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5458; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:6459; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:7460; ALIGNED-SDAG-NEXT: ds_read_u8 v9, v0 offset:8461; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9462; ALIGNED-SDAG-NEXT: ds_read_u8 v11, v0 offset:10463; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:11464; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v12, s1465; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)466; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v9 offset:8467; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)468; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v10 offset:9469; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v5 offset:4470; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v6 offset:5471; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v1472; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v2 offset:1473; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v3 offset:2474; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v4 offset:3475; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v7 offset:6476; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v8 offset:7477; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)478; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v11 offset:10479; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(11)480; ALIGNED-SDAG-NEXT: ds_write_b8 v12, v0 offset:11481; ALIGNED-SDAG-NEXT: s_endpgm482;483; ALIGNED-GISEL-LABEL: ds12align1:484; ALIGNED-GISEL: ; %bb.0:485; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24486; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)487; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0488; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0489; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:1490; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2491; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3492; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4493; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5494; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6495; ALIGNED-GISEL-NEXT: ds_read_u8 v8, v0 offset:7496; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)497; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2498; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1499; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8500; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0501; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)502; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3503; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)504; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4505; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24506; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16507; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:8508; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:9509; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2510; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)511; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6512; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0513; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5514; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8515; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:10516; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:11517; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2518; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)519; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7520; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)521; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v8522; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24523; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16524; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3525; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)526; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2527; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2528; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1529; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 8530; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3531; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)532; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3533; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)534; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v0535; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 24536; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16537; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4538; ALIGNED-GISEL-NEXT: s_and_b32 s5, 0xffff, s0539; ALIGNED-GISEL-NEXT: s_lshr_b32 s5, s5, 8540; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0541; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1542; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3543; ALIGNED-GISEL-NEXT: s_lshr_b32 s4, s0, 16544; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0545; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s5546; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:1547; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s0, 24548; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4549; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2550; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0551; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2552; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3553; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8554; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2555; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 16556; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:4557; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1558; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:5559; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s2, 24560; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0561; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:6562; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1563; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s3564; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:7565; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8566; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3567; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s3, 16568; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:8569; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1570; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:9571; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s3, 24572; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0573; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:10574; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1575; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:11576; ALIGNED-GISEL-NEXT: s_endpgm577;578; UNALIGNED-LABEL: ds12align1:579; UNALIGNED: ; %bb.0:580; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24581; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)582; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0583; UNALIGNED-NEXT: ds_read_b96 v[0:2], v0584; UNALIGNED-NEXT: v_mov_b32_e32 v3, s1585; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)586; UNALIGNED-NEXT: ds_write_b96 v3, v[0:2]587; UNALIGNED-NEXT: s_endpgm588 %val = load <3 x i32>, ptr addrspace(3) %in, align 1589 store <3 x i32> %val, ptr addrspace(3) %out, align 1590 ret void591}592 593define amdgpu_kernel void @ds12align2(ptr addrspace(3) %in, ptr addrspace(3) %out) {594; ALIGNED-SDAG-LABEL: ds12align2:595; ALIGNED-SDAG: ; %bb.0:596; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24597; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)598; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0599; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0600; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:8601; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4602; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:2603; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6604; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:10605; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1606; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)607; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:8608; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)609; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4610; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2611; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)612; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:2613; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)614; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:6615; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)616; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v0 offset:10617; ALIGNED-SDAG-NEXT: s_endpgm618;619; ALIGNED-GISEL-LABEL: ds12align2:620; ALIGNED-GISEL: ; %bb.0:621; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24622; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)623; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0624; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0625; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:2626; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4627; ALIGNED-GISEL-NEXT: ds_read_u16 v4, v0 offset:6628; ALIGNED-GISEL-NEXT: ds_read_u16 v5, v0 offset:8629; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:10630; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)631; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2632; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1633; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)634; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4635; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16636; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3637; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0638; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s4, 16639; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)640; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0641; ALIGNED-GISEL-NEXT: s_or_b32 s2, s2, s3642; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v5643; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16644; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3645; ALIGNED-GISEL-NEXT: s_lshr_b32 s4, s0, 16646; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0647; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1648; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0649; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4650; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:2651; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 16652; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2653; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:4654; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0655; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:6656; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s3, 16657; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3658; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:8659; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0660; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:10661; ALIGNED-GISEL-NEXT: s_endpgm662;663; UNALIGNED-LABEL: ds12align2:664; UNALIGNED: ; %bb.0:665; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24666; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)667; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0668; UNALIGNED-NEXT: ds_read_b96 v[0:2], v0669; UNALIGNED-NEXT: v_mov_b32_e32 v3, s1670; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)671; UNALIGNED-NEXT: ds_write_b96 v3, v[0:2]672; UNALIGNED-NEXT: s_endpgm673 %val = load <3 x i32>, ptr addrspace(3) %in, align 2674 store <3 x i32> %val, ptr addrspace(3) %out, align 2675 ret void676}677 678define amdgpu_kernel void @ds12align4(ptr addrspace(3) %in, ptr addrspace(3) %out) {679; ALIGNED-LABEL: ds12align4:680; ALIGNED: ; %bb.0:681; ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24682; ALIGNED-NEXT: s_waitcnt lgkmcnt(0)683; ALIGNED-NEXT: v_mov_b32_e32 v2, s0684; ALIGNED-NEXT: ds_read2_b32 v[0:1], v2 offset1:1685; ALIGNED-NEXT: ds_read_b32 v2, v2 offset:8686; ALIGNED-NEXT: v_mov_b32_e32 v3, s1687; ALIGNED-NEXT: s_waitcnt lgkmcnt(1)688; ALIGNED-NEXT: ds_write2_b32 v3, v0, v1 offset1:1689; ALIGNED-NEXT: s_waitcnt lgkmcnt(1)690; ALIGNED-NEXT: ds_write_b32 v3, v2 offset:8691; ALIGNED-NEXT: s_endpgm692;693; UNALIGNED-SDAG-LABEL: ds12align4:694; UNALIGNED-SDAG: ; %bb.0:695; UNALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24696; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)697; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v2, s0698; UNALIGNED-SDAG-NEXT: ds_read2_b32 v[0:1], v2 offset1:1699; UNALIGNED-SDAG-NEXT: ds_read_b32 v2, v2 offset:8700; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v3, s1701; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)702; UNALIGNED-SDAG-NEXT: ds_write2_b32 v3, v0, v1 offset1:1703; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)704; UNALIGNED-SDAG-NEXT: ds_write_b32 v3, v2 offset:8705; UNALIGNED-SDAG-NEXT: s_endpgm706;707; UNALIGNED-GISEL-LABEL: ds12align4:708; UNALIGNED-GISEL: ; %bb.0:709; UNALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24710; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)711; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0712; UNALIGNED-GISEL-NEXT: ds_read_b96 v[0:2], v0713; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v3, s1714; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)715; UNALIGNED-GISEL-NEXT: ds_write_b96 v3, v[0:2]716; UNALIGNED-GISEL-NEXT: s_endpgm717 %val = load <3 x i32>, ptr addrspace(3) %in, align 4718 store <3 x i32> %val, ptr addrspace(3) %out, align 4719 ret void720}721 722define amdgpu_kernel void @ds12align8(ptr addrspace(3) %in, ptr addrspace(3) %out) {723; ALIGNED-SDAG-LABEL: ds12align8:724; ALIGNED-SDAG: ; %bb.0:725; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24726; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)727; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v2, s0728; ALIGNED-SDAG-NEXT: ds_read_b64 v[0:1], v2729; ALIGNED-SDAG-NEXT: ds_read_b32 v2, v2 offset:8730; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v3, s1731; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)732; ALIGNED-SDAG-NEXT: ds_write_b64 v3, v[0:1]733; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)734; ALIGNED-SDAG-NEXT: ds_write_b32 v3, v2 offset:8735; ALIGNED-SDAG-NEXT: s_endpgm736;737; ALIGNED-GISEL-LABEL: ds12align8:738; ALIGNED-GISEL: ; %bb.0:739; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24740; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)741; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v2, s0742; ALIGNED-GISEL-NEXT: ds_read2_b32 v[0:1], v2 offset1:1743; ALIGNED-GISEL-NEXT: ds_read_b32 v2, v2 offset:8744; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v3, s1745; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)746; ALIGNED-GISEL-NEXT: ds_write2_b32 v3, v0, v1 offset1:1747; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)748; ALIGNED-GISEL-NEXT: ds_write_b32 v3, v2 offset:8749; ALIGNED-GISEL-NEXT: s_endpgm750;751; UNALIGNED-SDAG-LABEL: ds12align8:752; UNALIGNED-SDAG: ; %bb.0:753; UNALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24754; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)755; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0756; UNALIGNED-SDAG-NEXT: ds_read_b32 v2, v0 offset:8757; UNALIGNED-SDAG-NEXT: ds_read_b64 v[0:1], v0758; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v3, s1759; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)760; UNALIGNED-SDAG-NEXT: ds_write_b32 v3, v2 offset:8761; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)762; UNALIGNED-SDAG-NEXT: ds_write_b64 v3, v[0:1]763; UNALIGNED-SDAG-NEXT: s_endpgm764;765; UNALIGNED-GISEL-LABEL: ds12align8:766; UNALIGNED-GISEL: ; %bb.0:767; UNALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24768; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)769; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0770; UNALIGNED-GISEL-NEXT: ds_read_b96 v[0:2], v0771; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v3, s1772; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)773; UNALIGNED-GISEL-NEXT: ds_write_b96 v3, v[0:2]774; UNALIGNED-GISEL-NEXT: s_endpgm775 %val = load <3 x i32>, ptr addrspace(3) %in, align 8776 store <3 x i32> %val, ptr addrspace(3) %out, align 8777 ret void778}779 780define amdgpu_kernel void @ds12align16(ptr addrspace(3) %in, ptr addrspace(3) %out) {781; GCN-LABEL: ds12align16:782; GCN: ; %bb.0:783; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24784; GCN-NEXT: s_waitcnt lgkmcnt(0)785; GCN-NEXT: v_mov_b32_e32 v0, s0786; GCN-NEXT: ds_read_b96 v[0:2], v0787; GCN-NEXT: v_mov_b32_e32 v3, s1788; GCN-NEXT: s_waitcnt lgkmcnt(0)789; GCN-NEXT: ds_write_b96 v3, v[0:2]790; GCN-NEXT: s_endpgm791 %val = load <3 x i32>, ptr addrspace(3) %in, align 16792 store <3 x i32> %val, ptr addrspace(3) %out, align 16793 ret void794}795 796define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %out) {797; ALIGNED-SDAG-LABEL: ds16align1:798; ALIGNED-SDAG: ; %bb.0:799; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24800; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)801; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0802; ALIGNED-SDAG-NEXT: ds_read_u8 v1, v0803; ALIGNED-SDAG-NEXT: ds_read_u8 v2, v0 offset:1804; ALIGNED-SDAG-NEXT: ds_read_u8 v3, v0 offset:2805; ALIGNED-SDAG-NEXT: ds_read_u8 v4, v0 offset:3806; ALIGNED-SDAG-NEXT: ds_read_u8 v5, v0 offset:4807; ALIGNED-SDAG-NEXT: ds_read_u8 v6, v0 offset:5808; ALIGNED-SDAG-NEXT: ds_read_u8 v7, v0 offset:6809; ALIGNED-SDAG-NEXT: ds_read_u8 v8, v0 offset:7810; ALIGNED-SDAG-NEXT: ds_read_u8 v9, v0 offset:8811; ALIGNED-SDAG-NEXT: ds_read_u8 v10, v0 offset:9812; ALIGNED-SDAG-NEXT: ds_read_u8 v11, v0 offset:10813; ALIGNED-SDAG-NEXT: ds_read_u8 v12, v0 offset:11814; ALIGNED-SDAG-NEXT: ds_read_u8 v13, v0 offset:12815; ALIGNED-SDAG-NEXT: ds_read_u8 v14, v0 offset:13816; ALIGNED-SDAG-NEXT: ds_read_u8 v15, v0 offset:14817; ALIGNED-SDAG-NEXT: ds_read_u8 v0, v0 offset:15818; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v16, s1819; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)820; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v13 offset:12821; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(3)822; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v14 offset:13823; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v1824; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v2 offset:1825; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v5 offset:4826; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v6 offset:5827; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v9 offset:8828; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v10 offset:9829; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v3 offset:2830; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v4 offset:3831; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v7 offset:6832; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v8 offset:7833; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v11 offset:10834; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v12 offset:11835; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(14)836; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v15 offset:14837; ALIGNED-SDAG-NEXT: ds_write_b8 v16, v0 offset:15838; ALIGNED-SDAG-NEXT: s_endpgm839;840; ALIGNED-GISEL-LABEL: ds16align1:841; ALIGNED-GISEL: ; %bb.0:842; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24843; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)844; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0845; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0846; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:1847; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2848; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3849; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4850; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5851; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6852; ALIGNED-GISEL-NEXT: ds_read_u8 v8, v0 offset:7853; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)854; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2855; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1856; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8857; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0858; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)859; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3860; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)861; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4862; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24863; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16864; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2865; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)866; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6867; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0868; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5869; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8870; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2871; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)872; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7873; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)874; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v8875; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24876; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16877; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3878; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:8879; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:9880; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:10881; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:11882; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:12883; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:13884; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:14885; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:15886; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)887; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2888; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2889; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1890; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 8891; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3892; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)893; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3894; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)895; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v4896; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 24897; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16898; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4899; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)900; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v6901; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3902; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v5903; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 8904; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4905; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)906; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v7907; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)908; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s6, v0909; ALIGNED-GISEL-NEXT: s_lshl_b32 s6, s6, 24910; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 16911; ALIGNED-GISEL-NEXT: s_or_b32 s5, s6, s5912; ALIGNED-GISEL-NEXT: s_and_b32 s6, 0xffff, s0913; ALIGNED-GISEL-NEXT: s_lshr_b32 s6, s6, 8914; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0915; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s1916; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4917; ALIGNED-GISEL-NEXT: s_lshr_b32 s5, s0, 16918; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0919; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s6920; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:1921; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s0, 24922; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s5923; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:2924; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0925; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s2926; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:3927; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8928; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s2929; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 16930; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:4931; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1932; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:5933; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s2, 24934; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0935; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:6936; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1937; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s3938; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:7939; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8940; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s3941; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s3, 16942; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:8943; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1944; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:9945; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s3, 24946; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0947; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:10948; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1949; ALIGNED-GISEL-NEXT: s_and_b32 s1, 0xffff, s4950; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:11951; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s1, 8952; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s4953; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s4, 16954; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:12955; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1956; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:13957; ALIGNED-GISEL-NEXT: s_lshr_b32 s1, s4, 24958; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0959; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:14960; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s1961; ALIGNED-GISEL-NEXT: ds_write_b8 v1, v0 offset:15962; ALIGNED-GISEL-NEXT: s_endpgm963;964; UNALIGNED-LABEL: ds16align1:965; UNALIGNED: ; %bb.0:966; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24967; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)968; UNALIGNED-NEXT: v_mov_b32_e32 v0, s0969; UNALIGNED-NEXT: ds_read_b128 v[0:3], v0970; UNALIGNED-NEXT: v_mov_b32_e32 v4, s1971; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)972; UNALIGNED-NEXT: ds_write_b128 v4, v[0:3]973; UNALIGNED-NEXT: s_endpgm974 %val = load <4 x i32>, ptr addrspace(3) %in, align 1975 store <4 x i32> %val, ptr addrspace(3) %out, align 1976 ret void977}978 979define amdgpu_kernel void @ds16align2(ptr addrspace(3) %in, ptr addrspace(3) %out) {980; ALIGNED-SDAG-LABEL: ds16align2:981; ALIGNED-SDAG: ; %bb.0:982; ALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24983; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)984; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v0, s0985; ALIGNED-SDAG-NEXT: ds_read_u16 v2, v0986; ALIGNED-SDAG-NEXT: ds_read_u16 v3, v0 offset:2987; ALIGNED-SDAG-NEXT: ds_read_u16 v4, v0 offset:4988; ALIGNED-SDAG-NEXT: ds_read_u16 v5, v0 offset:12989; ALIGNED-SDAG-NEXT: ds_read_u16 v6, v0 offset:6990; ALIGNED-SDAG-NEXT: ds_read_u16 v7, v0 offset:8991; ALIGNED-SDAG-NEXT: ds_read_u16 v8, v0 offset:10992; ALIGNED-SDAG-NEXT: ds_read_u16 v0, v0 offset:14993; ALIGNED-SDAG-NEXT: v_mov_b32_e32 v1, s1994; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(4)995; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v5 offset:12996; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v2997; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v4 offset:4998; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(5)999; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v7 offset:81000; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v3 offset:21001; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v6 offset:61002; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)1003; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v8 offset:101004; ALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(7)1005; ALIGNED-SDAG-NEXT: ds_write_b16 v1, v0 offset:141006; ALIGNED-SDAG-NEXT: s_endpgm1007;1008; ALIGNED-GISEL-LABEL: ds16align2:1009; ALIGNED-GISEL: ; %bb.0:1010; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241011; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)1012; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01013; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v01014; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:21015; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:41016; ALIGNED-GISEL-NEXT: ds_read_u16 v4, v0 offset:61017; ALIGNED-GISEL-NEXT: ds_read_u16 v5, v0 offset:81018; ALIGNED-GISEL-NEXT: ds_read_u16 v6, v0 offset:101019; ALIGNED-GISEL-NEXT: ds_read_u16 v7, v0 offset:121020; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:141021; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)1022; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v21023; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v11024; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 161025; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)1026; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v41027; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s01028; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v31029; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 161030; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)1031; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v61032; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s21033; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v51034; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 161035; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)1036; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v01037; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s31038; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v71039; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 161040; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s41041; ALIGNED-GISEL-NEXT: s_lshr_b32 s5, s0, 161042; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01043; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v1, s11044; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v01045; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s51046; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:21047; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s2, 161048; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s21049; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:41050; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01051; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:61052; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s3, 161053; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s31054; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:81055; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01056; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:101057; ALIGNED-GISEL-NEXT: s_lshr_b32 s0, s4, 161058; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s41059; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:121060; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01061; ALIGNED-GISEL-NEXT: ds_write_b16 v1, v0 offset:141062; ALIGNED-GISEL-NEXT: s_endpgm1063;1064; UNALIGNED-LABEL: ds16align2:1065; UNALIGNED: ; %bb.0:1066; UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241067; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)1068; UNALIGNED-NEXT: v_mov_b32_e32 v0, s01069; UNALIGNED-NEXT: ds_read_b128 v[0:3], v01070; UNALIGNED-NEXT: v_mov_b32_e32 v4, s11071; UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)1072; UNALIGNED-NEXT: ds_write_b128 v4, v[0:3]1073; UNALIGNED-NEXT: s_endpgm1074 %val = load <4 x i32>, ptr addrspace(3) %in, align 21075 store <4 x i32> %val, ptr addrspace(3) %out, align 21076 ret void1077}1078 1079define amdgpu_kernel void @ds16align4(ptr addrspace(3) %in, ptr addrspace(3) %out) {1080; ALIGNED-LABEL: ds16align4:1081; ALIGNED: ; %bb.0:1082; ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241083; ALIGNED-NEXT: s_waitcnt lgkmcnt(0)1084; ALIGNED-NEXT: v_mov_b32_e32 v2, s01085; ALIGNED-NEXT: ds_read2_b32 v[0:1], v2 offset1:11086; ALIGNED-NEXT: ds_read2_b32 v[2:3], v2 offset0:2 offset1:31087; ALIGNED-NEXT: v_mov_b32_e32 v4, s11088; ALIGNED-NEXT: s_waitcnt lgkmcnt(1)1089; ALIGNED-NEXT: ds_write2_b32 v4, v0, v1 offset1:11090; ALIGNED-NEXT: s_waitcnt lgkmcnt(1)1091; ALIGNED-NEXT: ds_write2_b32 v4, v2, v3 offset0:2 offset1:31092; ALIGNED-NEXT: s_endpgm1093;1094; UNALIGNED-SDAG-LABEL: ds16align4:1095; UNALIGNED-SDAG: ; %bb.0:1096; UNALIGNED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241097; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(0)1098; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v2, s01099; UNALIGNED-SDAG-NEXT: ds_read2_b32 v[0:1], v2 offset0:2 offset1:31100; UNALIGNED-SDAG-NEXT: ds_read2_b32 v[2:3], v2 offset1:11101; UNALIGNED-SDAG-NEXT: v_mov_b32_e32 v4, s11102; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)1103; UNALIGNED-SDAG-NEXT: ds_write2_b32 v4, v0, v1 offset0:2 offset1:31104; UNALIGNED-SDAG-NEXT: s_waitcnt lgkmcnt(1)1105; UNALIGNED-SDAG-NEXT: ds_write2_b32 v4, v2, v3 offset1:11106; UNALIGNED-SDAG-NEXT: s_endpgm1107;1108; UNALIGNED-GISEL-LABEL: ds16align4:1109; UNALIGNED-GISEL: ; %bb.0:1110; UNALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241111; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)1112; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s01113; UNALIGNED-GISEL-NEXT: ds_read2_b64 v[0:3], v0 offset1:11114; UNALIGNED-GISEL-NEXT: v_mov_b32_e32 v4, s11115; UNALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)1116; UNALIGNED-GISEL-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:11117; UNALIGNED-GISEL-NEXT: s_endpgm1118 %val = load <4 x i32>, ptr addrspace(3) %in, align 41119 store <4 x i32> %val, ptr addrspace(3) %out, align 41120 ret void1121}1122 1123define amdgpu_kernel void @ds16align8(ptr addrspace(3) %in, ptr addrspace(3) %out) {1124; GCN-LABEL: ds16align8:1125; GCN: ; %bb.0:1126; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241127; GCN-NEXT: s_waitcnt lgkmcnt(0)1128; GCN-NEXT: v_mov_b32_e32 v0, s01129; GCN-NEXT: ds_read2_b64 v[0:3], v0 offset1:11130; GCN-NEXT: v_mov_b32_e32 v4, s11131; GCN-NEXT: s_waitcnt lgkmcnt(0)1132; GCN-NEXT: ds_write2_b64 v4, v[0:1], v[2:3] offset1:11133; GCN-NEXT: s_endpgm1134 %val = load <4 x i32>, ptr addrspace(3) %in, align 81135 store <4 x i32> %val, ptr addrspace(3) %out, align 81136 ret void1137}1138 1139define amdgpu_kernel void @ds16align16(ptr addrspace(3) %in, ptr addrspace(3) %out) {1140; GCN-LABEL: ds16align16:1141; GCN: ; %bb.0:1142; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241143; GCN-NEXT: s_waitcnt lgkmcnt(0)1144; GCN-NEXT: v_mov_b32_e32 v0, s01145; GCN-NEXT: ds_read_b128 v[0:3], v01146; GCN-NEXT: v_mov_b32_e32 v4, s11147; GCN-NEXT: s_waitcnt lgkmcnt(0)1148; GCN-NEXT: ds_write_b128 v4, v[0:3]1149; GCN-NEXT: s_endpgm1150 %val = load <4 x i32>, ptr addrspace(3) %in, align 161151 store <4 x i32> %val, ptr addrspace(3) %out, align 161152 ret void1153}1154