3969 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=SDAG %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefix=GISEL %s4 5; Note: if you're adding tests here, also add them to6; lower-buffer-fat-pointers-contents-legalization.ll to verify the IR produced by7; the lowering.8 9;;; Legal types. These are natively supported, no casts should be performed.10 11define i8 @load_i8(ptr addrspace(8) inreg %buf) {12; SDAG-LABEL: load_i8:13; SDAG: ; %bb.0:14; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)15; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 016; SDAG-NEXT: s_waitcnt vmcnt(0)17; SDAG-NEXT: s_setpc_b64 s[30:31]18;19; GISEL-LABEL: load_i8:20; GISEL: ; %bb.0:21; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)22; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 023; GISEL-NEXT: s_waitcnt vmcnt(0)24; GISEL-NEXT: s_setpc_b64 s[30:31]25 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)26 %ret = load i8, ptr addrspace(7) %p27 ret i8 %ret28}29 30define void @store_i8(i8 %data, ptr addrspace(8) inreg %buf) {31; SDAG-LABEL: store_i8:32; SDAG: ; %bb.0:33; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)34; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 035; SDAG-NEXT: s_waitcnt vmcnt(0)36; SDAG-NEXT: s_setpc_b64 s[30:31]37;38; GISEL-LABEL: store_i8:39; GISEL: ; %bb.0:40; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)41; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 042; GISEL-NEXT: s_waitcnt vmcnt(0)43; GISEL-NEXT: s_setpc_b64 s[30:31]44 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)45 store i8 %data, ptr addrspace(7) %p46 ret void47}48 49define i16 @load_i16(ptr addrspace(8) inreg %buf) {50; SDAG-LABEL: load_i16:51; SDAG: ; %bb.0:52; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)53; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 054; SDAG-NEXT: s_waitcnt vmcnt(0)55; SDAG-NEXT: s_setpc_b64 s[30:31]56;57; GISEL-LABEL: load_i16:58; GISEL: ; %bb.0:59; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)60; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 061; GISEL-NEXT: s_waitcnt vmcnt(0)62; GISEL-NEXT: s_setpc_b64 s[30:31]63 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)64 %ret = load i16, ptr addrspace(7) %p65 ret i16 %ret66}67 68define void @store_i16(i16 %data, ptr addrspace(8) inreg %buf) {69; SDAG-LABEL: store_i16:70; SDAG: ; %bb.0:71; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)72; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 073; SDAG-NEXT: s_waitcnt vmcnt(0)74; SDAG-NEXT: s_setpc_b64 s[30:31]75;76; GISEL-LABEL: store_i16:77; GISEL: ; %bb.0:78; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)79; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 080; GISEL-NEXT: s_waitcnt vmcnt(0)81; GISEL-NEXT: s_setpc_b64 s[30:31]82 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)83 store i16 %data, ptr addrspace(7) %p84 ret void85}86 87define i32 @load_i32(ptr addrspace(8) inreg %buf) {88; SDAG-LABEL: load_i32:89; SDAG: ; %bb.0:90; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)91; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 092; SDAG-NEXT: s_waitcnt vmcnt(0)93; SDAG-NEXT: s_setpc_b64 s[30:31]94;95; GISEL-LABEL: load_i32:96; GISEL: ; %bb.0:97; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)98; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 099; GISEL-NEXT: s_waitcnt vmcnt(0)100; GISEL-NEXT: s_setpc_b64 s[30:31]101 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)102 %ret = load i32, ptr addrspace(7) %p103 ret i32 %ret104}105 106define void @store_i32(i32 %data, ptr addrspace(8) inreg %buf) {107; SDAG-LABEL: store_i32:108; SDAG: ; %bb.0:109; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)110; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0111; SDAG-NEXT: s_waitcnt vmcnt(0)112; SDAG-NEXT: s_setpc_b64 s[30:31]113;114; GISEL-LABEL: store_i32:115; GISEL: ; %bb.0:116; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)117; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0118; GISEL-NEXT: s_waitcnt vmcnt(0)119; GISEL-NEXT: s_setpc_b64 s[30:31]120 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)121 store i32 %data, ptr addrspace(7) %p122 ret void123}124 125define i64 @load_i64(ptr addrspace(8) inreg %buf) {126; SDAG-LABEL: load_i64:127; SDAG: ; %bb.0:128; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)129; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0130; SDAG-NEXT: s_waitcnt vmcnt(0)131; SDAG-NEXT: s_setpc_b64 s[30:31]132;133; GISEL-LABEL: load_i64:134; GISEL: ; %bb.0:135; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)136; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0137; GISEL-NEXT: s_waitcnt vmcnt(0)138; GISEL-NEXT: s_setpc_b64 s[30:31]139 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)140 %ret = load i64, ptr addrspace(7) %p141 ret i64 %ret142}143 144define void @store_i64(i64 %data, ptr addrspace(8) inreg %buf) {145; SDAG-LABEL: store_i64:146; SDAG: ; %bb.0:147; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)148; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0149; SDAG-NEXT: s_waitcnt vmcnt(0)150; SDAG-NEXT: s_setpc_b64 s[30:31]151;152; GISEL-LABEL: store_i64:153; GISEL: ; %bb.0:154; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)155; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0156; GISEL-NEXT: s_waitcnt vmcnt(0)157; GISEL-NEXT: s_setpc_b64 s[30:31]158 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)159 store i64 %data, ptr addrspace(7) %p160 ret void161}162 163define i128 @load_i128(ptr addrspace(8) inreg %buf) {164; SDAG-LABEL: load_i128:165; SDAG: ; %bb.0:166; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)167; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0168; SDAG-NEXT: s_waitcnt vmcnt(0)169; SDAG-NEXT: s_setpc_b64 s[30:31]170;171; GISEL-LABEL: load_i128:172; GISEL: ; %bb.0:173; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)174; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0175; GISEL-NEXT: s_waitcnt vmcnt(0)176; GISEL-NEXT: s_setpc_b64 s[30:31]177 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)178 %ret = load i128, ptr addrspace(7) %p179 ret i128 %ret180}181 182define void @store_i128(i128 %data, ptr addrspace(8) inreg %buf) {183; SDAG-LABEL: store_i128:184; SDAG: ; %bb.0:185; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)186; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0187; SDAG-NEXT: s_waitcnt vmcnt(0)188; SDAG-NEXT: s_setpc_b64 s[30:31]189;190; GISEL-LABEL: store_i128:191; GISEL: ; %bb.0:192; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)193; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0194; GISEL-NEXT: s_waitcnt vmcnt(0)195; GISEL-NEXT: s_setpc_b64 s[30:31]196 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)197 store i128 %data, ptr addrspace(7) %p198 ret void199}200 201define <1 x i32> @load_v1i32(ptr addrspace(8) inreg %buf) {202; SDAG-LABEL: load_v1i32:203; SDAG: ; %bb.0:204; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)205; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0206; SDAG-NEXT: s_waitcnt vmcnt(0)207; SDAG-NEXT: s_setpc_b64 s[30:31]208;209; GISEL-LABEL: load_v1i32:210; GISEL: ; %bb.0:211; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)212; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0213; GISEL-NEXT: s_waitcnt vmcnt(0)214; GISEL-NEXT: s_setpc_b64 s[30:31]215 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)216 %ret = load <1 x i32>, ptr addrspace(7) %p217 ret <1 x i32> %ret218}219 220define void @store_v1i32(<1 x i32> %data, ptr addrspace(8) inreg %buf) {221; SDAG-LABEL: store_v1i32:222; SDAG: ; %bb.0:223; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)224; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0225; SDAG-NEXT: s_waitcnt vmcnt(0)226; SDAG-NEXT: s_setpc_b64 s[30:31]227;228; GISEL-LABEL: store_v1i32:229; GISEL: ; %bb.0:230; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)231; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0232; GISEL-NEXT: s_waitcnt vmcnt(0)233; GISEL-NEXT: s_setpc_b64 s[30:31]234 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)235 store <1 x i32> %data, ptr addrspace(7) %p236 ret void237}238 239define <2 x i32> @load_v2i32(ptr addrspace(8) inreg %buf) {240; SDAG-LABEL: load_v2i32:241; SDAG: ; %bb.0:242; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)243; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0244; SDAG-NEXT: s_waitcnt vmcnt(0)245; SDAG-NEXT: s_setpc_b64 s[30:31]246;247; GISEL-LABEL: load_v2i32:248; GISEL: ; %bb.0:249; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)250; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0251; GISEL-NEXT: s_waitcnt vmcnt(0)252; GISEL-NEXT: s_setpc_b64 s[30:31]253 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)254 %ret = load <2 x i32>, ptr addrspace(7) %p255 ret <2 x i32> %ret256}257 258define void @store_v2i32(<2 x i32> %data, ptr addrspace(8) inreg %buf) {259; SDAG-LABEL: store_v2i32:260; SDAG: ; %bb.0:261; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)262; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0263; SDAG-NEXT: s_waitcnt vmcnt(0)264; SDAG-NEXT: s_setpc_b64 s[30:31]265;266; GISEL-LABEL: store_v2i32:267; GISEL: ; %bb.0:268; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)269; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0270; GISEL-NEXT: s_waitcnt vmcnt(0)271; GISEL-NEXT: s_setpc_b64 s[30:31]272 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)273 store <2 x i32> %data, ptr addrspace(7) %p274 ret void275}276 277define <3 x i32> @load_v3i32(ptr addrspace(8) inreg %buf) {278; SDAG-LABEL: load_v3i32:279; SDAG: ; %bb.0:280; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)281; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 0282; SDAG-NEXT: s_waitcnt vmcnt(0)283; SDAG-NEXT: s_setpc_b64 s[30:31]284;285; GISEL-LABEL: load_v3i32:286; GISEL: ; %bb.0:287; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)288; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 0289; GISEL-NEXT: s_waitcnt vmcnt(0)290; GISEL-NEXT: s_setpc_b64 s[30:31]291 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)292 %ret = load <3 x i32>, ptr addrspace(7) %p293 ret <3 x i32> %ret294}295 296define void @store_v3i32(<3 x i32> %data, ptr addrspace(8) inreg %buf) {297; SDAG-LABEL: store_v3i32:298; SDAG: ; %bb.0:299; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)300; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 0301; SDAG-NEXT: s_waitcnt vmcnt(0)302; SDAG-NEXT: s_setpc_b64 s[30:31]303;304; GISEL-LABEL: store_v3i32:305; GISEL: ; %bb.0:306; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)307; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 0308; GISEL-NEXT: s_waitcnt vmcnt(0)309; GISEL-NEXT: s_setpc_b64 s[30:31]310 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)311 store <3 x i32> %data, ptr addrspace(7) %p312 ret void313}314 315define <4 x i32> @load_v4i32(ptr addrspace(8) inreg %buf) {316; SDAG-LABEL: load_v4i32:317; SDAG: ; %bb.0:318; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)319; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0320; SDAG-NEXT: s_waitcnt vmcnt(0)321; SDAG-NEXT: s_setpc_b64 s[30:31]322;323; GISEL-LABEL: load_v4i32:324; GISEL: ; %bb.0:325; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)326; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0327; GISEL-NEXT: s_waitcnt vmcnt(0)328; GISEL-NEXT: s_setpc_b64 s[30:31]329 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)330 %ret = load <4 x i32>, ptr addrspace(7) %p331 ret <4 x i32> %ret332}333 334define void @store_v4i32(<4 x i32> %data, ptr addrspace(8) inreg %buf) {335; SDAG-LABEL: store_v4i32:336; SDAG: ; %bb.0:337; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)338; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0339; SDAG-NEXT: s_waitcnt vmcnt(0)340; SDAG-NEXT: s_setpc_b64 s[30:31]341;342; GISEL-LABEL: store_v4i32:343; GISEL: ; %bb.0:344; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)345; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0346; GISEL-NEXT: s_waitcnt vmcnt(0)347; GISEL-NEXT: s_setpc_b64 s[30:31]348 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)349 store <4 x i32> %data, ptr addrspace(7) %p350 ret void351}352 353define <2 x i16> @load_v2i16(ptr addrspace(8) inreg %buf) {354; SDAG-LABEL: load_v2i16:355; SDAG: ; %bb.0:356; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)357; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0358; SDAG-NEXT: s_waitcnt vmcnt(0)359; SDAG-NEXT: s_setpc_b64 s[30:31]360;361; GISEL-LABEL: load_v2i16:362; GISEL: ; %bb.0:363; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)364; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0365; GISEL-NEXT: s_waitcnt vmcnt(0)366; GISEL-NEXT: s_setpc_b64 s[30:31]367 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)368 %ret = load <2 x i16>, ptr addrspace(7) %p369 ret <2 x i16> %ret370}371 372define void @store_v2i16(<2 x i16> %data, ptr addrspace(8) inreg %buf) {373; SDAG-LABEL: store_v2i16:374; SDAG: ; %bb.0:375; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)376; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0377; SDAG-NEXT: s_waitcnt vmcnt(0)378; SDAG-NEXT: s_setpc_b64 s[30:31]379;380; GISEL-LABEL: store_v2i16:381; GISEL: ; %bb.0:382; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)383; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0384; GISEL-NEXT: s_waitcnt vmcnt(0)385; GISEL-NEXT: s_setpc_b64 s[30:31]386 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)387 store <2 x i16> %data, ptr addrspace(7) %p388 ret void389}390 391define <4 x i16> @load_v4i16(ptr addrspace(8) inreg %buf) {392; SDAG-LABEL: load_v4i16:393; SDAG: ; %bb.0:394; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)395; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0396; SDAG-NEXT: s_waitcnt vmcnt(0)397; SDAG-NEXT: s_setpc_b64 s[30:31]398;399; GISEL-LABEL: load_v4i16:400; GISEL: ; %bb.0:401; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)402; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0403; GISEL-NEXT: s_waitcnt vmcnt(0)404; GISEL-NEXT: s_setpc_b64 s[30:31]405 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)406 %ret = load <4 x i16>, ptr addrspace(7) %p407 ret <4 x i16> %ret408}409 410define void @store_v4i16(<4 x i16> %data, ptr addrspace(8) inreg %buf) {411; SDAG-LABEL: store_v4i16:412; SDAG: ; %bb.0:413; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)414; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0415; SDAG-NEXT: s_waitcnt vmcnt(0)416; SDAG-NEXT: s_setpc_b64 s[30:31]417;418; GISEL-LABEL: store_v4i16:419; GISEL: ; %bb.0:420; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)421; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0422; GISEL-NEXT: s_waitcnt vmcnt(0)423; GISEL-NEXT: s_setpc_b64 s[30:31]424 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)425 store <4 x i16> %data, ptr addrspace(7) %p426 ret void427}428 429define <8 x i16> @load_v8i16(ptr addrspace(8) inreg %buf) {430; SDAG-LABEL: load_v8i16:431; SDAG: ; %bb.0:432; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)433; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0434; SDAG-NEXT: s_waitcnt vmcnt(0)435; SDAG-NEXT: s_setpc_b64 s[30:31]436;437; GISEL-LABEL: load_v8i16:438; GISEL: ; %bb.0:439; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)440; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0441; GISEL-NEXT: s_waitcnt vmcnt(0)442; GISEL-NEXT: s_setpc_b64 s[30:31]443 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)444 %ret = load <8 x i16>, ptr addrspace(7) %p445 ret <8 x i16> %ret446}447 448define void @store_v8i16(<8 x i16> %data, ptr addrspace(8) inreg %buf) {449; SDAG-LABEL: store_v8i16:450; SDAG: ; %bb.0:451; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)452; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0453; SDAG-NEXT: s_waitcnt vmcnt(0)454; SDAG-NEXT: s_setpc_b64 s[30:31]455;456; GISEL-LABEL: store_v8i16:457; GISEL: ; %bb.0:458; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)459; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0460; GISEL-NEXT: s_waitcnt vmcnt(0)461; GISEL-NEXT: s_setpc_b64 s[30:31]462 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)463 store <8 x i16> %data, ptr addrspace(7) %p464 ret void465}466 467define <2 x i64> @load_v2i64(ptr addrspace(8) inreg %buf) {468; SDAG-LABEL: load_v2i64:469; SDAG: ; %bb.0:470; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)471; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0472; SDAG-NEXT: s_waitcnt vmcnt(0)473; SDAG-NEXT: s_setpc_b64 s[30:31]474;475; GISEL-LABEL: load_v2i64:476; GISEL: ; %bb.0:477; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)478; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0479; GISEL-NEXT: s_waitcnt vmcnt(0)480; GISEL-NEXT: s_setpc_b64 s[30:31]481 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)482 %ret = load <2 x i64>, ptr addrspace(7) %p483 ret <2 x i64> %ret484}485 486define void @store_v2i64(<2 x i64> %data, ptr addrspace(8) inreg %buf) {487; SDAG-LABEL: store_v2i64:488; SDAG: ; %bb.0:489; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)490; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0491; SDAG-NEXT: s_waitcnt vmcnt(0)492; SDAG-NEXT: s_setpc_b64 s[30:31]493;494; GISEL-LABEL: store_v2i64:495; GISEL: ; %bb.0:496; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)497; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0498; GISEL-NEXT: s_waitcnt vmcnt(0)499; GISEL-NEXT: s_setpc_b64 s[30:31]500 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)501 store <2 x i64> %data, ptr addrspace(7) %p502 ret void503}504 505define half @load_f16(ptr addrspace(8) inreg %buf) {506; SDAG-LABEL: load_f16:507; SDAG: ; %bb.0:508; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)509; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 0510; SDAG-NEXT: s_waitcnt vmcnt(0)511; SDAG-NEXT: s_setpc_b64 s[30:31]512;513; GISEL-LABEL: load_f16:514; GISEL: ; %bb.0:515; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)516; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 0517; GISEL-NEXT: s_waitcnt vmcnt(0)518; GISEL-NEXT: s_setpc_b64 s[30:31]519 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)520 %ret = load half, ptr addrspace(7) %p521 ret half %ret522}523 524define void @store_f16(half %data, ptr addrspace(8) inreg %buf) {525; SDAG-LABEL: store_f16:526; SDAG: ; %bb.0:527; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)528; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 0529; SDAG-NEXT: s_waitcnt vmcnt(0)530; SDAG-NEXT: s_setpc_b64 s[30:31]531;532; GISEL-LABEL: store_f16:533; GISEL: ; %bb.0:534; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)535; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 0536; GISEL-NEXT: s_waitcnt vmcnt(0)537; GISEL-NEXT: s_setpc_b64 s[30:31]538 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)539 store half %data, ptr addrspace(7) %p540 ret void541}542 543define bfloat @load_bf16(ptr addrspace(8) inreg %buf) {544; SDAG-LABEL: load_bf16:545; SDAG: ; %bb.0:546; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)547; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 0548; SDAG-NEXT: s_waitcnt vmcnt(0)549; SDAG-NEXT: s_setpc_b64 s[30:31]550;551; GISEL-LABEL: load_bf16:552; GISEL: ; %bb.0:553; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)554; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 0555; GISEL-NEXT: s_waitcnt vmcnt(0)556; GISEL-NEXT: s_setpc_b64 s[30:31]557 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)558 %ret = load bfloat, ptr addrspace(7) %p559 ret bfloat %ret560}561 562define void @store_bf16(bfloat %data, ptr addrspace(8) inreg %buf) {563; SDAG-LABEL: store_bf16:564; SDAG: ; %bb.0:565; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)566; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 0567; SDAG-NEXT: s_waitcnt vmcnt(0)568; SDAG-NEXT: s_setpc_b64 s[30:31]569;570; GISEL-LABEL: store_bf16:571; GISEL: ; %bb.0:572; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)573; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 0574; GISEL-NEXT: s_waitcnt vmcnt(0)575; GISEL-NEXT: s_setpc_b64 s[30:31]576 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)577 store bfloat %data, ptr addrspace(7) %p578 ret void579}580 581define <2 x half> @load_v2f16(ptr addrspace(8) inreg %buf) {582; SDAG-LABEL: load_v2f16:583; SDAG: ; %bb.0:584; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)585; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0586; SDAG-NEXT: s_waitcnt vmcnt(0)587; SDAG-NEXT: s_setpc_b64 s[30:31]588;589; GISEL-LABEL: load_v2f16:590; GISEL: ; %bb.0:591; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)592; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0593; GISEL-NEXT: s_waitcnt vmcnt(0)594; GISEL-NEXT: s_setpc_b64 s[30:31]595 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)596 %ret = load <2 x half>, ptr addrspace(7) %p597 ret <2 x half> %ret598}599 600define void @store_v2f16(<2 x half> %data, ptr addrspace(8) inreg %buf) {601; SDAG-LABEL: store_v2f16:602; SDAG: ; %bb.0:603; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)604; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0605; SDAG-NEXT: s_waitcnt vmcnt(0)606; SDAG-NEXT: s_setpc_b64 s[30:31]607;608; GISEL-LABEL: store_v2f16:609; GISEL: ; %bb.0:610; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)611; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0612; GISEL-NEXT: s_waitcnt vmcnt(0)613; GISEL-NEXT: s_setpc_b64 s[30:31]614 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)615 store <2 x half> %data, ptr addrspace(7) %p616 ret void617}618 619define <4 x bfloat> @load_v4bf16(ptr addrspace(8) inreg %buf) {620; SDAG-LABEL: load_v4bf16:621; SDAG: ; %bb.0:622; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)623; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0624; SDAG-NEXT: s_waitcnt vmcnt(0)625; SDAG-NEXT: s_setpc_b64 s[30:31]626;627; GISEL-LABEL: load_v4bf16:628; GISEL: ; %bb.0:629; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)630; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0631; GISEL-NEXT: s_waitcnt vmcnt(0)632; GISEL-NEXT: s_setpc_b64 s[30:31]633 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)634 %ret = load <4 x bfloat>, ptr addrspace(7) %p635 ret <4 x bfloat> %ret636}637 638define void @store_v4bf16(<4 x bfloat> %data, ptr addrspace(8) inreg %buf) {639; SDAG-LABEL: store_v4bf16:640; SDAG: ; %bb.0:641; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)642; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0643; SDAG-NEXT: s_waitcnt vmcnt(0)644; SDAG-NEXT: s_setpc_b64 s[30:31]645;646; GISEL-LABEL: store_v4bf16:647; GISEL: ; %bb.0:648; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)649; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0650; GISEL-NEXT: s_waitcnt vmcnt(0)651; GISEL-NEXT: s_setpc_b64 s[30:31]652 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)653 store <4 x bfloat> %data, ptr addrspace(7) %p654 ret void655}656 657define <8 x half> @load_v8f16(ptr addrspace(8) inreg %buf) {658; SDAG-LABEL: load_v8f16:659; SDAG: ; %bb.0:660; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)661; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0662; SDAG-NEXT: s_waitcnt vmcnt(0)663; SDAG-NEXT: s_setpc_b64 s[30:31]664;665; GISEL-LABEL: load_v8f16:666; GISEL: ; %bb.0:667; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)668; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0669; GISEL-NEXT: s_waitcnt vmcnt(0)670; GISEL-NEXT: s_setpc_b64 s[30:31]671 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)672 %ret = load <8 x half>, ptr addrspace(7) %p673 ret <8 x half> %ret674}675 676define void @store_v8f16(<8 x half> %data, ptr addrspace(8) inreg %buf) {677; SDAG-LABEL: store_v8f16:678; SDAG: ; %bb.0:679; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)680; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0681; SDAG-NEXT: s_waitcnt vmcnt(0)682; SDAG-NEXT: s_setpc_b64 s[30:31]683;684; GISEL-LABEL: store_v8f16:685; GISEL: ; %bb.0:686; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)687; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0688; GISEL-NEXT: s_waitcnt vmcnt(0)689; GISEL-NEXT: s_setpc_b64 s[30:31]690 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)691 store <8 x half> %data, ptr addrspace(7) %p692 ret void693}694 695define float @load_f32(ptr addrspace(8) inreg %buf) {696; SDAG-LABEL: load_f32:697; SDAG: ; %bb.0:698; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)699; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0700; SDAG-NEXT: s_waitcnt vmcnt(0)701; SDAG-NEXT: s_setpc_b64 s[30:31]702;703; GISEL-LABEL: load_f32:704; GISEL: ; %bb.0:705; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)706; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0707; GISEL-NEXT: s_waitcnt vmcnt(0)708; GISEL-NEXT: s_setpc_b64 s[30:31]709 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)710 %ret = load float, ptr addrspace(7) %p711 ret float %ret712}713 714define void @store_f32(float %data, ptr addrspace(8) inreg %buf) {715; SDAG-LABEL: store_f32:716; SDAG: ; %bb.0:717; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)718; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0719; SDAG-NEXT: s_waitcnt vmcnt(0)720; SDAG-NEXT: s_setpc_b64 s[30:31]721;722; GISEL-LABEL: store_f32:723; GISEL: ; %bb.0:724; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)725; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0726; GISEL-NEXT: s_waitcnt vmcnt(0)727; GISEL-NEXT: s_setpc_b64 s[30:31]728 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)729 store float %data, ptr addrspace(7) %p730 ret void731}732 733define <2 x float> @load_v2f32(ptr addrspace(8) inreg %buf) {734; SDAG-LABEL: load_v2f32:735; SDAG: ; %bb.0:736; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)737; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0738; SDAG-NEXT: s_waitcnt vmcnt(0)739; SDAG-NEXT: s_setpc_b64 s[30:31]740;741; GISEL-LABEL: load_v2f32:742; GISEL: ; %bb.0:743; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)744; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0745; GISEL-NEXT: s_waitcnt vmcnt(0)746; GISEL-NEXT: s_setpc_b64 s[30:31]747 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)748 %ret = load <2 x float>, ptr addrspace(7) %p749 ret <2 x float> %ret750}751 752define void @store_v2f32(<2 x float> %data, ptr addrspace(8) inreg %buf) {753; SDAG-LABEL: store_v2f32:754; SDAG: ; %bb.0:755; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)756; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0757; SDAG-NEXT: s_waitcnt vmcnt(0)758; SDAG-NEXT: s_setpc_b64 s[30:31]759;760; GISEL-LABEL: store_v2f32:761; GISEL: ; %bb.0:762; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)763; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0764; GISEL-NEXT: s_waitcnt vmcnt(0)765; GISEL-NEXT: s_setpc_b64 s[30:31]766 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)767 store <2 x float> %data, ptr addrspace(7) %p768 ret void769}770 771define <3 x float> @load_v3f32(ptr addrspace(8) inreg %buf) {772; SDAG-LABEL: load_v3f32:773; SDAG: ; %bb.0:774; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)775; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 0776; SDAG-NEXT: s_waitcnt vmcnt(0)777; SDAG-NEXT: s_setpc_b64 s[30:31]778;779; GISEL-LABEL: load_v3f32:780; GISEL: ; %bb.0:781; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)782; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 0783; GISEL-NEXT: s_waitcnt vmcnt(0)784; GISEL-NEXT: s_setpc_b64 s[30:31]785 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)786 %ret = load <3 x float>, ptr addrspace(7) %p787 ret <3 x float> %ret788}789 790define void @store_v3f32(<3 x float> %data, ptr addrspace(8) inreg %buf) {791; SDAG-LABEL: store_v3f32:792; SDAG: ; %bb.0:793; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)794; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 0795; SDAG-NEXT: s_waitcnt vmcnt(0)796; SDAG-NEXT: s_setpc_b64 s[30:31]797;798; GISEL-LABEL: store_v3f32:799; GISEL: ; %bb.0:800; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)801; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 0802; GISEL-NEXT: s_waitcnt vmcnt(0)803; GISEL-NEXT: s_setpc_b64 s[30:31]804 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)805 store <3 x float> %data, ptr addrspace(7) %p806 ret void807}808 809define <4 x float> @load_v4f32(ptr addrspace(8) inreg %buf) {810; SDAG-LABEL: load_v4f32:811; SDAG: ; %bb.0:812; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)813; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0814; SDAG-NEXT: s_waitcnt vmcnt(0)815; SDAG-NEXT: s_setpc_b64 s[30:31]816;817; GISEL-LABEL: load_v4f32:818; GISEL: ; %bb.0:819; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)820; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 0821; GISEL-NEXT: s_waitcnt vmcnt(0)822; GISEL-NEXT: s_setpc_b64 s[30:31]823 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)824 %ret = load <4 x float>, ptr addrspace(7) %p825 ret <4 x float> %ret826}827 828define void @store_v4f32(<4 x float> %data, ptr addrspace(8) inreg %buf) {829; SDAG-LABEL: store_v4f32:830; SDAG: ; %bb.0:831; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)832; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0833; SDAG-NEXT: s_waitcnt vmcnt(0)834; SDAG-NEXT: s_setpc_b64 s[30:31]835;836; GISEL-LABEL: store_v4f32:837; GISEL: ; %bb.0:838; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)839; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0840; GISEL-NEXT: s_waitcnt vmcnt(0)841; GISEL-NEXT: s_setpc_b64 s[30:31]842 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)843 store <4 x float> %data, ptr addrspace(7) %p844 ret void845}846 847define ptr addrspace(0) @load_p0(ptr addrspace(8) inreg %buf) {848; SDAG-LABEL: load_p0:849; SDAG: ; %bb.0:850; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)851; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0852; SDAG-NEXT: s_waitcnt vmcnt(0)853; SDAG-NEXT: s_setpc_b64 s[30:31]854;855; GISEL-LABEL: load_p0:856; GISEL: ; %bb.0:857; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)858; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0859; GISEL-NEXT: s_waitcnt vmcnt(0)860; GISEL-NEXT: s_setpc_b64 s[30:31]861 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)862 %ret = load ptr addrspace(0), ptr addrspace(7) %p863 ret ptr addrspace(0) %ret864}865 866define void @store_p0(ptr addrspace(0) %data, ptr addrspace(8) inreg %buf) {867; SDAG-LABEL: store_p0:868; SDAG: ; %bb.0:869; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)870; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0871; SDAG-NEXT: s_waitcnt vmcnt(0)872; SDAG-NEXT: s_setpc_b64 s[30:31]873;874; GISEL-LABEL: store_p0:875; GISEL: ; %bb.0:876; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)877; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0878; GISEL-NEXT: s_waitcnt vmcnt(0)879; GISEL-NEXT: s_setpc_b64 s[30:31]880 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)881 store ptr addrspace(0) %data, ptr addrspace(7) %p882 ret void883}884 885define ptr addrspace(1) @load_p1(ptr addrspace(8) inreg %buf) {886; SDAG-LABEL: load_p1:887; SDAG: ; %bb.0:888; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)889; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0890; SDAG-NEXT: s_waitcnt vmcnt(0)891; SDAG-NEXT: s_setpc_b64 s[30:31]892;893; GISEL-LABEL: load_p1:894; GISEL: ; %bb.0:895; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)896; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0897; GISEL-NEXT: s_waitcnt vmcnt(0)898; GISEL-NEXT: s_setpc_b64 s[30:31]899 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)900 %ret = load ptr addrspace(1), ptr addrspace(7) %p901 ret ptr addrspace(1) %ret902}903 904define void @store_p1(ptr addrspace(1) %data, ptr addrspace(8) inreg %buf) {905; SDAG-LABEL: store_p1:906; SDAG: ; %bb.0:907; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)908; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0909; SDAG-NEXT: s_waitcnt vmcnt(0)910; SDAG-NEXT: s_setpc_b64 s[30:31]911;912; GISEL-LABEL: store_p1:913; GISEL: ; %bb.0:914; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)915; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0916; GISEL-NEXT: s_waitcnt vmcnt(0)917; GISEL-NEXT: s_setpc_b64 s[30:31]918 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)919 store ptr addrspace(1) %data, ptr addrspace(7) %p920 ret void921}922 923define ptr addrspace(2) @load_p2(ptr addrspace(8) inreg %buf) {924; SDAG-LABEL: load_p2:925; SDAG: ; %bb.0:926; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)927; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0928; SDAG-NEXT: s_waitcnt vmcnt(0)929; SDAG-NEXT: s_setpc_b64 s[30:31]930;931; GISEL-LABEL: load_p2:932; GISEL: ; %bb.0:933; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)934; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0935; GISEL-NEXT: s_waitcnt vmcnt(0)936; GISEL-NEXT: s_setpc_b64 s[30:31]937 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)938 %ret = load ptr addrspace(2), ptr addrspace(7) %p939 ret ptr addrspace(2) %ret940}941 942define void @store_p2(ptr addrspace(2) %data, ptr addrspace(8) inreg %buf) {943; SDAG-LABEL: store_p2:944; SDAG: ; %bb.0:945; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)946; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0947; SDAG-NEXT: s_waitcnt vmcnt(0)948; SDAG-NEXT: s_setpc_b64 s[30:31]949;950; GISEL-LABEL: store_p2:951; GISEL: ; %bb.0:952; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)953; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0954; GISEL-NEXT: s_waitcnt vmcnt(0)955; GISEL-NEXT: s_setpc_b64 s[30:31]956 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)957 store ptr addrspace(2) %data, ptr addrspace(7) %p958 ret void959}960 961define ptr addrspace(3) @load_p3(ptr addrspace(8) inreg %buf) {962; SDAG-LABEL: load_p3:963; SDAG: ; %bb.0:964; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)965; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0966; SDAG-NEXT: s_waitcnt vmcnt(0)967; SDAG-NEXT: s_setpc_b64 s[30:31]968;969; GISEL-LABEL: load_p3:970; GISEL: ; %bb.0:971; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)972; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0973; GISEL-NEXT: s_waitcnt vmcnt(0)974; GISEL-NEXT: s_setpc_b64 s[30:31]975 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)976 %ret = load ptr addrspace(3), ptr addrspace(7) %p977 ret ptr addrspace(3) %ret978}979 980define void @store_p3(ptr addrspace(3) %data, ptr addrspace(8) inreg %buf) {981; SDAG-LABEL: store_p3:982; SDAG: ; %bb.0:983; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)984; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 0985; SDAG-NEXT: s_waitcnt vmcnt(0)986; SDAG-NEXT: s_setpc_b64 s[30:31]987;988; GISEL-LABEL: store_p3:989; GISEL: ; %bb.0:990; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)991; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0992; GISEL-NEXT: s_waitcnt vmcnt(0)993; GISEL-NEXT: s_setpc_b64 s[30:31]994 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)995 store ptr addrspace(3) %data, ptr addrspace(7) %p996 ret void997}998 999define ptr addrspace(4) @load_p4(ptr addrspace(8) inreg %buf) {1000; SDAG-LABEL: load_p4:1001; SDAG: ; %bb.0:1002; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1003; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01004; SDAG-NEXT: s_waitcnt vmcnt(0)1005; SDAG-NEXT: s_setpc_b64 s[30:31]1006;1007; GISEL-LABEL: load_p4:1008; GISEL: ; %bb.0:1009; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1010; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01011; GISEL-NEXT: s_waitcnt vmcnt(0)1012; GISEL-NEXT: s_setpc_b64 s[30:31]1013 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1014 %ret = load ptr addrspace(4), ptr addrspace(7) %p1015 ret ptr addrspace(4) %ret1016}1017 1018define void @store_p4(ptr addrspace(4) %data, ptr addrspace(8) inreg %buf) {1019; SDAG-LABEL: store_p4:1020; SDAG: ; %bb.0:1021; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1022; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01023; SDAG-NEXT: s_waitcnt vmcnt(0)1024; SDAG-NEXT: s_setpc_b64 s[30:31]1025;1026; GISEL-LABEL: store_p4:1027; GISEL: ; %bb.0:1028; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1029; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01030; GISEL-NEXT: s_waitcnt vmcnt(0)1031; GISEL-NEXT: s_setpc_b64 s[30:31]1032 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1033 store ptr addrspace(4) %data, ptr addrspace(7) %p1034 ret void1035}1036 1037define ptr addrspace(5) @load_p5(ptr addrspace(8) inreg %buf) {1038; SDAG-LABEL: load_p5:1039; SDAG: ; %bb.0:1040; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1041; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 01042; SDAG-NEXT: s_waitcnt vmcnt(0)1043; SDAG-NEXT: s_setpc_b64 s[30:31]1044;1045; GISEL-LABEL: load_p5:1046; GISEL: ; %bb.0:1047; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1048; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 01049; GISEL-NEXT: s_waitcnt vmcnt(0)1050; GISEL-NEXT: s_setpc_b64 s[30:31]1051 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1052 %ret = load ptr addrspace(5), ptr addrspace(7) %p1053 ret ptr addrspace(5) %ret1054}1055 1056define void @store_p5(ptr addrspace(5) %data, ptr addrspace(8) inreg %buf) {1057; SDAG-LABEL: store_p5:1058; SDAG: ; %bb.0:1059; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1060; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 01061; SDAG-NEXT: s_waitcnt vmcnt(0)1062; SDAG-NEXT: s_setpc_b64 s[30:31]1063;1064; GISEL-LABEL: store_p5:1065; GISEL: ; %bb.0:1066; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1067; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 01068; GISEL-NEXT: s_waitcnt vmcnt(0)1069; GISEL-NEXT: s_setpc_b64 s[30:31]1070 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1071 store ptr addrspace(5) %data, ptr addrspace(7) %p1072 ret void1073}1074 1075define ptr addrspace(6) @load_p6(ptr addrspace(8) inreg %buf) {1076; SDAG-LABEL: load_p6:1077; SDAG: ; %bb.0:1078; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1079; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 01080; SDAG-NEXT: s_waitcnt vmcnt(0)1081; SDAG-NEXT: s_setpc_b64 s[30:31]1082;1083; GISEL-LABEL: load_p6:1084; GISEL: ; %bb.0:1085; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1086; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 01087; GISEL-NEXT: s_waitcnt vmcnt(0)1088; GISEL-NEXT: s_setpc_b64 s[30:31]1089 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1090 %ret = load ptr addrspace(6), ptr addrspace(7) %p1091 ret ptr addrspace(6) %ret1092}1093 1094define void @store_p6(ptr addrspace(6) %data, ptr addrspace(8) inreg %buf) {1095; SDAG-LABEL: store_p6:1096; SDAG: ; %bb.0:1097; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1098; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 01099; SDAG-NEXT: s_waitcnt vmcnt(0)1100; SDAG-NEXT: s_setpc_b64 s[30:31]1101;1102; GISEL-LABEL: store_p6:1103; GISEL: ; %bb.0:1104; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1105; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 01106; GISEL-NEXT: s_waitcnt vmcnt(0)1107; GISEL-NEXT: s_setpc_b64 s[30:31]1108 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1109 store ptr addrspace(6) %data, ptr addrspace(7) %p1110 ret void1111}1112 1113define ptr addrspace(8) @load_p8(ptr addrspace(8) inreg %buf) {1114; SDAG-LABEL: load_p8:1115; SDAG: ; %bb.0:1116; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1117; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01118; SDAG-NEXT: s_waitcnt vmcnt(0)1119; SDAG-NEXT: s_setpc_b64 s[30:31]1120;1121; GISEL-LABEL: load_p8:1122; GISEL: ; %bb.0:1123; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1124; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01125; GISEL-NEXT: s_waitcnt vmcnt(0)1126; GISEL-NEXT: s_setpc_b64 s[30:31]1127 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1128 %ret = load ptr addrspace(8), ptr addrspace(7) %p1129 ret ptr addrspace(8) %ret1130}1131 1132define void @store_p8(ptr addrspace(8) %data, ptr addrspace(8) inreg %buf) {1133; SDAG-LABEL: store_p8:1134; SDAG: ; %bb.0:1135; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1136; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01137; SDAG-NEXT: s_waitcnt vmcnt(0)1138; SDAG-NEXT: s_setpc_b64 s[30:31]1139;1140; GISEL-LABEL: store_p8:1141; GISEL: ; %bb.0:1142; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1143; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01144; GISEL-NEXT: s_waitcnt vmcnt(0)1145; GISEL-NEXT: s_setpc_b64 s[30:31]1146 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1147 store ptr addrspace(8) %data, ptr addrspace(7) %p1148 ret void1149}1150 1151define <2 x ptr addrspace(1)> @load_v2p1(ptr addrspace(8) inreg %buf) {1152; SDAG-LABEL: load_v2p1:1153; SDAG: ; %bb.0:1154; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1155; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01156; SDAG-NEXT: s_waitcnt vmcnt(0)1157; SDAG-NEXT: s_setpc_b64 s[30:31]1158;1159; GISEL-LABEL: load_v2p1:1160; GISEL: ; %bb.0:1161; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1162; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01163; GISEL-NEXT: s_waitcnt vmcnt(0)1164; GISEL-NEXT: s_setpc_b64 s[30:31]1165 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1166 %ret = load <2 x ptr addrspace(1)>, ptr addrspace(7) %p1167 ret <2 x ptr addrspace(1)> %ret1168}1169 1170define void @store_v2p1(<2 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {1171; SDAG-LABEL: store_v2p1:1172; SDAG: ; %bb.0:1173; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1174; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01175; SDAG-NEXT: s_waitcnt vmcnt(0)1176; SDAG-NEXT: s_setpc_b64 s[30:31]1177;1178; GISEL-LABEL: store_v2p1:1179; GISEL: ; %bb.0:1180; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1181; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01182; GISEL-NEXT: s_waitcnt vmcnt(0)1183; GISEL-NEXT: s_setpc_b64 s[30:31]1184 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1185 store <2 x ptr addrspace(1)> %data, ptr addrspace(7) %p1186 ret void1187}1188 1189define <2 x ptr addrspace(5)> @load_v2p5(ptr addrspace(8) inreg %buf) {1190; SDAG-LABEL: load_v2p5:1191; SDAG: ; %bb.0:1192; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1193; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01194; SDAG-NEXT: s_waitcnt vmcnt(0)1195; SDAG-NEXT: s_setpc_b64 s[30:31]1196;1197; GISEL-LABEL: load_v2p5:1198; GISEL: ; %bb.0:1199; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1200; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01201; GISEL-NEXT: s_waitcnt vmcnt(0)1202; GISEL-NEXT: s_setpc_b64 s[30:31]1203 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1204 %ret = load <2 x ptr addrspace(5)>, ptr addrspace(7) %p1205 ret <2 x ptr addrspace(5)> %ret1206}1207 1208define void @store_v2p5(<2 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {1209; SDAG-LABEL: store_v2p5:1210; SDAG: ; %bb.0:1211; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1212; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01213; SDAG-NEXT: s_waitcnt vmcnt(0)1214; SDAG-NEXT: s_setpc_b64 s[30:31]1215;1216; GISEL-LABEL: store_v2p5:1217; GISEL: ; %bb.0:1218; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1219; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01220; GISEL-NEXT: s_waitcnt vmcnt(0)1221; GISEL-NEXT: s_setpc_b64 s[30:31]1222 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1223 store <2 x ptr addrspace(5)> %data, ptr addrspace(7) %p1224 ret void1225}1226 1227define <3 x ptr addrspace(5)> @load_v3p5(ptr addrspace(8) inreg %buf) {1228; SDAG-LABEL: load_v3p5:1229; SDAG: ; %bb.0:1230; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1231; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01232; SDAG-NEXT: s_waitcnt vmcnt(0)1233; SDAG-NEXT: s_setpc_b64 s[30:31]1234;1235; GISEL-LABEL: load_v3p5:1236; GISEL: ; %bb.0:1237; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1238; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01239; GISEL-NEXT: s_waitcnt vmcnt(0)1240; GISEL-NEXT: s_setpc_b64 s[30:31]1241 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1242 %ret = load <3 x ptr addrspace(5)>, ptr addrspace(7) %p1243 ret <3 x ptr addrspace(5)> %ret1244}1245 1246define void @store_v3p5(<3 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {1247; SDAG-LABEL: store_v3p5:1248; SDAG: ; %bb.0:1249; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1250; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01251; SDAG-NEXT: s_waitcnt vmcnt(0)1252; SDAG-NEXT: s_setpc_b64 s[30:31]1253;1254; GISEL-LABEL: store_v3p5:1255; GISEL: ; %bb.0:1256; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1257; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01258; GISEL-NEXT: s_waitcnt vmcnt(0)1259; GISEL-NEXT: s_setpc_b64 s[30:31]1260 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1261 store <3 x ptr addrspace(5)> %data, ptr addrspace(7) %p1262 ret void1263}1264 1265define <4 x ptr addrspace(5)> @load_v4p5(ptr addrspace(8) inreg %buf) {1266; SDAG-LABEL: load_v4p5:1267; SDAG: ; %bb.0:1268; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1269; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01270; SDAG-NEXT: s_waitcnt vmcnt(0)1271; SDAG-NEXT: s_setpc_b64 s[30:31]1272;1273; GISEL-LABEL: load_v4p5:1274; GISEL: ; %bb.0:1275; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1276; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01277; GISEL-NEXT: s_waitcnt vmcnt(0)1278; GISEL-NEXT: s_setpc_b64 s[30:31]1279 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1280 %ret = load <4 x ptr addrspace(5)>, ptr addrspace(7) %p1281 ret <4 x ptr addrspace(5)> %ret1282}1283 1284define void @store_v4p5(<4 x ptr addrspace(5)> %data, ptr addrspace(8) inreg %buf) {1285; SDAG-LABEL: store_v4p5:1286; SDAG: ; %bb.0:1287; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1288; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01289; SDAG-NEXT: s_waitcnt vmcnt(0)1290; SDAG-NEXT: s_setpc_b64 s[30:31]1291;1292; GISEL-LABEL: store_v4p5:1293; GISEL: ; %bb.0:1294; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1295; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01296; GISEL-NEXT: s_waitcnt vmcnt(0)1297; GISEL-NEXT: s_setpc_b64 s[30:31]1298 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1299 store <4 x ptr addrspace(5)> %data, ptr addrspace(7) %p1300 ret void1301}1302 1303;;; 3 words in a short type. These need to be bitcast to <3 x i32> to be supported.1304 1305define <6 x half> @load_v6f16(ptr addrspace(8) inreg %buf) {1306; SDAG-LABEL: load_v6f16:1307; SDAG: ; %bb.0:1308; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1309; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01310; SDAG-NEXT: s_waitcnt vmcnt(0)1311; SDAG-NEXT: s_setpc_b64 s[30:31]1312;1313; GISEL-LABEL: load_v6f16:1314; GISEL: ; %bb.0:1315; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1316; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01317; GISEL-NEXT: s_waitcnt vmcnt(0)1318; GISEL-NEXT: s_setpc_b64 s[30:31]1319 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1320 %ret = load <6 x half>, ptr addrspace(7) %p1321 ret <6 x half> %ret1322}1323 1324define void @store_v6f16(<6 x half> %data, ptr addrspace(8) inreg %buf) {1325; SDAG-LABEL: store_v6f16:1326; SDAG: ; %bb.0:1327; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1328; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01329; SDAG-NEXT: s_waitcnt vmcnt(0)1330; SDAG-NEXT: s_setpc_b64 s[30:31]1331;1332; GISEL-LABEL: store_v6f16:1333; GISEL: ; %bb.0:1334; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1335; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01336; GISEL-NEXT: s_waitcnt vmcnt(0)1337; GISEL-NEXT: s_setpc_b64 s[30:31]1338 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1339 store <6 x half> %data, ptr addrspace(7) %p1340 ret void1341}1342 1343;;; Long types (32 bit elements). Must be split into multiple operations.1344 1345define <5 x float> @load_v5f32(ptr addrspace(8) inreg %buf) {1346; SDAG-LABEL: load_v5f32:1347; SDAG: ; %bb.0:1348; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1349; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01350; SDAG-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:161351; SDAG-NEXT: s_waitcnt vmcnt(0)1352; SDAG-NEXT: s_setpc_b64 s[30:31]1353;1354; GISEL-LABEL: load_v5f32:1355; GISEL: ; %bb.0:1356; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1357; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01358; GISEL-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:161359; GISEL-NEXT: s_waitcnt vmcnt(0)1360; GISEL-NEXT: s_setpc_b64 s[30:31]1361 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1362 %ret = load <5 x float>, ptr addrspace(7) %p1363 ret <5 x float> %ret1364}1365 1366define void @store_v5f32(<5 x float> %data, ptr addrspace(8) inreg %buf) {1367; SDAG-LABEL: store_v5f32:1368; SDAG: ; %bb.0:1369; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1370; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01371; SDAG-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:161372; SDAG-NEXT: s_waitcnt vmcnt(0)1373; SDAG-NEXT: s_setpc_b64 s[30:31]1374;1375; GISEL-LABEL: store_v5f32:1376; GISEL: ; %bb.0:1377; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1378; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01379; GISEL-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:161380; GISEL-NEXT: s_waitcnt vmcnt(0)1381; GISEL-NEXT: s_setpc_b64 s[30:31]1382 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1383 store <5 x float> %data, ptr addrspace(7) %p1384 ret void1385}1386 1387define <6 x float> @load_v6f32(ptr addrspace(8) inreg %buf) {1388; SDAG-LABEL: load_v6f32:1389; SDAG: ; %bb.0:1390; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1391; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01392; SDAG-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:161393; SDAG-NEXT: s_waitcnt vmcnt(0)1394; SDAG-NEXT: s_setpc_b64 s[30:31]1395;1396; GISEL-LABEL: load_v6f32:1397; GISEL: ; %bb.0:1398; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1399; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01400; GISEL-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:161401; GISEL-NEXT: s_waitcnt vmcnt(0)1402; GISEL-NEXT: s_setpc_b64 s[30:31]1403 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1404 %ret = load <6 x float>, ptr addrspace(7) %p1405 ret <6 x float> %ret1406}1407 1408define void @store_v6f32(<6 x float> %data, ptr addrspace(8) inreg %buf) {1409; SDAG-LABEL: store_v6f32:1410; SDAG: ; %bb.0:1411; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1412; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01413; SDAG-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:161414; SDAG-NEXT: s_waitcnt vmcnt(0)1415; SDAG-NEXT: s_setpc_b64 s[30:31]1416;1417; GISEL-LABEL: store_v6f32:1418; GISEL: ; %bb.0:1419; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1420; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01421; GISEL-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:161422; GISEL-NEXT: s_waitcnt vmcnt(0)1423; GISEL-NEXT: s_setpc_b64 s[30:31]1424 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1425 store <6 x float> %data, ptr addrspace(7) %p1426 ret void1427}1428 1429define <7 x float> @load_v7f32(ptr addrspace(8) inreg %buf) {1430; SDAG-LABEL: load_v7f32:1431; SDAG: ; %bb.0:1432; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1433; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01434; SDAG-NEXT: buffer_load_dwordx3 v[4:6], off, s[16:19], 0 offset:161435; SDAG-NEXT: s_waitcnt vmcnt(0)1436; SDAG-NEXT: s_setpc_b64 s[30:31]1437;1438; GISEL-LABEL: load_v7f32:1439; GISEL: ; %bb.0:1440; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1441; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01442; GISEL-NEXT: buffer_load_dwordx3 v[4:6], off, s[16:19], 0 offset:161443; GISEL-NEXT: s_waitcnt vmcnt(0)1444; GISEL-NEXT: s_setpc_b64 s[30:31]1445 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1446 %ret = load <7 x float>, ptr addrspace(7) %p1447 ret <7 x float> %ret1448}1449 1450define void @store_v7f32(<7 x float> %data, ptr addrspace(8) inreg %buf) {1451; SDAG-LABEL: store_v7f32:1452; SDAG: ; %bb.0:1453; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1454; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01455; SDAG-NEXT: buffer_store_dwordx3 v[4:6], off, s[16:19], 0 offset:161456; SDAG-NEXT: s_waitcnt vmcnt(0)1457; SDAG-NEXT: s_setpc_b64 s[30:31]1458;1459; GISEL-LABEL: store_v7f32:1460; GISEL: ; %bb.0:1461; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1462; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01463; GISEL-NEXT: buffer_store_dwordx3 v[4:6], off, s[16:19], 0 offset:161464; GISEL-NEXT: s_waitcnt vmcnt(0)1465; GISEL-NEXT: s_setpc_b64 s[30:31]1466 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1467 store <7 x float> %data, ptr addrspace(7) %p1468 ret void1469}1470 1471define <8 x float> @load_v8f32(ptr addrspace(8) inreg %buf) {1472; SDAG-LABEL: load_v8f32:1473; SDAG: ; %bb.0:1474; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1475; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01476; SDAG-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161477; SDAG-NEXT: s_waitcnt vmcnt(0)1478; SDAG-NEXT: s_setpc_b64 s[30:31]1479;1480; GISEL-LABEL: load_v8f32:1481; GISEL: ; %bb.0:1482; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1483; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01484; GISEL-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161485; GISEL-NEXT: s_waitcnt vmcnt(0)1486; GISEL-NEXT: s_setpc_b64 s[30:31]1487 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1488 %ret = load <8 x float>, ptr addrspace(7) %p1489 ret <8 x float> %ret1490}1491 1492define void @store_v8f32(<8 x float> %data, ptr addrspace(8) inreg %buf) {1493; SDAG-LABEL: store_v8f32:1494; SDAG: ; %bb.0:1495; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1496; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01497; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161498; SDAG-NEXT: s_waitcnt vmcnt(0)1499; SDAG-NEXT: s_setpc_b64 s[30:31]1500;1501; GISEL-LABEL: store_v8f32:1502; GISEL: ; %bb.0:1503; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1504; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01505; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161506; GISEL-NEXT: s_waitcnt vmcnt(0)1507; GISEL-NEXT: s_setpc_b64 s[30:31]1508 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1509 store <8 x float> %data, ptr addrspace(7) %p1510 ret void1511}1512 1513define <10 x float> @load_v10f32(ptr addrspace(8) inreg %buf) {1514; SDAG-LABEL: load_v10f32:1515; SDAG: ; %bb.0:1516; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1517; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01518; SDAG-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161519; SDAG-NEXT: buffer_load_dwordx2 v[8:9], off, s[16:19], 0 offset:321520; SDAG-NEXT: s_waitcnt vmcnt(0)1521; SDAG-NEXT: s_setpc_b64 s[30:31]1522;1523; GISEL-LABEL: load_v10f32:1524; GISEL: ; %bb.0:1525; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1526; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01527; GISEL-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161528; GISEL-NEXT: buffer_load_dwordx2 v[8:9], off, s[16:19], 0 offset:321529; GISEL-NEXT: s_waitcnt vmcnt(0)1530; GISEL-NEXT: s_setpc_b64 s[30:31]1531 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1532 %ret = load <10 x float>, ptr addrspace(7) %p1533 ret <10 x float> %ret1534}1535 1536define void @store_v10f32(<10 x float> %data, ptr addrspace(8) inreg %buf) {1537; SDAG-LABEL: store_v10f32:1538; SDAG: ; %bb.0:1539; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1540; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01541; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161542; SDAG-NEXT: buffer_store_dwordx2 v[8:9], off, s[16:19], 0 offset:321543; SDAG-NEXT: s_waitcnt vmcnt(0)1544; SDAG-NEXT: s_setpc_b64 s[30:31]1545;1546; GISEL-LABEL: store_v10f32:1547; GISEL: ; %bb.0:1548; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1549; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01550; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161551; GISEL-NEXT: buffer_store_dwordx2 v[8:9], off, s[16:19], 0 offset:321552; GISEL-NEXT: s_waitcnt vmcnt(0)1553; GISEL-NEXT: s_setpc_b64 s[30:31]1554 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1555 store <10 x float> %data, ptr addrspace(7) %p1556 ret void1557}1558 1559define <6 x i32> @load_v6i32(ptr addrspace(8) inreg %buf) {1560; SDAG-LABEL: load_v6i32:1561; SDAG: ; %bb.0:1562; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1563; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01564; SDAG-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:161565; SDAG-NEXT: s_waitcnt vmcnt(0)1566; SDAG-NEXT: s_setpc_b64 s[30:31]1567;1568; GISEL-LABEL: load_v6i32:1569; GISEL: ; %bb.0:1570; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1571; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01572; GISEL-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:161573; GISEL-NEXT: s_waitcnt vmcnt(0)1574; GISEL-NEXT: s_setpc_b64 s[30:31]1575 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1576 %ret = load <6 x i32>, ptr addrspace(7) %p1577 ret <6 x i32> %ret1578}1579 1580define void @store_v6i32(<6 x i32> %data, ptr addrspace(8) inreg %buf) {1581; SDAG-LABEL: store_v6i32:1582; SDAG: ; %bb.0:1583; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1584; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01585; SDAG-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:161586; SDAG-NEXT: s_waitcnt vmcnt(0)1587; SDAG-NEXT: s_setpc_b64 s[30:31]1588;1589; GISEL-LABEL: store_v6i32:1590; GISEL: ; %bb.0:1591; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1592; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01593; GISEL-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:161594; GISEL-NEXT: s_waitcnt vmcnt(0)1595; GISEL-NEXT: s_setpc_b64 s[30:31]1596 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1597 store <6 x i32> %data, ptr addrspace(7) %p1598 ret void1599}1600 1601define <4 x ptr addrspace(1)> @load_v4p1(ptr addrspace(8) inreg %buf) {1602; SDAG-LABEL: load_v4p1:1603; SDAG: ; %bb.0:1604; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1605; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01606; SDAG-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161607; SDAG-NEXT: s_waitcnt vmcnt(0)1608; SDAG-NEXT: s_setpc_b64 s[30:31]1609;1610; GISEL-LABEL: load_v4p1:1611; GISEL: ; %bb.0:1612; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1613; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01614; GISEL-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:161615; GISEL-NEXT: s_waitcnt vmcnt(0)1616; GISEL-NEXT: s_setpc_b64 s[30:31]1617 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1618 %ret = load <4 x ptr addrspace(1)>, ptr addrspace(7) %p1619 ret <4 x ptr addrspace(1)> %ret1620}1621 1622define void @store_v4p1(<4 x ptr addrspace(1)> %data, ptr addrspace(8) inreg %buf) {1623; SDAG-LABEL: store_v4p1:1624; SDAG: ; %bb.0:1625; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1626; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01627; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161628; SDAG-NEXT: s_waitcnt vmcnt(0)1629; SDAG-NEXT: s_setpc_b64 s[30:31]1630;1631; GISEL-LABEL: store_v4p1:1632; GISEL: ; %bb.0:1633; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1634; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01635; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:161636; GISEL-NEXT: s_waitcnt vmcnt(0)1637; GISEL-NEXT: s_setpc_b64 s[30:31]1638 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1639 store <4 x ptr addrspace(1)> %data, ptr addrspace(7) %p1640 ret void1641}1642 1643;;; Uneven types with 16-bit elements. Require splitting into multiple operations.1644 1645define <1 x i16> @load_v1i16(ptr addrspace(8) inreg %buf) {1646; SDAG-LABEL: load_v1i16:1647; SDAG: ; %bb.0:1648; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1649; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 01650; SDAG-NEXT: s_waitcnt vmcnt(0)1651; SDAG-NEXT: s_setpc_b64 s[30:31]1652;1653; GISEL-LABEL: load_v1i16:1654; GISEL: ; %bb.0:1655; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1656; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 01657; GISEL-NEXT: s_waitcnt vmcnt(0)1658; GISEL-NEXT: s_setpc_b64 s[30:31]1659 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1660 %ret = load <1 x i16>, ptr addrspace(7) %p1661 ret <1 x i16> %ret1662}1663 1664define void @store_v1i16(<1 x i16> %data, ptr addrspace(8) inreg %buf) {1665; SDAG-LABEL: store_v1i16:1666; SDAG: ; %bb.0:1667; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1668; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 01669; SDAG-NEXT: s_waitcnt vmcnt(0)1670; SDAG-NEXT: s_setpc_b64 s[30:31]1671;1672; GISEL-LABEL: store_v1i16:1673; GISEL: ; %bb.0:1674; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1675; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 01676; GISEL-NEXT: s_waitcnt vmcnt(0)1677; GISEL-NEXT: s_setpc_b64 s[30:31]1678 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1679 store <1 x i16> %data, ptr addrspace(7) %p1680 ret void1681}1682 1683define <3 x i16> @load_v3i16(ptr addrspace(8) inreg %buf) {1684; SDAG-LABEL: load_v3i16:1685; SDAG: ; %bb.0:1686; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1687; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 01688; SDAG-NEXT: buffer_load_ushort v1, off, s[16:19], 0 offset:41689; SDAG-NEXT: s_waitcnt vmcnt(0)1690; SDAG-NEXT: s_setpc_b64 s[30:31]1691;1692; GISEL-LABEL: load_v3i16:1693; GISEL: ; %bb.0:1694; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1695; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 01696; GISEL-NEXT: buffer_load_ushort v1, off, s[16:19], 0 offset:41697; GISEL-NEXT: s_waitcnt vmcnt(0)1698; GISEL-NEXT: s_setpc_b64 s[30:31]1699 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1700 %ret = load <3 x i16>, ptr addrspace(7) %p1701 ret <3 x i16> %ret1702}1703 1704define void @store_v3i16(<3 x i16> %data, ptr addrspace(8) inreg %buf) {1705; SDAG-LABEL: store_v3i16:1706; SDAG: ; %bb.0:1707; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1708; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 01709; SDAG-NEXT: buffer_store_short v1, off, s[16:19], 0 offset:41710; SDAG-NEXT: s_waitcnt vmcnt(0)1711; SDAG-NEXT: s_setpc_b64 s[30:31]1712;1713; GISEL-LABEL: store_v3i16:1714; GISEL: ; %bb.0:1715; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1716; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 01717; GISEL-NEXT: buffer_store_short v1, off, s[16:19], 0 offset:41718; GISEL-NEXT: s_waitcnt vmcnt(0)1719; GISEL-NEXT: s_setpc_b64 s[30:31]1720 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1721 store <3 x i16> %data, ptr addrspace(7) %p1722 ret void1723}1724 1725define <5 x i16> @load_v5i16(ptr addrspace(8) inreg %buf) {1726; SDAG-LABEL: load_v5i16:1727; SDAG: ; %bb.0:1728; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1729; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01730; SDAG-NEXT: buffer_load_ushort v2, off, s[16:19], 0 offset:81731; SDAG-NEXT: s_waitcnt vmcnt(0)1732; SDAG-NEXT: s_setpc_b64 s[30:31]1733;1734; GISEL-LABEL: load_v5i16:1735; GISEL: ; %bb.0:1736; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1737; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 01738; GISEL-NEXT: buffer_load_ushort v2, off, s[16:19], 0 offset:81739; GISEL-NEXT: s_waitcnt vmcnt(0)1740; GISEL-NEXT: s_setpc_b64 s[30:31]1741 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1742 %ret = load <5 x i16>, ptr addrspace(7) %p1743 ret <5 x i16> %ret1744}1745 1746define void @store_v5i16(<5 x i16> %data, ptr addrspace(8) inreg %buf) {1747; SDAG-LABEL: store_v5i16:1748; SDAG: ; %bb.0:1749; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1750; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01751; SDAG-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:81752; SDAG-NEXT: s_waitcnt vmcnt(0)1753; SDAG-NEXT: s_setpc_b64 s[30:31]1754;1755; GISEL-LABEL: store_v5i16:1756; GISEL: ; %bb.0:1757; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1758; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 01759; GISEL-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:81760; GISEL-NEXT: s_waitcnt vmcnt(0)1761; GISEL-NEXT: s_setpc_b64 s[30:31]1762 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1763 store <5 x i16> %data, ptr addrspace(7) %p1764 ret void1765}1766 1767define <6 x i16> @load_v6i16(ptr addrspace(8) inreg %buf) {1768; SDAG-LABEL: load_v6i16:1769; SDAG: ; %bb.0:1770; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1771; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01772; SDAG-NEXT: s_waitcnt vmcnt(0)1773; SDAG-NEXT: s_setpc_b64 s[30:31]1774;1775; GISEL-LABEL: load_v6i16:1776; GISEL: ; %bb.0:1777; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1778; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01779; GISEL-NEXT: s_waitcnt vmcnt(0)1780; GISEL-NEXT: s_setpc_b64 s[30:31]1781 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1782 %ret = load <6 x i16>, ptr addrspace(7) %p1783 ret <6 x i16> %ret1784}1785 1786define void @store_v6i16(<6 x i16> %data, ptr addrspace(8) inreg %buf) {1787; SDAG-LABEL: store_v6i16:1788; SDAG: ; %bb.0:1789; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1790; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01791; SDAG-NEXT: s_waitcnt vmcnt(0)1792; SDAG-NEXT: s_setpc_b64 s[30:31]1793;1794; GISEL-LABEL: store_v6i16:1795; GISEL: ; %bb.0:1796; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1797; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01798; GISEL-NEXT: s_waitcnt vmcnt(0)1799; GISEL-NEXT: s_setpc_b64 s[30:31]1800 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1801 store <6 x i16> %data, ptr addrspace(7) %p1802 ret void1803}1804 1805define <7 x i16> @load_v7i16(ptr addrspace(8) inreg %buf) {1806; SDAG-LABEL: load_v7i16:1807; SDAG: ; %bb.0:1808; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1809; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01810; SDAG-NEXT: buffer_load_ushort v3, off, s[16:19], 0 offset:121811; SDAG-NEXT: s_waitcnt vmcnt(0)1812; SDAG-NEXT: s_setpc_b64 s[30:31]1813;1814; GISEL-LABEL: load_v7i16:1815; GISEL: ; %bb.0:1816; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1817; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 01818; GISEL-NEXT: buffer_load_ushort v3, off, s[16:19], 0 offset:121819; GISEL-NEXT: s_waitcnt vmcnt(0)1820; GISEL-NEXT: s_setpc_b64 s[30:31]1821 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1822 %ret = load <7 x i16>, ptr addrspace(7) %p1823 ret <7 x i16> %ret1824}1825 1826define void @store_v7i16(<7 x i16> %data, ptr addrspace(8) inreg %buf) {1827; SDAG-LABEL: store_v7i16:1828; SDAG: ; %bb.0:1829; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1830; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01831; SDAG-NEXT: buffer_store_short v3, off, s[16:19], 0 offset:121832; SDAG-NEXT: s_waitcnt vmcnt(0)1833; SDAG-NEXT: s_setpc_b64 s[30:31]1834;1835; GISEL-LABEL: store_v7i16:1836; GISEL: ; %bb.0:1837; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1838; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 01839; GISEL-NEXT: buffer_store_short v3, off, s[16:19], 0 offset:121840; GISEL-NEXT: s_waitcnt vmcnt(0)1841; GISEL-NEXT: s_setpc_b64 s[30:31]1842 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1843 store <7 x i16> %data, ptr addrspace(7) %p1844 ret void1845}1846 1847define <9 x i16> @load_v9i16(ptr addrspace(8) inreg %buf) {1848; SDAG-LABEL: load_v9i16:1849; SDAG: ; %bb.0:1850; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1851; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01852; SDAG-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:161853; SDAG-NEXT: s_waitcnt vmcnt(0)1854; SDAG-NEXT: s_setpc_b64 s[30:31]1855;1856; GISEL-LABEL: load_v9i16:1857; GISEL: ; %bb.0:1858; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1859; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 01860; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:161861; GISEL-NEXT: s_waitcnt vmcnt(0)1862; GISEL-NEXT: s_setpc_b64 s[30:31]1863 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1864 %ret = load <9 x i16>, ptr addrspace(7) %p1865 ret <9 x i16> %ret1866}1867 1868define void @store_v9i16(<9 x i16> %data, ptr addrspace(8) inreg %buf) {1869; SDAG-LABEL: store_v9i16:1870; SDAG: ; %bb.0:1871; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1872; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01873; SDAG-NEXT: buffer_store_short v4, off, s[16:19], 0 offset:161874; SDAG-NEXT: s_waitcnt vmcnt(0)1875; SDAG-NEXT: s_setpc_b64 s[30:31]1876;1877; GISEL-LABEL: store_v9i16:1878; GISEL: ; %bb.0:1879; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1880; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 01881; GISEL-NEXT: buffer_store_short v4, off, s[16:19], 0 offset:161882; GISEL-NEXT: s_waitcnt vmcnt(0)1883; GISEL-NEXT: s_setpc_b64 s[30:31]1884 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1885 store <9 x i16> %data, ptr addrspace(7) %p1886 ret void1887}1888 1889;;; Byte vectors. Need to be1890;;; - Split into multiple operations1891;;; - Bitcast if they have a natively supported width1892 1893define <1 x i8> @load_v1i8(ptr addrspace(8) inreg %buf) {1894; SDAG-LABEL: load_v1i8:1895; SDAG: ; %bb.0:1896; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1897; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 01898; SDAG-NEXT: s_waitcnt vmcnt(0)1899; SDAG-NEXT: s_setpc_b64 s[30:31]1900;1901; GISEL-LABEL: load_v1i8:1902; GISEL: ; %bb.0:1903; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1904; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 01905; GISEL-NEXT: s_waitcnt vmcnt(0)1906; GISEL-NEXT: s_setpc_b64 s[30:31]1907 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1908 %ret = load <1 x i8>, ptr addrspace(7) %p1909 ret <1 x i8> %ret1910}1911 1912define void @store_v1i8(<1 x i8> %data, ptr addrspace(8) inreg %buf) {1913; SDAG-LABEL: store_v1i8:1914; SDAG: ; %bb.0:1915; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1916; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 01917; SDAG-NEXT: s_waitcnt vmcnt(0)1918; SDAG-NEXT: s_setpc_b64 s[30:31]1919;1920; GISEL-LABEL: store_v1i8:1921; GISEL: ; %bb.0:1922; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1923; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 01924; GISEL-NEXT: s_waitcnt vmcnt(0)1925; GISEL-NEXT: s_setpc_b64 s[30:31]1926 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1927 store <1 x i8> %data, ptr addrspace(7) %p1928 ret void1929}1930 1931define <2 x i8> @load_v2i8(ptr addrspace(8) inreg %buf) {1932; SDAG-LABEL: load_v2i8:1933; SDAG: ; %bb.0:1934; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1935; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 01936; SDAG-NEXT: s_waitcnt vmcnt(0)1937; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v01938; SDAG-NEXT: s_setpc_b64 s[30:31]1939;1940; GISEL-LABEL: load_v2i8:1941; GISEL: ; %bb.0:1942; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1943; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 01944; GISEL-NEXT: s_waitcnt vmcnt(0)1945; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v01946; GISEL-NEXT: s_setpc_b64 s[30:31]1947 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1948 %ret = load <2 x i8>, ptr addrspace(7) %p1949 ret <2 x i8> %ret1950}1951 1952define void @store_v2i8(<2 x i8> %data, ptr addrspace(8) inreg %buf) {1953; SDAG-LABEL: store_v2i8:1954; SDAG: ; %bb.0:1955; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1956; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v11957; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD1958; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 01959; SDAG-NEXT: s_waitcnt vmcnt(0)1960; SDAG-NEXT: s_setpc_b64 s[30:31]1961;1962; GISEL-LABEL: store_v2i8:1963; GISEL: ; %bb.0:1964; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1965; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v11966; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v11967; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD1968; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 01969; GISEL-NEXT: s_waitcnt vmcnt(0)1970; GISEL-NEXT: s_setpc_b64 s[30:31]1971 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1972 store <2 x i8> %data, ptr addrspace(7) %p1973 ret void1974}1975 1976define <3 x i8> @load_v3i8(ptr addrspace(8) inreg %buf) {1977; SDAG-LABEL: load_v3i8:1978; SDAG: ; %bb.0:1979; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1980; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 01981; SDAG-NEXT: buffer_load_ubyte v2, off, s[16:19], 0 offset:21982; SDAG-NEXT: s_waitcnt vmcnt(1)1983; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v01984; SDAG-NEXT: s_waitcnt vmcnt(0)1985; SDAG-NEXT: s_setpc_b64 s[30:31]1986;1987; GISEL-LABEL: load_v3i8:1988; GISEL: ; %bb.0:1989; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1990; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 01991; GISEL-NEXT: buffer_load_ubyte v2, off, s[16:19], 0 offset:21992; GISEL-NEXT: s_waitcnt vmcnt(1)1993; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v01994; GISEL-NEXT: s_waitcnt vmcnt(0)1995; GISEL-NEXT: s_setpc_b64 s[30:31]1996 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)1997 %ret = load <3 x i8>, ptr addrspace(7) %p1998 ret <3 x i8> %ret1999}2000 2001define void @store_v3i8(<3 x i8> %data, ptr addrspace(8) inreg %buf) {2002; SDAG-LABEL: store_v3i8:2003; SDAG: ; %bb.0:2004; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2005; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12006; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2007; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 02008; SDAG-NEXT: buffer_store_byte v2, off, s[16:19], 0 offset:22009; SDAG-NEXT: s_waitcnt vmcnt(0)2010; SDAG-NEXT: s_setpc_b64 s[30:31]2011;2012; GISEL-LABEL: store_v3i8:2013; GISEL: ; %bb.0:2014; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2015; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v12016; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v12017; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2018; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 02019; GISEL-NEXT: buffer_store_byte v2, off, s[16:19], 0 offset:22020; GISEL-NEXT: s_waitcnt vmcnt(0)2021; GISEL-NEXT: s_setpc_b64 s[30:31]2022 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2023 store <3 x i8> %data, ptr addrspace(7) %p2024 ret void2025}2026 2027define <4 x i8> @load_v4i8(ptr addrspace(8) inreg %buf) {2028; SDAG-LABEL: load_v4i8:2029; SDAG: ; %bb.0:2030; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2031; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02032; SDAG-NEXT: s_waitcnt vmcnt(0)2033; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v02034; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v02035; SDAG-NEXT: v_lshrrev_b32_e32 v3, 24, v02036; SDAG-NEXT: s_setpc_b64 s[30:31]2037;2038; GISEL-LABEL: load_v4i8:2039; GISEL: ; %bb.0:2040; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2041; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02042; GISEL-NEXT: s_waitcnt vmcnt(0)2043; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v02044; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v02045; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02046; GISEL-NEXT: s_setpc_b64 s[30:31]2047 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2048 %ret = load <4 x i8>, ptr addrspace(7) %p2049 ret <4 x i8> %ret2050}2051 2052define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {2053; SDAG-LABEL: store_v4i8:2054; SDAG: ; %bb.0:2055; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2056; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12057; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2058; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32059; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2060; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2061; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02062; SDAG-NEXT: s_waitcnt vmcnt(0)2063; SDAG-NEXT: s_setpc_b64 s[30:31]2064;2065; GISEL-LABEL: store_v4i8:2066; GISEL: ; %bb.0:2067; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2068; GISEL-NEXT: v_mov_b32_e32 v5, 82069; GISEL-NEXT: v_mov_b32_e32 v4, 0xff2070; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02071; GISEL-NEXT: v_and_or_b32 v0, v0, v4, v12072; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22073; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32074; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12075; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22076; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22077; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02078; GISEL-NEXT: s_waitcnt vmcnt(0)2079; GISEL-NEXT: s_setpc_b64 s[30:31]2080 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2081 store <4 x i8> %data, ptr addrspace(7) %p2082 ret void2083}2084 2085define <5 x i8> @load_v5i8(ptr addrspace(8) inreg %buf) {2086; SDAG-LABEL: load_v5i8:2087; SDAG: ; %bb.0:2088; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2089; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02090; SDAG-NEXT: buffer_load_ubyte v4, off, s[16:19], 0 offset:42091; SDAG-NEXT: s_waitcnt vmcnt(1)2092; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v02093; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v02094; SDAG-NEXT: v_lshrrev_b32_e32 v3, 24, v02095; SDAG-NEXT: s_waitcnt vmcnt(0)2096; SDAG-NEXT: s_setpc_b64 s[30:31]2097;2098; GISEL-LABEL: load_v5i8:2099; GISEL: ; %bb.0:2100; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2101; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02102; GISEL-NEXT: buffer_load_ubyte v4, off, s[16:19], 0 offset:42103; GISEL-NEXT: s_waitcnt vmcnt(1)2104; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v02105; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v02106; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02107; GISEL-NEXT: s_waitcnt vmcnt(0)2108; GISEL-NEXT: s_setpc_b64 s[30:31]2109 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2110 %ret = load <5 x i8>, ptr addrspace(7) %p2111 ret <5 x i8> %ret2112}2113 2114define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {2115; SDAG-LABEL: store_v5i8:2116; SDAG: ; %bb.0:2117; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2118; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12119; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2120; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32121; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2122; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2123; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02124; SDAG-NEXT: buffer_store_byte v4, off, s[16:19], 0 offset:42125; SDAG-NEXT: s_waitcnt vmcnt(0)2126; SDAG-NEXT: s_setpc_b64 s[30:31]2127;2128; GISEL-LABEL: store_v5i8:2129; GISEL: ; %bb.0:2130; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2131; GISEL-NEXT: v_mov_b32_e32 v6, 82132; GISEL-NEXT: v_mov_b32_e32 v5, 0xff2133; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02134; GISEL-NEXT: v_and_or_b32 v0, v0, v5, v12135; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22136; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32137; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12138; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22139; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22140; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02141; GISEL-NEXT: buffer_store_byte v4, off, s[16:19], 0 offset:42142; GISEL-NEXT: s_waitcnt vmcnt(0)2143; GISEL-NEXT: s_setpc_b64 s[30:31]2144 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2145 store <5 x i8> %data, ptr addrspace(7) %p2146 ret void2147}2148 2149define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {2150; SDAG-LABEL: load_v6i8:2151; SDAG: ; %bb.0:2152; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2153; SDAG-NEXT: buffer_load_ushort v6, off, s[16:19], 0 offset:42154; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02155; SDAG-NEXT: s_waitcnt vmcnt(1)2156; SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v62157; SDAG-NEXT: s_waitcnt vmcnt(0)2158; SDAG-NEXT: v_lshrrev_b32_e32 v7, 8, v02159; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]2160; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v02161; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v12162; SDAG-NEXT: v_mov_b32_e32 v4, v62163; SDAG-NEXT: v_mov_b32_e32 v1, v72164; SDAG-NEXT: s_setpc_b64 s[30:31]2165;2166; GISEL-LABEL: load_v6i8:2167; GISEL: ; %bb.0:2168; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2169; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02170; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:42171; GISEL-NEXT: s_waitcnt vmcnt(1)2172; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v02173; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v02174; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02175; GISEL-NEXT: s_waitcnt vmcnt(0)2176; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v42177; GISEL-NEXT: s_setpc_b64 s[30:31]2178 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2179 %ret = load <6 x i8>, ptr addrspace(7) %p2180 ret <6 x i8> %ret2181}2182 2183define void @store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {2184; SDAG-LABEL: store_v6i8:2185; SDAG: ; %bb.0:2186; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2187; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12188; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2189; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32190; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2191; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v52192; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2193; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2194; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02195; SDAG-NEXT: buffer_store_short v4, off, s[16:19], 0 offset:42196; SDAG-NEXT: s_waitcnt vmcnt(0)2197; SDAG-NEXT: s_setpc_b64 s[30:31]2198;2199; GISEL-LABEL: store_v6i8:2200; GISEL: ; %bb.0:2201; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2202; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v12203; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v12204; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2205; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v32206; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v12207; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2208; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v52209; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v02210; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v22211; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v02212; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2213; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02214; GISEL-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:42215; GISEL-NEXT: s_waitcnt vmcnt(0)2216; GISEL-NEXT: s_setpc_b64 s[30:31]2217 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2218 store <6 x i8> %data, ptr addrspace(7) %p2219 ret void2220}2221 2222define <7 x i8> @load_v7i8(ptr addrspace(8) inreg %buf) {2223; SDAG-LABEL: load_v7i8:2224; SDAG: ; %bb.0:2225; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2226; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02227; SDAG-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:42228; SDAG-NEXT: buffer_load_ubyte v6, off, s[16:19], 0 offset:62229; SDAG-NEXT: s_waitcnt vmcnt(2)2230; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v02231; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v02232; SDAG-NEXT: v_lshrrev_b32_e32 v3, 24, v02233; SDAG-NEXT: s_waitcnt vmcnt(1)2234; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v42235; SDAG-NEXT: s_waitcnt vmcnt(0)2236; SDAG-NEXT: s_setpc_b64 s[30:31]2237;2238; GISEL-LABEL: load_v7i8:2239; GISEL: ; %bb.0:2240; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2241; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02242; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:42243; GISEL-NEXT: buffer_load_ubyte v6, off, s[16:19], 0 offset:62244; GISEL-NEXT: s_waitcnt vmcnt(2)2245; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v02246; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v02247; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02248; GISEL-NEXT: s_waitcnt vmcnt(1)2249; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v42250; GISEL-NEXT: s_waitcnt vmcnt(0)2251; GISEL-NEXT: s_setpc_b64 s[30:31]2252 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2253 %ret = load <7 x i8>, ptr addrspace(7) %p2254 ret <7 x i8> %ret2255}2256 2257define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {2258; SDAG-LABEL: store_v7i8:2259; SDAG: ; %bb.0:2260; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2261; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12262; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2263; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32264; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2265; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2266; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02267; SDAG-NEXT: v_lshlrev_b16_e32 v0, 8, v52268; SDAG-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2269; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 0 offset:42270; SDAG-NEXT: buffer_store_byte v6, off, s[16:19], 0 offset:62271; SDAG-NEXT: s_waitcnt vmcnt(0)2272; SDAG-NEXT: s_setpc_b64 s[30:31]2273;2274; GISEL-LABEL: store_v7i8:2275; GISEL: ; %bb.0:2276; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2277; GISEL-NEXT: v_mov_b32_e32 v8, 82278; GISEL-NEXT: v_mov_b32_e32 v7, 0xff2279; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02280; GISEL-NEXT: v_and_or_b32 v0, v0, v7, v12281; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22282; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32283; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12284; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22285; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22286; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02287; GISEL-NEXT: v_and_b32_e32 v0, 0xff, v52288; GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v02289; GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2290; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 0 offset:42291; GISEL-NEXT: buffer_store_byte v6, off, s[16:19], 0 offset:62292; GISEL-NEXT: s_waitcnt vmcnt(0)2293; GISEL-NEXT: s_setpc_b64 s[30:31]2294 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2295 store <7 x i8> %data, ptr addrspace(7) %p2296 ret void2297}2298 2299define <8 x i8> @load_v8i8(ptr addrspace(8) inreg %buf) {2300; SDAG-LABEL: load_v8i8:2301; SDAG: ; %bb.0:2302; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2303; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 02304; SDAG-NEXT: s_waitcnt vmcnt(0)2305; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]2306; SDAG-NEXT: v_lshrrev_b32_e32 v8, 8, v02307; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v02308; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v12309; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v12310; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v12311; SDAG-NEXT: v_mov_b32_e32 v4, v12312; SDAG-NEXT: v_mov_b32_e32 v1, v82313; SDAG-NEXT: s_setpc_b64 s[30:31]2314;2315; GISEL-LABEL: load_v8i8:2316; GISEL: ; %bb.0:2317; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2318; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 02319; GISEL-NEXT: s_waitcnt vmcnt(0)2320; GISEL-NEXT: v_lshrrev_b32_e32 v8, 8, v02321; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v02322; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02323; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v12324; GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v12325; GISEL-NEXT: v_lshrrev_b32_e32 v7, 24, v12326; GISEL-NEXT: v_mov_b32_e32 v4, v12327; GISEL-NEXT: v_mov_b32_e32 v1, v82328; GISEL-NEXT: s_setpc_b64 s[30:31]2329 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2330 %ret = load <8 x i8>, ptr addrspace(7) %p2331 ret <8 x i8> %ret2332}2333 2334define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {2335; SDAG-LABEL: store_v8i8:2336; SDAG: ; %bb.0:2337; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2338; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v52339; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12340; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2341; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v72342; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2343; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32344; SDAG-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2345; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2346; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2347; SDAG-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2348; SDAG-NEXT: buffer_store_dwordx2 v[3:4], off, s[16:19], 02349; SDAG-NEXT: s_waitcnt vmcnt(0)2350; SDAG-NEXT: s_setpc_b64 s[30:31]2351;2352; GISEL-LABEL: store_v8i8:2353; GISEL: ; %bb.0:2354; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2355; GISEL-NEXT: v_mov_b32_e32 v9, 82356; GISEL-NEXT: v_mov_b32_e32 v8, 0xff2357; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02358; GISEL-NEXT: v_and_or_b32 v0, v0, v8, v12359; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22360; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32361; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12362; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22363; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22364; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02365; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v62366; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v72367; GISEL-NEXT: v_and_or_b32 v1, v4, v8, v12368; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v22369; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v32370; GISEL-NEXT: v_or3_b32 v1, v1, v2, v32371; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 02372; GISEL-NEXT: s_waitcnt vmcnt(0)2373; GISEL-NEXT: s_setpc_b64 s[30:31]2374 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2375 store <8 x i8> %data, ptr addrspace(7) %p2376 ret void2377}2378 2379define <12 x i8> @load_v12i8(ptr addrspace(8) inreg %buf) {2380; SDAG-LABEL: load_v12i8:2381; SDAG: ; %bb.0:2382; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2383; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 02384; SDAG-NEXT: s_waitcnt vmcnt(0)2385; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]2386; SDAG-NEXT: v_lshrrev_b32_e32 v14, 8, v02387; SDAG-NEXT: v_lshrrev_b32_e32 v13, 16, v02388; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[2:3]2389; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v12390; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v12391; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v12392; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v22393; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v22394; SDAG-NEXT: v_mov_b32_e32 v4, v12395; SDAG-NEXT: v_mov_b32_e32 v8, v22396; SDAG-NEXT: v_mov_b32_e32 v1, v142397; SDAG-NEXT: v_mov_b32_e32 v2, v132398; SDAG-NEXT: s_setpc_b64 s[30:31]2399;2400; GISEL-LABEL: load_v12i8:2401; GISEL: ; %bb.0:2402; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2403; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 02404; GISEL-NEXT: s_waitcnt vmcnt(0)2405; GISEL-NEXT: v_lshrrev_b32_e32 v13, 8, v02406; GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v02407; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v02408; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v12409; GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v12410; GISEL-NEXT: v_lshrrev_b32_e32 v7, 24, v12411; GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v22412; GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v22413; GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v22414; GISEL-NEXT: v_mov_b32_e32 v4, v12415; GISEL-NEXT: v_mov_b32_e32 v8, v22416; GISEL-NEXT: v_mov_b32_e32 v1, v132417; GISEL-NEXT: v_mov_b32_e32 v2, v122418; GISEL-NEXT: s_setpc_b64 s[30:31]2419 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2420 %ret = load <12 x i8>, ptr addrspace(7) %p2421 ret <12 x i8> %ret2422}2423 2424define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {2425; SDAG-LABEL: store_v12i8:2426; SDAG: ; %bb.0:2427; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2428; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v92429; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v52430; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12431; SDAG-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2432; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v112433; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2434; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v72435; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2436; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32437; SDAG-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2438; SDAG-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2439; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2440; SDAG-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2441; SDAG-NEXT: v_or_b32_sdwa v7, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2442; SDAG-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2443; SDAG-NEXT: buffer_store_dwordx3 v[6:8], off, s[16:19], 02444; SDAG-NEXT: s_waitcnt vmcnt(0)2445; SDAG-NEXT: s_setpc_b64 s[30:31]2446;2447; GISEL-LABEL: store_v12i8:2448; GISEL: ; %bb.0:2449; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2450; GISEL-NEXT: v_mov_b32_e32 v13, 82451; GISEL-NEXT: v_mov_b32_e32 v12, 0xff2452; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02453; GISEL-NEXT: v_and_or_b32 v0, v0, v12, v12454; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22455; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32456; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12457; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22458; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22459; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v13, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02460; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v62461; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v72462; GISEL-NEXT: v_and_or_b32 v1, v4, v12, v12463; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v22464; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v32465; GISEL-NEXT: v_or3_b32 v1, v1, v2, v32466; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02467; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v102468; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v112469; GISEL-NEXT: v_and_or_b32 v2, v8, v12, v22470; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v32471; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v42472; GISEL-NEXT: v_or3_b32 v2, v2, v3, v42473; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 02474; GISEL-NEXT: s_waitcnt vmcnt(0)2475; GISEL-NEXT: s_setpc_b64 s[30:31]2476 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2477 store <12 x i8> %data, ptr addrspace(7) %p2478 ret void2479}2480 2481define <16 x i8> @load_v16i8(ptr addrspace(8) inreg %buf) {2482; SDAG-LABEL: load_v16i8:2483; SDAG: ; %bb.0:2484; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2485; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 02486; SDAG-NEXT: s_waitcnt vmcnt(0)2487; SDAG-NEXT: v_lshrrev_b64 v[18:19], 24, v[0:1]2488; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[2:3]2489; SDAG-NEXT: v_lshrrev_b32_e32 v17, 8, v02490; SDAG-NEXT: v_lshrrev_b32_e32 v16, 16, v02491; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v12492; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v12493; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v12494; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v22495; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v22496; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v32497; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v32498; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v32499; SDAG-NEXT: v_mov_b32_e32 v4, v12500; SDAG-NEXT: v_mov_b32_e32 v8, v22501; SDAG-NEXT: v_mov_b32_e32 v12, v32502; SDAG-NEXT: v_mov_b32_e32 v1, v172503; SDAG-NEXT: v_mov_b32_e32 v2, v162504; SDAG-NEXT: v_mov_b32_e32 v3, v182505; SDAG-NEXT: s_setpc_b64 s[30:31]2506;2507; GISEL-LABEL: load_v16i8:2508; GISEL: ; %bb.0:2509; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2510; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 02511; GISEL-NEXT: s_waitcnt vmcnt(0)2512; GISEL-NEXT: v_lshrrev_b32_e32 v16, 8, v02513; GISEL-NEXT: v_lshrrev_b32_e32 v17, 16, v02514; GISEL-NEXT: v_lshrrev_b32_e32 v18, 24, v02515; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v12516; GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v12517; GISEL-NEXT: v_lshrrev_b32_e32 v7, 24, v12518; GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v22519; GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v22520; GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v22521; GISEL-NEXT: v_lshrrev_b32_e32 v13, 8, v32522; GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v32523; GISEL-NEXT: v_lshrrev_b32_e32 v15, 24, v32524; GISEL-NEXT: v_mov_b32_e32 v4, v12525; GISEL-NEXT: v_mov_b32_e32 v8, v22526; GISEL-NEXT: v_mov_b32_e32 v12, v32527; GISEL-NEXT: v_mov_b32_e32 v1, v162528; GISEL-NEXT: v_mov_b32_e32 v2, v172529; GISEL-NEXT: v_mov_b32_e32 v3, v182530; GISEL-NEXT: s_setpc_b64 s[30:31]2531 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2532 %ret = load <16 x i8>, ptr addrspace(7) %p2533 ret <16 x i8> %ret2534}2535 2536define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {2537; SDAG-LABEL: store_v16i8:2538; SDAG: ; %bb.0:2539; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2540; SDAG-NEXT: v_lshlrev_b16_e32 v13, 8, v132541; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v92542; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v52543; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12544; SDAG-NEXT: v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2545; SDAG-NEXT: v_lshlrev_b16_e32 v13, 8, v152546; SDAG-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2547; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v112548; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2549; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v72550; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2551; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v32552; SDAG-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2553; SDAG-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2554; SDAG-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2555; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2556; SDAG-NEXT: v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2557; SDAG-NEXT: v_or_b32_sdwa v11, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2558; SDAG-NEXT: v_or_b32_sdwa v10, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2559; SDAG-NEXT: v_or_b32_sdwa v9, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2560; SDAG-NEXT: buffer_store_dwordx4 v[9:12], off, s[16:19], 02561; SDAG-NEXT: s_waitcnt vmcnt(0)2562; SDAG-NEXT: s_setpc_b64 s[30:31]2563;2564; GISEL-LABEL: store_v16i8:2565; GISEL: ; %bb.0:2566; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2567; GISEL-NEXT: v_mov_b32_e32 v17, 82568; GISEL-NEXT: v_mov_b32_e32 v16, 0xff2569; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v17, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02570; GISEL-NEXT: v_and_or_b32 v0, v0, v16, v12571; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v22572; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v32573; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v12574; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v22575; GISEL-NEXT: v_or3_b32 v0, v0, v1, v22576; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02577; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v62578; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v72579; GISEL-NEXT: v_and_or_b32 v1, v4, v16, v12580; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v22581; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v32582; GISEL-NEXT: v_or3_b32 v1, v1, v2, v32583; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02584; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v102585; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v112586; GISEL-NEXT: v_and_or_b32 v2, v8, v16, v22587; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v32588; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v42589; GISEL-NEXT: v_or3_b32 v2, v2, v3, v42590; GISEL-NEXT: v_lshlrev_b32_sdwa v3, v17, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02591; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v142592; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v152593; GISEL-NEXT: v_and_or_b32 v3, v12, v16, v32594; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v42595; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v52596; GISEL-NEXT: v_or3_b32 v3, v3, v4, v52597; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 02598; GISEL-NEXT: s_waitcnt vmcnt(0)2599; GISEL-NEXT: s_setpc_b64 s[30:31]2600 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2601 store <16 x i8> %data, ptr addrspace(7) %p2602 ret void2603}2604 2605define <32 x i8> @load_v32i8(ptr addrspace(8) inreg %buf) {2606; SDAG-LABEL: load_v32i8:2607; SDAG: ; %bb.0:2608; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2609; SDAG-NEXT: buffer_load_dwordx4 v[36:39], off, s[16:19], 02610; SDAG-NEXT: buffer_load_dwordx4 v[32:35], off, s[16:19], 0 offset:162611; SDAG-NEXT: s_waitcnt vmcnt(1)2612; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]2613; SDAG-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]2614; SDAG-NEXT: s_waitcnt vmcnt(0)2615; SDAG-NEXT: v_lshrrev_b64 v[19:20], 24, v[32:33]2616; SDAG-NEXT: v_lshrrev_b64 v[27:28], 24, v[34:35]2617; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v362618; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v362619; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v372620; SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v372621; SDAG-NEXT: v_lshrrev_b32_e32 v7, 24, v372622; SDAG-NEXT: v_lshrrev_b32_e32 v9, 8, v382623; SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v382624; SDAG-NEXT: v_lshrrev_b32_e32 v13, 8, v392625; SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v392626; SDAG-NEXT: v_lshrrev_b32_e32 v15, 24, v392627; SDAG-NEXT: v_lshrrev_b32_e32 v17, 8, v322628; SDAG-NEXT: v_lshrrev_b32_e32 v18, 16, v322629; SDAG-NEXT: v_lshrrev_b32_e32 v21, 8, v332630; SDAG-NEXT: v_lshrrev_b32_e32 v22, 16, v332631; SDAG-NEXT: v_lshrrev_b32_e32 v23, 24, v332632; SDAG-NEXT: v_lshrrev_b32_e32 v25, 8, v342633; SDAG-NEXT: v_lshrrev_b32_e32 v26, 16, v342634; SDAG-NEXT: v_lshrrev_b32_e32 v29, 8, v352635; SDAG-NEXT: v_lshrrev_b32_e32 v30, 16, v352636; SDAG-NEXT: v_lshrrev_b32_e32 v31, 24, v352637; SDAG-NEXT: v_mov_b32_e32 v0, v362638; SDAG-NEXT: v_mov_b32_e32 v4, v372639; SDAG-NEXT: v_mov_b32_e32 v8, v382640; SDAG-NEXT: v_mov_b32_e32 v12, v392641; SDAG-NEXT: v_mov_b32_e32 v16, v322642; SDAG-NEXT: v_mov_b32_e32 v20, v332643; SDAG-NEXT: v_mov_b32_e32 v24, v342644; SDAG-NEXT: v_mov_b32_e32 v28, v352645; SDAG-NEXT: s_setpc_b64 s[30:31]2646;2647; GISEL-LABEL: load_v32i8:2648; GISEL: ; %bb.0:2649; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2650; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 02651; GISEL-NEXT: buffer_load_dwordx4 v[16:19], off, s[16:19], 0 offset:162652; GISEL-NEXT: s_waitcnt vmcnt(1)2653; GISEL-NEXT: v_lshrrev_b32_e32 v35, 8, v02654; GISEL-NEXT: v_lshrrev_b32_e32 v36, 16, v02655; GISEL-NEXT: v_lshrrev_b32_e32 v37, 24, v02656; GISEL-NEXT: s_waitcnt vmcnt(0)2657; GISEL-NEXT: v_lshrrev_b32_e32 v32, 8, v162658; GISEL-NEXT: v_lshrrev_b32_e32 v33, 16, v162659; GISEL-NEXT: v_lshrrev_b32_e32 v34, 24, v162660; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v12661; GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v12662; GISEL-NEXT: v_lshrrev_b32_e32 v7, 24, v12663; GISEL-NEXT: v_lshrrev_b32_e32 v9, 8, v22664; GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v22665; GISEL-NEXT: v_lshrrev_b32_e32 v11, 24, v22666; GISEL-NEXT: v_lshrrev_b32_e32 v13, 8, v32667; GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v32668; GISEL-NEXT: v_lshrrev_b32_e32 v15, 24, v32669; GISEL-NEXT: v_lshrrev_b32_e32 v21, 8, v172670; GISEL-NEXT: v_lshrrev_b32_e32 v22, 16, v172671; GISEL-NEXT: v_lshrrev_b32_e32 v23, 24, v172672; GISEL-NEXT: v_lshrrev_b32_e32 v25, 8, v182673; GISEL-NEXT: v_lshrrev_b32_e32 v26, 16, v182674; GISEL-NEXT: v_lshrrev_b32_e32 v27, 24, v182675; GISEL-NEXT: v_lshrrev_b32_e32 v29, 8, v192676; GISEL-NEXT: v_lshrrev_b32_e32 v30, 16, v192677; GISEL-NEXT: v_lshrrev_b32_e32 v31, 24, v192678; GISEL-NEXT: v_mov_b32_e32 v4, v12679; GISEL-NEXT: v_mov_b32_e32 v8, v22680; GISEL-NEXT: v_mov_b32_e32 v12, v32681; GISEL-NEXT: v_mov_b32_e32 v20, v172682; GISEL-NEXT: v_mov_b32_e32 v24, v182683; GISEL-NEXT: v_mov_b32_e32 v28, v192684; GISEL-NEXT: v_mov_b32_e32 v1, v352685; GISEL-NEXT: v_mov_b32_e32 v2, v362686; GISEL-NEXT: v_mov_b32_e32 v3, v372687; GISEL-NEXT: v_mov_b32_e32 v17, v322688; GISEL-NEXT: v_mov_b32_e32 v18, v332689; GISEL-NEXT: v_mov_b32_e32 v19, v342690; GISEL-NEXT: s_setpc_b64 s[30:31]2691 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2692 %ret = load <32 x i8>, ptr addrspace(7) %p2693 ret <32 x i8> %ret2694}2695 2696define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {2697; SDAG-LABEL: store_v32i8:2698; SDAG: ; %bb.0:2699; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2700; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v92701; SDAG-NEXT: v_or_b32_sdwa v8, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2702; SDAG-NEXT: v_lshlrev_b16_e32 v9, 8, v112703; SDAG-NEXT: v_or_b32_sdwa v9, v10, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2704; SDAG-NEXT: buffer_load_ubyte v10, off, s[0:3], s322705; SDAG-NEXT: v_lshlrev_b16_e32 v13, 8, v132706; SDAG-NEXT: v_or_b32_sdwa v12, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2707; SDAG-NEXT: v_lshlrev_b16_e32 v13, 8, v152708; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v52709; SDAG-NEXT: v_lshlrev_b16_e32 v7, 8, v72710; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v12711; SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v32712; SDAG-NEXT: v_or_b32_sdwa v13, v14, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2713; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2714; SDAG-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2715; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2716; SDAG-NEXT: v_or_b32_sdwa v6, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2717; SDAG-NEXT: v_or_b32_sdwa v3, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2718; SDAG-NEXT: v_or_b32_sdwa v2, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2719; SDAG-NEXT: v_or_b32_sdwa v1, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2720; SDAG-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2721; SDAG-NEXT: v_lshlrev_b16_e32 v11, 8, v292722; SDAG-NEXT: v_lshlrev_b16_e32 v14, 8, v252723; SDAG-NEXT: v_lshlrev_b16_e32 v15, 8, v272724; SDAG-NEXT: v_lshlrev_b16_e32 v21, 8, v212725; SDAG-NEXT: v_lshlrev_b16_e32 v23, 8, v232726; SDAG-NEXT: v_lshlrev_b16_e32 v17, 8, v172727; SDAG-NEXT: v_lshlrev_b16_e32 v19, 8, v192728; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 02729; SDAG-NEXT: v_or_b32_sdwa v7, v28, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2730; SDAG-NEXT: v_or_b32_sdwa v11, v24, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2731; SDAG-NEXT: v_or_b32_sdwa v14, v26, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2732; SDAG-NEXT: v_or_b32_sdwa v15, v20, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2733; SDAG-NEXT: v_or_b32_sdwa v20, v22, v23 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2734; SDAG-NEXT: v_or_b32_sdwa v16, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2735; SDAG-NEXT: v_or_b32_sdwa v17, v18, v19 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2736; SDAG-NEXT: v_or_b32_sdwa v5, v11, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2737; SDAG-NEXT: v_or_b32_sdwa v4, v15, v20 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2738; SDAG-NEXT: v_or_b32_sdwa v3, v16, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2739; SDAG-NEXT: s_waitcnt vmcnt(1)2740; SDAG-NEXT: v_lshlrev_b16_e32 v0, 8, v102741; SDAG-NEXT: v_or_b32_sdwa v0, v30, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD2742; SDAG-NEXT: v_or_b32_sdwa v6, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD2743; SDAG-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:162744; SDAG-NEXT: s_waitcnt vmcnt(0)2745; SDAG-NEXT: s_setpc_b64 s[30:31]2746;2747; GISEL-LABEL: store_v32i8:2748; GISEL: ; %bb.0:2749; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2750; GISEL-NEXT: v_mov_b32_e32 v31, 82751; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02752; GISEL-NEXT: v_mov_b32_e32 v32, 0xff2753; GISEL-NEXT: v_and_or_b32 v0, v0, v32, v12754; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v31, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02755; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v72756; GISEL-NEXT: buffer_load_ubyte v7, off, s[0:3], s322757; GISEL-NEXT: v_and_or_b32 v1, v4, v32, v12758; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v22759; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v32760; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v62761; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v22762; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v32763; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v42764; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v52765; GISEL-NEXT: v_or3_b32 v0, v0, v2, v32766; GISEL-NEXT: v_or3_b32 v1, v1, v4, v52767; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v31, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02768; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v102769; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v112770; GISEL-NEXT: v_and_or_b32 v2, v8, v32, v22771; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v32772; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v42773; GISEL-NEXT: v_or3_b32 v2, v2, v3, v42774; GISEL-NEXT: v_lshlrev_b32_sdwa v3, v31, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02775; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v142776; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v152777; GISEL-NEXT: v_and_or_b32 v3, v12, v32, v32778; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v42779; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v52780; GISEL-NEXT: v_or3_b32 v3, v3, v4, v52781; GISEL-NEXT: v_lshlrev_b32_sdwa v4, v31, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02782; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v182783; GISEL-NEXT: v_and_b32_e32 v6, 0xff, v192784; GISEL-NEXT: v_and_or_b32 v4, v16, v32, v42785; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v52786; GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v62787; GISEL-NEXT: v_lshlrev_b32_sdwa v8, v31, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02788; GISEL-NEXT: v_or3_b32 v4, v4, v5, v62789; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v222790; GISEL-NEXT: v_and_b32_e32 v6, 0xff, v232791; GISEL-NEXT: v_and_or_b32 v8, v20, v32, v82792; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v52793; GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v62794; GISEL-NEXT: v_or3_b32 v5, v8, v5, v62795; GISEL-NEXT: v_lshlrev_b32_sdwa v6, v31, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02796; GISEL-NEXT: v_and_b32_e32 v8, 0xff, v262797; GISEL-NEXT: v_and_b32_e32 v9, 0xff, v272798; GISEL-NEXT: v_and_or_b32 v6, v24, v32, v62799; GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v82800; GISEL-NEXT: v_lshlrev_b32_e32 v9, 24, v92801; GISEL-NEXT: v_or3_b32 v6, v6, v8, v92802; GISEL-NEXT: v_lshlrev_b32_sdwa v8, v31, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_02803; GISEL-NEXT: v_and_b32_e32 v9, 0xff, v302804; GISEL-NEXT: v_and_or_b32 v8, v28, v32, v82805; GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v92806; GISEL-NEXT: s_waitcnt vmcnt(0)2807; GISEL-NEXT: v_lshlrev_b32_e32 v7, 24, v72808; GISEL-NEXT: v_or3_b32 v7, v8, v9, v72809; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 02810; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:162811; GISEL-NEXT: s_waitcnt vmcnt(0)2812; GISEL-NEXT: s_setpc_b64 s[30:31]2813 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2814 store <32 x i8> %data, ptr addrspace(7) %p2815 ret void2816}2817 2818;;; Arrays. Need to become vectors.2819 2820define [1 x i32] @load_a1i32(ptr addrspace(8) inreg %buf) {2821; SDAG-LABEL: load_a1i32:2822; SDAG: ; %bb.0:2823; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2824; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02825; SDAG-NEXT: s_waitcnt vmcnt(0)2826; SDAG-NEXT: s_setpc_b64 s[30:31]2827;2828; GISEL-LABEL: load_a1i32:2829; GISEL: ; %bb.0:2830; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2831; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02832; GISEL-NEXT: s_waitcnt vmcnt(0)2833; GISEL-NEXT: s_setpc_b64 s[30:31]2834 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2835 %ret = load [1 x i32], ptr addrspace(7) %p2836 ret [1 x i32] %ret2837}2838 2839define void @store_a1i32([1 x i32] %data, ptr addrspace(8) inreg %buf) {2840; SDAG-LABEL: store_a1i32:2841; SDAG: ; %bb.0:2842; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2843; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02844; SDAG-NEXT: s_waitcnt vmcnt(0)2845; SDAG-NEXT: s_setpc_b64 s[30:31]2846;2847; GISEL-LABEL: store_a1i32:2848; GISEL: ; %bb.0:2849; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2850; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02851; GISEL-NEXT: s_waitcnt vmcnt(0)2852; GISEL-NEXT: s_setpc_b64 s[30:31]2853 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2854 store [1 x i32] %data, ptr addrspace(7) %p2855 ret void2856}2857 2858define [2 x i32] @load_a2i32(ptr addrspace(8) inreg %buf) {2859; SDAG-LABEL: load_a2i32:2860; SDAG: ; %bb.0:2861; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2862; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 02863; SDAG-NEXT: s_waitcnt vmcnt(0)2864; SDAG-NEXT: s_setpc_b64 s[30:31]2865;2866; GISEL-LABEL: load_a2i32:2867; GISEL: ; %bb.0:2868; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2869; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 02870; GISEL-NEXT: s_waitcnt vmcnt(0)2871; GISEL-NEXT: s_setpc_b64 s[30:31]2872 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2873 %ret = load [2 x i32], ptr addrspace(7) %p2874 ret [2 x i32] %ret2875}2876 2877define void @store_a2i32([2 x i32] %data, ptr addrspace(8) inreg %buf) {2878; SDAG-LABEL: store_a2i32:2879; SDAG: ; %bb.0:2880; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2881; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 02882; SDAG-NEXT: s_waitcnt vmcnt(0)2883; SDAG-NEXT: s_setpc_b64 s[30:31]2884;2885; GISEL-LABEL: store_a2i32:2886; GISEL: ; %bb.0:2887; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2888; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 02889; GISEL-NEXT: s_waitcnt vmcnt(0)2890; GISEL-NEXT: s_setpc_b64 s[30:31]2891 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2892 store [2 x i32] %data, ptr addrspace(7) %p2893 ret void2894}2895 2896define [2 x half] @load_a2f16(ptr addrspace(8) inreg %buf) {2897; SDAG-LABEL: load_a2f16:2898; SDAG: ; %bb.0:2899; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2900; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02901; SDAG-NEXT: s_waitcnt vmcnt(0)2902; SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v02903; SDAG-NEXT: s_setpc_b64 s[30:31]2904;2905; GISEL-LABEL: load_a2f16:2906; GISEL: ; %bb.0:2907; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2908; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02909; GISEL-NEXT: s_waitcnt vmcnt(0)2910; GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v02911; GISEL-NEXT: s_setpc_b64 s[30:31]2912 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2913 %ret = load [2 x half], ptr addrspace(7) %p2914 ret [2 x half] %ret2915}2916 2917define void @store_a2f16([2 x half] %data, ptr addrspace(8) inreg %buf) {2918; SDAG-LABEL: store_a2f16:2919; SDAG: ; %bb.0:2920; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2921; SDAG-NEXT: s_mov_b32 s4, 0x50401002922; SDAG-NEXT: v_perm_b32 v0, v1, v0, s42923; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 02924; SDAG-NEXT: s_waitcnt vmcnt(0)2925; SDAG-NEXT: s_setpc_b64 s[30:31]2926;2927; GISEL-LABEL: store_a2f16:2928; GISEL: ; %bb.0:2929; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2930; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v02931; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v02932; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 02933; GISEL-NEXT: s_waitcnt vmcnt(0)2934; GISEL-NEXT: s_setpc_b64 s[30:31]2935 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2936 store [2 x half] %data, ptr addrspace(7) %p2937 ret void2938}2939 2940define [2 x ptr addrspace(1)] @load_a2p1(ptr addrspace(8) inreg %buf) {2941; SDAG-LABEL: load_a2p1:2942; SDAG: ; %bb.0:2943; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2944; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 02945; SDAG-NEXT: s_waitcnt vmcnt(0)2946; SDAG-NEXT: s_setpc_b64 s[30:31]2947;2948; GISEL-LABEL: load_a2p1:2949; GISEL: ; %bb.0:2950; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2951; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 02952; GISEL-NEXT: s_waitcnt vmcnt(0)2953; GISEL-NEXT: s_setpc_b64 s[30:31]2954 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2955 %ret = load [2 x ptr addrspace(1)], ptr addrspace(7) %p2956 ret [2 x ptr addrspace(1)] %ret2957}2958 2959define void @store_a2p1([2 x ptr addrspace(1)] %data, ptr addrspace(8) inreg %buf) {2960; SDAG-LABEL: store_a2p1:2961; SDAG: ; %bb.0:2962; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2963; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 02964; SDAG-NEXT: s_waitcnt vmcnt(0)2965; SDAG-NEXT: s_setpc_b64 s[30:31]2966;2967; GISEL-LABEL: store_a2p1:2968; GISEL: ; %bb.0:2969; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2970; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 02971; GISEL-NEXT: s_waitcnt vmcnt(0)2972; GISEL-NEXT: s_setpc_b64 s[30:31]2973 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)2974 store [2 x ptr addrspace(1)] %data, ptr addrspace(7) %p2975 ret void2976}2977 2978;;; Scalars of atypical width. Need to be cast to vectors and split.2979 2980define i40 @load_i40(ptr addrspace(8) inreg %buf) {2981; SDAG-LABEL: load_i40:2982; SDAG: ; %bb.0:2983; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2984; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 02985; SDAG-NEXT: buffer_load_ubyte v1, off, s[16:19], 0 offset:42986; SDAG-NEXT: s_waitcnt vmcnt(0)2987; SDAG-NEXT: s_setpc_b64 s[30:31]2988;2989; GISEL-LABEL: load_i40:2990; GISEL: ; %bb.0:2991; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2992; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 02993; GISEL-NEXT: buffer_load_ubyte v1, off, s[16:19], 0 offset:42994; GISEL-NEXT: v_mov_b32_e32 v2, 0xff2995; GISEL-NEXT: s_waitcnt vmcnt(1)2996; GISEL-NEXT: v_lshrrev_b32_e32 v3, 8, v02997; GISEL-NEXT: v_lshrrev_b32_e32 v4, 24, v02998; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v32999; GISEL-NEXT: v_and_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD3000; GISEL-NEXT: v_lshlrev_b16_e32 v4, 8, v43001; GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v33002; GISEL-NEXT: v_or_b32_e32 v2, v2, v43003; GISEL-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3004; GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v23005; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v03006; GISEL-NEXT: v_lshl_or_b32 v0, v2, 16, v03007; GISEL-NEXT: s_waitcnt vmcnt(0)3008; GISEL-NEXT: s_setpc_b64 s[30:31]3009 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3010 %ret = load i40, ptr addrspace(7) %p3011 ret i40 %ret3012}3013 3014define void @store_i40(i40 %data, ptr addrspace(8) inreg %buf) {3015; SDAG-LABEL: store_i40:3016; SDAG: ; %bb.0:3017; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3018; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03019; SDAG-NEXT: buffer_store_byte v1, off, s[16:19], 0 offset:43020; SDAG-NEXT: s_waitcnt vmcnt(0)3021; SDAG-NEXT: s_setpc_b64 s[30:31]3022;3023; GISEL-LABEL: store_i40:3024; GISEL: ; %bb.0:3025; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3026; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03027; GISEL-NEXT: buffer_store_byte v1, off, s[16:19], 0 offset:43028; GISEL-NEXT: s_waitcnt vmcnt(0)3029; GISEL-NEXT: s_setpc_b64 s[30:31]3030 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3031 store i40 %data, ptr addrspace(7) %p3032 ret void3033}3034 3035define i96 @load_i96(ptr addrspace(8) inreg %buf) {3036; SDAG-LABEL: load_i96:3037; SDAG: ; %bb.0:3038; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3039; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 03040; SDAG-NEXT: s_waitcnt vmcnt(0)3041; SDAG-NEXT: s_setpc_b64 s[30:31]3042;3043; GISEL-LABEL: load_i96:3044; GISEL: ; %bb.0:3045; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3046; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 03047; GISEL-NEXT: s_waitcnt vmcnt(0)3048; GISEL-NEXT: s_setpc_b64 s[30:31]3049 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3050 %ret = load i96, ptr addrspace(7) %p3051 ret i96 %ret3052}3053 3054define void @store_i96(i96 %data, ptr addrspace(8) inreg %buf) {3055; SDAG-LABEL: store_i96:3056; SDAG: ; %bb.0:3057; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3058; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 03059; SDAG-NEXT: s_waitcnt vmcnt(0)3060; SDAG-NEXT: s_setpc_b64 s[30:31]3061;3062; GISEL-LABEL: store_i96:3063; GISEL: ; %bb.0:3064; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3065; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 03066; GISEL-NEXT: s_waitcnt vmcnt(0)3067; GISEL-NEXT: s_setpc_b64 s[30:31]3068 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3069 store i96 %data, ptr addrspace(7) %p3070 ret void3071}3072 3073define i160 @load_i160(ptr addrspace(8) inreg %buf) {3074; SDAG-LABEL: load_i160:3075; SDAG: ; %bb.0:3076; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3077; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03078; SDAG-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:163079; SDAG-NEXT: s_waitcnt vmcnt(0)3080; SDAG-NEXT: s_setpc_b64 s[30:31]3081;3082; GISEL-LABEL: load_i160:3083; GISEL: ; %bb.0:3084; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3085; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03086; GISEL-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:163087; GISEL-NEXT: s_waitcnt vmcnt(0)3088; GISEL-NEXT: s_setpc_b64 s[30:31]3089 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3090 %ret = load i160, ptr addrspace(7) %p3091 ret i160 %ret3092}3093 3094define void @store_i160(i160 %data, ptr addrspace(8) inreg %buf) {3095; SDAG-LABEL: store_i160:3096; SDAG: ; %bb.0:3097; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3098; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03099; SDAG-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:163100; SDAG-NEXT: s_waitcnt vmcnt(0)3101; SDAG-NEXT: s_setpc_b64 s[30:31]3102;3103; GISEL-LABEL: store_i160:3104; GISEL: ; %bb.0:3105; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3106; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03107; GISEL-NEXT: buffer_store_dword v4, off, s[16:19], 0 offset:163108; GISEL-NEXT: s_waitcnt vmcnt(0)3109; GISEL-NEXT: s_setpc_b64 s[30:31]3110 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3111 store i160 %data, ptr addrspace(7) %p3112 ret void3113}3114 3115define i256 @load_i256(ptr addrspace(8) inreg %buf) {3116; SDAG-LABEL: load_i256:3117; SDAG: ; %bb.0:3118; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3119; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03120; SDAG-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:163121; SDAG-NEXT: s_waitcnt vmcnt(0)3122; SDAG-NEXT: s_setpc_b64 s[30:31]3123;3124; GISEL-LABEL: load_i256:3125; GISEL: ; %bb.0:3126; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3127; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03128; GISEL-NEXT: buffer_load_dwordx4 v[4:7], off, s[16:19], 0 offset:163129; GISEL-NEXT: s_waitcnt vmcnt(0)3130; GISEL-NEXT: s_setpc_b64 s[30:31]3131 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3132 %ret = load i256, ptr addrspace(7) %p3133 ret i256 %ret3134}3135 3136define void @store_i256(i256 %data, ptr addrspace(8) inreg %buf) {3137; SDAG-LABEL: store_i256:3138; SDAG: ; %bb.0:3139; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3140; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03141; SDAG-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:163142; SDAG-NEXT: s_waitcnt vmcnt(0)3143; SDAG-NEXT: s_setpc_b64 s[30:31]3144;3145; GISEL-LABEL: store_i256:3146; GISEL: ; %bb.0:3147; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3148; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03149; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:163150; GISEL-NEXT: s_waitcnt vmcnt(0)3151; GISEL-NEXT: s_setpc_b64 s[30:31]3152 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3153 store i256 %data, ptr addrspace(7) %p3154 ret void3155}3156 3157;;; Non-byte-sized scalars. Require zero-extension.3158 3159define i7 @load_i7(ptr addrspace(8) inreg %buf) {3160; SDAG-LABEL: load_i7:3161; SDAG: ; %bb.0:3162; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3163; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 03164; SDAG-NEXT: s_waitcnt vmcnt(0)3165; SDAG-NEXT: s_setpc_b64 s[30:31]3166;3167; GISEL-LABEL: load_i7:3168; GISEL: ; %bb.0:3169; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3170; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 03171; GISEL-NEXT: s_waitcnt vmcnt(0)3172; GISEL-NEXT: s_setpc_b64 s[30:31]3173 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3174 %ret = load i7, ptr addrspace(7) %p3175 ret i7 %ret3176}3177 3178define void @store_i7(i7 %data, ptr addrspace(8) inreg %buf) {3179; SDAG-LABEL: store_i7:3180; SDAG: ; %bb.0:3181; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3182; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v03183; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 03184; SDAG-NEXT: s_waitcnt vmcnt(0)3185; SDAG-NEXT: s_setpc_b64 s[30:31]3186;3187; GISEL-LABEL: store_i7:3188; GISEL: ; %bb.0:3189; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3190; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v03191; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 03192; GISEL-NEXT: s_waitcnt vmcnt(0)3193; GISEL-NEXT: s_setpc_b64 s[30:31]3194 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3195 store i7 %data, ptr addrspace(7) %p3196 ret void3197}3198 3199define i4 @load_i4(ptr addrspace(8) inreg %buf) {3200; SDAG-LABEL: load_i4:3201; SDAG: ; %bb.0:3202; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3203; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 03204; SDAG-NEXT: s_waitcnt vmcnt(0)3205; SDAG-NEXT: s_setpc_b64 s[30:31]3206;3207; GISEL-LABEL: load_i4:3208; GISEL: ; %bb.0:3209; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3210; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 03211; GISEL-NEXT: s_waitcnt vmcnt(0)3212; GISEL-NEXT: s_setpc_b64 s[30:31]3213 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3214 %ret = load i4, ptr addrspace(7) %p3215 ret i4 %ret3216}3217 3218define void @store_i4(i4 %data, ptr addrspace(8) inreg %buf) {3219; SDAG-LABEL: store_i4:3220; SDAG: ; %bb.0:3221; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3222; SDAG-NEXT: v_and_b32_e32 v0, 15, v03223; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 03224; SDAG-NEXT: s_waitcnt vmcnt(0)3225; SDAG-NEXT: s_setpc_b64 s[30:31]3226;3227; GISEL-LABEL: store_i4:3228; GISEL: ; %bb.0:3229; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3230; GISEL-NEXT: v_and_b32_e32 v0, 15, v03231; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 03232; GISEL-NEXT: s_waitcnt vmcnt(0)3233; GISEL-NEXT: s_setpc_b64 s[30:31]3234 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3235 store i4 %data, ptr addrspace(7) %p3236 ret void3237}3238 3239 3240;;; Byte-sized vectors of i4. Require casts.3241 3242define <2 x i4> @load_v2i4(ptr addrspace(8) inreg %buf) {3243; SDAG-LABEL: load_v2i4:3244; SDAG: ; %bb.0:3245; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3246; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 03247; SDAG-NEXT: s_waitcnt vmcnt(0)3248; SDAG-NEXT: buffer_store_byte v0, off, s[0:3], s323249; SDAG-NEXT: buffer_load_ubyte v1, off, s[0:3], s323250; SDAG-NEXT: s_waitcnt vmcnt(0)3251; SDAG-NEXT: v_and_b32_e32 v0, 15, v13252; SDAG-NEXT: v_lshrrev_b16_e32 v1, 4, v13253; SDAG-NEXT: s_setpc_b64 s[30:31]3254;3255; GISEL-LABEL: load_v2i4:3256; GISEL: ; %bb.0:3257; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3258; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 03259; GISEL-NEXT: s_waitcnt vmcnt(0)3260; GISEL-NEXT: v_lshrrev_b32_e32 v1, 4, v03261; GISEL-NEXT: s_setpc_b64 s[30:31]3262 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3263 %ret = load <2 x i4>, ptr addrspace(7) %p3264 ret <2 x i4> %ret3265}3266 3267define void @store_v2i4(<2 x i4> %data, ptr addrspace(8) inreg %buf) {3268; SDAG-LABEL: store_v2i4:3269; SDAG: ; %bb.0:3270; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3271; SDAG-NEXT: v_lshlrev_b16_e32 v1, 4, v13272; SDAG-NEXT: v_and_b32_e32 v0, 15, v03273; SDAG-NEXT: v_or_b32_e32 v0, v0, v13274; SDAG-NEXT: buffer_store_byte v0, off, s[0:3], s323275; SDAG-NEXT: buffer_load_ubyte v0, off, s[0:3], s323276; SDAG-NEXT: s_waitcnt vmcnt(0)3277; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 03278; SDAG-NEXT: s_waitcnt vmcnt(0)3279; SDAG-NEXT: s_setpc_b64 s[30:31]3280;3281; GISEL-LABEL: store_v2i4:3282; GISEL: ; %bb.0:3283; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3284; GISEL-NEXT: v_and_b32_e32 v1, 15, v13285; GISEL-NEXT: v_and_b32_e32 v0, 15, v03286; GISEL-NEXT: v_lshlrev_b16_e32 v1, 4, v13287; GISEL-NEXT: v_or_b32_e32 v0, v0, v13288; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 03289; GISEL-NEXT: s_waitcnt vmcnt(0)3290; GISEL-NEXT: s_setpc_b64 s[30:31]3291 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3292 store <2 x i4> %data, ptr addrspace(7) %p3293 ret void3294}3295 3296define <4 x i4> @load_v4i4(ptr addrspace(8) inreg %buf) {3297; SDAG-LABEL: load_v4i4:3298; SDAG: ; %bb.0:3299; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3300; SDAG-NEXT: buffer_load_ushort v0, off, s[16:19], 03301; SDAG-NEXT: v_mov_b32_e32 v2, 153302; SDAG-NEXT: s_waitcnt vmcnt(0)3303; SDAG-NEXT: buffer_store_short v0, off, s[0:3], s323304; SDAG-NEXT: buffer_load_ushort v1, off, s[0:3], s323305; SDAG-NEXT: s_waitcnt vmcnt(0)3306; SDAG-NEXT: v_lshrrev_b16_e32 v4, 4, v13307; SDAG-NEXT: v_and_b32_e32 v0, 15, v13308; SDAG-NEXT: v_lshrrev_b16_e32 v3, 12, v13309; SDAG-NEXT: v_and_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD3310; SDAG-NEXT: v_and_b32_e32 v1, 15, v43311; SDAG-NEXT: s_setpc_b64 s[30:31]3312;3313; GISEL-LABEL: load_v4i4:3314; GISEL: ; %bb.0:3315; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3316; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 03317; GISEL-NEXT: s_waitcnt vmcnt(0)3318; GISEL-NEXT: v_lshrrev_b32_e32 v1, 4, v03319; GISEL-NEXT: v_lshrrev_b32_e32 v2, 8, v03320; GISEL-NEXT: v_lshrrev_b32_e32 v3, 12, v03321; GISEL-NEXT: s_setpc_b64 s[30:31]3322 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3323 %ret = load <4 x i4>, ptr addrspace(7) %p3324 ret <4 x i4> %ret3325}3326 3327define void @store_v4i4(<4 x i4> %data, ptr addrspace(8) inreg %buf) {3328; SDAG-LABEL: store_v4i4:3329; SDAG: ; %bb.0:3330; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3331; SDAG-NEXT: v_and_b32_e32 v1, 15, v13332; SDAG-NEXT: v_and_b32_e32 v0, 15, v03333; SDAG-NEXT: v_lshlrev_b16_e32 v1, 4, v13334; SDAG-NEXT: v_or_b32_e32 v0, v0, v13335; SDAG-NEXT: v_mov_b32_e32 v1, 153336; SDAG-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3337; SDAG-NEXT: v_or_b32_e32 v0, v0, v13338; SDAG-NEXT: v_lshlrev_b16_e32 v1, 12, v33339; SDAG-NEXT: v_or_b32_e32 v0, v0, v13340; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 03341; SDAG-NEXT: s_waitcnt vmcnt(0)3342; SDAG-NEXT: s_setpc_b64 s[30:31]3343;3344; GISEL-LABEL: store_v4i4:3345; GISEL: ; %bb.0:3346; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3347; GISEL-NEXT: v_and_b32_e32 v1, 15, v13348; GISEL-NEXT: v_and_b32_e32 v0, 15, v03349; GISEL-NEXT: v_lshlrev_b16_e32 v1, 4, v13350; GISEL-NEXT: v_or_b32_e32 v0, v0, v13351; GISEL-NEXT: v_mov_b32_e32 v1, 153352; GISEL-NEXT: v_and_b32_sdwa v1, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3353; GISEL-NEXT: v_or_b32_e32 v0, v0, v13354; GISEL-NEXT: v_and_b32_e32 v1, 15, v33355; GISEL-NEXT: v_lshlrev_b16_e32 v1, 12, v13356; GISEL-NEXT: v_or_b32_e32 v0, v0, v13357; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 03358; GISEL-NEXT: s_waitcnt vmcnt(0)3359; GISEL-NEXT: s_setpc_b64 s[30:31]3360 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3361 store <4 x i4> %data, ptr addrspace(7) %p3362 ret void3363}3364 3365define <8 x i4> @load_v8i4(ptr addrspace(8) inreg %buf) {3366; SDAG-LABEL: load_v8i4:3367; SDAG: ; %bb.0:3368; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3369; SDAG-NEXT: buffer_load_dword v7, off, s[16:19], 03370; SDAG-NEXT: s_waitcnt vmcnt(0)3371; SDAG-NEXT: v_and_b32_e32 v0, 15, v73372; SDAG-NEXT: v_bfe_u32 v1, v7, 4, 43373; SDAG-NEXT: v_bfe_u32 v2, v7, 8, 43374; SDAG-NEXT: v_bfe_u32 v3, v7, 12, 43375; SDAG-NEXT: v_bfe_u32 v4, v7, 16, 43376; SDAG-NEXT: v_bfe_u32 v5, v7, 20, 43377; SDAG-NEXT: v_bfe_u32 v6, v7, 24, 43378; SDAG-NEXT: v_lshrrev_b32_e32 v7, 28, v73379; SDAG-NEXT: s_setpc_b64 s[30:31]3380;3381; GISEL-LABEL: load_v8i4:3382; GISEL: ; %bb.0:3383; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3384; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 03385; GISEL-NEXT: s_waitcnt vmcnt(0)3386; GISEL-NEXT: v_lshrrev_b32_e32 v1, 4, v03387; GISEL-NEXT: v_lshrrev_b32_e32 v2, 8, v03388; GISEL-NEXT: v_lshrrev_b32_e32 v3, 12, v03389; GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v03390; GISEL-NEXT: v_lshrrev_b32_e32 v5, 20, v03391; GISEL-NEXT: v_lshrrev_b32_e32 v6, 24, v03392; GISEL-NEXT: v_lshrrev_b32_e32 v7, 28, v03393; GISEL-NEXT: s_setpc_b64 s[30:31]3394 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3395 %ret = load <8 x i4>, ptr addrspace(7) %p3396 ret <8 x i4> %ret3397}3398 3399define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {3400; SDAG-LABEL: store_v8i4:3401; SDAG: ; %bb.0:3402; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3403; SDAG-NEXT: v_and_b32_e32 v1, 15, v13404; SDAG-NEXT: v_lshlrev_b32_e32 v1, 4, v13405; SDAG-NEXT: v_and_or_b32 v0, v0, 15, v13406; SDAG-NEXT: v_and_b32_e32 v1, 15, v33407; SDAG-NEXT: v_and_b32_e32 v2, 15, v23408; SDAG-NEXT: v_lshlrev_b32_e32 v1, 12, v13409; SDAG-NEXT: v_lshlrev_b32_e32 v2, 8, v23410; SDAG-NEXT: v_or3_b32 v0, v0, v2, v13411; SDAG-NEXT: v_and_b32_e32 v1, 15, v53412; SDAG-NEXT: v_mov_b32_e32 v2, 153413; SDAG-NEXT: v_lshlrev_b32_e32 v1, 20, v13414; SDAG-NEXT: v_and_b32_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3415; SDAG-NEXT: v_or3_b32 v0, v0, v3, v13416; SDAG-NEXT: v_lshlrev_b32_e32 v1, 28, v73417; SDAG-NEXT: v_and_b32_sdwa v2, v6, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3418; SDAG-NEXT: v_or3_b32 v0, v0, v2, v13419; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03420; SDAG-NEXT: s_waitcnt vmcnt(0)3421; SDAG-NEXT: s_setpc_b64 s[30:31]3422;3423; GISEL-LABEL: store_v8i4:3424; GISEL: ; %bb.0:3425; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3426; GISEL-NEXT: v_and_b32_e32 v1, 15, v13427; GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v13428; GISEL-NEXT: v_and_or_b32 v0, v0, 15, v13429; GISEL-NEXT: v_and_b32_e32 v1, 15, v23430; GISEL-NEXT: v_and_b32_e32 v2, 15, v33431; GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v13432; GISEL-NEXT: v_lshlrev_b32_e32 v2, 12, v23433; GISEL-NEXT: v_or3_b32 v0, v0, v1, v23434; GISEL-NEXT: v_mov_b32_e32 v1, 153435; GISEL-NEXT: v_and_b32_e32 v3, 15, v53436; GISEL-NEXT: v_and_b32_sdwa v2, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3437; GISEL-NEXT: v_lshlrev_b32_e32 v3, 20, v33438; GISEL-NEXT: v_or3_b32 v0, v0, v2, v33439; GISEL-NEXT: v_and_b32_e32 v2, 15, v73440; GISEL-NEXT: v_and_b32_sdwa v1, v6, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3441; GISEL-NEXT: v_lshlrev_b32_e32 v2, 28, v23442; GISEL-NEXT: v_or3_b32 v0, v0, v1, v23443; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03444; GISEL-NEXT: s_waitcnt vmcnt(0)3445; GISEL-NEXT: s_setpc_b64 s[30:31]3446 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3447 store <8 x i4> %data, ptr addrspace(7) %p3448 ret void3449}3450 3451;;; Vectors of non-byte-sized integers.3452 3453define <2 x i6> @load_v2i6(ptr addrspace(8) inreg %buf) {3454; SDAG-LABEL: load_v2i6:3455; SDAG: ; %bb.0:3456; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3457; SDAG-NEXT: buffer_load_ushort v1, off, s[16:19], 03458; SDAG-NEXT: s_waitcnt vmcnt(0)3459; SDAG-NEXT: v_and_b32_e32 v0, 63, v13460; SDAG-NEXT: v_bfe_u32 v1, v1, 6, 63461; SDAG-NEXT: s_setpc_b64 s[30:31]3462;3463; GISEL-LABEL: load_v2i6:3464; GISEL: ; %bb.0:3465; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3466; GISEL-NEXT: buffer_load_ushort v0, off, s[16:19], 03467; GISEL-NEXT: s_waitcnt vmcnt(0)3468; GISEL-NEXT: v_lshrrev_b16_e32 v1, 6, v03469; GISEL-NEXT: s_setpc_b64 s[30:31]3470 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3471 %ret = load <2 x i6>, ptr addrspace(7) %p3472 ret <2 x i6> %ret3473}3474 3475define void @store_v2i6(<2 x i6> %data, ptr addrspace(8) inreg %buf) {3476; SDAG-LABEL: store_v2i6:3477; SDAG: ; %bb.0:3478; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3479; SDAG-NEXT: v_lshlrev_b16_e32 v1, 6, v13480; SDAG-NEXT: v_and_b32_e32 v0, 63, v03481; SDAG-NEXT: v_or_b32_e32 v0, v0, v13482; SDAG-NEXT: v_and_b32_e32 v0, 0xfff, v03483; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v03484; SDAG-NEXT: buffer_store_short v0, off, s[16:19], 03485; SDAG-NEXT: s_waitcnt vmcnt(0)3486; SDAG-NEXT: s_setpc_b64 s[30:31]3487;3488; GISEL-LABEL: store_v2i6:3489; GISEL: ; %bb.0:3490; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3491; GISEL-NEXT: v_and_b32_e32 v1, 63, v13492; GISEL-NEXT: v_and_b32_e32 v0, 63, v03493; GISEL-NEXT: v_lshlrev_b16_e32 v1, 6, v13494; GISEL-NEXT: v_or_b32_e32 v0, v0, v13495; GISEL-NEXT: v_and_b32_e32 v0, 0xfff, v03496; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 03497; GISEL-NEXT: s_waitcnt vmcnt(0)3498; GISEL-NEXT: s_setpc_b64 s[30:31]3499 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3500 store <2 x i6> %data, ptr addrspace(7) %p3501 ret void3502}3503 3504;; Blocks of fp6 elements3505define <6 x i32> @load_v32i6(ptr addrspace(8) inreg %buf) {3506; SDAG-LABEL: load_v32i6:3507; SDAG: ; %bb.0:3508; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3509; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03510; SDAG-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:163511; SDAG-NEXT: s_waitcnt vmcnt(0)3512; SDAG-NEXT: s_setpc_b64 s[30:31]3513;3514; GISEL-LABEL: load_v32i6:3515; GISEL: ; %bb.0:3516; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3517; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03518; GISEL-NEXT: buffer_load_dwordx2 v[4:5], off, s[16:19], 0 offset:163519; GISEL-NEXT: s_waitcnt vmcnt(0)3520; GISEL-NEXT: s_setpc_b64 s[30:31]3521 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3522 %ret = load <32 x i6>, ptr addrspace(7) %p3523 %ret.cast = bitcast <32 x i6> %ret to <6 x i32>3524 ret <6 x i32> %ret.cast3525}3526 3527define void @store_v32i6(<6 x i32> %data.abi, ptr addrspace(8) inreg %buf) {3528; SDAG-LABEL: store_v32i6:3529; SDAG: ; %bb.0:3530; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3531; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03532; SDAG-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:163533; SDAG-NEXT: s_waitcnt vmcnt(0)3534; SDAG-NEXT: s_setpc_b64 s[30:31]3535;3536; GISEL-LABEL: store_v32i6:3537; GISEL: ; %bb.0:3538; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3539; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03540; GISEL-NEXT: buffer_store_dwordx2 v[4:5], off, s[16:19], 0 offset:163541; GISEL-NEXT: s_waitcnt vmcnt(0)3542; GISEL-NEXT: s_setpc_b64 s[30:31]3543 %data = bitcast <6 x i32> %data.abi to <32 x i6>3544 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3545 store <32 x i6> %data, ptr addrspace(7) %p3546 ret void3547}3548 3549;;; Modifiers3550 3551define <4 x i8> @volatile_load_v4i8(ptr addrspace(8) inreg %buf) {3552; SDAG-LABEL: volatile_load_v4i8:3553; SDAG: ; %bb.0:3554; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3555; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0 glc3556; SDAG-NEXT: s_waitcnt vmcnt(0)3557; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v03558; SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v03559; SDAG-NEXT: v_lshrrev_b32_e32 v3, 24, v03560; SDAG-NEXT: s_setpc_b64 s[30:31]3561;3562; GISEL-LABEL: volatile_load_v4i8:3563; GISEL: ; %bb.0:3564; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3565; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0 glc3566; GISEL-NEXT: s_waitcnt vmcnt(0)3567; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v03568; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v03569; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v03570; GISEL-NEXT: s_setpc_b64 s[30:31]3571 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3572 %ret = load volatile <4 x i8>, ptr addrspace(7) %p3573 ret <4 x i8> %ret3574}3575 3576define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {3577; SDAG-LABEL: volatile_store_v4i8:3578; SDAG: ; %bb.0:3579; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3580; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v13581; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3582; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v33583; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3584; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD3585; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03586; SDAG-NEXT: s_waitcnt vmcnt(0)3587; SDAG-NEXT: s_setpc_b64 s[30:31]3588;3589; GISEL-LABEL: volatile_store_v4i8:3590; GISEL: ; %bb.0:3591; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3592; GISEL-NEXT: v_mov_b32_e32 v5, 83593; GISEL-NEXT: v_mov_b32_e32 v4, 0xff3594; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_03595; GISEL-NEXT: v_and_or_b32 v0, v0, v4, v13596; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v23597; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v33598; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v13599; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v23600; GISEL-NEXT: v_or3_b32 v0, v0, v1, v23601; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03602; GISEL-NEXT: s_waitcnt vmcnt(0)3603; GISEL-NEXT: s_setpc_b64 s[30:31]3604 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3605 store volatile <4 x i8> %data, ptr addrspace(7) %p3606 ret void3607}3608 3609define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {3610; SDAG-LABEL: volatile_load_v6i8:3611; SDAG: ; %bb.0:3612; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3613; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 0 glc3614; SDAG-NEXT: s_waitcnt vmcnt(0)3615; SDAG-NEXT: buffer_load_ushort v6, off, s[16:19], 0 offset:4 glc3616; SDAG-NEXT: s_waitcnt vmcnt(0)3617; SDAG-NEXT: v_lshrrev_b32_e32 v7, 8, v03618; SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v63619; SDAG-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]3620; SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v03621; SDAG-NEXT: v_lshrrev_b32_e32 v5, 8, v13622; SDAG-NEXT: v_mov_b32_e32 v4, v63623; SDAG-NEXT: v_mov_b32_e32 v1, v73624; SDAG-NEXT: s_setpc_b64 s[30:31]3625;3626; GISEL-LABEL: volatile_load_v6i8:3627; GISEL: ; %bb.0:3628; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3629; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0 glc3630; GISEL-NEXT: s_waitcnt vmcnt(0)3631; GISEL-NEXT: buffer_load_ushort v4, off, s[16:19], 0 offset:4 glc3632; GISEL-NEXT: s_waitcnt vmcnt(0)3633; GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v03634; GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v03635; GISEL-NEXT: v_lshrrev_b32_e32 v3, 24, v03636; GISEL-NEXT: v_lshrrev_b32_e32 v5, 8, v43637; GISEL-NEXT: s_setpc_b64 s[30:31]3638 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3639 %ret = load volatile <6 x i8>, ptr addrspace(7) %p3640 ret <6 x i8> %ret3641}3642 3643define void @volatile_store_v6i8(<6 x i8> %data, ptr addrspace(8) inreg %buf) {3644; SDAG-LABEL: volatile_store_v6i8:3645; SDAG: ; %bb.0:3646; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3647; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v13648; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3649; SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v33650; SDAG-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3651; SDAG-NEXT: v_lshlrev_b16_e32 v5, 8, v53652; SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD3653; SDAG-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3654; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03655; SDAG-NEXT: s_waitcnt vmcnt(0)3656; SDAG-NEXT: buffer_store_short v4, off, s[16:19], 0 offset:43657; SDAG-NEXT: s_waitcnt vmcnt(0)3658; SDAG-NEXT: s_setpc_b64 s[30:31]3659;3660; GISEL-LABEL: volatile_store_v6i8:3661; GISEL: ; %bb.0:3662; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3663; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v13664; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v13665; GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3666; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v33667; GISEL-NEXT: v_lshlrev_b16_e32 v1, 8, v13668; GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3669; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v53670; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v03671; GISEL-NEXT: v_lshlrev_b16_e32 v2, 8, v23672; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v03673; GISEL-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD3674; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03675; GISEL-NEXT: s_waitcnt vmcnt(0)3676; GISEL-NEXT: buffer_store_short v2, off, s[16:19], 0 offset:43677; GISEL-NEXT: s_waitcnt vmcnt(0)3678; GISEL-NEXT: s_setpc_b64 s[30:31]3679 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3680 store volatile <6 x i8> %data, ptr addrspace(7) %p3681 ret void3682}3683 3684define [2 x [2 x i32]] @load_a2a2i32(ptr addrspace(8) inreg %buf) {3685; SDAG-LABEL: load_a2a2i32:3686; SDAG: ; %bb.0:3687; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3688; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03689; SDAG-NEXT: s_waitcnt vmcnt(0)3690; SDAG-NEXT: s_setpc_b64 s[30:31]3691;3692; GISEL-LABEL: load_a2a2i32:3693; GISEL: ; %bb.0:3694; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3695; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03696; GISEL-NEXT: s_waitcnt vmcnt(0)3697; GISEL-NEXT: s_setpc_b64 s[30:31]3698 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3699 %ret = load [2 x [2 x i32]], ptr addrspace(7) %p3700 ret [2 x [2 x i32]] %ret3701}3702 3703define void @store_a2a2i32([2 x [2 x i32]] %data, ptr addrspace(8) inreg %buf) {3704; SDAG-LABEL: store_a2a2i32:3705; SDAG: ; %bb.0:3706; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3707; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03708; SDAG-NEXT: s_waitcnt vmcnt(0)3709; SDAG-NEXT: s_setpc_b64 s[30:31]3710;3711; GISEL-LABEL: store_a2a2i32:3712; GISEL: ; %bb.0:3713; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3714; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03715; GISEL-NEXT: s_waitcnt vmcnt(0)3716; GISEL-NEXT: s_setpc_b64 s[30:31]3717 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3718 store [2 x [2 x i32]] %data, ptr addrspace(7) %p3719 ret void3720}3721 3722define [2 x <2 x i32>] @load_a2v2i32(ptr addrspace(8) inreg %buf) {3723; SDAG-LABEL: load_a2v2i32:3724; SDAG: ; %bb.0:3725; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3726; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03727; SDAG-NEXT: s_waitcnt vmcnt(0)3728; SDAG-NEXT: s_setpc_b64 s[30:31]3729;3730; GISEL-LABEL: load_a2v2i32:3731; GISEL: ; %bb.0:3732; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3733; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[16:19], 03734; GISEL-NEXT: s_waitcnt vmcnt(0)3735; GISEL-NEXT: s_setpc_b64 s[30:31]3736 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3737 %ret = load [2 x <2 x i32>], ptr addrspace(7) %p3738 ret [2 x <2 x i32>] %ret3739}3740 3741define void @store_a2v2i32([2 x <2 x i32>] %data, ptr addrspace(8) inreg %buf) {3742; SDAG-LABEL: store_a2v2i32:3743; SDAG: ; %bb.0:3744; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3745; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03746; SDAG-NEXT: s_waitcnt vmcnt(0)3747; SDAG-NEXT: s_setpc_b64 s[30:31]3748;3749; GISEL-LABEL: store_a2v2i32:3750; GISEL: ; %bb.0:3751; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3752; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 03753; GISEL-NEXT: s_waitcnt vmcnt(0)3754; GISEL-NEXT: s_setpc_b64 s[30:31]3755 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3756 store [2 x <2 x i32>] %data, ptr addrspace(7) %p3757 ret void3758}3759 3760define { i32 } @load_sl_i32s(ptr addrspace(8) inreg %buf) {3761; SDAG-LABEL: load_sl_i32s:3762; SDAG: ; %bb.0:3763; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3764; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 03765; SDAG-NEXT: s_waitcnt vmcnt(0)3766; SDAG-NEXT: s_setpc_b64 s[30:31]3767;3768; GISEL-LABEL: load_sl_i32s:3769; GISEL: ; %bb.0:3770; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3771; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 03772; GISEL-NEXT: s_waitcnt vmcnt(0)3773; GISEL-NEXT: s_setpc_b64 s[30:31]3774 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3775 %ret = load { i32 }, ptr addrspace(7) %p3776 ret { i32 } %ret3777}3778 3779define void @store_sl_i32s({ i32 } %data, ptr addrspace(8) inreg %buf) {3780; SDAG-LABEL: store_sl_i32s:3781; SDAG: ; %bb.0:3782; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3783; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03784; SDAG-NEXT: s_waitcnt vmcnt(0)3785; SDAG-NEXT: s_setpc_b64 s[30:31]3786;3787; GISEL-LABEL: store_sl_i32s:3788; GISEL: ; %bb.0:3789; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3790; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03791; GISEL-NEXT: s_waitcnt vmcnt(0)3792; GISEL-NEXT: s_setpc_b64 s[30:31]3793 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3794 store { i32 } %data, ptr addrspace(7) %p3795 ret void3796}3797 3798define { { float } } @load_sl_sl_f32ss(ptr addrspace(8) inreg %buf) {3799; SDAG-LABEL: load_sl_sl_f32ss:3800; SDAG: ; %bb.0:3801; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3802; SDAG-NEXT: buffer_load_dword v0, off, s[16:19], 03803; SDAG-NEXT: s_waitcnt vmcnt(0)3804; SDAG-NEXT: s_setpc_b64 s[30:31]3805;3806; GISEL-LABEL: load_sl_sl_f32ss:3807; GISEL: ; %bb.0:3808; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3809; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 03810; GISEL-NEXT: s_waitcnt vmcnt(0)3811; GISEL-NEXT: s_setpc_b64 s[30:31]3812 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3813 %ret = load { { float } }, ptr addrspace(7) %p3814 ret { { float } } %ret3815}3816 3817define void @store_sl_sl_f32ss({ { float } } %data, ptr addrspace(8) inreg %buf) {3818; SDAG-LABEL: store_sl_sl_f32ss:3819; SDAG: ; %bb.0:3820; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3821; SDAG-NEXT: buffer_store_dword v0, off, s[16:19], 03822; SDAG-NEXT: s_waitcnt vmcnt(0)3823; SDAG-NEXT: s_setpc_b64 s[30:31]3824;3825; GISEL-LABEL: store_sl_sl_f32ss:3826; GISEL: ; %bb.0:3827; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3828; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 03829; GISEL-NEXT: s_waitcnt vmcnt(0)3830; GISEL-NEXT: s_setpc_b64 s[30:31]3831 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3832 store { { float } } %data, ptr addrspace(7) %p3833 ret void3834}3835 3836define { <2 x i32> } @load_sl_v2i32s(ptr addrspace(8) inreg %buf) {3837; SDAG-LABEL: load_sl_v2i32s:3838; SDAG: ; %bb.0:3839; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3840; SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 03841; SDAG-NEXT: s_waitcnt vmcnt(0)3842; SDAG-NEXT: s_setpc_b64 s[30:31]3843;3844; GISEL-LABEL: load_sl_v2i32s:3845; GISEL: ; %bb.0:3846; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3847; GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 03848; GISEL-NEXT: s_waitcnt vmcnt(0)3849; GISEL-NEXT: s_setpc_b64 s[30:31]3850 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3851 %ret = load { <2 x i32> }, ptr addrspace(7) %p3852 ret { <2 x i32> } %ret3853}3854 3855define void @store_sl_v2i32s({ <2 x i32> } %data, ptr addrspace(8) inreg %buf) {3856; SDAG-LABEL: store_sl_v2i32s:3857; SDAG: ; %bb.0:3858; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3859; SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 03860; SDAG-NEXT: s_waitcnt vmcnt(0)3861; SDAG-NEXT: s_setpc_b64 s[30:31]3862;3863; GISEL-LABEL: store_sl_v2i32s:3864; GISEL: ; %bb.0:3865; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3866; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 03867; GISEL-NEXT: s_waitcnt vmcnt(0)3868; GISEL-NEXT: s_setpc_b64 s[30:31]3869 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3870 store { <2 x i32> } %data, ptr addrspace(7) %p3871 ret void3872}3873 3874define { i64, i32 } @load_sl_i64i32s(ptr addrspace(8) inreg %buf) {3875; SDAG-LABEL: load_sl_i64i32s:3876; SDAG: ; %bb.0:3877; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3878; SDAG-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 03879; SDAG-NEXT: s_waitcnt vmcnt(0)3880; SDAG-NEXT: s_setpc_b64 s[30:31]3881;3882; GISEL-LABEL: load_sl_i64i32s:3883; GISEL: ; %bb.0:3884; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3885; GISEL-NEXT: buffer_load_dwordx3 v[0:2], off, s[16:19], 03886; GISEL-NEXT: s_waitcnt vmcnt(0)3887; GISEL-NEXT: s_setpc_b64 s[30:31]3888 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3889 %ret = load { i64, i32 }, ptr addrspace(7) %p3890 ret { i64, i32 } %ret3891}3892 3893define void @store_sl_i64i32s({ i64, i32 } %data, ptr addrspace(8) inreg %buf) {3894; SDAG-LABEL: store_sl_i64i32s:3895; SDAG: ; %bb.0:3896; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3897; SDAG-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 03898; SDAG-NEXT: s_waitcnt vmcnt(0)3899; SDAG-NEXT: s_setpc_b64 s[30:31]3900;3901; GISEL-LABEL: store_sl_i64i32s:3902; GISEL: ; %bb.0:3903; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3904; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 03905; GISEL-NEXT: s_waitcnt vmcnt(0)3906; GISEL-NEXT: s_setpc_b64 s[30:31]3907 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3908 store { i64, i32 } %data, ptr addrspace(7) %p3909 ret void3910}3911 3912define [4 x i7] @load_a4i7(ptr addrspace(8) inreg %buf) {3913; SDAG-LABEL: load_a4i7:3914; SDAG: ; %bb.0:3915; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3916; SDAG-NEXT: buffer_load_ubyte v0, off, s[16:19], 03917; SDAG-NEXT: buffer_load_ubyte v1, off, s[16:19], 0 offset:13918; SDAG-NEXT: buffer_load_ubyte v2, off, s[16:19], 0 offset:23919; SDAG-NEXT: buffer_load_ubyte v3, off, s[16:19], 0 offset:33920; SDAG-NEXT: s_waitcnt vmcnt(0)3921; SDAG-NEXT: s_setpc_b64 s[30:31]3922;3923; GISEL-LABEL: load_a4i7:3924; GISEL: ; %bb.0:3925; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3926; GISEL-NEXT: buffer_load_ubyte v0, off, s[16:19], 03927; GISEL-NEXT: buffer_load_ubyte v1, off, s[16:19], 0 offset:13928; GISEL-NEXT: buffer_load_ubyte v2, off, s[16:19], 0 offset:23929; GISEL-NEXT: buffer_load_ubyte v3, off, s[16:19], 0 offset:33930; GISEL-NEXT: s_waitcnt vmcnt(0)3931; GISEL-NEXT: s_setpc_b64 s[30:31]3932 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3933 %ret = load [4 x i7], ptr addrspace(7) %p3934 ret [4 x i7] %ret3935}3936 3937define void @store_a4i7([4 x i7] %data, ptr addrspace(8) inreg %buf) {3938; SDAG-LABEL: store_a4i7:3939; SDAG: ; %bb.0:3940; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3941; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v03942; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 03943; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v13944; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:13945; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v23946; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:23947; SDAG-NEXT: v_and_b32_e32 v0, 0x7f, v33948; SDAG-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:33949; SDAG-NEXT: s_waitcnt vmcnt(0)3950; SDAG-NEXT: s_setpc_b64 s[30:31]3951;3952; GISEL-LABEL: store_a4i7:3953; GISEL: ; %bb.0:3954; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3955; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v03956; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 03957; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v13958; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:13959; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v23960; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:23961; GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v33962; GISEL-NEXT: buffer_store_byte v0, off, s[16:19], 0 offset:33963; GISEL-NEXT: s_waitcnt vmcnt(0)3964; GISEL-NEXT: s_setpc_b64 s[30:31]3965 %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)3966 store [4 x i7] %data, ptr addrspace(7) %p3967 ret void3968}3969