601 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 %s -o - | FileCheck %s3 4%struct.S = type { [32 x i32] }5 6@shared = addrspace(3) global %struct.S poison, align 47 8define amdgpu_kernel void @memcpy_p0_p0_minsize(ptr %dest, ptr readonly %src) #0 {9; CHECK-LABEL: memcpy_p0_p0_minsize:10; CHECK: ; %bb.0: ; %entry11; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x012; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s1713; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 014; CHECK-NEXT: s_waitcnt lgkmcnt(0)15; CHECK-NEXT: v_mov_b32_e32 v9, s316; CHECK-NEXT: v_mov_b32_e32 v8, s217; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[8:9] offset:3218; CHECK-NEXT: flat_load_dwordx2 v[12:13], v[8:9] offset:3919; CHECK-NEXT: flat_load_dwordx4 v[0:3], v[8:9]20; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[8:9] offset:1621; CHECK-NEXT: v_mov_b32_e32 v9, s122; CHECK-NEXT: v_mov_b32_e32 v8, s023; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)24; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[10:11] offset:3225; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[12:13] offset:3926; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[0:3]27; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[4:7] offset:1628; CHECK-NEXT: s_endpgm29entry:30 tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)31 ret void32}33 34define amdgpu_kernel void @memcpy_p1_p1_minsize(ptr addrspace(1) %dest, ptr addrspace(1) %src) #0 {35; CHECK-LABEL: memcpy_p1_p1_minsize:36; CHECK: ; %bb.0: ; %entry37; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x038; CHECK-NEXT: v_mov_b32_e32 v12, 039; CHECK-NEXT: s_waitcnt lgkmcnt(0)40; CHECK-NEXT: global_load_dwordx2 v[8:9], v12, s[2:3] offset:3241; CHECK-NEXT: global_load_dwordx2 v[10:11], v12, s[2:3] offset:3942; CHECK-NEXT: global_load_dwordx4 v[0:3], v12, s[2:3]43; CHECK-NEXT: global_load_dwordx4 v[4:7], v12, s[2:3] offset:1644; CHECK-NEXT: s_waitcnt vmcnt(3)45; CHECK-NEXT: global_store_dwordx2 v12, v[8:9], s[0:1] offset:3246; CHECK-NEXT: s_waitcnt vmcnt(3)47; CHECK-NEXT: global_store_dwordx2 v12, v[10:11], s[0:1] offset:3948; CHECK-NEXT: s_waitcnt vmcnt(3)49; CHECK-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1]50; CHECK-NEXT: s_waitcnt vmcnt(3)51; CHECK-NEXT: global_store_dwordx4 v12, v[4:7], s[0:1] offset:1652; CHECK-NEXT: s_endpgm53entry:54 tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 47, i1 false)55 ret void56}57 58define amdgpu_kernel void @memcpy_p1_p4_minsize(ptr addrspace(1) %global, ptr addrspace(4) %0) #0 {59; CHECK-LABEL: memcpy_p1_p4_minsize:60; CHECK: ; %bb.0: ; %entry61; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x062; CHECK-NEXT: v_mov_b32_e32 v32, 063; CHECK-NEXT: s_waitcnt lgkmcnt(0)64; CHECK-NEXT: global_load_dwordx4 v[0:3], v32, s[2:3]65; CHECK-NEXT: global_load_dwordx4 v[4:7], v32, s[2:3] offset:1666; CHECK-NEXT: global_load_dwordx4 v[8:11], v32, s[2:3] offset:3267; CHECK-NEXT: global_load_dwordx4 v[12:15], v32, s[2:3] offset:4868; CHECK-NEXT: global_load_dwordx4 v[16:19], v32, s[2:3] offset:6469; CHECK-NEXT: global_load_dwordx4 v[20:23], v32, s[2:3] offset:8070; CHECK-NEXT: global_load_dwordx4 v[24:27], v32, s[2:3] offset:9671; CHECK-NEXT: global_load_dwordx4 v[28:31], v32, s[2:3] offset:11272; CHECK-NEXT: s_waitcnt vmcnt(7)73; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]74; CHECK-NEXT: s_waitcnt vmcnt(7)75; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:1676; CHECK-NEXT: s_waitcnt vmcnt(7)77; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:3278; CHECK-NEXT: s_waitcnt vmcnt(7)79; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:4880; CHECK-NEXT: s_waitcnt vmcnt(7)81; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:6482; CHECK-NEXT: s_waitcnt vmcnt(7)83; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:8084; CHECK-NEXT: s_waitcnt vmcnt(7)85; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:9686; CHECK-NEXT: s_waitcnt vmcnt(7)87; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:11288; CHECK-NEXT: s_endpgm89entry:90 tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) %global, ptr addrspace(4) %0, i64 128, i1 false)91 ret void92}93 94define amdgpu_kernel void @memcpy_p5_p4_minsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #0 {95; CHECK-LABEL: memcpy_p5_p4_minsize:96; CHECK: ; %bb.0: ; %entry97; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]98; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]99; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8100; CHECK-NEXT: s_load_dword s2, s[8:9], 0x0101; CHECK-NEXT: v_mov_b32_e32 v24, 0102; CHECK-NEXT: s_add_u32 s20, s20, s17103; CHECK-NEXT: s_waitcnt lgkmcnt(0)104; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:112105; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:96106; CHECK-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:80107; CHECK-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:64108; CHECK-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:48109; CHECK-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:32110; CHECK-NEXT: s_addc_u32 s21, s21, 0111; CHECK-NEXT: v_mov_b32_e32 v25, s2112; CHECK-NEXT: s_waitcnt vmcnt(5)113; CHECK-NEXT: buffer_store_dword v3, v25, s[20:23], 0 offen offset:124114; CHECK-NEXT: buffer_store_dword v2, v25, s[20:23], 0 offen offset:120115; CHECK-NEXT: buffer_store_dword v1, v25, s[20:23], 0 offen offset:116116; CHECK-NEXT: buffer_store_dword v0, v25, s[20:23], 0 offen offset:112117; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:16118; CHECK-NEXT: s_waitcnt vmcnt(9)119; CHECK-NEXT: buffer_store_dword v7, v25, s[20:23], 0 offen offset:108120; CHECK-NEXT: buffer_store_dword v6, v25, s[20:23], 0 offen offset:104121; CHECK-NEXT: buffer_store_dword v5, v25, s[20:23], 0 offen offset:100122; CHECK-NEXT: buffer_store_dword v4, v25, s[20:23], 0 offen offset:96123; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1]124; CHECK-NEXT: s_waitcnt vmcnt(13)125; CHECK-NEXT: buffer_store_dword v11, v25, s[20:23], 0 offen offset:92126; CHECK-NEXT: buffer_store_dword v10, v25, s[20:23], 0 offen offset:88127; CHECK-NEXT: buffer_store_dword v9, v25, s[20:23], 0 offen offset:84128; CHECK-NEXT: buffer_store_dword v8, v25, s[20:23], 0 offen offset:80129; CHECK-NEXT: s_waitcnt vmcnt(16)130; CHECK-NEXT: buffer_store_dword v15, v25, s[20:23], 0 offen offset:76131; CHECK-NEXT: buffer_store_dword v14, v25, s[20:23], 0 offen offset:72132; CHECK-NEXT: buffer_store_dword v13, v25, s[20:23], 0 offen offset:68133; CHECK-NEXT: buffer_store_dword v12, v25, s[20:23], 0 offen offset:64134; CHECK-NEXT: s_waitcnt vmcnt(19)135; CHECK-NEXT: buffer_store_dword v19, v25, s[20:23], 0 offen offset:60136; CHECK-NEXT: buffer_store_dword v18, v25, s[20:23], 0 offen offset:56137; CHECK-NEXT: buffer_store_dword v17, v25, s[20:23], 0 offen offset:52138; CHECK-NEXT: buffer_store_dword v16, v25, s[20:23], 0 offen offset:48139; CHECK-NEXT: s_waitcnt vmcnt(22)140; CHECK-NEXT: buffer_store_dword v23, v25, s[20:23], 0 offen offset:44141; CHECK-NEXT: buffer_store_dword v22, v25, s[20:23], 0 offen offset:40142; CHECK-NEXT: buffer_store_dword v21, v25, s[20:23], 0 offen offset:36143; CHECK-NEXT: buffer_store_dword v20, v25, s[20:23], 0 offen offset:32144; CHECK-NEXT: s_waitcnt vmcnt(21)145; CHECK-NEXT: buffer_store_dword v3, v25, s[20:23], 0 offen offset:28146; CHECK-NEXT: buffer_store_dword v2, v25, s[20:23], 0 offen offset:24147; CHECK-NEXT: buffer_store_dword v1, v25, s[20:23], 0 offen offset:20148; CHECK-NEXT: buffer_store_dword v0, v25, s[20:23], 0 offen offset:16149; CHECK-NEXT: s_waitcnt vmcnt(20)150; CHECK-NEXT: buffer_store_dword v7, v25, s[20:23], 0 offen offset:12151; CHECK-NEXT: buffer_store_dword v6, v25, s[20:23], 0 offen offset:8152; CHECK-NEXT: buffer_store_dword v5, v25, s[20:23], 0 offen offset:4153; CHECK-NEXT: buffer_store_dword v4, v25, s[20:23], 0 offen154; CHECK-NEXT: s_endpgm155entry:156 tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)157 ret void158}159 160define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %src) #0 {161; CHECK-LABEL: memcpy_p0_p5_minsize:162; CHECK: ; %bb.0: ; %entry163; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]164; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]165; CHECK-NEXT: s_load_dword s0, s[8:9], 0x8166; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17167; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0168; CHECK-NEXT: s_add_u32 s20, s20, s17169; CHECK-NEXT: s_addc_u32 s21, s21, 0170; CHECK-NEXT: s_waitcnt lgkmcnt(0)171; CHECK-NEXT: v_mov_b32_e32 v26, s0172; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:124173; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:120174; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:116175; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen offset:112176; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:108177; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:104178; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:100179; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:96180; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0181; CHECK-NEXT: buffer_load_dword v11, v26, s[20:23], 0 offen offset:92182; CHECK-NEXT: buffer_load_dword v10, v26, s[20:23], 0 offen offset:88183; CHECK-NEXT: buffer_load_dword v9, v26, s[20:23], 0 offen offset:84184; CHECK-NEXT: buffer_load_dword v8, v26, s[20:23], 0 offen offset:80185; CHECK-NEXT: buffer_load_dword v15, v26, s[20:23], 0 offen offset:76186; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72187; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68188; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64189; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32190; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36191; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40192; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44193; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48194; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52195; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56196; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60197; CHECK-NEXT: s_waitcnt lgkmcnt(0)198; CHECK-NEXT: v_mov_b32_e32 v25, s1199; CHECK-NEXT: v_mov_b32_e32 v24, s0200; CHECK-NEXT: s_waitcnt vmcnt(20)201; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112202; CHECK-NEXT: s_waitcnt vmcnt(0)203; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96204; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen205; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4206; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8207; CHECK-NEXT: s_nop 0208; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16209; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20210; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24211; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28212; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12213; CHECK-NEXT: s_nop 0214; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80215; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[12:15] offset:64216; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[20:23] offset:48217; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[16:19] offset:32218; CHECK-NEXT: s_waitcnt vmcnt(0)219; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:16220; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3]221; CHECK-NEXT: s_endpgm222entry:223 tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)224 ret void225}226 227define amdgpu_kernel void @memcpy_p3_p4_minsize(ptr addrspace(4) %0) #0 {228; CHECK-LABEL: memcpy_p3_p4_minsize:229; CHECK: ; %bb.0: ; %entry230; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0231; CHECK-NEXT: v_mov_b32_e32 v24, 0232; CHECK-NEXT: s_waitcnt lgkmcnt(0)233; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1]234; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:16235; CHECK-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:32236; CHECK-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:48237; CHECK-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:64238; CHECK-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:80239; CHECK-NEXT: s_waitcnt vmcnt(5)240; CHECK-NEXT: ds_write2_b64 v24, v[0:1], v[2:3] offset1:1241; CHECK-NEXT: s_waitcnt vmcnt(4)242; CHECK-NEXT: ds_write2_b64 v24, v[4:5], v[6:7] offset0:2 offset1:3243; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:96244; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:112245; CHECK-NEXT: s_waitcnt vmcnt(5)246; CHECK-NEXT: ds_write2_b64 v24, v[8:9], v[10:11] offset0:4 offset1:5247; CHECK-NEXT: s_waitcnt vmcnt(4)248; CHECK-NEXT: ds_write2_b64 v24, v[12:13], v[14:15] offset0:6 offset1:7249; CHECK-NEXT: s_waitcnt vmcnt(3)250; CHECK-NEXT: ds_write2_b64 v24, v[16:17], v[18:19] offset0:8 offset1:9251; CHECK-NEXT: s_waitcnt vmcnt(2)252; CHECK-NEXT: ds_write2_b64 v24, v[20:21], v[22:23] offset0:10 offset1:11253; CHECK-NEXT: s_waitcnt vmcnt(1)254; CHECK-NEXT: ds_write2_b64 v24, v[0:1], v[2:3] offset0:12 offset1:13255; CHECK-NEXT: s_waitcnt vmcnt(0)256; CHECK-NEXT: ds_write2_b64 v24, v[4:5], v[6:7] offset0:14 offset1:15257; CHECK-NEXT: s_endpgm258entry:259 tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) @shared, ptr addrspace(4) %0, i64 128, i1 false)260 ret void261}262 263define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {264; CHECK-LABEL: memcpy_p0_p3_minsize:265; CHECK: ; %bb.0: ; %entry266; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0267; CHECK-NEXT: v_mov_b32_e32 v16, 0268; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset1:1269; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:2 offset1:3270; CHECK-NEXT: ds_read2_b64 v[8:11], v16 offset0:4 offset1:5271; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7272; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17273; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0274; CHECK-NEXT: s_waitcnt lgkmcnt(0)275; CHECK-NEXT: v_mov_b32_e32 v21, s1276; CHECK-NEXT: v_mov_b32_e32 v20, s0277; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3]278; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:16279; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:32280; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset0:8 offset1:9281; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:10 offset1:11282; CHECK-NEXT: ds_read_b128 v[8:11], v16 offset:96283; CHECK-NEXT: ds_read_b128 v[16:19], v16 offset:112284; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48285; CHECK-NEXT: s_waitcnt lgkmcnt(0)286; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3] offset:64287; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:80288; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:96289; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[16:19] offset:112290; CHECK-NEXT: s_endpgm291entry:292 tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)293 ret void294}295 296define amdgpu_kernel void @memcpy_p0_p0_optsize(ptr %dest, ptr %src) #1 {297; CHECK-LABEL: memcpy_p0_p0_optsize:298; CHECK: ; %bb.0: ; %entry299; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0300; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17301; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0302; CHECK-NEXT: s_waitcnt lgkmcnt(0)303; CHECK-NEXT: v_mov_b32_e32 v9, s3304; CHECK-NEXT: v_mov_b32_e32 v8, s2305; CHECK-NEXT: flat_load_dwordx2 v[10:11], v[8:9] offset:32306; CHECK-NEXT: flat_load_dwordx2 v[12:13], v[8:9] offset:39307; CHECK-NEXT: flat_load_dwordx4 v[0:3], v[8:9]308; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[8:9] offset:16309; CHECK-NEXT: v_mov_b32_e32 v9, s1310; CHECK-NEXT: v_mov_b32_e32 v8, s0311; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)312; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[10:11] offset:32313; CHECK-NEXT: flat_store_dwordx2 v[8:9], v[12:13] offset:39314; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[0:3]315; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[4:7] offset:16316; CHECK-NEXT: s_endpgm317entry:318 tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)319 ret void320}321 322define amdgpu_kernel void @memcpy_p1_p1_optsize(ptr addrspace(1) %dest, ptr addrspace(1) %src) #1 {323; CHECK-LABEL: memcpy_p1_p1_optsize:324; CHECK: ; %bb.0: ; %entry325; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0326; CHECK-NEXT: v_mov_b32_e32 v12, 0327; CHECK-NEXT: s_waitcnt lgkmcnt(0)328; CHECK-NEXT: global_load_dwordx2 v[8:9], v12, s[2:3] offset:32329; CHECK-NEXT: global_load_dwordx2 v[10:11], v12, s[2:3] offset:39330; CHECK-NEXT: global_load_dwordx4 v[0:3], v12, s[2:3]331; CHECK-NEXT: global_load_dwordx4 v[4:7], v12, s[2:3] offset:16332; CHECK-NEXT: s_waitcnt vmcnt(3)333; CHECK-NEXT: global_store_dwordx2 v12, v[8:9], s[0:1] offset:32334; CHECK-NEXT: s_waitcnt vmcnt(3)335; CHECK-NEXT: global_store_dwordx2 v12, v[10:11], s[0:1] offset:39336; CHECK-NEXT: s_waitcnt vmcnt(3)337; CHECK-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1]338; CHECK-NEXT: s_waitcnt vmcnt(3)339; CHECK-NEXT: global_store_dwordx4 v12, v[4:7], s[0:1] offset:16340; CHECK-NEXT: s_endpgm341entry:342 tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 47, i1 false)343 ret void344}345 346define amdgpu_kernel void @memcpy_p1_p4_optsize(ptr addrspace(1) %global, ptr addrspace(4) %0) #1 {347; CHECK-LABEL: memcpy_p1_p4_optsize:348; CHECK: ; %bb.0: ; %entry349; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0350; CHECK-NEXT: v_mov_b32_e32 v32, 0351; CHECK-NEXT: s_waitcnt lgkmcnt(0)352; CHECK-NEXT: global_load_dwordx4 v[0:3], v32, s[2:3]353; CHECK-NEXT: global_load_dwordx4 v[4:7], v32, s[2:3] offset:16354; CHECK-NEXT: global_load_dwordx4 v[8:11], v32, s[2:3] offset:32355; CHECK-NEXT: global_load_dwordx4 v[12:15], v32, s[2:3] offset:48356; CHECK-NEXT: global_load_dwordx4 v[16:19], v32, s[2:3] offset:64357; CHECK-NEXT: global_load_dwordx4 v[20:23], v32, s[2:3] offset:80358; CHECK-NEXT: global_load_dwordx4 v[24:27], v32, s[2:3] offset:96359; CHECK-NEXT: global_load_dwordx4 v[28:31], v32, s[2:3] offset:112360; CHECK-NEXT: s_waitcnt vmcnt(7)361; CHECK-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]362; CHECK-NEXT: s_waitcnt vmcnt(7)363; CHECK-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16364; CHECK-NEXT: s_waitcnt vmcnt(7)365; CHECK-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32366; CHECK-NEXT: s_waitcnt vmcnt(7)367; CHECK-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48368; CHECK-NEXT: s_waitcnt vmcnt(7)369; CHECK-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64370; CHECK-NEXT: s_waitcnt vmcnt(7)371; CHECK-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80372; CHECK-NEXT: s_waitcnt vmcnt(7)373; CHECK-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96374; CHECK-NEXT: s_waitcnt vmcnt(7)375; CHECK-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112376; CHECK-NEXT: s_endpgm377entry:378 tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) %global, ptr addrspace(4) %0, i64 128, i1 false)379 ret void380}381 382define amdgpu_kernel void @memcpy_p5_p4_optsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #1 {383; CHECK-LABEL: memcpy_p5_p4_optsize:384; CHECK: ; %bb.0: ; %entry385; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]386; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]387; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8388; CHECK-NEXT: s_load_dword s2, s[8:9], 0x0389; CHECK-NEXT: v_mov_b32_e32 v24, 0390; CHECK-NEXT: s_add_u32 s20, s20, s17391; CHECK-NEXT: s_waitcnt lgkmcnt(0)392; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:112393; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:96394; CHECK-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:80395; CHECK-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:64396; CHECK-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:48397; CHECK-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:32398; CHECK-NEXT: s_addc_u32 s21, s21, 0399; CHECK-NEXT: v_mov_b32_e32 v25, s2400; CHECK-NEXT: s_waitcnt vmcnt(5)401; CHECK-NEXT: buffer_store_dword v3, v25, s[20:23], 0 offen offset:124402; CHECK-NEXT: buffer_store_dword v2, v25, s[20:23], 0 offen offset:120403; CHECK-NEXT: buffer_store_dword v1, v25, s[20:23], 0 offen offset:116404; CHECK-NEXT: buffer_store_dword v0, v25, s[20:23], 0 offen offset:112405; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:16406; CHECK-NEXT: s_waitcnt vmcnt(9)407; CHECK-NEXT: buffer_store_dword v7, v25, s[20:23], 0 offen offset:108408; CHECK-NEXT: buffer_store_dword v6, v25, s[20:23], 0 offen offset:104409; CHECK-NEXT: buffer_store_dword v5, v25, s[20:23], 0 offen offset:100410; CHECK-NEXT: buffer_store_dword v4, v25, s[20:23], 0 offen offset:96411; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1]412; CHECK-NEXT: s_waitcnt vmcnt(13)413; CHECK-NEXT: buffer_store_dword v11, v25, s[20:23], 0 offen offset:92414; CHECK-NEXT: buffer_store_dword v10, v25, s[20:23], 0 offen offset:88415; CHECK-NEXT: buffer_store_dword v9, v25, s[20:23], 0 offen offset:84416; CHECK-NEXT: buffer_store_dword v8, v25, s[20:23], 0 offen offset:80417; CHECK-NEXT: s_waitcnt vmcnt(16)418; CHECK-NEXT: buffer_store_dword v15, v25, s[20:23], 0 offen offset:76419; CHECK-NEXT: buffer_store_dword v14, v25, s[20:23], 0 offen offset:72420; CHECK-NEXT: buffer_store_dword v13, v25, s[20:23], 0 offen offset:68421; CHECK-NEXT: buffer_store_dword v12, v25, s[20:23], 0 offen offset:64422; CHECK-NEXT: s_waitcnt vmcnt(19)423; CHECK-NEXT: buffer_store_dword v19, v25, s[20:23], 0 offen offset:60424; CHECK-NEXT: buffer_store_dword v18, v25, s[20:23], 0 offen offset:56425; CHECK-NEXT: buffer_store_dword v17, v25, s[20:23], 0 offen offset:52426; CHECK-NEXT: buffer_store_dword v16, v25, s[20:23], 0 offen offset:48427; CHECK-NEXT: s_waitcnt vmcnt(22)428; CHECK-NEXT: buffer_store_dword v23, v25, s[20:23], 0 offen offset:44429; CHECK-NEXT: buffer_store_dword v22, v25, s[20:23], 0 offen offset:40430; CHECK-NEXT: buffer_store_dword v21, v25, s[20:23], 0 offen offset:36431; CHECK-NEXT: buffer_store_dword v20, v25, s[20:23], 0 offen offset:32432; CHECK-NEXT: s_waitcnt vmcnt(21)433; CHECK-NEXT: buffer_store_dword v3, v25, s[20:23], 0 offen offset:28434; CHECK-NEXT: buffer_store_dword v2, v25, s[20:23], 0 offen offset:24435; CHECK-NEXT: buffer_store_dword v1, v25, s[20:23], 0 offen offset:20436; CHECK-NEXT: buffer_store_dword v0, v25, s[20:23], 0 offen offset:16437; CHECK-NEXT: s_waitcnt vmcnt(20)438; CHECK-NEXT: buffer_store_dword v7, v25, s[20:23], 0 offen offset:12439; CHECK-NEXT: buffer_store_dword v6, v25, s[20:23], 0 offen offset:8440; CHECK-NEXT: buffer_store_dword v5, v25, s[20:23], 0 offen offset:4441; CHECK-NEXT: buffer_store_dword v4, v25, s[20:23], 0 offen442; CHECK-NEXT: s_endpgm443entry:444 tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)445 ret void446}447 448define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %src) #1 {449; CHECK-LABEL: memcpy_p0_p5_optsize:450; CHECK: ; %bb.0: ; %entry451; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]452; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]453; CHECK-NEXT: s_load_dword s0, s[8:9], 0x8454; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17455; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0456; CHECK-NEXT: s_add_u32 s20, s20, s17457; CHECK-NEXT: s_addc_u32 s21, s21, 0458; CHECK-NEXT: s_waitcnt lgkmcnt(0)459; CHECK-NEXT: v_mov_b32_e32 v26, s0460; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:124461; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:120462; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:116463; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen offset:112464; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:108465; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:104466; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:100467; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:96468; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0469; CHECK-NEXT: buffer_load_dword v11, v26, s[20:23], 0 offen offset:92470; CHECK-NEXT: buffer_load_dword v10, v26, s[20:23], 0 offen offset:88471; CHECK-NEXT: buffer_load_dword v9, v26, s[20:23], 0 offen offset:84472; CHECK-NEXT: buffer_load_dword v8, v26, s[20:23], 0 offen offset:80473; CHECK-NEXT: buffer_load_dword v15, v26, s[20:23], 0 offen offset:76474; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72475; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68476; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64477; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32478; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36479; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40480; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44481; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48482; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52483; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56484; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60485; CHECK-NEXT: s_waitcnt lgkmcnt(0)486; CHECK-NEXT: v_mov_b32_e32 v25, s1487; CHECK-NEXT: v_mov_b32_e32 v24, s0488; CHECK-NEXT: s_waitcnt vmcnt(20)489; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112490; CHECK-NEXT: s_waitcnt vmcnt(0)491; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96492; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen493; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4494; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8495; CHECK-NEXT: s_nop 0496; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16497; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20498; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24499; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28500; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12501; CHECK-NEXT: s_nop 0502; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80503; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[12:15] offset:64504; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[20:23] offset:48505; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[16:19] offset:32506; CHECK-NEXT: s_waitcnt vmcnt(0)507; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:16508; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3]509; CHECK-NEXT: s_endpgm510entry:511 tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)512 ret void513}514 515define amdgpu_kernel void @memcpy_p3_p4_optsize(ptr addrspace(4) %0) #1 {516; CHECK-LABEL: memcpy_p3_p4_optsize:517; CHECK: ; %bb.0: ; %entry518; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0519; CHECK-NEXT: v_mov_b32_e32 v24, 0520; CHECK-NEXT: s_waitcnt lgkmcnt(0)521; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1]522; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:16523; CHECK-NEXT: global_load_dwordx4 v[8:11], v24, s[0:1] offset:32524; CHECK-NEXT: global_load_dwordx4 v[12:15], v24, s[0:1] offset:48525; CHECK-NEXT: global_load_dwordx4 v[16:19], v24, s[0:1] offset:64526; CHECK-NEXT: global_load_dwordx4 v[20:23], v24, s[0:1] offset:80527; CHECK-NEXT: s_waitcnt vmcnt(5)528; CHECK-NEXT: ds_write2_b64 v24, v[0:1], v[2:3] offset1:1529; CHECK-NEXT: s_waitcnt vmcnt(4)530; CHECK-NEXT: ds_write2_b64 v24, v[4:5], v[6:7] offset0:2 offset1:3531; CHECK-NEXT: global_load_dwordx4 v[0:3], v24, s[0:1] offset:96532; CHECK-NEXT: global_load_dwordx4 v[4:7], v24, s[0:1] offset:112533; CHECK-NEXT: s_waitcnt vmcnt(5)534; CHECK-NEXT: ds_write2_b64 v24, v[8:9], v[10:11] offset0:4 offset1:5535; CHECK-NEXT: s_waitcnt vmcnt(4)536; CHECK-NEXT: ds_write2_b64 v24, v[12:13], v[14:15] offset0:6 offset1:7537; CHECK-NEXT: s_waitcnt vmcnt(3)538; CHECK-NEXT: ds_write2_b64 v24, v[16:17], v[18:19] offset0:8 offset1:9539; CHECK-NEXT: s_waitcnt vmcnt(2)540; CHECK-NEXT: ds_write2_b64 v24, v[20:21], v[22:23] offset0:10 offset1:11541; CHECK-NEXT: s_waitcnt vmcnt(1)542; CHECK-NEXT: ds_write2_b64 v24, v[0:1], v[2:3] offset0:12 offset1:13543; CHECK-NEXT: s_waitcnt vmcnt(0)544; CHECK-NEXT: ds_write2_b64 v24, v[4:5], v[6:7] offset0:14 offset1:15545; CHECK-NEXT: s_endpgm546entry:547 tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) @shared, ptr addrspace(4) %0, i64 128, i1 false)548 ret void549}550 551define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {552; CHECK-LABEL: memcpy_p0_p3_optsize:553; CHECK: ; %bb.0: ; %entry554; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0555; CHECK-NEXT: v_mov_b32_e32 v16, 0556; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset1:1557; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:2 offset1:3558; CHECK-NEXT: ds_read2_b64 v[8:11], v16 offset0:4 offset1:5559; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7560; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17561; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0562; CHECK-NEXT: s_waitcnt lgkmcnt(0)563; CHECK-NEXT: v_mov_b32_e32 v21, s1564; CHECK-NEXT: v_mov_b32_e32 v20, s0565; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3]566; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:16567; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:32568; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset0:8 offset1:9569; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:10 offset1:11570; CHECK-NEXT: ds_read_b128 v[8:11], v16 offset:96571; CHECK-NEXT: ds_read_b128 v[16:19], v16 offset:112572; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48573; CHECK-NEXT: s_waitcnt lgkmcnt(0)574; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3] offset:64575; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:80576; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:96577; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[16:19] offset:112578; CHECK-NEXT: s_endpgm579entry:580 tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)581 ret void582}583 584declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2585 586declare void @llvm.memcpy.p0.p5.i64(ptr noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #2587 588declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #2589 590declare void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2591 592declare void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2593 594declare void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2595 596declare void @llvm.memcpy.p0.p3.i64(ptr noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #2597 598attributes #0 = { minsize "amdgpu-flat-work-group-size"="1024,1024" }599attributes #1 = { optsize "amdgpu-flat-work-group-size"="1024,1024" }600attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }601